checkpoint

Cargo.toml

@@ -11,8 +11,8 @@ metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]}
 # minio = "0.1.0"
 minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"}
 reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] }
+rusqlite = { version = "0.34.0", features = ["bundled"] }
 serde = { version = "1.0", features = ["derive"] }
-surrealdb = "2.2"
 tokio = { version="1.41.0", features = ["full"] }
 toml = "0.8.20"
 tracing = "0.1"

Crawler.toml

@@ -3,7 +3,7 @@ surreal_url = "localhost:8000"
 surreal_username = "root"
 surreal_password = "root"
 surreal_ns = "test"
-surreal_db = "v1.18.1"
+surreal_db = "v1.19.2"
 
 # Crawler config
 crawl_filter = "en.wikipedia.com" 

README.md

@@ -17,6 +17,8 @@ This ment we stored 1000 pages, 142,997 urls, and 1,425,798 links between the tw
 
 3/20/25: Took 5min to crawl 1000 pages
 
+3/21/25: Took 3min to crawl 1000 pages
+
 # About
 
 ![Screenshot](/pngs/graphana.png)

docker/compose.yml

@@ -14,22 +14,6 @@ services:
       - --pass
       - root
       - rocksdb:/mydata/database.db
-  minio:
-    image: quay.io/minio/minio
-    ports:
-      - 9000:9000
-      - 9001:9001
-    environment:
-      - MINIO_ROOT_USER=root
-      - MINIO_ROOT_PASSWORD=an8charpassword
-      - MINIO_PROMETHEUS_AUTH_TYPE=public
-    volumes:
-      - minio_storage:/data
-    command:
-      - server
-      - /data
-      - --console-address
-      - ":9001"
 
   alloy:
     image: grafana/alloy:latest

docker/prometheus.yaml

@@ -7,15 +7,11 @@ scrape_configs:
     static_configs:
     # change this your machine's ip, localhost won't work
     # because localhost refers to the docker container.
-    #      - targets: ['172.20.239.48:2500']
-      - targets: ['192.168.8.209:2500']
+      - targets: ['172.20.239.48:2500']
+        #- targets: ['192.168.8.209:2500']
   - job_name: loki
     static_configs:
       - targets: ['loki:3100']
   - job_name: prometheus
     static_configs:
       - targets: ['localhost:9090']
-  - job_name: minio
-    metrics_path: /minio/v2/metrics/cluster
-    static_configs:
-      - targets: ['minio:9000']

src/db.rs

@@ -1,21 +1,12 @@
 use metrics::counter;
+use rusqlite::Connection;
+use std::fmt::Debug;
 use serde::{Deserialize, Serialize};
-use std::{fmt::Debug, sync::LazyLock, time::Instant};
-use surrealdb::{
-    engine::remote::ws::{Client, Ws},
-    opt::auth::Root,
-    sql::Thing,
-    Surreal,
-};
-use tokio::sync::Mutex;
 use tracing::{error, instrument, trace};
 use url::Url;
 
 use crate::Config;
 
-static LOCK: LazyLock<Mutex<bool>> = LazyLock::new(|| Mutex::new(true));
-
-const TIME_SPENT_ON_LOCK: &str = "surql_lock_waiting_ms";
 const STORE: &str = "surql_store_calls";
 
 #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
@@ -53,17 +44,15 @@ impl Website {
 
     // Insert ever item in the vec into surreal, crawled state will be preserved as TRUE
     // if already in the database as such or incoming data is TRUE.
-    pub async fn store_all(all: Vec<Self>, db: &Surreal<Client>) -> Vec<Thing> {
+    pub async fn store_all(all: Vec<Self>, db: &Connection) {
         counter!(STORE).increment(1);
         let mut things = Vec::with_capacity(all.len());
 
-        // TODO this only allows for one thread to be in the database at a time.
-        // This is currently required since otherwise we get write errors.
-        // If the default `crawled` is set to false, we might not need to write any more
-        // than just the name. `accessed_at` is fun but not needed.
-        let now = Instant::now();
-        let lock = LOCK.lock().await;
-        counter!(TIME_SPENT_ON_LOCK).increment(now.elapsed().as_millis() as u64);
+        rusqlite::ParamsFromIter;
+
+        db.execute("",
+            params![]
+        );
 
         match db
             .query(
@@ -85,7 +74,6 @@ impl Website {
                 error!("{:?}", err);
             }
         }
-        drop(lock);
         things
     }
 }
@@ -102,32 +90,10 @@ pub struct Record {
     pub id: Thing,
 }
 
-#[instrument(skip_all, name = "SurrealDB")]
-pub async fn connect(config: &Config) -> surrealdb::Result<Surreal<Client>> {
-    trace!("Establishing connection to surreal...");
+#[instrument(skip_all, name = "sqlite_connect")]
+pub async fn connect(config: &Config) -> Result<Connection, rusqlite::Error> {
+    trace!("Establishing connection to sqlite...");
     // Connect to the server
-    let db = Surreal::new::<Ws>(&config.surreal_url).await?;
-
-    trace!("Logging in...");
-    // Signin as a namespace, database, or root user
-    db.signin(Root {
-        username: &config.surreal_username,
-        password: &config.surreal_password,
-    })
-    .await?;
-
-    // Select a specific namespace / database
-    db.use_ns(&config.surreal_ns)
-        .use_db(&config.surreal_db)
-        .await?;
-
-    let setup = include_bytes!("setup.surql");
-    let file = setup.iter().map(|c| *c as char).collect::<String>();
-
-    db.query(file)
-        .await
-        .expect("Failed to setup surreal tables.");
-
-    Ok(db)
+    Connection::open("./squeelite.db")
 }
 

src/filesystem.rs

@@ -1,24 +1,70 @@
-use std::path::PathBuf;
+use std::{ffi::OsStr, path::PathBuf};
 
 use tokio::fs;
-use tracing::{error, instrument, trace};
+use tracing::{debug, error, instrument, trace, warn};
 use url::Url;
 
 #[instrument(skip(data))]
 pub async fn store(data: &str, url: &Url) {
-    let path = PathBuf::from("./downloaded".to_string() + url.path());
-    let basepath = path.ancestors().skip(1).take(1).collect::<PathBuf>();
-    trace!("Save path: {:?} and base path: {:?}", &path, &basepath);
-    if let Err(err) = fs::create_dir_all(&basepath).await {
-        let ex = path.ancestors().fold(String::new(), |mut s, item| {
-            s += ", ";
-            s += &item.to_string_lossy().to_string();
-            s
-        });
-        error!("Dir creation: {err} {:?} {ex}", basepath);
+    // extract data from url to save it accurately
+    let url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
+
+    // if it's a file
+    let (basepath, filename) = if url_path.extension().filter(valid_file_extension).is_some() {
+        // get everything up till the file
+        let basepath = url_path.ancestors().skip(1).take(1).collect::<PathBuf>();
+        // get the file name
+        let filename = url_path.file_name().expect("This should exist").to_string_lossy();
+        trace!("Save path: {:?} and base path: {:?}", &url_path, &basepath);
+        (basepath, filename.to_string())
     } else {
-        if let Err(err) = fs::write(&path, data).await {
-            error!("File creation: {err} {:?}", path);
+        (url_path.clone(), "index.html".into())
+    };
+
+    debug!("Writing at: {:?} {:?}", basepath, filename);
+
+    // create the folders
+    if let Err(err) = fs::create_dir_all(&basepath).await {
+        error!("Dir creation: {err} {:?}", basepath);
+    } else {
+        // FIXME I don't think this handles index.html files well...
+        // TODO this should probably append .html to non-described files
+        // create the file if that was successful
+        if let Err(err) = fs::write(&basepath.join(filename), data).await {
+            error!("File creation: {err} {:?}", url_path);
         }
     }
 }
+
+fn valid_file_extension(take: &&OsStr) -> bool {
+    let los = take.to_string_lossy();
+    let all = los.split('.');
+    match all.last() {
+        Some(s) => {
+            match s.to_lowercase().as_str() {
+                "html" => true,
+                "css" => true,
+                "js" => true,
+                "ts" => true,
+                "otf" => true, // font
+
+                "png" => true,
+                "svg" => true,
+                "jpg" => true,
+                "jpeg" => true,
+                "mp4" => true,
+                "mp3" => true,
+                "webp" => true,
+
+                "pdf" => true,
+                "json" => true,
+                "xml" => true,
+                _ => {
+                    warn!("Might be forgetting a file extension: {s}");
+                    false
+                }
+            }
+        },
+        None => false,
+    }
+}

src/main.rs

@@ -167,6 +167,7 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
             .text()
             .await
             .expect("Failed to read http response's body!");
+
         // Store document
         filesystem::store(&data, &site.site).await;
 

src/parser.rs

@@ -19,8 +19,7 @@ impl TokenSink for Website {
             TagToken(tag) => {
                 if tag.kind == StartTag {
                     match tag.name {
-                        // this should be all the html
-                        // elements that have links
+                        // this should be all the html elements that have links
                         local_name!("a")
                         | local_name!("audio")
                         | local_name!("area")
@@ -35,37 +34,9 @@ impl TokenSink for Website {
                                 let attr_name = attr.name.local.to_string();
                                 if attr_name == "src" || attr_name == "href" || attr_name == "data"
                                 {
-                                    let url: Option<Url> = match Url::parse(&attr.value) {
-                                        Ok(ok) => {
-                                            trace!("Found `{}` in the html on `{}` tag", ok.to_string(), tag.name);
-                                            Some(ok)
-                                        },
-                                        Err(e) => {
-                                            if attr.value.starts_with('#') {
-                                                trace!("Rejecting # url");
-                                                None
-                                            } else {
-                                                match e {
-                                                    url::ParseError::RelativeUrlWithoutBase => {
-                                                        let origin = self.site.origin().ascii_serialization();
-                                                        let url = origin.clone() + &attr.value;
-                                                        trace!("Built `{url}` from `{origin} + {}`", &attr.value.to_string());
-                                                        if let Ok(url) = Url::parse(&url) {
-                                                            trace!("Saved relative url `{}` AS: `{}`", &attr.value, url);
-                                                            Some(url)
-                                                        } else {
-                                                            error!("Failed to reconstruct a url from relative url: `{}` on site: `{}`", &attr.value, self.site.to_string());
-                                                            None
-                                                        }
-                                                    }, 
-                                                    _ => {
-                                                        error!("MISC error: {:?} {:?}", e, &attr.value);
-                                                        None
-                                                    },
-                                                }
-                                            }
-                                        },
-                                    };
+                                    trace!("Found `{}` in html `{}` tag", &attr.value, tag.name);
+                                    let url = try_get_url(&self.site, &attr.value);
+
                                     if let Some(mut parsed) = url {
                                         parsed.set_query(None);
                                         parsed.set_fragment(None);
@@ -119,3 +90,56 @@ pub async fn parse(site: &Website, data: &str) -> Vec<Website> {
     other_sites
 }
 
+#[instrument]
+fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
+    match Url::parse(link) {
+        Ok(ok) => Some(ok),
+        Err(e) => {
+            if link.starts_with('#') {
+                trace!("Rejecting # url");
+                None
+            } else if link.starts_with("//") {
+                // if a url starts with "//" is assumed that it will adopt
+                // the same scheme as it's parent
+                // https://stackoverflow.com/questions/9646407/two-forward-slashes-in-a-url-src-href-attribute
+                let scheme = parent.scheme();
+
+                match Url::parse(&format!("{scheme}://{}", link)) {
+                    Ok(url) => Some(url),
+                    Err(err) => {
+                        error!("Failed parsing realative scheme url: {}", err);
+                        None
+                    }
+                }
+            } else {
+                // # This is some sort of realative url, gonna try patching it up into an absolute
+                // url
+                match e {
+                    url::ParseError::RelativeUrlWithoutBase => {
+                        // Is: scheme://host:port
+                        let origin = parent.origin().ascii_serialization();
+                        let url = origin.clone() + link;
+
+                        trace!("Built `{url}` from `{origin} + {}`", link.to_string());
+
+                        if let Ok(url) = Url::parse(&url) {
+                            trace!("Saved relative url `{}` AS: `{}`", link, url);
+                            Some(url)
+                        } else {
+                            error!(
+                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`",
+                                link,
+                                parent.to_string()
+                            );
+                            None
+                        }
+                    }
+                    _ => {
+                        error!("MISC error: {:?} {:?}", e, link);
+                        None
+                    }
+                }
+            }
+        }
+    }
+}