checkpoint

.vscode/launch.json

@@ -7,15 +7,18 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug executable 'internet_mapper'",
+            "name": "Debug executable 'surreal_spider'",
+            "env": {
+                "RUST_LOG": "surreal_spider=trace,reqwest=info",
+            },
             "cargo": {
                 "args": [
                     "build",
-                    "--bin=internet_mapper",
-                    "--package=internet_mapper"
+                    "--bin=surreal_spider",
+                    "--package=surreal_spider"
                 ],
                 "filter": {
-                    "name": "internet_mapper",
+                    "name": "surreal_spider",
                     "kind": "bin"
                 }
             },
@@ -25,16 +28,16 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug unit tests in executable 'internet_mapper'",
+            "name": "Debug unit tests in executable 'surreal_spider'",
             "cargo": {
                 "args": [
                     "test",
                     "--no-run",
-                    "--bin=internet_mapper",
-                    "--package=internet_mapper"
+                    "--bin=surreal_spider",
+                    "--package=surreal_spider"
                 ],
                 "filter": {
-                    "name": "internet_mapper",
+                    "name": "surreal_spider",
                     "kind": "bin"
                 }
             },

.vscode/settings.json

@@ -3,6 +3,6 @@
         "creds",
         "reqwest",
         "rustls",
-        "surql",
+        "surql"
     ]
 }

Cargo.toml

@@ -5,15 +5,14 @@ edition = "2021"
 
 [dependencies]
 base64 = "0.22.1"
-futures-util = "0.3.31"
 html5ever = "0.29"
 metrics = "0.24.1"
 metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]}
 # minio = "0.1.0"
 minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"}
-reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls", "stream"] }
+reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] }
+rusqlite = { version = "0.34.0", features = ["bundled"] }
 serde = { version = "1.0", features = ["derive"] }
-surrealdb = "2.2"
 tokio = { version="1.41.0", features = ["full"] }
 toml = "0.8.20"
 tracing = "0.1"

Crawler.toml

@@ -3,10 +3,8 @@ surreal_url = "localhost:8000"
 surreal_username = "root"
 surreal_password = "root"
 surreal_ns = "test"
-surreal_db = "v1.21.1"
+surreal_db = "v1.19.2"
 
 # Crawler config
-crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI" 
-start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI"
-budget = 10000
-batch_size = 50
+crawl_filter = "en.wikipedia.com" 
+budget = 1000

README.md

@@ -2,43 +2,13 @@
 
 Crawls sites saving all the found links to a surrealdb database. It then proceeds to take batches of 100 uncrawled links untill the crawl budget is reached. It saves the data of each site in a minio database.
 
-## How to use
-
-1. Clone the repo and `cd` into it.
-2. Build the repo with `cargo build -r`
-3. Start the docker conatiners
-	1. cd into the docker folder `cd docker`
-	2. Bring up the docker containers `docker compose up -d`
-4. From the project's root, edit the `Crawler.toml` file to your liking.
-5. Run with `./target/release/internet_mapper`
-
-You can view stats of the project at `http://<your-ip>:3000/dashboards`
-
-```bash
-# Untested script but probably works
-git clone https://git.oliveratkinson.net/Oliver/internet_mapper.git
-cd internet_mapper
-
-cargo build -r
-
-cd docker
-docker compose up -d
-cd ..
-
-$EDITOR Crawler.toml
-
-./target/release/internet_mapper
-
-```
-
 ### TODO
 
-- [x] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
+- [ ] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
 - [ ] Conditionally save content - based on filename or file contents
 - [x] GUI / TUI ? - Graphana
 - [x] Better asynchronous getting of the sites. Currently it all happens serially.
-- [x] Allow for storing asynchronously - dropping the "links to" logic fixes this need
-- [x] Control crawler via config file (no recompliation needed)
+- [ ] Allow for storing asynchronously
 
 3/17/25: Took >1hr to crawl 100 pages
 

docker/compose.yml

@@ -66,3 +66,4 @@ volumes:
   grafana_storage:
   alloy_storage:
   surrealdb_storage:
+  minio_storage:

docker/prometheus.yaml

@@ -7,7 +7,7 @@ scrape_configs:
     static_configs:
     # change this your machine's ip, localhost won't work
     # because localhost refers to the docker container.
-      - targets: ['192.168.1.200:2500']
+      - targets: ['172.20.239.48:2500']
         #- targets: ['192.168.8.209:2500']
   - job_name: loki
     static_configs:

src/db.rs

@@ -1,12 +1,7 @@
 use metrics::counter;
+use rusqlite::Connection;
 use std::fmt::Debug;
 use serde::{Deserialize, Serialize};
-use surrealdb::{
-    engine::remote::ws::{Client, Ws},
-    opt::auth::Root,
-    sql::Thing,
-    Surreal,
-};
 use tracing::{error, instrument, trace};
 use url::Url;
 
@@ -20,18 +15,12 @@ pub struct Website {
     pub site: Url,
     /// Wether or not this link has been crawled yet
     pub crawled: bool,
-    /// 200, 404, etc
-    pub status_code: u16,
 }
 
 // manual impl to make tracing look nicer
 impl Debug for Website {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Website")
-            .field("host", &self.site.host())
-            .field("path", &self.site.path())
-            .field("status_code", &self.status_code)
-            .finish()
+        f.debug_struct("Website").field("site", &self.site).finish()
     }
 }
 
@@ -44,25 +33,32 @@ impl Website {
         };
         Self {
             crawled,
-            site,
-            status_code: 0,
+            site
         }
     }
 
+    pub fn set_crawled(&mut self) {
+        trace!("Set crawled to true");
+        self.crawled = true
+    }
+
     // Insert ever item in the vec into surreal, crawled state will be preserved as TRUE
     // if already in the database as such or incoming data is TRUE.
-    #[instrument(skip(db))]
-    pub async fn store_all(all: Vec<Self>, db: &Surreal<Client>) -> Vec<Thing> {
+    pub async fn store_all(all: Vec<Self>, db: &Connection) {
         counter!(STORE).increment(1);
         let mut things = Vec::with_capacity(all.len());
 
-        // FIXME failes *sometimes* because "Resource Busy"
+        rusqlite::ParamsFromIter;
+
+        db.execute("",
+            params![]
+        );
+
         match db
             .query(
                 "INSERT INTO website $array
                     ON DUPLICATE KEY UPDATE
                         accessed_at = time::now(),
-                        status_code = $input.status_code,
                         crawled = crawled OR $input.crawled
                     RETURN VALUE id;
                  ",
@@ -94,32 +90,10 @@ pub struct Record {
     pub id: Thing,
 }
 
-#[instrument(skip_all, name = "SurrealDB")]
-pub async fn connect(config: &Config) -> surrealdb::Result<Surreal<Client>> {
-    trace!("Establishing connection to surreal...");
+#[instrument(skip_all, name = "sqlite_connect")]
+pub async fn connect(config: &Config) -> Result<Connection, rusqlite::Error> {
+    trace!("Establishing connection to sqlite...");
     // Connect to the server
-    let db = Surreal::new::<Ws>(&config.surreal_url).await?;
-
-    trace!("Logging in...");
-    // Signin as a namespace, database, or root user
-    db.signin(Root {
-        username: &config.surreal_username,
-        password: &config.surreal_password,
-    })
-    .await?;
-
-    // Select a specific namespace / database
-    db.use_ns(&config.surreal_ns)
-        .use_db(&config.surreal_db)
-        .await?;
-
-    let setup = include_bytes!("setup.surql");
-    let file = setup.iter().map(|c| *c as char).collect::<String>();
-
-    db.query(file)
-        .await
-        .expect("Failed to setup surreal tables.");
-
-    Ok(db)
+    Connection::open("./squeelite.db")
 }
 

src/filesystem.rs

@@ -1,66 +1,70 @@
-use std::{io::ErrorKind, path::PathBuf};
+use std::{ffi::OsStr, path::PathBuf};
 
-use reqwest::header::HeaderValue;
 use tokio::fs;
-use tracing::{error, trace, warn};
+use tracing::{debug, error, instrument, trace, warn};
 use url::Url;
 
-pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
+#[instrument(skip(data))]
+pub async fn store(data: &str, url: &Url) {
     // extract data from url to save it accurately
-    let mut url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
+    let url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
 
-    if let Ok(header) = content_type.to_str() {
-        // text/html; charset=UTF-8; option=value
-        let ttype = if let Some((t, _)) = header.split_once(';') {
-            t
+    // if it's a file
+    let (basepath, filename) = if url_path.extension().filter(valid_file_extension).is_some() {
+        // get everything up till the file
+        let basepath = url_path.ancestors().skip(1).take(1).collect::<PathBuf>();
+        // get the file name
+        let filename = url_path.file_name().expect("This should exist").to_string_lossy();
+        trace!("Save path: {:?} and base path: {:?}", &url_path, &basepath);
+        (basepath, filename.to_string())
     } else {
-            header
+        (url_path.clone(), "index.html".into())
     };
 
-        if let Some((ttype, subtype)) = ttype.split_once('/') {
-            trace!("Found Content-Type to be: {ttype}/{subtype} for {}", url.to_string());
-            // If the Content-Type header is "*/html" (most likely "text/html") and the path's
-            // extension is anything but html:
-            if subtype=="html" && !url_path.extension().is_some_and(|f| f=="html" || f=="htm" ) {
-                // time to slap a index.html to the end of that path there!
-                url_path = url_path.join("index.html");
-            }
-        }
-    } else {
-        warn!("Header: {:?} couldn't be parsed into a string!", content_type);
-    }
-    trace!("Final path for {} is: {:?}", url, url_path);
+    debug!("Writing at: {:?} {:?}", basepath, filename);
 
-    url_path
-}
-
-pub async fn init(filename: &PathBuf) -> Option<fs::File> {
-    let file = async || tokio::fs::OpenOptions::new()
-        .append(true)
-        .create(true)
-        .open(&filename).await;
-
-    match file().await {
-        Ok(ok) => Some(ok),
-        Err(err) => {
-            // the file/folder isn't found
-            if err.kind() == ErrorKind::NotFound {
-                if let Some(parent ) = &filename.parent() {
     // create the folders
-                    if let Err(err) = fs::create_dir_all(&parent).await {
-                        error!("Dir creation: {err} {:?}", filename);
-                        eprintln!("{}", err)
-                    } else if let Ok(ok) = file().await {
-                        return Some(ok);
-                    }
+    if let Err(err) = fs::create_dir_all(&basepath).await {
+        error!("Dir creation: {err} {:?}", basepath);
     } else {
-                    error!("Couldn't get file's parents: {:?}", &filename);
-                }
-            } else {
-                error!("File open error: {err} {:?}", filename);
-            }
-            // we don't care about other errors, we can't/shouldn't fix them
-            None
+        // FIXME I don't think this handles index.html files well...
+        // TODO this should probably append .html to non-described files
+        // create the file if that was successful
+        if let Err(err) = fs::write(&basepath.join(filename), data).await {
+            error!("File creation: {err} {:?}", url_path);
         }
     }
 }
+
+fn valid_file_extension(take: &&OsStr) -> bool {
+    let los = take.to_string_lossy();
+    let all = los.split('.');
+    match all.last() {
+        Some(s) => {
+            match s.to_lowercase().as_str() {
+                "html" => true,
+                "css" => true,
+                "js" => true,
+                "ts" => true,
+                "otf" => true, // font
+
+                "png" => true,
+                "svg" => true,
+                "jpg" => true,
+                "jpeg" => true,
+                "mp4" => true,
+                "mp3" => true,
+                "webp" => true,
+
+                "pdf" => true,
+                "json" => true,
+                "xml" => true,
+                _ => {
+                    warn!("Might be forgetting a file extension: {s}");
+                    false
+                }
+            }
+        },
+        None => false,
+    }
+}

src/main.rs

@@ -1,16 +1,9 @@
 #![feature(ip_from)]
-#![warn(clippy::expect_used)]
-#![deny(clippy::unwrap_used)]
 
 extern crate html5ever;
 
-use futures_util::StreamExt;
-
 use std::{
-    collections::HashSet,
-    fs::File,
-    io::Read,
-    net::{IpAddr, Ipv4Addr},
+    collections::HashSet, fs::File, io::Read, net::{IpAddr, Ipv4Addr}
 };
 
 use db::{connect, Website};
@@ -18,13 +11,13 @@ use metrics::{counter, gauge};
 use metrics_exporter_prometheus::PrometheusBuilder;
 use serde::Deserialize;
 use surrealdb::{engine::remote::ws::Client, Surreal};
-use tokio::{io::{AsyncWriteExt, BufWriter}, task::JoinSet};
-use tracing::{debug, debug_span, error, info, instrument, level_filters::LevelFilter, trace, trace_span, warn};
+use tokio::task::JoinSet;
+use tracing::{debug, error, info, instrument, level_filters::LevelFilter, trace, trace_span};
 use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry};
 
 mod db;
-mod filesystem;
 mod parser;
+mod filesystem;
 
 const GET_METRIC: &str = "total_gets";
 const GET_IN_FLIGHT: &str = "gets_in_flight";
@@ -40,15 +33,11 @@ struct Config {
     surreal_password: String,
 
     crawl_filter: String,
-    start_url: String,
     budget: usize,
-    batch_size: usize,
 }
 
 #[tokio::main]
 async fn main() {
-    println!("Logs and metrics are provided to the Grafana dashboard");
-
     let writer = std::fs::OpenOptions::new()
         .append(true)
         .create(true)
@@ -81,7 +70,8 @@ async fn main() {
         .expect("failed to install recorder/exporter");
 
     info!("Starting...");
-
+    // Would probably take these in as parameters from a cli
+    let starting_url = "https://en.wikipedia.org/";
     // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
     // let crawl_filter = "en.wikipedia.org/";
     // let budget = 50;
@@ -92,7 +82,6 @@ async fn main() {
     let _ = file.read_to_string(&mut buf);
 
     let config: Config = toml::from_str(&buf).expect("Failed to parse Crawler.toml");
-    let starting_url = &config.start_url;
 
     let db = connect(&config)
         .await
@@ -117,8 +106,13 @@ async fn main() {
     let span = trace_span!("Loop");
     let span = span.enter();
     while crawled < config.budget {
-        let uncrawled =
-            get_uncrawled_links(&db, config.budget - crawled, config.crawl_filter.clone(), &config).await;
+        let get_num = if config.budget - crawled < 100 {
+            config.budget - crawled
+        } else {
+            100
+        };
+
+        let uncrawled = get_uncrawled_links(&db, get_num, config.crawl_filter.clone()).await;
         if uncrawled.is_empty() {
             info!("Had more budget but finished crawling everything.");
             return;
@@ -144,16 +138,6 @@ async fn main() {
     }
     drop(span);
 
-    if let Ok(mut ok) = db
-        .query("count(select id from website where crawled = true)")
-        .await
-    {
-        let res = ok.take::<Option<usize>>(0);
-        if let Ok(Some(n)) = res {
-            info!("Total crawled pages now equals {n}");
-        }
-    }
-
     info!("Done");
 }
 
@@ -161,6 +145,7 @@ async fn main() {
 /// Downloads and crawls and stores a webpage.
 /// It is acceptable to clone `db`, `reqwest`, and `s3` because they all use `Arc`s internally. - Noted by Oliver
 async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client) {
+    
     // METRICS
     trace!("Process: {}", &site.site);
     // Build the request
@@ -172,64 +157,27 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
 
     // Send the http request (get)
     if let Ok(response) = request_builder.send().await {
-        let headers = response.headers();
-        let code = response.status();
 
-        #[allow(non_snake_case)]
-        let CT = headers.get("Content-Type");
-        let ct = headers.get("content-type");
+        // METRICS
+        g.decrement(1);
+        counter!(GET_METRIC).increment(1);
 
-        let ct = match (CT,ct) {
-            (None, None) => {
-                warn!("Server did not respond with Content-Type header. Url: {} Headers: ({:?})", site.site.to_string(), headers);
-                return
-            },
-            (None, Some(a)) => a,
-            (Some(a), None) => a,
-            (Some(a), Some(_)) => a,
-        };
-
-        // create filepath (handles / -> /index.html)
-        let path = filesystem::as_path(&site.site, ct);
-
-        // make sure that the file is good to go
-        if let Some(file) = filesystem::init(&path).await {
         // Get body from response
-            // stream the response onto the disk
-            let mut stream = response.bytes_stream();
+        let data = response
+            .text()
+            .await
+            .expect("Failed to read http response's body!");
 
-            let should_parse = path.to_string_lossy().ends_with(".html");
-            let mut writer = BufWriter::new(file);
-            let mut buf: Vec<u8> = Vec::new();
-
-            // Write file to disk
-            info!("Writing at: {:?}", path);
-            while let Some(data) = stream.next().await {
-                match data {
-                    Ok(data) => {
-                        let _ = writer.write_all(&data).await;
-                        // If we are going to parse this file later, we will save it
-                        // into memory as well as the disk.
-                        // We do this because the data here might be incomplete
-                        if should_parse {
-                            data.iter().for_each(|f| buf.push(*f));
-                        }
-                    },
-                    Err(err) => {
-                        eprintln!("{}", err)
-                    },
-                }
-            }
-            let _ = writer.flush();
-
-
-            // (If needed) Parse the file
-            if should_parse {
-                let span = debug_span!("Should Parse");
-                let enter = span.enter();
+        // Store document
+        filesystem::store(&data, &site.site).await;
 
         // Parse document and get relationships
-                let sites = parser::parse(&site, &buf).await;
+        let sites = parser::parse(&site, &data).await;
+        
+        // update self in db
+        site.set_crawled();
+        Website::store_all(vec![site], &db).await;
+
         // De-duplicate this list
         let prev_len = sites.len();
         let set = sites.into_iter().fold(HashSet::new(), |mut set,item| {
@@ -239,43 +187,26 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
         let de_dupe_sites: Vec<Website> = set.into_iter().collect();
         let diff = prev_len - de_dupe_sites.len();
         trace!("Saved {diff} from being entered into the db by de-duping");
+
         // Store all the other sites so that we can link to them.
         let _ = Website::store_all(de_dupe_sites, &db).await;
 
-                drop(enter);
-            }
-
-            // METRICS
-            g.decrement(1);
-            counter!(GET_METRIC).increment(1);
-
-            // update self in db
-            site.crawled = true;
-            site.status_code = code.as_u16();
-            Website::store_all(vec![site.clone()], &db).await;
-        } else {
-            error!("File failed to cooperate: {:?}", path);
-        }
-
-        trace!("Done processing: {}", &site.site);
     } else {
         error!("Failed to get: {}", &site.site);
     }
 }
 
 /// Returns uncrawled links
-#[instrument(skip(db, config))]
+#[instrument(skip(db))]
 async fn get_uncrawled_links(
     db: &Surreal<Client>,
     mut count: usize,
     filter: String,
-    config: &Config,
 ) -> Vec<Website> {
-    if count > config.batch_size {
-        count = config.batch_size;
+    if count > 100 {
+        count = 100
     }
-
-    debug!("Getting {} uncrawled links", count);
+    debug!("Getting uncrawled links");
 
     let mut response = db
         .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;")
@@ -287,3 +218,4 @@ async fn get_uncrawled_links(
         .take(0)
         .expect("Returned websites couldn't be parsed")
 }
+

src/parser.rs

@@ -1,4 +1,5 @@
 use std::default::Default;
+use std::str::FromStr;
 
 use html5ever::tokenizer::{BufferQueue, TokenizerResult};
 use html5ever::tokenizer::{StartTag, TagToken};
@@ -39,7 +40,7 @@ impl TokenSink for Website {
                                     if let Some(mut parsed) = url {
                                         parsed.set_query(None);
                                         parsed.set_fragment(None);
-                                        trace!("Final cleaned URL: `{}`", parsed.to_string());
+                                        debug!("Final cleaned URL: `{}`", parsed.to_string());
                                         let web = Website::new(&parsed.to_string(), false);
                                         links.push(web);
                                     }
@@ -62,14 +63,12 @@ impl TokenSink for Website {
 
 #[instrument(skip_all)]
 /// Parses the passed site and returns all the sites it links to.
-pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
-    debug!("Parsing {}", site.site.to_string());
+pub async fn parse(site: &Website, data: &str) -> Vec<Website> {
     // prep work
     let mut other_sites: Vec<Website> = Vec::new();
 
     // change data into something that can be tokenized
-    let s: Result<Tendril<fmt::UTF8>, ()> = Tendril::try_from_byte_slice(data);
-    if let Ok(chunk) = s {
+    let chunk = Tendril::from_str(data).expect("Failed to parse string into Tendril!");
     // create buffer of tokens and push our input into it
     let token_buffer = BufferQueue::default();
     token_buffer.push_back(
@@ -87,9 +86,6 @@ pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
     }
     assert!(token_buffer.is_empty());
     tokenizer.end();
-    } else {
-        warn!("Tendril failed to parse on: {}", site.site.to_string());
-    }
 
     other_sites
 }
@@ -111,7 +107,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
                 match Url::parse(&format!("{scheme}://{}", link)) {
                     Ok(url) => Some(url),
                     Err(err) => {
-                        error!("Failed parsing relative scheme url: {}", err);
+                        error!("Failed parsing realative scheme url: {}", err);
                         None
                     }
                 }
@@ -121,21 +117,19 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
                 match e {
                     url::ParseError::RelativeUrlWithoutBase => {
                         // Is: scheme://host:port
-                        let mut origin = parent.origin().ascii_serialization();
-                        if !origin.ends_with('/') && !link.starts_with('/') {
-                            origin += "/";
-                        }
+                        let origin = parent.origin().ascii_serialization();
                         let url = origin.clone() + link;
 
+                        trace!("Built `{url}` from `{origin} + {}`", link.to_string());
+
                         if let Ok(url) = Url::parse(&url) {
-                            trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
+                            trace!("Saved relative url `{}` AS: `{}`", link, url);
                             Some(url)
                         } else {
                             error!(
-                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`. Failed url was: {}",
+                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`",
                                 link,
-                                parent.to_string(),
-                                url
+                                parent.to_string()
                             );
                             None
                         }