not yet

.gitignore

@@ -4,5 +4,4 @@
 perf.data
 flamegraph.svg
 perf.data.old
-/docker/logs/*
+/docker/logs/*
-/downloaded

.vscode/launch.json

@@ -7,15 +7,18 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug executable 'internet_mapper'",
+            "name": "Debug executable 'surreal_spider'",
+            "env": {
+                "RUST_LOG": "surreal_spider=trace,reqwest=info",
+            },
             "cargo": {
                 "args": [
                     "build",
-                    "--bin=internet_mapper",
+                    "--bin=surreal_spider",
-                    "--package=internet_mapper"
+                    "--package=surreal_spider"
                 ],
                 "filter": {
-                    "name": "internet_mapper",
+                    "name": "surreal_spider",
                     "kind": "bin"
                 }
             },
@@ -25,16 +28,16 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug unit tests in executable 'internet_mapper'",
+            "name": "Debug unit tests in executable 'surreal_spider'",
             "cargo": {
                 "args": [
                     "test",
                     "--no-run",
-                    "--bin=internet_mapper",
+                    "--bin=surreal_spider",
-                    "--package=internet_mapper"
+                    "--package=surreal_spider"
                 ],
                 "filter": {
-                    "name": "internet_mapper",
+                    "name": "surreal_spider",
                     "kind": "bin"
                 }
             },

.vscode/settings.json

@@ -3,6 +3,6 @@
         "creds",
         "reqwest",
         "rustls",
-        "surql",
+        "surql"
     ]
 }

Cargo.lock

@@ -1966,7 +1966,6 @@ name = "internet_mapper"
 version = "0.1.0"
 dependencies = [
  "base64 0.22.1",
- "futures-util",
  "html5ever 0.29.1",
  "metrics",
  "metrics-exporter-prometheus",

Cargo.toml

@@ -5,13 +5,12 @@ edition = "2021"
 
 [dependencies]
 base64 = "0.22.1"
-futures-util = "0.3.31"
 html5ever = "0.29"
 metrics = "0.24.1"
 metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]}
 # minio = "0.1.0"
 minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"}
-reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls", "stream"] }
+reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] }
 serde = { version = "1.0", features = ["derive"] }
 surrealdb = "2.2"
 tokio = { version="1.41.0", features = ["full"] }

Crawler.toml

@@ -3,12 +3,14 @@ surreal_url = "localhost:8000"
 surreal_username = "root"
 surreal_password = "root"
 surreal_ns = "test"
-surreal_db = "v1.21.1"
+surreal_db = "v1.16"
+
+# Minio config
+s3_bucket = "v1.16"
+s3_url = "http://localhost:9000"
+s3_access_key = "DwJfDDVIbmCmfAblwSqp"
+s3_secret_key = "V4UqvC1Vm4AwLE5FAhu2gxlMfvexTBQnDxuy8uZx"
 
 # Crawler config
-# crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/" 
+crawl_filter = "en.wikipedia.com" 
-crawl_filter = "https://oliveratkinson.net" 
-# start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/"
-start_url = "https://oliveratkinson.net"
 budget = 1000
-batch_size = 500

README.md

@@ -2,54 +2,14 @@
 
 Crawls sites saving all the found links to a surrealdb database. It then proceeds to take batches of 100 uncrawled links untill the crawl budget is reached. It saves the data of each site in a minio database.
 
-## How to use
-
-1. Clone the repo and `cd` into it.
-2. Build the repo with `cargo build -r`
-3. Start the docker conatiners
-	1. cd into the docker folder `cd docker`
-	2. Bring up the docker containers `docker compose up -d`
-4. From the project's root, edit the `Crawler.toml` file to your liking.
-5. Run with `./target/release/internet_mapper`
-
-You can view stats of the project at `http://<your-ip>:3000/dashboards`
-
-```bash
-# Untested script but probably works
-git clone https://git.oliveratkinson.net/Oliver/internet_mapper.git
-cd internet_mapper
-
-cargo build -r
-
-cd docker
-docker compose up -d
-cd ..
-
-$EDITOR Crawler.toml
-
-./target/release/internet_mapper
-
-```
 
 ### TODO
 
-- [x] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
+- [ ] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
 - [ ] Conditionally save content - based on filename or file contents
 - [x] GUI / TUI ? - Graphana
 - [x] Better asynchronous getting of the sites. Currently it all happens serially.
-- [x] Allow for storing asynchronously - dropping the "links to" logic fixes this need
+- [ ] Allow for storing asynchronously
-- [x] Control crawler via config file (no recompliation needed)
-
-3/17/25: Took >1hr to crawl 100 pages
-
-3/19/25: Took 20min to crawl 1000 pages
-This ment we stored 1000 pages, 142,997 urls, and 1,425,798 links between the two.
-
-3/20/25: Took 5min to crawl 1000 pages
-
-3/21/25: Took 3min to crawl 1000 pages
-
-# About
-
-![Screenshot](/pngs/graphana.png)
 
+3/19/25: Took 20min to crawl 100 pages
+This ment we stored 100 pages, 142,997 urls, and 1,425,798 links between the two.

docker/compose.yml

@@ -14,6 +14,22 @@ services:
       - --pass
       - root
       - rocksdb:/mydata/database.db
+  minio:
+    image: quay.io/minio/minio
+    ports:
+      - 9000:9000
+      - 9001:9001
+    environment:
+      - MINIO_ROOT_USER=root
+      - MINIO_ROOT_PASSWORD=an8charpassword
+      - MINIO_PROMETHEUS_AUTH_TYPE=public
+    volumes:
+      - minio_storage:/data
+    command:
+      - server
+      - /data
+      - --console-address
+      - ":9001"
 
   alloy:
     image: grafana/alloy:latest
@@ -66,3 +82,4 @@ volumes:
   grafana_storage:
   alloy_storage:
   surrealdb_storage:
+  minio_storage:

docker/prometheus.yaml

@@ -8,10 +8,13 @@ scrape_configs:
     # change this your machine's ip, localhost won't work
     # because localhost refers to the docker container.
       - targets: ['172.20.239.48:2500']
-        #- targets: ['192.168.8.209:2500']
   - job_name: loki
     static_configs:
       - targets: ['loki:3100']
   - job_name: prometheus
     static_configs:
       - targets: ['localhost:9090']
+  - job_name: minio
+    metrics_path: /minio/v2/metrics/cluster
+    static_configs:
+      - targets: ['minio:9000']

src/db.rs

@@ -1,37 +1,59 @@
+use base64::{
+    alphabet,
+    engine::{self, general_purpose},
+    Engine,
+};
 use metrics::counter;
-use std::fmt::Debug;
+use serde::{ser::SerializeStruct, Deserialize, Serialize};
-use serde::{Deserialize, Serialize};
+use std::{collections::HashSet, fmt::Debug, sync::LazyLock, time::Instant};
 use surrealdb::{
     engine::remote::ws::{Client, Ws},
     opt::auth::Root,
     sql::Thing,
-    Surreal,
+    Error::Api,
+    Response, Surreal,
 };
-use tracing::{error, instrument, trace};
+use tokio::sync::Mutex;
+use tracing::{error, instrument, trace, warn};
 use url::Url;
 
-use crate::Config;
+use crate::{Config, Timer};
 
-const STORE: &str = "surql_store_calls";
+// static LOCK: LazyLock<Arc<Mutex<bool>>> = LazyLock::new(|| Arc::new(Mutex::new(true)));
+static LOCK: LazyLock<Mutex<bool>> = LazyLock::new(|| Mutex::new(true));
 
-#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
+const CUSTOM_ENGINE: engine::GeneralPurpose =
+    engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD);
+
+const TIME_SPENT_ON_LOCK: &'static str = "surql_lock_waiting_ms";
+const STORE: &'static str = "surql_store_calls";
+const LINK: &'static str = "surql_link_calls";
+
+#[derive(Deserialize, Clone, Hash, Eq, PartialEq)]
 pub struct Website {
     /// The url that this data is found at
     pub site: Url,
     /// Wether or not this link has been crawled yet
     pub crawled: bool,
-    /// 200, 404, etc
+}
-    pub status_code: u16,
+
+impl Serialize for Website {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let mut state = serializer.serialize_struct("Website", 2)?;
+        state.serialize_field("crawled", &self.crawled)?;
+        // to_string() calls the correct naming of site
+        state.serialize_field("site", &self.site.to_string())?;
+        state.end()
+    }
 }
 
 // manual impl to make tracing look nicer
 impl Debug for Website {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Website")
+        f.debug_struct("Website").field("site", &self.site).finish()
-            .field("host", &self.site.host())
-            .field("path", &self.site.path())
-            .field("status_code", &self.status_code)
-            .finish()
     }
 }
 
@@ -42,43 +64,141 @@ impl Website {
             Ok(a) => a,
             Err(_) => todo!(),
         };
-        Self {
+        Self { crawled, site }
-            crawled,
+    }
-            site,
+
-            status_code: 0,
+    pub fn set_crawled(&mut self) {
+        trace!("Set crawled to true");
+        self.crawled = true
+    }
+
+    pub fn get_url_as_string(site: &Url) -> String {
+        let domain = match site.domain() {
+            Some(s) => s.to_string(),
+            None => {
+                warn!(
+                    "Failed to get domain of URL: {}, falling back to 'localhost'",
+                    site.to_string()
+                );
+                "localhost".to_string()
+            }
+        };
+        let path = site.path();
+
+        domain + path
+    }
+
+    pub fn get_url_as_b64_path(site: &Url) -> String {
+        let domain = site.domain().unwrap_or("DOMAIN").to_string();
+        let path = &CUSTOM_ENGINE.encode(site.path());
+
+        domain + path
+    }
+
+    #[instrument(skip_all)]
+    pub async fn links_to(&self, other: Vec<Thing>, db: &Surreal<Client>) {
+        let len = other.len();
+        if len == 0 {
+            return;
         }
+
+        let from = &self.site;
+
+        // let to = other.site.to_string();
+        trace!("Linking {} pages to {from}", other.len());
+        let msg = format!("Linked {len} pages to {from}");
+        let timer = Timer::start(&msg);
+        // prevent the timer from being dropped instantly.
+        let _ = timer;
+        counter!(LINK).increment(1);
+        match db
+            .query("COUNT(RELATE (SELECT id FROM website WHERE site = $in) -> links_to -> $out)")
+            .bind(("in", from.clone()))
+            .bind(("out", other))
+            .await
+        {
+            Ok(mut e) => {
+                // The relate could technically "fail" (not relate anything), this just means that
+                // the query was ok.
+                let _: Response = e;
+                if let Ok(vec) = e.take(0) {
+                    let _: Vec<usize> = vec;
+                    if let Some(num) = vec.get(0) {
+                        if *num == len {
+                            trace!("Link for {from} OK - {num}/{len}");
+                            return;
+                        } else {
+                            error!(
+                                "Didn't link all the records. {num}/{len}. Surreal response: {:?}",
+                                e
+                            );
+                            return;
+                        }
+                    }
+                }
+                error!("Linking request succeeded but couldn't verify the results.");
+            }
+            Err(e) => {
+                error!("{}", e.to_string());
+            }
+        }
+    }
+
+    pub async fn store_self(&self, db: &Surreal<Client>) {
+        counter!(STORE).increment(1);
+
+        db.query(
+            "INSERT INTO website $self
+                    ON DUPLICATE KEY UPDATE
+                        crawled = crawled OR $input.crawled
+                    RETURN VALUE id;
+            ",
+        )
+        .await
+        .expect("Failed to store self");
     }
 
     // Insert ever item in the vec into surreal, crawled state will be preserved as TRUE
     // if already in the database as such or incoming data is TRUE.
-    #[instrument(skip(db))]
+    pub async fn store_all(all: HashSet<Self>, db: &Surreal<Client>) -> Vec<Thing> {
-    pub async fn store_all(all: Vec<Self>, db: &Surreal<Client>) -> Vec<Thing> {
+        // NOTES:
+        // * all incoming Websites come in as !crawled
+        // * there are potentially duplicates in all
+        
         counter!(STORE).increment(1);
-        let mut things = Vec::with_capacity(all.len());
 
-        // FIXME failes *sometimes* because "Resource Busy"
+        let now = Instant::now();
+        let lock = LOCK.lock().await;
+        counter!(TIME_SPENT_ON_LOCK).increment(now.elapsed().as_millis() as u64);
+
+        let mut results = Vec::with_capacity(all.len());
         match db
             .query(
+                // TODO making this an upsert would make sense, but
+                // upserts seem to be broken.
+                //
+                // Doesn't look like upsert can take in an array, so insert
+                // it is...
+                //
                 "INSERT INTO website $array
                     ON DUPLICATE KEY UPDATE
-                        accessed_at = time::now(),
+                        last_write = time::now()
-                        status_code = $input.status_code,
+                    RETURN VALUE id;"
-                        crawled = crawled OR $input.crawled
+                ,
-                    RETURN VALUE id;
-                 ",
             )
             .bind(("array", all))
             .await
         {
             Ok(mut id) => match id.take::<Vec<Thing>>(0) {
-                Ok(mut x) => things.append(&mut x),
+                Ok(mut x) => results.append(&mut x),
-                Err(err) => error!("{:?}", err),
+                Err(err) => {
+                    error!("{:?}", err);
+                }
             },
-            Err(err) => {
+            Err(err) => error!("{:?}", err),
-                error!("{:?}", err);
-            }
         }
-        things
+        drop(lock);
+        results
     }
 }
 

src/filesystem.rs

@@ -1,66 +0,0 @@
-use std::{io::ErrorKind, path::PathBuf};
-
-use reqwest::header::HeaderValue;
-use tokio::fs;
-use tracing::{error, trace, warn};
-use url::Url;
-
-pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
-    // extract data from url to save it accurately
-    let mut url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
-
-    if let Ok(header) = content_type.to_str() {
-        // text/html; charset=UTF-8; option=value
-        let ttype = if let Some((t, _)) = header.split_once(';') {
-            t
-        } else {
-            header
-        };
-
-        if let Some((ttype, subtype)) = ttype.split_once('/') {
-            trace!("Found Content-Type to be: {ttype}/{subtype} for {}", url.to_string());
-            // If the Content-Type header is "*/html" (most likely "text/html") and the path's
-            // extension is anything but html:
-            if subtype=="html" && !url_path.extension().is_some_and(|f| f=="html" || f=="htm" ) {
-                // time to slap a index.html to the end of that path there!
-                url_path = url_path.join("index.html");
-            }
-        }
-    } else {
-        warn!("Header: {:?} couldn't be parsed into a string!", content_type);
-    }
-    trace!("Final path for {} is: {:?}", url, url_path);
-
-    url_path
-}
-
-pub async fn init(filename: &PathBuf) -> Option<fs::File> {
-    let file = async || tokio::fs::OpenOptions::new()
-        .append(true)
-        .create(true)
-        .open(&filename).await;
-
-    match file().await {
-        Ok(ok) => Some(ok),
-        Err(err) => {
-            // the file/folder isn't found
-            if err.kind() == ErrorKind::NotFound {
-                if let Some(parent ) = &filename.parent() {
-                    // create the folders
-                    if let Err(err) = fs::create_dir_all(&parent).await {
-                        error!("Dir creation: {err} {:?}", filename);
-                        eprintln!("{}", err)
-                    } else if let Ok(ok) = file().await {
-                        return Some(ok);
-                    }
-                } else {
-                    error!("Couldn't get file's parents: {:?}", &filename);
-                }
-            } else {
-                error!("File open error: {err} {:?}", filename);
-            }
-            // we don't care about other errors, we can't/shouldn't fix them
-            None
-        }
-    }
-}

src/main.rs

@@ -1,35 +1,29 @@
 #![feature(ip_from)]
-#![warn(clippy::expect_used)]
-#![deny(clippy::unwrap_used)]
 
 extern crate html5ever;
 
-use futures_util::StreamExt;
-
 use std::{
-    collections::HashSet,
+    collections::HashSet, fs::File, io::Read, net::{IpAddr, Ipv4Addr}, time::Instant
-    fs::File,
-    io::Read,
-    net::{IpAddr, Ipv4Addr},
 };
 
 use db::{connect, Website};
 use metrics::{counter, gauge};
 use metrics_exporter_prometheus::PrometheusBuilder;
+use s3::S3;
 use serde::Deserialize;
 use surrealdb::{engine::remote::ws::Client, Surreal};
-use tokio::{io::{AsyncWriteExt, BufWriter}, task::JoinSet};
+use tokio::task::JoinSet;
-use tracing::{debug, debug_span, error, info, instrument, level_filters::LevelFilter, trace, trace_span, warn};
+use tracing::{debug, error, info, instrument, trace, trace_span, warn};
 use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry};
 
 mod db;
-mod filesystem;
 mod parser;
+mod s3;
 
-const GET_METRIC: &str = "total_gets";
+const GET_METRIC: &'static str = "total_gets";
-const GET_IN_FLIGHT: &str = "gets_in_flight";
+const GET_IN_FLIGHT: &'static str = "gets_in_flight";
-const SITES_CRAWLED: &str = "pages_crawled";
+const SITES_CRAWLED: &'static str = "pages_crawled";
-const BEING_PROCESSED: &str = "pages_being_processed";
+const BEING_PROCESSED: &'static str = "pages_being_processed";
 
 #[derive(Deserialize)]
 struct Config {
@@ -39,15 +33,18 @@ struct Config {
     surreal_username: String,
     surreal_password: String,
 
+    s3_url: String,
+    s3_bucket: String,
+    s3_access_key: String,
+    s3_secret_key: String,
+
     crawl_filter: String,
-    start_url: String,
     budget: usize,
-    batch_size: usize,
 }
 
 #[tokio::main]
 async fn main() {
-    println!("Logs and metrics are provided to the Grafana dashboard");
+    let total_runtime = Timer::start("Completed");
 
     let writer = std::fs::OpenOptions::new()
         .append(true)
@@ -55,9 +52,7 @@ async fn main() {
         .open("./docker/logs/tracing.log")
         .expect("Couldn't make log file!");
 
-    let filter = EnvFilter::builder()
+    let filter = EnvFilter::from_default_env();
-        .with_default_directive(LevelFilter::DEBUG.into())
-        .from_env_lossy();
 
     let registry = Registry::default().with(
         fmt::layer()
@@ -80,8 +75,9 @@ async fn main() {
         .install()
         .expect("failed to install recorder/exporter");
 
-    info!("Starting...");
+    debug!("Starting...");
-
+    // Would probably take these in as parameters from a cli
+    let starting_url = "https://en.wikipedia.org/";
     // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
     // let crawl_filter = "en.wikipedia.org/";
     // let budget = 50;
@@ -92,11 +88,13 @@ async fn main() {
     let _ = file.read_to_string(&mut buf);
 
     let config: Config = toml::from_str(&buf).expect("Failed to parse Crawler.toml");
-    let starting_url = &config.start_url;
 
     let db = connect(&config)
         .await
         .expect("Failed to connect to surreal, aborting.");
+    let s3 = S3::connect(&config)
+        .await
+        .expect("Failed to connect to minio, aborting.\n\nThis probably means you need to login to the minio console and get a new access key!\n\n(Probably here) http://localhost:9001/access-keys/new-account\n\n");
 
     let reqwest = reqwest::Client::builder()
         // .use_rustls_tls()
@@ -109,33 +107,43 @@ async fn main() {
     let span = trace_span!("Pre-Loop");
     let pre_loop_span = span.enter();
     // Download the site
-    let site = Website::new(starting_url, false);
+    let site = Website::new(&starting_url, false);
-    process(site, db.clone(), reqwest.clone()).await;
+    process(site, db.clone(), reqwest.clone(), s3.clone()).await;
 
     drop(pre_loop_span);
 
     let span = trace_span!("Loop");
     let span = span.enter();
     while crawled < config.budget {
-        let uncrawled =
+        let get_num = if config.budget - crawled < 100 {
-            get_uncrawled_links(&db, config.budget - crawled, config.crawl_filter.clone(), &config).await;
+            config.budget - crawled
-        if uncrawled.is_empty() {
+        } else {
+            100
+        };
+
+        let uncrawled = get_uncrawled_links(&db, get_num, config.crawl_filter.clone()).await;
+        if uncrawled.len() == 0 {
             info!("Had more budget but finished crawling everything.");
             return;
         }
+        debug!("Crawling {} pages...", uncrawled.len());
+
+        let span = trace_span!("Crawling");
+        let _ = span.enter();
 
         {
             let mut futures = JoinSet::new();
             for site in uncrawled {
                 gauge!(BEING_PROCESSED).increment(1);
-                futures.spawn(process(site, db.clone(), reqwest.clone()));
+                futures.spawn(process(site, db.clone(), reqwest.clone(), s3.clone()));
                 // let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32);
                 // info!("Crawled {crawled} out of {budget} pages. ({percent})");
             }
+            debug!("Joining {} futures...", futures.len());
 
             let c = counter!(SITES_CRAWLED);
             // As futures complete runs code in while block
-            while futures.join_next().await.is_some() {
+            while let Some(_) = futures.join_next().await {
                 c.increment(1);
                 gauge!(BEING_PROCESSED).decrement(1);
                 crawled += 1;
@@ -144,137 +152,79 @@ async fn main() {
     }
     drop(span);
 
-    if let Ok(mut ok) = db
+    debug!("Done");
-        .query("count(select id from website where crawled = true)")
+    drop(total_runtime);
-        .await
-    {
-        let res = ok.take::<Option<usize>>(0);
-        if let Ok(Some(n)) = res {
-            info!("Total crawled pages now equals {n}");
-        }
-    }
-
-    info!("Done");
 }
 
-#[instrument(skip(db, reqwest))]
+#[instrument(skip(db, s3, reqwest))]
 /// Downloads and crawls and stores a webpage.
 /// It is acceptable to clone `db`, `reqwest`, and `s3` because they all use `Arc`s internally. - Noted by Oliver
-async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client) {
+async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client, s3: S3) {
+    
     // METRICS
     trace!("Process: {}", &site.site);
+    let timer = Timer::start("Built request");
     // Build the request
-    let request_builder = reqwest.get(site.site.to_string());
+    let request_builder = reqwest.get(&site.site.to_string());
+    // METRICS
+    timer.stop();
 
     // METRICS
     let g = gauge!(GET_IN_FLIGHT);
     g.increment(1);
+    counter!(GET_METRIC).increment(1);
+    let timer = Timer::start("Got page");
 
     // Send the http request (get)
     if let Ok(response) = request_builder.send().await {
-        let headers = response.headers();
-        let code = response.status();
 
-        #[allow(non_snake_case)]
+        // METRICS
-        let CT = headers.get("Content-Type");
+        timer.stop();
-        let ct = headers.get("content-type");
+        g.decrement(1);
 
-        let ct = match (CT,ct) {
+        // Get body from response
-            (None, None) => {
+        let data = response
-                warn!("Server did not respond with Content-Type header. Url: {} Headers: ({:?})", site.site.to_string(), headers);
+            .text()
-                return
+            .await
-            },
+            .expect("Failed to read http response's body!");
-            (None, Some(a)) => a,
+        // Store document
-            (Some(a), None) => a,
+        s3.store(&data, &site.site).await;
-            (Some(a), Some(_)) => a,
-        };
 
-        // create filepath (handles / -> /index.html)
+        // Parse document and get relationships
-        let path = filesystem::as_path(&site.site, ct);
+        let sites = parser::parse(&site, &data).await;
+        
+        // update self in db
+        site.set_crawled();
+        site.store_self(&db).await;
 
-        // make sure that the file is good to go
+        // de duplicate this list
-        if let Some(file) = filesystem::init(&path).await {
+        let set = sites.iter().fold(HashSet::new(), |mut set, item| {
-            // Get body from response
+            // TODO seems kinda dumb to clone everything.
-            // stream the response onto the disk
+            set.insert(item.clone());
-            let mut stream = response.bytes_stream();
+            set
+        });
+        trace!("Shrunk items to store from {} to {}", sites.len(), set.len());
 
-            let should_parse = path.to_string_lossy().ends_with(".html");
+        // Store all the other sites so that we can link to them.
-            let mut writer = BufWriter::new(file);
+        let others = Website::store_all(set, &db).await;
-            let mut buf: Vec<u8> = Vec::new();
 
-            // Write file to disk
+        // Make the database's links reflect the html links between sites
-            info!("Writing at: {:?}", path);
+        site.links_to(others, &db).await;
-            while let Some(data) = stream.next().await {
-                match data {
-                    Ok(data) => {
-                        let _ = writer.write_all(&data).await;
-                        // If we are going to parse this file later, we will save it
-                        // into memory as well as the disk.
-                        // We do this because the data here might be incomplete
-                        if should_parse {
-                            data.iter().for_each(|f| buf.push(*f));
-                        }
-                    },
-                    Err(err) => {
-                        eprintln!("{}", err)
-                    },
-                }
-            }
-            let _ = writer.flush();
-
-
-            // (If needed) Parse the file
-            if should_parse {
-                let span = debug_span!("Should Parse");
-                let enter = span.enter();
-
-                // Parse document and get relationships
-                let sites = parser::parse(&site, &buf).await;
-                // De-duplicate this list
-                let prev_len = sites.len();
-                let set = sites.into_iter().fold(HashSet::new(), |mut set, item| {
-                    set.insert(item);
-                    set
-                });
-                let de_dupe_sites: Vec<Website> = set.into_iter().collect();
-                let diff = prev_len - de_dupe_sites.len();
-                trace!("Saved {diff} from being entered into the db by de-duping");
-                // Store all the other sites so that we can link to them.
-                let _ = Website::store_all(de_dupe_sites, &db).await;
-
-                drop(enter);
-            }
-
-            // METRICS
-            g.decrement(1);
-            counter!(GET_METRIC).increment(1);
-
-            // update self in db
-            site.crawled = true;
-            site.status_code = code.as_u16();
-            Website::store_all(vec![site.clone()], &db).await;
-        } else {
-            error!("File failed to cooperate: {:?}", path);
-        }
-
-        trace!("Done processing: {}", &site.site);
     } else {
         error!("Failed to get: {}", &site.site);
     }
 }
 
 /// Returns uncrawled links
-#[instrument(skip(db, config))]
+#[instrument(skip(db))]
 async fn get_uncrawled_links(
     db: &Surreal<Client>,
     mut count: usize,
     filter: String,
-    config: &Config,
 ) -> Vec<Website> {
-    if count > config.batch_size {
+    if count > 100 {
-        count = config.batch_size;
+        count = 100
     }
-
     debug!("Getting uncrawled links");
 
     let mut response = db
@@ -287,3 +237,34 @@ async fn get_uncrawled_links(
         .take(0)
         .expect("Returned websites couldn't be parsed")
 }
+
+pub struct Timer<'a> {
+    start: Instant,
+    msg: &'a str,
+}
+
+impl<'a> Timer<'a> {
+    #[inline]
+    pub fn start(msg: &'a str) -> Self {
+        Self {
+            start: Instant::now(),
+            msg,
+        }
+    }
+    pub fn stop(&self) -> f64 {
+        let dif = self.start.elapsed().as_micros();
+        let ms = dif as f64 / 1000.;
+
+        if ms > 200. {
+            warn!("{}", format!("{} in {:.3}ms", self.msg, ms));
+        }
+
+        ms
+    }
+}
+
+impl Drop for Timer<'_> {
+    fn drop(&mut self) {
+        self.stop();
+    }
+}

src/parser.rs

@@ -1,24 +1,23 @@
 use std::default::Default;
+use std::str::FromStr;
 
 use html5ever::tokenizer::{BufferQueue, TokenizerResult};
 use html5ever::tokenizer::{StartTag, TagToken};
 use html5ever::tokenizer::{Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts};
 use html5ever::{local_name, tendril::*};
-use tracing::{debug, error, instrument, trace, warn};
+use tracing::instrument;
-use url::Url;
 
 use crate::db::Website;
+use crate::Timer;
 
 impl TokenSink for Website {
     type Handle = Vec<Website>;
 
-    #[instrument(skip(token, _line_number))]
     fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<Self::Handle> {
         match token {
             TagToken(tag) => {
                 if tag.kind == StartTag {
                     match tag.name {
-                        // this should be all the html elements that have links
                         local_name!("a")
                         | local_name!("audio")
                         | local_name!("area")
@@ -33,18 +32,23 @@ impl TokenSink for Website {
                                 let attr_name = attr.name.local.to_string();
                                 if attr_name == "src" || attr_name == "href" || attr_name == "data"
                                 {
-                                    trace!("Found `{}` in html `{}` tag", &attr.value, tag.name);
+                                    // Get clone of the current site object
-                                    let url = try_get_url(&self.site, &attr.value);
+                                    let mut web = self.clone();
 
-                                    if let Some(mut parsed) = url {
+                                    // Set url
-                                        parsed.set_query(None);
+                                    let mut url = web.site;
-                                        parsed.set_fragment(None);
+                                    url.set_fragment(None); // removes #xyz
-                                        trace!("Final cleaned URL: `{}`", parsed.to_string());
+                                    let joined = url
-                                        let web = Website::new(&parsed.to_string(), false);
+                                        .join(&attr.value)
-                                        links.push(web);
+                                        .expect("Failed to join url during parsing!");
-                                    }
+                                    web.site = joined;
+
+                                    web.crawled = false;
+
+                                    links.push(web);
                                 }
                             }
+
                             return TokenSinkResult::Script(links);
                         }
                         local_name!("button") | local_name!("meta") | local_name!("iframe") => {
@@ -62,90 +66,30 @@ impl TokenSink for Website {
 
 #[instrument(skip_all)]
 /// Parses the passed site and returns all the sites it links to.
-pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
+pub async fn parse(site: &Website, data: &str) -> Vec<Website> {
-    debug!("Parsing {}", site.site.to_string());
     // prep work
     let mut other_sites: Vec<Website> = Vec::new();
+    let _t = Timer::start("Parsed page");
 
     // change data into something that can be tokenized
-    let s: Result<Tendril<fmt::UTF8>, ()> = Tendril::try_from_byte_slice(data);
+    let chunk = Tendril::from_str(&data).expect("Failed to parse string into Tendril!");
-    if let Ok(chunk) = s {
+    // create buffer of tokens and push our input into it
-        // create buffer of tokens and push our input into it
+    let mut token_buffer = BufferQueue::default();
-        let token_buffer = BufferQueue::default();
+    token_buffer.push_back(
-        token_buffer.push_back(
+        chunk
-            chunk
+            .try_reinterpret::<fmt::UTF8>()
-                .try_reinterpret::<fmt::UTF8>()
+            .expect("Failed to reinterprt chunk!"),
-                .expect("Failed to reinterpret chunk!"),
+    );
-        );
+    // create the tokenizer
-        // create the tokenizer
+    let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
-        let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
 
-        // go thru buffer
+    // go thru buffer
-        while let TokenizerResult::Script(mut sites) = tokenizer.feed(&token_buffer) {
+    while let TokenizerResult::Script(mut sites) = tokenizer.feed(&mut token_buffer) {
-            other_sites.append(&mut sites);
+        other_sites.append(&mut sites);
-            // other_sites.push(sites);
+        // other_sites.push(sites);
-        }
-        assert!(token_buffer.is_empty());
-        tokenizer.end();
-    } else {
-        warn!("Tendril failed to parse on: {}", site.site.to_string());
     }
+    assert!(token_buffer.is_empty());
+    tokenizer.end();
 
     other_sites
 }
-
-#[instrument]
-fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
-    match Url::parse(link) {
-        Ok(ok) => Some(ok),
-        Err(e) => {
-            if link.starts_with('#') {
-                trace!("Rejecting # url");
-                None
-            } else if link.starts_with("//") {
-                // if a url starts with "//" is assumed that it will adopt
-                // the same scheme as it's parent
-                // https://stackoverflow.com/questions/9646407/two-forward-slashes-in-a-url-src-href-attribute
-                let scheme = parent.scheme();
-
-                match Url::parse(&format!("{scheme}://{}", link)) {
-                    Ok(url) => Some(url),
-                    Err(err) => {
-                        error!("Failed parsing relative scheme url: {}", err);
-                        None
-                    }
-                }
-            } else {
-                // # This is some sort of realative url, gonna try patching it up into an absolute
-                // url
-                match e {
-                    url::ParseError::RelativeUrlWithoutBase => {
-                        // Is: scheme://host:port
-                        let mut origin = parent.origin().ascii_serialization();
-                        if !origin.ends_with('/') && !link.starts_with('/') {
-                            origin += "/";
-                        }
-                        let url = origin.clone() + link;
-
-                        if let Ok(url) = Url::parse(&url) {
-                            trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
-                            Some(url)
-                        } else {
-                            error!(
-                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`. Failed url was: {}",
-                                link,
-                                parent.to_string(),
-                                url
-                            );
-                            None
-                        }
-                    }
-                    _ => {
-                        error!("MISC error: {:?} {:?}", e, link);
-                        None
-                    }
-                }
-            }
-        }
-    }
-}

src/s3.rs

@@ -0,0 +1,101 @@
+use metrics::counter;
+use minio::s3::{
+    args::{BucketExistsArgs, MakeBucketArgs},
+    client::ClientBuilder,
+    creds::StaticProvider,
+    error::Error,
+    http::BaseUrl,
+    Client,
+};
+use tracing::{instrument, trace, warn};
+use url::Url;
+
+use crate::{db::Website, Config, Timer};
+
+const S3_ROUND_TRIP_METRIC: &'static str = "s3_trips";
+
+#[derive(Clone)]
+pub struct S3 {
+    bucket_name: String,
+    client: Client,
+}
+
+impl S3 {
+    #[instrument(skip_all, name = "S3")]
+    pub async fn connect(config: &Config) -> Result<Self, Error> {
+        let base_url = config
+            .s3_url
+            .parse::<BaseUrl>()
+            .expect("Failed to parse url into BaseUrl");
+
+        let static_provider =
+            StaticProvider::new(&config.s3_access_key, &config.s3_secret_key, None);
+
+        let client = ClientBuilder::new(base_url)
+            .provider(Some(Box::new(static_provider)))
+            .build()?;
+
+        trace!("Checking bucket...");
+        let exists = client
+            .bucket_exists(
+                &BucketExistsArgs::new(&config.s3_bucket)
+                    .expect("Failed to check if bucket exists"),
+            )
+            .await?;
+        counter!(S3_ROUND_TRIP_METRIC).increment(1);
+
+        if !exists {
+            trace!("Creating bucket...");
+            client
+                .make_bucket(
+                    &MakeBucketArgs::new(&config.s3_bucket).expect("Failed to create bucket!"),
+                )
+                .await?;
+        }
+        counter!(S3_ROUND_TRIP_METRIC).increment(1);
+
+        trace!("Connection successful");
+
+        Ok(Self {
+            bucket_name: config.s3_bucket.to_owned(),
+            client,
+        })
+    }
+
+    #[instrument(name = "s3_store", skip_all)]
+    pub async fn store(&self, data: &str, url: &Url) {
+        let counter = counter!(S3_ROUND_TRIP_METRIC);
+        let t = Timer::start("Stored page");
+        let _ = t; // prevent compiler drop
+
+        let filename = Website::get_url_as_string(url);
+        trace!("Storing {} as {filename}", url.to_string());
+
+        counter.increment(1);
+        let _ = match &self
+            .client
+            .put_object_content(&self.bucket_name, &filename, data.to_owned())
+            .send()
+            .await
+        {
+            Ok(_) => {}
+            Err(err) => match err {
+                Error::InvalidObjectName(_) => {
+                    // This code will really only run if the url has non-english chars
+                    warn!("Tried storing invalid object name, retrying with Base64 encoding. Last try.");
+
+                    let filename: String = Website::get_url_as_b64_path(url);
+
+                    counter.increment(1);
+                    let _ = &self
+                        .client
+                        .put_object_content(&self.bucket_name, &filename, data.to_owned())
+                        .send()
+                        .await
+                        .unwrap();
+                }
+                _ => {}
+            },
+        };
+    }
+}

src/setup.surql

@@ -5,5 +5,4 @@ DEFINE INDEX IF NOT EXISTS idx ON TABLE website COLUMNS site UNIQUE;
 
 DEFINE FIELD IF NOT EXISTS crawled ON TABLE website TYPE bool;
 
-DEFINE FIELD IF NOT EXISTS accessed_at ON TABLE website VALUE time::now();
+DEFINE FIELD IF NOT EXISTS created ON TABLE website VALUE time::now();
-DEFINE FIELD IF NOT EXISTS first_accessed_at ON TABLE website VALUE time::now();