Merge pull request 'traces and new metrics work' (#13) from better_metrics into main

Reviewed-on: #13

.gitignore

@@ -6,3 +6,4 @@ flamegraph.svg
 perf.data.old
 /docker/logs/*
 /downloaded
+/Crawler.toml

.vscode/launch.json

@@ -7,18 +7,15 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug executable 'surreal_spider'",
-            "env": {
-                "RUST_LOG": "surreal_spider=trace,reqwest=info",
-            },
+            "name": "Debug executable 'internet_mapper'",
             "cargo": {
                 "args": [
                     "build",
-                    "--bin=surreal_spider",
-                    "--package=surreal_spider"
+                    "--bin=internet_mapper",
+                    "--package=internet_mapper"
                 ],
                 "filter": {
-                    "name": "surreal_spider",
+                    "name": "internet_mapper",
                     "kind": "bin"
                 }
             },
@@ -28,16 +25,16 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug unit tests in executable 'surreal_spider'",
+            "name": "Debug unit tests in executable 'internet_mapper'",
             "cargo": {
                 "args": [
                     "test",
                     "--no-run",
-                    "--bin=surreal_spider",
-                    "--package=surreal_spider"
+                    "--bin=internet_mapper",
+                    "--package=internet_mapper"
                 ],
                 "filter": {
-                    "name": "surreal_spider",
+                    "name": "internet_mapper",
                     "kind": "bin"
                 }
             },

.vscode/settings.json

@@ -3,6 +3,6 @@
         "creds",
         "reqwest",
         "rustls",
-        "surql"
+        "surql",
     ]
 }

Cargo.toml

@@ -5,14 +5,16 @@ edition = "2021"
 
 [dependencies]
 base64 = "0.22.1"
+futures-util = "0.3.31"
 html5ever = "0.29"
 metrics = "0.24.1"
 metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]}
-# minio = "0.1.0"
-minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"}
-reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] }
-rusqlite = { version = "0.34.0", features = ["bundled"] }
+opentelemetry = "0.30.0"
+opentelemetry-otlp = { version = "0.30.0", features = ["metrics", "trace", "logs", "grpc-tonic"] }
+opentelemetry_sdk = "0.30.0"
+reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls", "stream"] }
 serde = { version = "1.0", features = ["derive"] }
+surrealdb = "2.2"
 tokio = { version="1.41.0", features = ["full"] }
 toml = "0.8.20"
 tracing = "0.1"

Crawler.toml

@@ -3,8 +3,12 @@ surreal_url = "localhost:8000"
 surreal_username = "root"
 surreal_password = "root"
 surreal_ns = "test"
-surreal_db = "v1.19.2"
+surreal_db = "v1.21.1"
 
 # Crawler config
-crawl_filter = "en.wikipedia.com" 
+# crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/" 
+crawl_filter = "https://oliveratkinson.net" 
+# start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/"
+start_url = "https://oliveratkinson.net"
 budget = 1000
+batch_size = 500

README.md

@@ -2,13 +2,43 @@
 
 Crawls sites saving all the found links to a surrealdb database. It then proceeds to take batches of 100 uncrawled links untill the crawl budget is reached. It saves the data of each site in a minio database.
 
+## How to use
+
+1. Clone the repo and `cd` into it.
+2. Build the repo with `cargo build -r`
+3. Start the docker conatiners
+	1. cd into the docker folder `cd docker`
+	2. Bring up the docker containers `docker compose up -d`
+4. From the project's root, edit the `Crawler.toml` file to your liking.
+5. Run with `./target/release/internet_mapper`
+
+You can view stats of the project at `http://<your-ip>:3000/dashboards`
+
+```bash
+# Untested script but probably works
+git clone https://git.oliveratkinson.net/Oliver/internet_mapper.git
+cd internet_mapper
+
+cargo build -r
+
+cd docker
+docker compose up -d
+cd ..
+
+$EDITOR Crawler.toml
+
+./target/release/internet_mapper
+
+```
+
 ### TODO
 
-- [ ] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
+- [x] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
 - [ ] Conditionally save content - based on filename or file contents
 - [x] GUI / TUI ? - Graphana
 - [x] Better asynchronous getting of the sites. Currently it all happens serially.
-- [ ] Allow for storing asynchronously
+- [x] Allow for storing asynchronously - dropping the "links to" logic fixes this need
+- [x] Control crawler via config file (no recompliation needed)
 
 3/17/25: Took >1hr to crawl 100 pages
 

docker/alloy.conf

@@ -3,8 +3,8 @@ local.file_match "tmplogs" {
 }
 
 loki.source.file "local_files" {
-    targets    = local.file_match.tmplogs.targets
-    forward_to = [loki.write.local_loki.receiver]
+    targets    	= local.file_match.tmplogs.targets
+    forward_to 	= [loki.write.local_loki.receiver]
 }
 
 loki.write "local_loki" {
@@ -12,3 +12,25 @@ loki.write "local_loki" {
         url = "http://loki:3100/loki/api/v1/push"
     }
 }
+
+otelcol.receiver.otlp "otlp_receiver" {
+  grpc {
+    endpoint = "0.0.0.0:4317"
+  }
+  http {
+    endpoint = "0.0.0.0:4318"
+  }
+
+  output {
+    traces = [otelcol.exporter.otlp.tempo.input,]
+  }
+}
+
+otelcol.exporter.otlp "tempo" {
+  client {
+    endpoint = "tempo:4317"
+    tls {
+			insecure = true
+		}
+  }
+}

docker/compose.yml

@@ -1,4 +1,6 @@
 services:
+
+  # Database
   surreal:
     image: surrealdb/surrealdb:latest-dev
     ports:
@@ -15,6 +17,18 @@ services:
       - root
       - rocksdb:/mydata/database.db
 
+  # Tracing
+  tempo:
+    image: grafana/tempo:latest
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml
+      - tempo_storage:/var/tempo
+    ports:
+      - 3200:3200 # self metrics for prometheus
+      - 4317:4317 # otlp grpc - (alloy) 
+
+  # Log scraper
   alloy:
     image: grafana/alloy:latest
     ports:
@@ -24,9 +38,13 @@ services:
       - ./logs/:/tmp/alloy-logs
       - ./alloy.conf:/etc/alloy/config.alloy
       - alloy_storage:/var/lib/alloy
-    command:  run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
 
-  #logs
+  # Log storage / analysis
   loki:
     image: grafana/loki:latest
     ports:
@@ -35,16 +53,20 @@ services:
     volumes:
       - ./loki.yaml:/etc/loki/local-config.yaml
 
-  # Metrics collector
+  # Metrics
   prometheus:
     image: prom/prometheus:latest
-    expose:
-      - 9090
+    ports:
+      - 9090:9090
     volumes:
       - ./prometheus.yaml:/etc/prometheus/prometheus.yml
       # persist data
-      - prometheus_storage:/prometheus
-    command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
+      # - prometheus_storage:/prometheus
+    command:
+      - --enable-feature=native-histograms
+      - --web.enable-remote-write-receiver
+      - --web.enable-lifecycle
+      - --config.file=/etc/prometheus/prometheus.yml
 
   # Everything viewer
   grafana:
@@ -66,4 +88,4 @@ volumes:
   grafana_storage:
   alloy_storage:
   surrealdb_storage:
-  minio_storage:
+  tempo_storage:

docker/grafana.yaml

@@ -22,3 +22,20 @@ datasources:
   editable: false
   jsonData:
     httpMethod: GET
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: true
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+    httpMethod: GET
+    serviceMap:
+      datasourceUid: prometheus
+    streamingEnabled:
+      search: true

docker/prometheus.yaml

@@ -1,17 +1,15 @@
 global:
-  scrape_interval: 5s 
+  scrape_interval: 60s 
   query_log_file: /etc/prometheus/query.log
 
 scrape_configs:
-  - job_name: crawler
+  # Crawler configs get pushed with OTLP
+#  - job_name: 'loki'
+#    static_configs:
+#      - targets: ['loki:3100']
+#  - job_name: 'prometheus'
+#    static_configs:
+#      - targets: ['localhost:9090']
+  - job_name: 'tempo'
     static_configs:
-    # change this your machine's ip, localhost won't work
-    # because localhost refers to the docker container.
-      - targets: ['172.20.239.48:2500']
-        #- targets: ['192.168.8.209:2500']
-  - job_name: loki
-    static_configs:
-      - targets: ['loki:3100']
-  - job_name: prometheus
-    static_configs:
-      - targets: ['localhost:9090']
+      - targets: ['tempo:3200']

docker/tempo.yaml

@@ -0,0 +1,48 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+      duration_slo: 5s
+      throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "tempo:4317"
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local # backend configuration to use
+    wal:
+      path: /var/tempo/wal # where to store the wal locally
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
+      generate_native_histograms: both

src/db.rs

@@ -1,7 +1,12 @@
 use metrics::counter;
-use rusqlite::Connection;
 use std::fmt::Debug;
 use serde::{Deserialize, Serialize};
+use surrealdb::{
+    engine::remote::ws::{Client, Ws},
+    opt::auth::Root,
+    sql::Thing,
+    Surreal,
+};
 use tracing::{error, instrument, trace};
 use url::Url;
 
@@ -15,12 +20,18 @@ pub struct Website {
     pub site: Url,
     /// Wether or not this link has been crawled yet
     pub crawled: bool,
+    /// 200, 404, etc
+    pub status_code: u16,
 }
 
 // manual impl to make tracing look nicer
 impl Debug for Website {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Website").field("site", &self.site).finish()
+        f.debug_struct("Website")
+            .field("host", &self.site.host())
+            .field("path", &self.site.path())
+            .field("status_code", &self.status_code)
+            .finish()
     }
 }
 
@@ -33,32 +44,25 @@ impl Website {
         };
         Self {
             crawled,
-            site
+            site,
+            status_code: 0,
         }
     }
 
-    pub fn set_crawled(&mut self) {
-        trace!("Set crawled to true");
-        self.crawled = true
-    }
-
     // Insert ever item in the vec into surreal, crawled state will be preserved as TRUE
     // if already in the database as such or incoming data is TRUE.
-    pub async fn store_all(all: Vec<Self>, db: &Connection) {
+    #[instrument(skip(db))]
+    pub async fn store_all(all: Vec<Self>, db: &Surreal<Client>) -> Vec<Thing> {
         counter!(STORE).increment(1);
         let mut things = Vec::with_capacity(all.len());
 
-        rusqlite::ParamsFromIter;
-
-        db.execute("",
-            params![]
-        );
-
+        // FIXME failes *sometimes* because "Resource Busy"
         match db
             .query(
                 "INSERT INTO website $array
                     ON DUPLICATE KEY UPDATE
                         accessed_at = time::now(),
+                        status_code = $input.status_code,
                         crawled = crawled OR $input.crawled
                     RETURN VALUE id;
                  ",
@@ -90,10 +94,32 @@ pub struct Record {
     pub id: Thing,
 }
 
-#[instrument(skip_all, name = "sqlite_connect")]
-pub async fn connect(config: &Config) -> Result<Connection, rusqlite::Error> {
-    trace!("Establishing connection to sqlite...");
+#[instrument(skip_all, name = "SurrealDB")]
+pub async fn connect(config: &Config) -> surrealdb::Result<Surreal<Client>> {
+    trace!("Establishing connection to surreal...");
     // Connect to the server
-    Connection::open("./squeelite.db")
+    let db = Surreal::new::<Ws>(&config.surreal_url).await?;
+
+    trace!("Logging in...");
+    // Signin as a namespace, database, or root user
+    db.signin(Root {
+        username: &config.surreal_username,
+        password: &config.surreal_password,
+    })
+    .await?;
+
+    // Select a specific namespace / database
+    db.use_ns(&config.surreal_ns)
+        .use_db(&config.surreal_db)
+        .await?;
+
+    let setup = include_bytes!("setup.surql");
+    let file = setup.iter().map(|c| *c as char).collect::<String>();
+
+    db.query(file)
+        .await
+        .expect("Failed to setup surreal tables.");
+
+    Ok(db)
 }
 

src/filesystem.rs

@@ -1,70 +1,66 @@
-use std::{ffi::OsStr, path::PathBuf};
+use std::{io::ErrorKind, path::PathBuf};
 
+use reqwest::header::HeaderValue;
 use tokio::fs;
-use tracing::{debug, error, instrument, trace, warn};
+use tracing::{error, trace, warn};
 use url::Url;
 
-#[instrument(skip(data))]
-pub async fn store(data: &str, url: &Url) {
+pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
     // extract data from url to save it accurately
-    let url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
+    let mut url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
 
-    // if it's a file
-    let (basepath, filename) = if url_path.extension().filter(valid_file_extension).is_some() {
-        // get everything up till the file
-        let basepath = url_path.ancestors().skip(1).take(1).collect::<PathBuf>();
-        // get the file name
-        let filename = url_path.file_name().expect("This should exist").to_string_lossy();
-        trace!("Save path: {:?} and base path: {:?}", &url_path, &basepath);
-        (basepath, filename.to_string())
+    if let Ok(header) = content_type.to_str() {
+        // text/html; charset=UTF-8; option=value
+        let ttype = if let Some((t, _)) = header.split_once(';') {
+            t
+        } else {
+            header
+        };
+
+        if let Some((ttype, subtype)) = ttype.split_once('/') {
+            trace!("Found Content-Type to be: {ttype}/{subtype} for {}", url.to_string());
+            // If the Content-Type header is "*/html" (most likely "text/html") and the path's
+            // extension is anything but html:
+            if subtype=="html" && !url_path.extension().is_some_and(|f| f=="html" || f=="htm" ) {
+                // time to slap a index.html to the end of that path there!
+                url_path = url_path.join("index.html");
+            }
+        }
     } else {
-        (url_path.clone(), "index.html".into())
-    };
+        warn!("Header: {:?} couldn't be parsed into a string!", content_type);
+    }
+    trace!("Final path for {} is: {:?}", url, url_path);
 
-    debug!("Writing at: {:?} {:?}", basepath, filename);
+    url_path
+}
 
-    // create the folders
-    if let Err(err) = fs::create_dir_all(&basepath).await {
-        error!("Dir creation: {err} {:?}", basepath);
-    } else {
-        // FIXME I don't think this handles index.html files well...
-        // TODO this should probably append .html to non-described files
-        // create the file if that was successful
-        if let Err(err) = fs::write(&basepath.join(filename), data).await {
-            error!("File creation: {err} {:?}", url_path);
+pub async fn init(filename: &PathBuf) -> Option<fs::File> {
+    let file = async || tokio::fs::OpenOptions::new()
+        .append(true)
+        .create(true)
+        .open(&filename).await;
+
+    match file().await {
+        Ok(ok) => Some(ok),
+        Err(err) => {
+            // the file/folder isn't found
+            if err.kind() == ErrorKind::NotFound {
+                if let Some(parent ) = &filename.parent() {
+                    // create the folders
+                    if let Err(err) = fs::create_dir_all(&parent).await {
+                        error!("Dir creation: {err} {:?}", filename);
+                        eprintln!("{}", err)
+                    } else if let Ok(ok) = file().await {
+                        return Some(ok);
+                    }
+                } else {
+                    error!("Couldn't get file's parents: {:?}", &filename);
+                }
+            } else {
+                error!("File open error: {err} {:?}", filename);
+            }
+            // we don't care about other errors, we can't/shouldn't fix them
+            None
         }
     }
 }
-
-fn valid_file_extension(take: &&OsStr) -> bool {
-    let los = take.to_string_lossy();
-    let all = los.split('.');
-    match all.last() {
-        Some(s) => {
-            match s.to_lowercase().as_str() {
-                "html" => true,
-                "css" => true,
-                "js" => true,
-                "ts" => true,
-                "otf" => true, // font
-
-                "png" => true,
-                "svg" => true,
-                "jpg" => true,
-                "jpeg" => true,
-                "mp4" => true,
-                "mp3" => true,
-                "webp" => true,
-
-                "pdf" => true,
-                "json" => true,
-                "xml" => true,
-                _ => {
-                    warn!("Might be forgetting a file extension: {s}");
-                    false
-                }
-            }
-        },
-        None => false,
-    }
-}

src/main.rs

@@ -1,28 +1,62 @@
 #![feature(ip_from)]
+#![warn(clippy::expect_used)]
+#![deny(clippy::unwrap_used)]
 
 extern crate html5ever;
 
 use std::{
-    collections::HashSet, fs::File, io::Read, net::{IpAddr, Ipv4Addr}
+    collections::HashSet, fs::File, io::Read, net::{IpAddr, Ipv4Addr}, sync::LazyLock, time::Instant
 };
 
+use futures_util::StreamExt;
+use opentelemetry::{global::{self, BoxedTracer}, metrics::{Counter, Meter, UpDownCounter}, trace::{Span, Tracer}};
+use opentelemetry_otlp::{Protocol, WithExportConfig};
 use db::{connect, Website};
-use metrics::{counter, gauge};
-use metrics_exporter_prometheus::PrometheusBuilder;
 use serde::Deserialize;
 use surrealdb::{engine::remote::ws::Client, Surreal};
-use tokio::task::JoinSet;
-use tracing::{debug, error, info, instrument, level_filters::LevelFilter, trace, trace_span};
+use tokio::{io::{AsyncWriteExt, BufWriter}, task::JoinSet};
+use tracing::{debug, error, info, instrument, level_filters::LevelFilter, trace, warn};
 use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry};
 
 mod db;
-mod parser;
 mod filesystem;
+mod parser;
 
-const GET_METRIC: &str = "total_gets";
-const GET_IN_FLIGHT: &str = "gets_in_flight";
-const SITES_CRAWLED: &str = "pages_crawled";
-const BEING_PROCESSED: &str = "pages_being_processed";
+// TODO prefix all of these with "crawler" or something
+
+static METER: LazyLock<Meter> = LazyLock::new(|| global::meter("Internet_Mapper"));
+static BEING_PROCESSED: LazyLock<UpDownCounter<i64>> = LazyLock::new(||
+    METER
+        .i64_up_down_counter("pages_being_processed")
+        .build()
+);
+static BEING_PARSED: LazyLock<UpDownCounter<i64>> = LazyLock::new(||
+    METER
+        .i64_up_down_counter("pages_being_parsed")
+        .build()
+);
+static BEING_STREAMED: LazyLock<UpDownCounter<i64>> = LazyLock::new(|| 
+    METER
+        .i64_up_down_counter("pages_being_streamed")
+        .build()
+);
+static GET_IN_FLIGHT: LazyLock<UpDownCounter<i64>> = LazyLock::new(|| 
+    METER
+        .i64_up_down_counter("gets_in_flight")
+        .build()
+);
+static TOTAL_BYTES_DOWN: LazyLock<Counter<u64>> = LazyLock::new(|| 
+    METER
+        .u64_counter("total_bytes_down")
+        .build()
+);
+static SITES_CRAWLED: LazyLock<Counter<u64>> = LazyLock::new(|| 
+    METER
+        .u64_counter("total_sites_crawled")
+        .build()
+);
+
+static TRACER: LazyLock<BoxedTracer> = LazyLock::new(|| global::tracer("Internet_Mapper"));
 
 #[derive(Deserialize)]
 struct Config {
@@ -33,11 +67,52 @@ struct Config {
     surreal_password: String,
 
     crawl_filter: String,
+    start_url: String,
     budget: usize,
+    batch_size: usize,
 }
 
 #[tokio::main]
 async fn main() {
+    println!("Logs and metrics are provided to the Grafana dashboard");
+
+    // Start TRACE / LOGGING / METRICS
+//    let otlp_log = opentelemetry_otlp::LogExporter::builder()
+//        .with_tonic()
+//        .with_endpoint(endpoint)
+//        .build()
+//        .unwrap();
+    //  Send metrics to Prometheus
+    let otlp_metrics = opentelemetry_otlp::MetricExporter::builder()
+        .with_http()
+        .with_protocol(Protocol::HttpBinary)
+        .with_endpoint("http://localhost:9090/api/v1/otlp/v1/metrics")
+        .build()
+        .unwrap();
+    // Send spans to Alloy (which will send them to Tempo)
+    let otlp_span = opentelemetry_otlp::SpanExporter::builder()
+        .with_tonic()
+        .with_endpoint("http://localhost:4317")
+        .build()
+        .unwrap();
+
+    let tracer_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
+        .with_simple_exporter(otlp_span)
+        .build();
+    // let logger_provider = opentelemetry_sdk::logs::SdkLoggerProvider::builder()
+    //    .with_simple_exporter(otlp_log)
+    //    .build();
+    let metrics_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()
+        .with_periodic_exporter(otlp_metrics) // default delay is 60s, turn down to like 15
+        .build();
+
+    global::set_tracer_provider(tracer_provider);
+    global::set_meter_provider(metrics_provider);
+    // How to set logger?
+
+    // End TRACE
+
+    // Start LOGGING
     let writer = std::fs::OpenOptions::new()
         .append(true)
         .create(true)
@@ -59,19 +134,10 @@ async fn main() {
     );
 
     tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber");
-
-    let builder = PrometheusBuilder::new();
-    builder
-        .with_http_listener(std::net::SocketAddr::new(
-            IpAddr::V4(Ipv4Addr::from_octets([0, 0, 0, 0])),
-            2500,
-        ))
-        .install()
-        .expect("failed to install recorder/exporter");
+    // End LOGGING
 
     info!("Starting...");
-    // Would probably take these in as parameters from a cli
-    let starting_url = "https://en.wikipedia.org/";
+
     // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
     // let crawl_filter = "en.wikipedia.org/";
     // let budget = 50;
@@ -82,6 +148,7 @@ async fn main() {
     let _ = file.read_to_string(&mut buf);
 
     let config: Config = toml::from_str(&buf).expect("Failed to parse Crawler.toml");
+    let starting_url = &config.start_url;
 
     let db = connect(&config)
         .await
@@ -95,24 +162,16 @@ async fn main() {
 
     // Kick off the whole machine - This Website object doesn't matter, it's just to allow for
     // get() to work.
-    let span = trace_span!("Pre-Loop");
-    let pre_loop_span = span.enter();
-    // Download the site
+    let mut span = TRACER.start("Pre-Loop");
     let site = Website::new(starting_url, false);
     process(site, db.clone(), reqwest.clone()).await;
+    span.end();
+    // Download the site
 
-    drop(pre_loop_span);
-
-    let span = trace_span!("Loop");
-    let span = span.enter();
+    let mut main_loop_span= TRACER.start("Main-Loop");
     while crawled < config.budget {
-        let get_num = if config.budget - crawled < 100 {
-            config.budget - crawled
-        } else {
-            100
-        };
-
-        let uncrawled = get_uncrawled_links(&db, get_num, config.crawl_filter.clone()).await;
+        let uncrawled =
+            get_uncrawled_links(&db, config.budget - crawled, config.crawl_filter.clone(), &config).await;
         if uncrawled.is_empty() {
             info!("Had more budget but finished crawling everything.");
             return;
@@ -121,22 +180,27 @@ async fn main() {
         {
             let mut futures = JoinSet::new();
             for site in uncrawled {
-                gauge!(BEING_PROCESSED).increment(1);
                 futures.spawn(process(site, db.clone(), reqwest.clone()));
-                // let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32);
-                // info!("Crawled {crawled} out of {budget} pages. ({percent})");
             }
 
-            let c = counter!(SITES_CRAWLED);
             // As futures complete runs code in while block
             while futures.join_next().await.is_some() {
-                c.increment(1);
-                gauge!(BEING_PROCESSED).decrement(1);
+                SITES_CRAWLED.add(1, &[]);
                 crawled += 1;
             }
         }
     }
-    drop(span);
+    main_loop_span.end();
+
+    if let Ok(mut ok) = db
+        .query("count(select id from website where crawled = true)")
+        .await
+    {
+        let res = ok.take::<Option<usize>>(0);
+        if let Ok(Some(n)) = res {
+            info!("Total crawled pages now equals {n}");
+        }
+    }
 
     info!("Done");
 }
@@ -148,65 +212,123 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
 
     // METRICS
     trace!("Process: {}", &site.site);
+    BEING_PROCESSED.add(1, &[]);
+    let mut process_span = TRACER.start("Process");
+
     // Build the request
     let request_builder = reqwest.get(site.site.to_string());
     
-    // METRICS
-    let g = gauge!(GET_IN_FLIGHT);
-    g.increment(1);
-
     // Send the http request (get)
+    GET_IN_FLIGHT.add(1, &[]);
     if let Ok(response) = request_builder.send().await {
+        GET_IN_FLIGHT.add(-1, &[]);
 
-        // METRICS
-        g.decrement(1);
-        counter!(GET_METRIC).increment(1);
+        let headers = response.headers();
+        let code = response.status();
 
-        // Get body from response
-        let data = response
-            .text()
-            .await
-            .expect("Failed to read http response's body!");
+        #[allow(non_snake_case)]
+        let CT = headers.get("Content-Type");
+        let ct = headers.get("content-type");
 
-        // Store document
-        filesystem::store(&data, &site.site).await;
+        let ct = match (CT,ct) {
+            (None, None) => {
+                warn!("Server did not respond with Content-Type header. Url: {} Headers: ({:?})", site.site.to_string(), headers);
+                return
+            },
+            (None, Some(a)) => a,
+            (Some(a), None) => a,
+            (Some(a), Some(_)) => a,
+        };
 
-        // Parse document and get relationships
-        let sites = parser::parse(&site, &data).await;
+        // create filepath (handles / -> /index.html)
+        let path = filesystem::as_path(&site.site, ct);
 
-        // update self in db
-        site.set_crawled();
-        Website::store_all(vec![site], &db).await;
+        // make sure that the file is good to go
+        if let Some(file) = filesystem::init(&path).await {
+            // Get body from response
+            // stream the response onto the disk
+            let mut stream = response.bytes_stream();
 
-        // De-duplicate this list
-        let prev_len = sites.len();
-        let set = sites.into_iter().fold(HashSet::new(), |mut set,item| {
-            set.insert(item);
-            set
-        });
-        let de_dupe_sites: Vec<Website> = set.into_iter().collect();
-        let diff = prev_len - de_dupe_sites.len();
-        trace!("Saved {diff} from being entered into the db by de-duping");
+            let should_parse = path.to_string_lossy().ends_with(".html");
+            let mut writer = BufWriter::new(file);
+            let mut buf: Vec<u8> = Vec::new();
 
-        // Store all the other sites so that we can link to them.
-        let _ = Website::store_all(de_dupe_sites, &db).await;
+            // Write file to disk
+            trace!("Writing at: {:?}", path);
+            BEING_STREAMED.add(1, &[]);
+            let mut stream_span = TRACER.start("Stream");
+            while let Some(data) = stream.next().await {
+                match data {
+                    Ok(data) => {
+                        TOTAL_BYTES_DOWN.add(data.len() as u64, &[]);
+                        let _ = writer.write_all(&data).await;
+                        // If we are going to parse this file later, we will save it
+                        // into memory as well as the disk.
+                        // We do this because the data here might be incomplete
+                        if should_parse {
+                            data.iter().for_each(|f| buf.push(*f));
+                        }
+                    },
+                    Err(err) => {
+                        eprintln!("{}", err)
+                    },
+                }
+            }
+            stream_span.end();
+            BEING_STREAMED.add(-1, &[]);
+            let _ = writer.flush();
 
+            // (If needed) Parse the file
+            if should_parse {
+                BEING_PARSED.add(1, &[]);
+                let mut parsing_span = TRACER.start("Parsing");
+
+                // Parse document and get relationships
+                let sites = parser::parse(&site, &buf).await;
+                // De-duplicate this list
+                let prev_len = sites.len();
+                let set = sites.into_iter().fold(HashSet::new(), |mut set, item| {
+                    set.insert(item);
+                    set
+                });
+                let de_dupe_sites: Vec<Website> = set.into_iter().collect();
+                let diff = prev_len - de_dupe_sites.len();
+                trace!("Saved {diff} from being entered into the db by de-duping");
+                // Store all the other sites so that we can link to them.
+                let _ = Website::store_all(de_dupe_sites, &db).await;
+
+                parsing_span.end();
+                BEING_PARSED.add(-1, &[]);
+            }
+
+            // update self in db
+            site.crawled = true;
+            site.status_code = code.as_u16();
+            Website::store_all(vec![site.clone()], &db).await;
+        } else {
+            error!("File failed to cooperate: {:?}", path);
+        }
     } else {
         error!("Failed to get: {}", &site.site);
     }
+    
+    process_span.end();
+    BEING_PROCESSED.add(-1, &[]);
 }
 
 /// Returns uncrawled links
-#[instrument(skip(db))]
+#[instrument(skip(db, config))]
 async fn get_uncrawled_links(
     db: &Surreal<Client>,
     mut count: usize,
     filter: String,
+    config: &Config,
 ) -> Vec<Website> {
-    if count > 100 {
-        count = 100
+    if count > config.batch_size {
+        count = config.batch_size;
     }
-    debug!("Getting uncrawled links");
+
+    debug!("Getting {} uncrawled links from DB", count);
 
     let mut response = db
         .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;")
@@ -218,4 +340,3 @@ async fn get_uncrawled_links(
         .take(0)
         .expect("Returned websites couldn't be parsed")
 }
-

src/parser.rs

@@ -1,5 +1,4 @@
 use std::default::Default;
-use std::str::FromStr;
 
 use html5ever::tokenizer::{BufferQueue, TokenizerResult};
 use html5ever::tokenizer::{StartTag, TagToken};
@@ -40,7 +39,7 @@ impl TokenSink for Website {
                                     if let Some(mut parsed) = url {
                                         parsed.set_query(None);
                                         parsed.set_fragment(None);
-                                        debug!("Final cleaned URL: `{}`", parsed.to_string());
+                                        trace!("Final cleaned URL: `{}`", parsed.to_string());
                                         let web = Website::new(&parsed.to_string(), false);
                                         links.push(web);
                                     }
@@ -63,29 +62,34 @@ impl TokenSink for Website {
 
 #[instrument(skip_all)]
 /// Parses the passed site and returns all the sites it links to.
-pub async fn parse(site: &Website, data: &str) -> Vec<Website> {
+pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
+    trace!("Parsing {}", site.site.to_string());
     // prep work
     let mut other_sites: Vec<Website> = Vec::new();
 
     // change data into something that can be tokenized
-    let chunk = Tendril::from_str(data).expect("Failed to parse string into Tendril!");
-    // create buffer of tokens and push our input into it
-    let token_buffer = BufferQueue::default();
-    token_buffer.push_back(
-        chunk
-            .try_reinterpret::<fmt::UTF8>()
-            .expect("Failed to reinterpret chunk!"),
-    );
-    // create the tokenizer
-    let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
+    let s: Result<Tendril<fmt::UTF8>, ()> = Tendril::try_from_byte_slice(data);
+    if let Ok(chunk) = s {
+        // create buffer of tokens and push our input into it
+        let token_buffer = BufferQueue::default();
+        token_buffer.push_back(
+            chunk
+                .try_reinterpret::<fmt::UTF8>()
+                .expect("Failed to reinterpret chunk!"),
+        );
+        // create the tokenizer
+        let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
 
-    // go thru buffer
-    while let TokenizerResult::Script(mut sites) = tokenizer.feed(&token_buffer) {
-        other_sites.append(&mut sites);
-        // other_sites.push(sites);
+        // go thru buffer
+        while let TokenizerResult::Script(mut sites) = tokenizer.feed(&token_buffer) {
+            other_sites.append(&mut sites);
+            // other_sites.push(sites);
+        }
+        assert!(token_buffer.is_empty());
+        tokenizer.end();
+    } else {
+        warn!("Tendril failed to parse on: {}", site.site.to_string());
     }
-    assert!(token_buffer.is_empty());
-    tokenizer.end();
 
     other_sites
 }
@@ -107,7 +111,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
                 match Url::parse(&format!("{scheme}://{}", link)) {
                     Ok(url) => Some(url),
                     Err(err) => {
-                        error!("Failed parsing realative scheme url: {}", err);
+                        error!("Failed parsing relative scheme url: {}", err);
                         None
                     }
                 }
@@ -117,19 +121,21 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
                 match e {
                     url::ParseError::RelativeUrlWithoutBase => {
                         // Is: scheme://host:port
-                        let origin = parent.origin().ascii_serialization();
+                        let mut origin = parent.origin().ascii_serialization();
+                        if !origin.ends_with('/') && !link.starts_with('/') {
+                            origin += "/";
+                        }
                         let url = origin.clone() + link;
 
-                        trace!("Built `{url}` from `{origin} + {}`", link.to_string());
-
                         if let Ok(url) = Url::parse(&url) {
-                            trace!("Saved relative url `{}` AS: `{}`", link, url);
+                            trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
                             Some(url)
                         } else {
                             error!(
-                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`",
+                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`. Failed url was: {}",
                                 link,
-                                parent.to_string()
+                                parent.to_string(),
+                                url
                             );
                             None
                         }