it works now

.vscode/launch.json

@@ -9,7 +9,7 @@
             "request": "launch",
             "name": "Debug executable 'surreal_spider'",
             "env": {
-                "RUST_LOG": "surreal_spider=trace",
+                "RUST_LOG": "surreal_spider=trace,reqwest=trace",
             },
             "cargo": {
                 "args": [

compose.yml

@@ -3,6 +3,8 @@ services:
     image: surrealdb/surrealdb:latest-dev
     ports:
     - 8000:8000
+    volumes:
+      - ./.surrealdb/:/mydata
     command:
       - start
       - --log
@@ -11,3 +13,4 @@ services:
       - root
       - --pass
       - root
+      - rocksdb:/mydata/database.db

src/db.rs

@@ -3,126 +3,117 @@ use surrealdb::{
     engine::remote::ws::{Client, Ws},
     opt::auth::Root,
     sql::Thing,
-    Surreal,
+    Response, Surreal,
 };
-use tracing::{debug, error, info, instrument};
+use tracing::{debug, error, info, instrument, trace};
 use url::Url;
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Website {
     /// The url that this data is found at
     site: Url,
-    /// The url as defined in the <a> tag
-    href: Url,
     /// Wether or not this link has been crawled yet
     crawled: bool,
-    /// Wether or not the href was doctored
+    #[serde(skip_serializing)]
-    doctored_href: bool,
+    id: Option<Thing>,
-    original_href: Option<String>,
 }
 
 impl Website {
     /// Creates a blank site (assumes that url param is site's root)
-    pub fn new(url: &str, href: &str, crawled: bool) -> Self {
+    pub fn new(url: &str, crawled: bool) -> Self {
-        let mut new = Self::from(url);
+        let site = match Url::parse(url) {
-        new.crawled = crawled;
+            Ok(a) => a,
-        new.original_href = Some(href.to_string());
+            Err(_) => todo!(),
-        new.href =
+        };
-            match Url::parse(href) {
+        Self {
-                Ok(e) => e,
+            id: None,
-                Err(e) => {
+            crawled,
-                    match e {
+            site,
-                        url::ParseError::RelativeUrlWithoutBase => {
+        }
-                            // Try to combine the scheme_host and href to get a useable domain
-                            new.doctored_href = true;
-
-                            let url = if !url.ends_with('/') && !href.starts_with('/') {
-                                format!("{url}/{href}")
-                            } else {
-                                format!("{url}{href}")
-                            };
-
-                            // paste the domain onto the begining of the href
-                            Url::parse(&url).map_or_else(|err| {
-                                debug!("Parsing {url} with {href}");
-                                error!("{err} Failed to parse href into url on second try. Aborting");
-                                panic!("See error logs for more info.");
-                            }, |ok| ok)
-                        }
-                        _ => {
-                            error!("{e}");
-                            panic!("See error logs for more info.");
-                        }
-                    }
-                }
-            };
-        new
     }
-    pub fn crawled(&mut self) {
+
+    pub fn set_crawled(&mut self) {
+        trace!("Set crawled to true");
         self.crawled = true
     }
-    pub fn href_str(&self) -> &str {
+
-        self.href.as_str()
+    pub fn crawled(&mut self) -> &mut bool {
+        &mut self.crawled
     }
-    pub fn site(&self) -> String {
+
-        self.site.to_string()
+    pub fn mut_url(&mut self) -> &mut Url {
+        &mut self.site
     }
+
     pub fn domain_str(&self) -> &str {
         self.site.as_str()
     }
+
+    #[instrument(skip_all)]
+    pub async fn links_to(&self, other: &Self, db: &Surreal<Client>) {
+        let from = self.site.to_string();
+        let to = other.site.to_string();
+        trace!("Linking {from} to {to}");
+
+        match db
+            .query("RELATE (SELECT id FROM website WHERE site = $in) -> links_to -> (SELECT id FROM website WHERE site = $out)")
+            .bind(("in", from))
+            .bind(("out", to))
+            .await
+        {
+            Ok(e) => {
+                let _: Response = e;
+            },
+            Err(e) => {
+                error!("{}", e.to_string());
+            },
+        }
+    }
+
     #[instrument(skip_all)]
     pub async fn store(&mut self, db: &Surreal<Client>) {
-        // is root record?
+        // check if it's been gone thru before
-        if self.href.path() == "/" {
+        let mut response = db
-            // Upsert is create or update
+            .query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1")
-            // Whereas Update is just update
+            .bind(("site", self.site.to_string()))
-            let record = ("website", &self.href.to_string());
+            .await
+            .unwrap();
 
-            let crawled = if let Some(old) = db.select(record).await.unwrap() {
+        if let Some(old) = response.take::<Option<Website>>(0).unwrap() {
-                let old: Website = old; // infer type
+            // site exists already
-                old.crawled
+            if let Some(id) = old.id {
-            } else {false};
+                let mut new = self.clone();
+                new.crawled = old.crawled | new.crawled;
 
-            if !self.crawled {self.crawled = crawled};
+                match db.upsert((id.tb, id.id.to_string())).content(new).await {
-
+                    Ok(e) => {
-            match db.upsert(record).content(self.clone()).await {
+                        if let Some(a) = &e {
-                Ok(e) => {
+                            let _: &Record = a;
-                    if let Some(a) = &e {
+                        }
-                        let _: &Record = a;
                     }
-                }
+                    Err(e) => {
-                Err(e) => {
+                        error!("{}", e);
-                    error!("{}", e);
+                    }
-                },
+                };
-            };
+            }
         } else {
-            let _: Option<Record> = match db.create("website").content(self.clone()).await {
+            // sites hasn't existed yet
+            match db.create("website").content(self.clone()).await {
                 Ok(e) => {
+                    let _: Option<Record> = e;
                     if let Some(a) = &e {
                         let _: &Record = a;
                     }
-                    e
                 }
-                Err(_) => todo!(),
+                Err(a) => error!("{:?}", a),
             };
         }
     }
 }
 
-impl From<&str> for Website {
+impl ToString for Website {
-    /// site == href, crawled = false
+    fn to_string(&self) -> String {
-    fn from(value: &str) -> Self {
+        self.site.to_string()
-        let site = match Url::parse(value) {
-            Ok(a) => a,
-            Err(_) => todo!(),
-        };
-        Self {
-            href: site.clone(),
-            crawled: false,
-            site,
-            doctored_href: false,
-            original_href: None,
-        }
     }
 }
 
@@ -149,7 +140,7 @@ pub async fn connect() -> surrealdb::Result<Surreal<Client>> {
     .await?;
 
     // Select a specific namespace / database
-    db.use_ns("test").use_db("test").await?;
+    db.use_ns("test").use_db("time").await?;
 
     Ok(db)
 }

src/main.rs

@@ -6,7 +6,7 @@ use db::{connect, Website};
 use html5ever::{parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, ParseOpts};
 use rcdom::{Node, RcDom};
 use surrealdb::{engine::remote::ws::Client, Surreal};
-use tracing::{debug, info, instrument};
+use tracing::{debug, info, instrument, trace, trace_span, warn};
 use tracing_subscriber::EnvFilter;
 
 mod db;
@@ -29,24 +29,37 @@ async fn main() {
 
     // Kick off the whole machine - This Website object doesn't matter, it's just to allow for
     // get() to work.
-    let mut site = Website::from(url);
+    let span = trace_span!("Pre-Loop");
+    let pre_loop_span = span.enter();
+    let mut site = Website::new(&url, false);
     let dom = get(&mut site, &db).await.expect("Inital page returned None.");
     crawled += 1;
     walk(&dom, &db, &site).await;
+    drop(pre_loop_span);
 
+    let span = trace_span!("Loop");
+    let span = span.enter();
+    // Can go upto 49 above budget because the reterival function gets 50 no matter what
     while crawled < budget {
-        let uncrawled = get_uncrawled_links(&db).await;
+        let uncrawled = get_uncrawled_links(&db, 100).await;
         debug!("Crawling {} pages...", uncrawled.len());
 
+        let span = trace_span!("Crawling");
+        let _ = span.enter();
+
         for mut site in uncrawled {
             if let Some(dom) = get(&mut site, &db).await {
+                trace!("pre-walk checkpoint");
                 walk(&dom, &db, &site).await;
                 crawled += 1;
                 let percent = format!("{:.2}%", (crawled as f32/budget as f32) * 100f32);
                 info!("Crawled {crawled} out of {budget} pages. ({percent})");
+            } else {
+                warn!("Failed to get {}", site.to_string());
             }
         }
     }
+    drop(span);
 
     info!("Done");
 }
@@ -54,7 +67,8 @@ async fn main() {
 #[instrument(skip_all)]
 /// A quick helper function for downloading a url
 async fn get(site: &mut Website, db: &Surreal<Client>) -> Option<Rc<Node>> {
-    if let Ok(response) = reqwest::get(site.href_str()).await {
+    trace!("Get: {}", site.to_string());
+    if let Ok(response) = reqwest::get(site.to_string()).await {
         let data = response.text().await.unwrap();
         let opts = ParseOpts {
             tree_builder: TreeBuilderOpts {
@@ -69,29 +83,42 @@ async fn get(site: &mut Website, db: &Surreal<Client>) -> Option<Rc<Node>> {
             .read_from(&mut data.as_bytes())
             .unwrap();
 
-        site.crawled();
+        site.set_crawled();
         site.store(db).await;
+        trace!("Got: {}", site.to_string());
         return Some(dom.document);
     }
+    trace!("Failed to get: {}", site.to_string());
     None
 }
 
 /// Walks the givin site, placing it's findings in the database
-async fn walk(node: &rcdom::Handle, db: &Surreal<Client> , site_name: &Website) {
+async fn walk(node: &rcdom::Handle, db: &Surreal<Client> , site: &Website) {
-    // Insert Or Update
+    let span = trace_span!("Walk");
-    // create_root(site_name, db).await;
+    let span = span.enter();
 
     match &node.data {
         rcdom::NodeData::Element { name, attrs, template_contents, mathml_annotation_xml_integration_point } => {
             for attr in attrs.borrow().clone() {
                 if name.local.to_string() == "a" {
                     if attr.value.starts_with("mailto") {
+                        trace!("Is mailto");
                         // mailto link, lol
                         let _created: Option<db::Record> = db.create("email").content(db::Email {
                             email: attr.value.to_string()
                         }).await.unwrap();
                     } else {
-                        let mut web = Website::new(&site_name.site(), &attr.value, false);
+                        let mut web = site.clone();
+                        let url = web.mut_url();
+
+                        // TODO remove #xyz
+                        let joined = url.join(&attr.value).unwrap();
+                        *url = joined;
+
+                        let crawled = web.crawled();
+                        *crawled = false;
+
+                        site.links_to(&web, &db).await;
                         web.store(db).await;
                     }
                 }
@@ -99,15 +126,23 @@ async fn walk(node: &rcdom::Handle, db: &Surreal<Client> , site_name: &Website)
         },
         _ => {},
     };
-
+    drop(span);
     for child in node.children.borrow().iter() {
-        Box::pin(walk(child, db, site_name)).await; 
+        Box::pin(walk(child, db, site)).await; 
     }
 }
 
-/// Returns 0-50 uncrawled links (LIMIT = 50)
+/// Returns uncrawled links
-async fn get_uncrawled_links(db: &Surreal<Client>) -> Vec<Website> {
+async fn get_uncrawled_links(db: &Surreal<Client>, mut count: usize) -> Vec<Website> {
-    let mut response = db.query("SELECT * FROM website WHERE crawled = false LIMIT 50").await.expect("Hard-coded query failed..?");
+    if count > 100 {
+        count = 100
+    }
+
+    let mut response = db
+        .query("SELECT * FROM website WHERE crawled = false LIMIT $count")
+        .bind(("count", count))
+        .await
+        .expect("Hard-coded query failed..?");
     response.take(0).expect("Returned websites couldn't be parsed")
 }