src/db.rs

@@ -98,7 +98,7 @@ impl Website {
             .await
             .expect("Failed to check surreal for duplicates!");
 
-        if let Some(old) = response.take::<Option<Website>>(0).expect("Failed to read response from surreal for duplicates.") {
+        if let Some(old) = response.take::<Option<Website>>(0).expect("Failed to read reponse from surreal for duplicates.") {
             // site exists already
             if let Some(id) = old.id {
                 // make sure to preserve the "crawled status"
@@ -119,7 +119,7 @@ impl Website {
                                 match error {
                                     Db::QueryCancelled => todo!(),
                                     Db::QueryNotExecuted => todo!(),
-                                    Db::QueryNotExecutedDetail { message: _ } => todo!(),
+                                    Db::QueryNotExecutedDetail { message } => todo!(),
                                    _=>{},
                                 }
                             },

src/parser.rs

@@ -71,7 +71,7 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
     site.store(db).await;
 
     // prep work
-    let mut other_sites: Vec<Website> = Vec::new();
+    let mut other_sites = Vec::new();
     { // using blocks to prevent compiler's async worries
         let _t = Timer::start("Parsed page");
 
@@ -84,9 +84,8 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
         let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
 
         // go thru buffer
-        while let TokenizerResult::Script(mut sites) = tokenizer.feed(&mut token_buffer) {
-            other_sites.append(&mut sites);
-            // other_sites.push(sites);
+        while let TokenizerResult::Script(sites) = tokenizer.feed(&mut token_buffer) {
+            other_sites.push(sites);
         }
 
         assert!(token_buffer.is_empty());
@@ -94,12 +93,16 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
     }
 
     {
-        let mut links_to = Vec::with_capacity(other_sites.len());
+        let mut links_to = Vec::new();
 
+        // this is a 2d vec accidentally
         for a in other_sites {
-            let other = a.store(db).await;
-            if let Some(o) = other {
-                links_to.push(o);
+            for b in a {
+                // TODO this can become a JoinSet later
+                let other = b.store(db).await;
+                if let Some(o) = other {
+                    links_to.push(o);
+                }
             }
         }