src/parser.rs

@@ -71,7 +71,7 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
     site.store(db).await;
 
     // prep work
-    let mut other_sites = Vec::new();
+    let mut other_sites: Vec<Website> = Vec::new();
     { // using blocks to prevent compiler's async worries
         let _t = Timer::start("Parsed page");
 
@@ -84,8 +84,9 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
         let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
 
         // go thru buffer
-        while let TokenizerResult::Script(sites) = tokenizer.feed(&mut token_buffer) {
-            other_sites.push(sites);
+        while let TokenizerResult::Script(mut sites) = tokenizer.feed(&mut token_buffer) {
+            other_sites.append(&mut sites);
+            // other_sites.push(sites);
         }
 
         assert!(token_buffer.is_empty());
@@ -93,16 +94,12 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
     }
 
     {
-        let mut links_to = Vec::new();
+        let mut links_to = Vec::with_capacity(other_sites.len());
 
-        // this is a 2d vec accidentally
         for a in other_sites {
-            for b in a {
-                // TODO this can become a JoinSet later
-                let other = b.store(db).await;
-                if let Some(o) = other {
-                    links_to.push(o);
-                }
+            let other = a.store(db).await;
+            if let Some(o) = other {
+                links_to.push(o);
             }
         }