Restart threads that prematurely ended

src/db.rs

@@ -102,7 +102,7 @@ pub async fn get_next(db: &Surreal<Client>, config: &Config) -> Option<Website>
             Ok(ok) => ok,
             Err(_err) => {
                 // basically just CSMA/CA
-                let delay = rand::random_range(10..500);
+                let delay = rand::random_range(10..10_000);
                 sleep(Duration::from_millis(delay)).await;
                 fails += 1;
                 // Don't get stuck here forever, failing...

src/main.rs

@@ -125,6 +125,16 @@ async fn main() {
     for _ in 0..CONFIG.batch_size {
         futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
     }
+
+    while let Some(_) = futures.join_next().await {
+        // Budget - Threads - This thread (1)
+        // Would roughly be the acceptable amount at which a thread should exit
+        if *(crawled.read().await) < CONFIG.budget - CONFIG.batch_size -1 {
+            warn!("Thread terminated early, restarting");
+            futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
+        }
+    }
+
     futures.join_all().await;
     // main_loop_span.end();