Restart threads that prematurely ended

src/main.rs

@@ -125,6 +125,16 @@ async fn main() {
     for _ in 0..CONFIG.batch_size {
         futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
     }
+
+    while let Some(_) = futures.join_next().await {
+        // Budget - Threads - This thread (1)
+        // Would roughly be the acceptable amount at which a thread should exit
+        if *(crawled.read().await) < CONFIG.budget - CONFIG.batch_size -1 {
+            warn!("Thread terminated early, restarting");
+            futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
+        }
+    }
+
     futures.join_all().await;
     // main_loop_span.end();