From 95b8af0356081d198e7626a46fce26ffb4faf438 Mon Sep 17 00:00:00 2001
From: Oliver <oliveratkinsoneng@gmail.com>
Date: Fri, 8 Aug 2025 23:35:01 -0600
Subject: [PATCH] Restart threads that prematurely ended

---
 src/main.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/main.rs b/src/main.rs
index 8cc9675..11cb136 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -125,6 +125,16 @@ async fn main() {
     for _ in 0..CONFIG.batch_size {
         futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
     }
+
+    while let Some(_) = futures.join_next().await {
+        // Budget - Threads - This thread (1)
+        // Would roughly be the acceptable amount at which a thread should exit
+        if *(crawled.read().await) < CONFIG.budget - CONFIG.batch_size -1 {
+            warn!("Thread terminated early, restarting");
+            futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
+        }
+    }
+
     futures.join_all().await;
     // main_loop_span.end();