diff --git a/src/main.rs b/src/main.rs index 8cc9675..11cb136 100644 --- a/src/main.rs +++ b/src/main.rs @@ -125,6 +125,16 @@ async fn main() { for _ in 0..CONFIG.batch_size { futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone())); } + + while let Some(_) = futures.join_next().await { + // Budget - Threads - This thread (1) + // Would roughly be the acceptable amount at which a thread should exit + if *(crawled.read().await) < CONFIG.budget - CONFIG.batch_size -1 { + warn!("Thread terminated early, restarting"); + futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone())); + } + } + futures.join_all().await; // main_loop_span.end();