Oliver/internet_mapper
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

Restart threads that prematurely ended

Browse Source
This commit is contained in:
Oliver 2025-08-08 23:35:01 -06:00
parent ad8d7c606d
commit 95b8af0356
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main.rs
View File

@ -125,6 +125,16 @@ async fn main() {
for _ in 0..CONFIG.batch_size { for _ in 0..CONFIG.batch_size {
futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone())); futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
} }
while let Some(_) = futures.join_next().await {
// Budget - Threads - This thread (1)
// Would roughly be the acceptable amount at which a thread should exit
if *(crawled.read().await) < CONFIG.budget - CONFIG.batch_size -1 {
warn!("Thread terminated early, restarting");
futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
}
}
futures.join_all().await; futures.join_all().await;
// main_loop_span.end(); // main_loop_span.end();