src/db.rs

@@ -102,7 +102,7 @@ pub async fn get_next(db: &Surreal<Client>, config: &Config) -> Option<Website>
             Ok(ok) => ok,
             Err(_err) => {
                 // basically just CSMA/CA
-                let delay = rand::random_range(10..10_000);
+                let delay = rand::random_range(10..500);
                 sleep(Duration::from_millis(delay)).await;
                 fails += 1;
                 // Don't get stuck here forever, failing...

src/main.rs

@@ -125,16 +125,6 @@ async fn main() {
     for _ in 0..CONFIG.batch_size {
         futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
     }
-
-    while let Some(_) = futures.join_next().await {
-        // Budget - Threads - This thread (1)
-        // Would roughly be the acceptable amount at which a thread should exit
-        if *(crawled.read().await) < CONFIG.budget - CONFIG.batch_size -1 {
-            warn!("Thread terminated early, restarting");
-            futures.spawn(process_single_thread(&CONFIG, db.clone(), reqwest.clone(), crawled.clone()));
-        }
-    }
-
     futures.join_all().await;
     // main_loop_span.end();