Merge pull request 'status_codes' (#8) from status_codes into main

Reviewed-on: #8

Crawler.toml

@@ -6,7 +6,9 @@ surreal_ns = "test"
 surreal_db = "v1.21.1"
 
 # Crawler config
-crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI" 
+# crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/" 
-start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI"
+crawl_filter = "https://oliveratkinson.net" 
-budget = 10000
+# start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/"
-batch_size = 50
+start_url = "https://oliveratkinson.net"
+budget = 1000
+batch_size = 500

docker/prometheus.yaml

@@ -7,7 +7,7 @@ scrape_configs:
     static_configs:
     # change this your machine's ip, localhost won't work
     # because localhost refers to the docker container.
-      - targets: ['192.168.1.200:2500']
+      - targets: ['172.20.239.48:2500']
         #- targets: ['192.168.8.209:2500']
   - job_name: loki
     static_configs:

src/main.rs

@@ -275,7 +275,7 @@ async fn get_uncrawled_links(
         count = config.batch_size;
     }
 
-    debug!("Getting {} uncrawled links", count);
+    debug!("Getting uncrawled links");
 
     let mut response = db
         .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;")