From 9bfa8f91084ae31454c0f231d9ea243169d1b153 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Tue, 15 Apr 2025 13:38:28 -0600 Subject: [PATCH] batch_size --- Crawler.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Crawler.toml b/Crawler.toml index 0048238..e57d08b 100644 --- a/Crawler.toml +++ b/Crawler.toml @@ -3,9 +3,12 @@ surreal_url = "localhost:8000" surreal_username = "root" surreal_password = "root" surreal_ns = "test" -surreal_db = "v1.19.5" +surreal_db = "v1.20.3" # Crawler config -crawl_filter = "en.wikipedia.org" -start_url = "https://en.wikipedia.org" +crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/" +# crawl_filter = "https://oliveratkinson.net" +start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/" +# start_url = "https://oliveratkinson.net" budget = 100 +batch_size = 5