Oliver/internet_mapper
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: Oliver:main
Branches Tags
Oliver:main Oliver:status_codes Oliver:foss_storage
...
pull from: Oliver:status_codes
Branches Tags
Oliver:main Oliver:status_codes Oliver:foss_storage

1 Commits
main ... status_cod

Author SHA1 Message Date
table
f7a3ca8fd7 changes 2025-08-04 23:02:05 +00:00
3 changed files with 6 additions and 8 deletions
Expand all files Collapse all files

10
Crawler.toml
View File

@@ -6,9 +6,7 @@ surreal_ns = "test"
surreal_db = "v1.21.1" surreal_db = "v1.21.1"
# Crawler config # Crawler config
# crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/" crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI"
crawl_filter = "https://oliveratkinson.net" start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI"
# start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI/Imagery/2023_NAIP/UTM_County_Mosaics/" budget = 10000
start_url = "https://oliveratkinson.net" batch_size = 50
budget = 1000
batch_size = 500

2
docker/prometheus.yaml
View File

@@ -7,7 +7,7 @@ scrape_configs:
static_configs: static_configs:
# change this your machine's ip, localhost won't work # change this your machine's ip, localhost won't work
# because localhost refers to the docker container. # because localhost refers to the docker container.
- targets: ['172.20.239.48:2500'] - targets: ['192.168.1.200:2500']
#- targets: ['192.168.8.209:2500'] #- targets: ['192.168.8.209:2500']
- job_name: loki - job_name: loki
static_configs: static_configs:

2
src/main.rs
View File

@@ -275,7 +275,7 @@ async fn get_uncrawled_links(
count = config.batch_size; count = config.batch_size;
} }
debug!("Getting uncrawled links"); debug!("Getting {} uncrawled links", count);
let mut response = db let mut response = db
.query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;") .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;")