Oliver/internet_mapper
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

closes #11

Browse Source
This commit is contained in:
Oliver 2025-08-08 23:39:44 -06:00
parent 95b8af0356
commit 5ade5e36df
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main.rs
View File

@ -26,6 +26,11 @@ mod filesystem;
mod parser; mod parser;
static METER: LazyLock<Meter> = LazyLock::new(|| global::meter("Internet_Mapper")); static METER: LazyLock<Meter> = LazyLock::new(|| global::meter("Internet_Mapper"));
static BATCH_SIZE: LazyLock<Counter<u64>> = LazyLock::new(||
METER
.u64_counter("crawler_batch_size")
.build()
);
static BEING_PROCESSED: LazyLock<UpDownCounter<i64>> = LazyLock::new(|| static BEING_PROCESSED: LazyLock<UpDownCounter<i64>> = LazyLock::new(||
METER METER
.i64_up_down_counter("crawler_pages_being_processed") .i64_up_down_counter("crawler_pages_being_processed")
@ -96,6 +101,8 @@ async fn main() {
global::set_tracer_provider(load_tracing(&CONFIG)); global::set_tracer_provider(load_tracing(&CONFIG));
global::set_meter_provider(load_metrics(&CONFIG)); global::set_meter_provider(load_metrics(&CONFIG));
BATCH_SIZE.add(CONFIG.batch_size as u64, &[]);
// When getting uncrawled pages, name must contain this variable. "" will effectively get ignored. // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
// let crawl_filter = "en.wikipedia.org/"; // let crawl_filter = "en.wikipedia.org/";
// let budget = 50; // let budget = 50;