diff --git a/Cargo.lock b/Cargo.lock index 8c6a73f..dd31000 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -398,6 +398,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-lc-rs" +version = "1.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -453,6 +476,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.11.0", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.100", + "which", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -614,6 +660,8 @@ version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -674,6 +722,15 @@ dependencies = [ "unicode-security", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -738,6 +795,26 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -769,6 +846,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1002,6 +1089,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "earcutr" version = "0.4.3" @@ -1166,6 +1259,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fst" version = "0.4.7" @@ -1388,6 +1487,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "h2" version = "0.4.8" @@ -1614,6 +1719,7 @@ dependencies = [ "hyper", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1924,6 +2030,15 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -1986,6 +2101,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lexicmp" version = "0.1.0" @@ -2001,6 +2122,16 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.11" @@ -2029,6 +2160,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.9.3" @@ -2149,6 +2286,53 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "metrics" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7deb012b3b2767169ff203fadb4c6b0b82b947512e5eb9e0b78c2e186ad9e3" +dependencies = [ + "ahash 0.8.11", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" +dependencies = [ + "base64 0.22.1", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "indexmap 2.8.0", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd4884b1dd24f7d6628274a2f5ae22465c337c5ba065ec9b6edccddf8acc673" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.2", + "metrics", + "quanta", + "rand 0.8.5", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "miette" version = "5.10.0" @@ -2188,6 +2372,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minio" version = "0.2.0-alpha" @@ -2295,7 +2485,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -2353,6 +2543,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntapi" version = "0.4.1" @@ -2757,6 +2957,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "prettyplease" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +dependencies = [ + "proc-macro2", + "syn 2.0.100", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -2810,6 +3020,21 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quanta" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick_cache" version = "0.5.2" @@ -2833,7 +3058,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "socket2", "thiserror 2.0.12", @@ -2852,7 +3077,7 @@ dependencies = [ "getrandom 0.3.2", "rand 0.9.0", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", @@ -2968,6 +3193,24 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" +dependencies = [ + "bitflags", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -3296,6 +3539,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -3320,6 +3569,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.0.3" @@ -3329,7 +3591,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.9.3", "windows-sys 0.59.0", ] @@ -3339,6 +3601,7 @@ version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -3348,6 +3611,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -3372,6 +3647,7 @@ version = "0.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3447,7 +3723,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -3520,6 +3809,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3632,6 +3930,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +[[package]] +name = "sketches-ddsketch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" + [[package]] name = "slab" version = "0.4.9" @@ -3814,11 +4118,14 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "html5ever 0.29.1", + "metrics", + "metrics-exporter-prometheus", "minio", "reqwest", "serde", "surrealdb", "tokio", + "toml", "tracing", "tracing-subscriber", "url", @@ -4015,7 +4322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4044,7 +4351,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix", + "rustix 1.0.3", "windows-sys 0.59.0", ] @@ -4277,11 +4584,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -4290,6 +4612,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap 2.8.0", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] @@ -4750,6 +5074,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 8499bea..6a1d88e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,12 +6,15 @@ edition = "2021" [dependencies] base64 = "0.22.1" html5ever = "0.29" +metrics = "0.24.1" +metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]} # minio = "0.1.0" minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"} reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } surrealdb = "2.2" tokio = { version="1.41.0", features = ["full"] } +toml = "0.8.20" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time", "json"] } url = { version = "2.5", features = ["serde"] } diff --git a/Crawler.toml b/Crawler.toml new file mode 100644 index 0000000..29625a8 --- /dev/null +++ b/Crawler.toml @@ -0,0 +1,16 @@ +# Surreal config +surreal_url = "localhost:8000" +surreal_username = "root" +surreal_password = "root" +surreal_ns = "test" +surreal_db = "v1.12" + +# Minio config +s3_bucket = "v1.12" +s3_url = "http://localhost:9000" +s3_access_key = "jLDPKGuu513VENc8kJwX" +s3_secret_key = "4T1nymEzsGYOlKSAb1WX7V3stnQn9a5ZoTQjDfcL" + +# Crawler config +crawl_filter = "en.wikipedia.com" +budget = 200 \ No newline at end of file diff --git a/docker/compose.yml b/docker/compose.yml index 48c1a90..1f3faf3 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -50,6 +50,17 @@ services: volumes: - ./loki.yaml:/etc/loki/local-config.yaml + # Metrics collector + prometheus: + image: prom/prometheus:latest + expose: + - 9090 + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + # persist data + - prometheus_storage:/prometheus + command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml + # Everything viewer grafana: image: grafana/grafana:latest @@ -63,8 +74,9 @@ services: - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor ports: - 3000:3000 - + volumes: + prometheus_storage: grafana_storage: alloy_storage: surrealdb_storage: diff --git a/docker/grafana.yaml b/docker/grafana.yaml index be79737..ac68607 100644 --- a/docker/grafana.yaml +++ b/docker/grafana.yaml @@ -10,3 +10,15 @@ datasources: isDefault: true version: 1 editable: false +- name: Prometheus + type: prometheus + uid: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: false + version: 1 + editable: false + jsonData: + httpMethod: GET diff --git a/docker/prometheus.yaml b/docker/prometheus.yaml new file mode 100644 index 0000000..b8f5992 --- /dev/null +++ b/docker/prometheus.yaml @@ -0,0 +1,16 @@ +global: + scrape_interval: 5s + query_log_file: /etc/prometheus/query.log + +scrape_configs: + - job_name: crawler + static_configs: + # change this your machine's ip, localhost won't work + # because localhost refers to the docker container. + - targets: ['192.168.8.209:2500'] + - job_name: loki + static_configs: + - targets: ['loki:3100'] + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/src/db.rs b/src/db.rs index 9b14d89..bb0edac 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,4 +1,5 @@ use std::fmt::Debug; +use metrics::counter; use serde::{Deserialize, Serialize}; use surrealdb::{ engine::remote::ws::{Client, Ws}, error::Db, opt::auth::Root, sql::Thing, Response, Surreal @@ -8,6 +9,10 @@ use url::Url; use crate::{Config, Timer}; +const ROUND_TRIP_METRIC: &'static str = "surql_trips"; +const STORE: &'static str = "surql_store_calls"; +const LINK: &'static str = "surql_link_calls"; + #[derive(Serialize, Deserialize, Clone)] pub struct Website { /// The url that this data is found at @@ -47,6 +52,7 @@ impl Website { #[instrument(skip_all)] pub async fn links_to(&self, other: Vec, db: &Surreal) { + let len = other.len(); if len == 0 {return} @@ -57,6 +63,8 @@ impl Website { let timer = Timer::start(&msg); // prevent the timer from being dropped instantly. let _ = timer; + counter!(ROUND_TRIP_METRIC).increment(1); + counter!(LINK).increment(1); match db .query("COUNT(RELATE (SELECT id FROM website WHERE site = $in) -> links_to -> $out)") .bind(("in", from)) @@ -89,8 +97,12 @@ impl Website { #[instrument(name = "surql_store", skip_all)] pub async fn store(&self, db: &Surreal) -> Option { + counter!(STORE).increment(1); + let counter = counter!(ROUND_TRIP_METRIC); let t = Timer::start("Stored link"); let _ = t; + counter.increment(1); + // check if it's been gone thru before let mut response = db .query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1") @@ -105,6 +117,7 @@ impl Website { let mut new = self.clone(); new.crawled = old.crawled | new.crawled; + counter.increment(1); // update the record match db.upsert((id.tb, id.id.to_string())).content(new).await { Ok(e) => { @@ -130,6 +143,7 @@ impl Website { }; } } else { + counter.increment(1); // sites hasn't existed yet match db.create("website").content(self.clone()).await { Ok(e) => { @@ -165,23 +179,23 @@ pub struct Record { } #[instrument(skip_all, name = "SurrealDB")] -pub async fn connect(config: &Config<'_>) -> surrealdb::Result> { +pub async fn connect(config: &Config) -> surrealdb::Result> { trace!("Establishing connection to surreal..."); // Connect to the server - let db = Surreal::new::(config.surreal_url).await?; + let db = Surreal::new::(&config.surreal_url).await?; trace!("Logging in..."); // Signin as a namespace, database, or root user db.signin(Root { - username: config.surreal_username, - password: config.surreal_password, + username: &config.surreal_username, + password: &config.surreal_password, }) .await?; // Select a specific namespace / database db - .use_ns(config.surreal_ns) - .use_db(config.surreal_db) + .use_ns(&config.surreal_ns) + .use_db(&config.surreal_db) .await?; let setup = include_bytes!("setup.surql"); diff --git a/src/main.rs b/src/main.rs index bd97892..42f65ae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,32 +1,43 @@ -#![feature(type_alias_impl_trait)] -#![feature(const_async_blocks)] +#![feature(ip_from)] extern crate html5ever; -use std::time::Instant; +use std::{fs::File, io::Read, net::{IpAddr, Ipv4Addr}, time::Instant}; use db::{connect, Website}; +use metrics::{counter, gauge}; +use metrics_exporter_prometheus::PrometheusBuilder; use s3::S3; +use serde::Deserialize; use surrealdb::{engine::remote::ws::Client, Surreal}; -use tokio::{task::JoinSet}; +use tokio::task::JoinSet; use tracing::{debug, info, instrument, trace, trace_span, warn}; -use tracing_subscriber::{fmt::{self, time::LocalTime}, layer::{Filter, SubscriberExt}, EnvFilter, Layer, Registry}; +use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry}; mod db; mod parser; mod s3; -struct Config<'a> { - surreal_ns: &'a str, - surreal_db: &'a str, - surreal_url: &'a str, - surreal_username: &'a str, - surreal_password: &'a str, +const GET_METRIC: &'static str = "total_gets"; +const GET_IN_FLIGHT: &'static str = "gets_in_flight"; +const SITES_CRAWLED: &'static str = "pages_crawled"; +const BEING_PROCESSED: &'static str = "pages_being_processed"; - s3_url: &'a str, - s3_bucket: &'a str, - s3_access_key: &'a str, - s3_secret_key: &'a str, +#[derive(Deserialize)] +struct Config { + surreal_ns: String, + surreal_db: String, + surreal_url: String, + surreal_username: String, + surreal_password: String, + + s3_url: String, + s3_bucket: String, + s3_access_key: String, + s3_secret_key: String, + + crawl_filter: String, + budget: usize, } #[tokio::main] @@ -48,38 +59,33 @@ async fn main() { // .with_timer(LocalTime::rfc_3339()) // Loki or alloy does this automatically .json() .with_writer(writer) + // .with_filter(EnvFilter::from_default_env()) ); tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber"); - // tracing_subscriber::fmt() - // .with_env_filter(EnvFilter::from_default_env()) - // .with_line_number(true) - // .with_thread_ids(true) - // .with_file(true) - // .with_timer(LocalTime::rfc_3339()) - // .init(); + + let builder = PrometheusBuilder::new(); + builder.with_http_listener( + std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::from_octets([0,0,0,0])), 2500) + ) + .install() + .expect("failed to install recorder/exporter"); debug!("Starting..."); - // Would probably take these in as parameters from a cli + // Would probably take these in as parameters from a cli let starting_url = "https://en.wikipedia.org/"; // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored. - let crawl_filter = "en.wikipedia.org/"; - let budget = 50; + // let crawl_filter = "en.wikipedia.org/"; + // let budget = 50; let mut crawled = 0; - let config = Config { - surreal_url: "localhost:8000", - surreal_username: "root", - surreal_password: "root", - surreal_ns: "test", - surreal_db: "v1.12", - s3_bucket: "v1.12", - s3_url: "http://localhost:9000", - s3_access_key: "p8gXIZEO2FnWqWBiJYwo", - s3_secret_key: "1mRO0EYA2YAQ0xsKrlbkIIz4AT8KNXy6QIQPtxUu", - }; - + + let mut file = File::open("./Crawler.toml").expect("Failed to read Crawler.toml"); + let mut buf = String::new(); + let _ = file.read_to_string(&mut buf); + + let config: Config = toml::from_str(&buf).expect("Failed to parse Crawler.toml"); let db = connect(&config) .await @@ -106,14 +112,14 @@ async fn main() { let span = trace_span!("Loop"); let span = span.enter(); - while crawled < budget { - let get_num = if budget - crawled < 100 { - budget - crawled + while crawled < config.budget { + let get_num = if config.budget - crawled < 100 { + config.budget - crawled } else { 100 }; - let uncrawled = get_uncrawled_links(&db, get_num, crawl_filter.to_string()).await; + let uncrawled = get_uncrawled_links(&db, get_num, config.crawl_filter.clone()).await; if uncrawled.len() == 0 { info!("Had more budget but finished crawling everything."); return; @@ -126,16 +132,20 @@ async fn main() { { let mut futures = JoinSet::new(); for site in uncrawled { + gauge!(BEING_PROCESSED).increment(1); futures.spawn(get(site, db.clone(), reqwest.clone(), s3.clone())); - // technically the site hasn't be crawled *yet*, but the future - // where it is crawled has been set up. - crawled += 1; // let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32); // info!("Crawled {crawled} out of {budget} pages. ({percent})"); } debug!("Joining {} futures...", futures.len()); - // join all the gets together - let _ = futures.join_all().await; + + let c = counter!(SITES_CRAWLED); + // As futures complete runs code in while block + while let Some(_) = futures.join_next().await { + c.increment(1); + gauge!(BEING_PROCESSED).decrement(1); + crawled += 1; + } } } drop(span); @@ -154,9 +164,15 @@ async fn get(mut site: Website, db: Surreal, reqwest: reqwest::Client, s let request_builder = reqwest.get(site.to_string()); timer.stop(); + let g = gauge!(GET_IN_FLIGHT); + g.increment(1); let timer = Timer::start("Got page"); + if let Ok(response) = request_builder.send().await { + timer.stop(); + g.decrement(1); + counter!(GET_METRIC).increment(1); debug!("Getting body..."); // Get body diff --git a/src/parser.rs b/src/parser.rs index 4c4a1ca..0e5dc38 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -97,6 +97,7 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: &str) { let mut links_to = Vec::with_capacity(other_sites.len()); for a in other_sites { + let other = a.store(db).await; if let Some(o) = other { links_to.push(o); diff --git a/src/s3.rs b/src/s3.rs index 4439df2..99bd91a 100644 --- a/src/s3.rs +++ b/src/s3.rs @@ -1,4 +1,5 @@ use base64::{alphabet, engine::{self, general_purpose}, Engine}; +use metrics::counter; use minio::s3::{ args::{BucketExistsArgs, MakeBucketArgs}, client::ClientBuilder, @@ -14,6 +15,8 @@ use crate::{Config, Timer}; const CUSTOM_ENGINE: engine::GeneralPurpose = engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD); +const ROUND_TRIP_METRIC: &'static str = "s3_trips"; + #[derive(Clone)] pub struct S3 { bucket_name: String, @@ -22,7 +25,7 @@ pub struct S3 { impl S3 { #[instrument(skip_all, name = "S3")] - pub async fn connect(config: &Config<'_>) -> Result { + pub async fn connect(config: &Config) -> Result { let base_url = config .s3_url .parse::() @@ -62,6 +65,7 @@ impl S3 { #[instrument(name = "s3_store", skip_all)] pub async fn store(&self, data: &str, url: &Url) { + let counter = counter!(ROUND_TRIP_METRIC); let t = Timer::start("Stored page"); let _ = t; // prevent compiler drop if let Some(domain) = url.domain() { @@ -69,6 +73,7 @@ impl S3 { trace!("Created filename: {filename} from raw: {}", url.to_string()); + counter.increment(1); let _ = match &self .client .put_object_content(&self.bucket_name, &filename, data.to_owned()) @@ -83,6 +88,7 @@ impl S3 { let filename: String = domain.to_owned() + &CUSTOM_ENGINE.encode(url.path()); + counter.increment(1); let _ = &self .client .put_object_content(&self.bucket_name, &filename, data.to_owned())