epic metrics
This commit is contained in:
parent
de80418c00
commit
9aa34b3eee
350
Cargo.lock
generated
350
Cargo.lock
generated
@ -398,6 +398,29 @@ version = "1.4.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
|
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aws-lc-rs"
|
||||||
|
version = "1.12.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01"
|
||||||
|
dependencies = [
|
||||||
|
"aws-lc-sys",
|
||||||
|
"zeroize",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aws-lc-sys"
|
||||||
|
version = "0.27.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f"
|
||||||
|
dependencies = [
|
||||||
|
"bindgen",
|
||||||
|
"cc",
|
||||||
|
"cmake",
|
||||||
|
"dunce",
|
||||||
|
"fs_extra",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "backtrace"
|
name = "backtrace"
|
||||||
version = "0.3.74"
|
version = "0.3.74"
|
||||||
@ -453,6 +476,29 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bindgen"
|
||||||
|
version = "0.69.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"cexpr",
|
||||||
|
"clang-sys",
|
||||||
|
"itertools 0.11.0",
|
||||||
|
"lazy_static",
|
||||||
|
"lazycell",
|
||||||
|
"log",
|
||||||
|
"prettyplease",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"regex",
|
||||||
|
"rustc-hash 1.1.0",
|
||||||
|
"shlex",
|
||||||
|
"syn 2.0.100",
|
||||||
|
"which",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bit-set"
|
name = "bit-set"
|
||||||
version = "0.5.3"
|
version = "0.5.3"
|
||||||
@ -614,6 +660,8 @@ version = "1.2.16"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
|
checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"jobserver",
|
||||||
|
"libc",
|
||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
||||||
@ -674,6 +722,15 @@ dependencies = [
|
|||||||
"unicode-security",
|
"unicode-security",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cexpr"
|
||||||
|
version = "0.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||||
|
dependencies = [
|
||||||
|
"nom",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cfg-if"
|
name = "cfg-if"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
@ -738,6 +795,26 @@ dependencies = [
|
|||||||
"inout",
|
"inout",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clang-sys"
|
||||||
|
version = "1.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
|
||||||
|
dependencies = [
|
||||||
|
"glob",
|
||||||
|
"libc",
|
||||||
|
"libloading",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cmake"
|
||||||
|
version = "0.1.54"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "colorchoice"
|
name = "colorchoice"
|
||||||
version = "1.0.3"
|
version = "1.0.3"
|
||||||
@ -769,6 +846,16 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "core-foundation"
|
||||||
|
version = "0.10.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63"
|
||||||
|
dependencies = [
|
||||||
|
"core-foundation-sys",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "core-foundation-sys"
|
name = "core-foundation-sys"
|
||||||
version = "0.8.7"
|
version = "0.8.7"
|
||||||
@ -1002,6 +1089,12 @@ version = "0.3.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
|
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dunce"
|
||||||
|
version = "1.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "earcutr"
|
name = "earcutr"
|
||||||
version = "0.4.3"
|
version = "0.4.3"
|
||||||
@ -1166,6 +1259,12 @@ dependencies = [
|
|||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fs_extra"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fst"
|
name = "fst"
|
||||||
version = "0.4.7"
|
version = "0.4.7"
|
||||||
@ -1388,6 +1487,12 @@ version = "0.31.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
|
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "glob"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "h2"
|
name = "h2"
|
||||||
version = "0.4.8"
|
version = "0.4.8"
|
||||||
@ -1614,6 +1719,7 @@ dependencies = [
|
|||||||
"hyper",
|
"hyper",
|
||||||
"hyper-util",
|
"hyper-util",
|
||||||
"rustls",
|
"rustls",
|
||||||
|
"rustls-native-certs",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
@ -1924,6 +2030,15 @@ dependencies = [
|
|||||||
"syn 2.0.100",
|
"syn 2.0.100",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jobserver"
|
||||||
|
version = "0.1.32"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "js-sys"
|
name = "js-sys"
|
||||||
version = "0.3.77"
|
version = "0.3.77"
|
||||||
@ -1986,6 +2101,12 @@ version = "1.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lazycell"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lexicmp"
|
name = "lexicmp"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@ -2001,6 +2122,16 @@ version = "0.2.171"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
|
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libloading"
|
||||||
|
version = "0.8.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libm"
|
name = "libm"
|
||||||
version = "0.2.11"
|
version = "0.2.11"
|
||||||
@ -2029,6 +2160,12 @@ dependencies = [
|
|||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "linux-raw-sys"
|
||||||
|
version = "0.4.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "linux-raw-sys"
|
name = "linux-raw-sys"
|
||||||
version = "0.9.3"
|
version = "0.9.3"
|
||||||
@ -2149,6 +2286,53 @@ version = "2.7.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "metrics"
|
||||||
|
version = "0.24.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7a7deb012b3b2767169ff203fadb4c6b0b82b947512e5eb9e0b78c2e186ad9e3"
|
||||||
|
dependencies = [
|
||||||
|
"ahash 0.8.11",
|
||||||
|
"portable-atomic",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "metrics-exporter-prometheus"
|
||||||
|
version = "0.16.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034"
|
||||||
|
dependencies = [
|
||||||
|
"base64 0.22.1",
|
||||||
|
"http-body-util",
|
||||||
|
"hyper",
|
||||||
|
"hyper-rustls",
|
||||||
|
"hyper-util",
|
||||||
|
"indexmap 2.8.0",
|
||||||
|
"ipnet",
|
||||||
|
"metrics",
|
||||||
|
"metrics-util",
|
||||||
|
"quanta",
|
||||||
|
"thiserror 1.0.69",
|
||||||
|
"tokio",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "metrics-util"
|
||||||
|
version = "0.19.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dbd4884b1dd24f7d6628274a2f5ae22465c337c5ba065ec9b6edccddf8acc673"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-epoch",
|
||||||
|
"crossbeam-utils",
|
||||||
|
"hashbrown 0.15.2",
|
||||||
|
"metrics",
|
||||||
|
"quanta",
|
||||||
|
"rand 0.8.5",
|
||||||
|
"rand_xoshiro",
|
||||||
|
"sketches-ddsketch",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "miette"
|
name = "miette"
|
||||||
version = "5.10.0"
|
version = "5.10.0"
|
||||||
@ -2188,6 +2372,12 @@ dependencies = [
|
|||||||
"unicase",
|
"unicase",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "minimal-lexical"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "minio"
|
name = "minio"
|
||||||
version = "0.2.0-alpha"
|
version = "0.2.0-alpha"
|
||||||
@ -2295,7 +2485,7 @@ dependencies = [
|
|||||||
"openssl-probe",
|
"openssl-probe",
|
||||||
"openssl-sys",
|
"openssl-sys",
|
||||||
"schannel",
|
"schannel",
|
||||||
"security-framework",
|
"security-framework 2.11.1",
|
||||||
"security-framework-sys",
|
"security-framework-sys",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
]
|
]
|
||||||
@ -2353,6 +2543,16 @@ dependencies = [
|
|||||||
"num-traits",
|
"num-traits",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nom"
|
||||||
|
version = "7.1.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
"minimal-lexical",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ntapi"
|
name = "ntapi"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
@ -2757,6 +2957,16 @@ version = "0.1.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "prettyplease"
|
||||||
|
version = "0.2.31"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"syn 2.0.100",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro-crate"
|
name = "proc-macro-crate"
|
||||||
version = "3.3.0"
|
version = "3.3.0"
|
||||||
@ -2810,6 +3020,21 @@ dependencies = [
|
|||||||
"syn 1.0.109",
|
"syn 1.0.109",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quanta"
|
||||||
|
version = "0.12.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-utils",
|
||||||
|
"libc",
|
||||||
|
"once_cell",
|
||||||
|
"raw-cpuid",
|
||||||
|
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||||
|
"web-sys",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quick_cache"
|
name = "quick_cache"
|
||||||
version = "0.5.2"
|
version = "0.5.2"
|
||||||
@ -2833,7 +3058,7 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"quinn-proto",
|
"quinn-proto",
|
||||||
"quinn-udp",
|
"quinn-udp",
|
||||||
"rustc-hash",
|
"rustc-hash 2.1.1",
|
||||||
"rustls",
|
"rustls",
|
||||||
"socket2",
|
"socket2",
|
||||||
"thiserror 2.0.12",
|
"thiserror 2.0.12",
|
||||||
@ -2852,7 +3077,7 @@ dependencies = [
|
|||||||
"getrandom 0.3.2",
|
"getrandom 0.3.2",
|
||||||
"rand 0.9.0",
|
"rand 0.9.0",
|
||||||
"ring",
|
"ring",
|
||||||
"rustc-hash",
|
"rustc-hash 2.1.1",
|
||||||
"rustls",
|
"rustls",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"slab",
|
"slab",
|
||||||
@ -2968,6 +3193,24 @@ dependencies = [
|
|||||||
"getrandom 0.3.2",
|
"getrandom 0.3.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_xoshiro"
|
||||||
|
version = "0.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
|
||||||
|
dependencies = [
|
||||||
|
"rand_core 0.6.4",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "raw-cpuid"
|
||||||
|
version = "11.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rawpointer"
|
name = "rawpointer"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
@ -3296,6 +3539,12 @@ version = "0.1.24"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustc-hash"
|
||||||
|
version = "1.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc-hash"
|
name = "rustc-hash"
|
||||||
version = "2.1.1"
|
version = "2.1.1"
|
||||||
@ -3320,6 +3569,19 @@ dependencies = [
|
|||||||
"semver",
|
"semver",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustix"
|
||||||
|
version = "0.38.44"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"errno",
|
||||||
|
"libc",
|
||||||
|
"linux-raw-sys 0.4.15",
|
||||||
|
"windows-sys 0.59.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustix"
|
name = "rustix"
|
||||||
version = "1.0.3"
|
version = "1.0.3"
|
||||||
@ -3329,7 +3591,7 @@ dependencies = [
|
|||||||
"bitflags",
|
"bitflags",
|
||||||
"errno",
|
"errno",
|
||||||
"libc",
|
"libc",
|
||||||
"linux-raw-sys",
|
"linux-raw-sys 0.9.3",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
@ -3339,6 +3601,7 @@ version = "0.23.25"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c"
|
checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"aws-lc-rs",
|
||||||
"log",
|
"log",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"ring",
|
"ring",
|
||||||
@ -3348,6 +3611,18 @@ dependencies = [
|
|||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustls-native-certs"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3"
|
||||||
|
dependencies = [
|
||||||
|
"openssl-probe",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"schannel",
|
||||||
|
"security-framework 3.2.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls-pemfile"
|
name = "rustls-pemfile"
|
||||||
version = "2.2.0"
|
version = "2.2.0"
|
||||||
@ -3372,6 +3647,7 @@ version = "0.103.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f"
|
checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"aws-lc-rs",
|
||||||
"ring",
|
"ring",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"untrusted",
|
"untrusted",
|
||||||
@ -3447,7 +3723,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
|
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bitflags",
|
"bitflags",
|
||||||
"core-foundation",
|
"core-foundation 0.9.4",
|
||||||
|
"core-foundation-sys",
|
||||||
|
"libc",
|
||||||
|
"security-framework-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "security-framework"
|
||||||
|
version = "3.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"core-foundation 0.10.0",
|
||||||
"core-foundation-sys",
|
"core-foundation-sys",
|
||||||
"libc",
|
"libc",
|
||||||
"security-framework-sys",
|
"security-framework-sys",
|
||||||
@ -3520,6 +3809,15 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_spanned"
|
||||||
|
version = "0.6.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde_urlencoded"
|
name = "serde_urlencoded"
|
||||||
version = "0.7.1"
|
version = "0.7.1"
|
||||||
@ -3632,6 +3930,12 @@ version = "1.0.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
|
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sketches-ddsketch"
|
||||||
|
version = "0.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "slab"
|
name = "slab"
|
||||||
version = "0.4.9"
|
version = "0.4.9"
|
||||||
@ -3814,11 +4118,14 @@ version = "0.1.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"html5ever 0.29.1",
|
"html5ever 0.29.1",
|
||||||
|
"metrics",
|
||||||
|
"metrics-exporter-prometheus",
|
||||||
"minio",
|
"minio",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"surrealdb",
|
"surrealdb",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"toml",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
"url",
|
"url",
|
||||||
@ -4015,7 +4322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
|
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bitflags",
|
"bitflags",
|
||||||
"core-foundation",
|
"core-foundation 0.9.4",
|
||||||
"system-configuration-sys",
|
"system-configuration-sys",
|
||||||
]
|
]
|
||||||
|
|
||||||
@ -4044,7 +4351,7 @@ dependencies = [
|
|||||||
"fastrand",
|
"fastrand",
|
||||||
"getrandom 0.3.2",
|
"getrandom 0.3.2",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"rustix",
|
"rustix 1.0.3",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
@ -4277,11 +4584,26 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "toml"
|
||||||
|
version = "0.8.20"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
"serde_spanned",
|
||||||
|
"toml_datetime",
|
||||||
|
"toml_edit",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "toml_datetime"
|
name = "toml_datetime"
|
||||||
version = "0.6.8"
|
version = "0.6.8"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
|
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "toml_edit"
|
name = "toml_edit"
|
||||||
@ -4290,6 +4612,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
|
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"indexmap 2.8.0",
|
"indexmap 2.8.0",
|
||||||
|
"serde",
|
||||||
|
"serde_spanned",
|
||||||
"toml_datetime",
|
"toml_datetime",
|
||||||
"winnow",
|
"winnow",
|
||||||
]
|
]
|
||||||
@ -4750,6 +5074,18 @@ dependencies = [
|
|||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "which"
|
||||||
|
version = "4.4.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
|
||||||
|
dependencies = [
|
||||||
|
"either",
|
||||||
|
"home",
|
||||||
|
"once_cell",
|
||||||
|
"rustix 0.38.44",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "winapi"
|
name = "winapi"
|
||||||
version = "0.3.9"
|
version = "0.3.9"
|
||||||
|
|||||||
@ -6,12 +6,15 @@ edition = "2021"
|
|||||||
[dependencies]
|
[dependencies]
|
||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
html5ever = "0.29"
|
html5ever = "0.29"
|
||||||
|
metrics = "0.24.1"
|
||||||
|
metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]}
|
||||||
# minio = "0.1.0"
|
# minio = "0.1.0"
|
||||||
minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"}
|
minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"}
|
||||||
reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] }
|
reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] }
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
surrealdb = "2.2"
|
surrealdb = "2.2"
|
||||||
tokio = { version="1.41.0", features = ["full"] }
|
tokio = { version="1.41.0", features = ["full"] }
|
||||||
|
toml = "0.8.20"
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time", "json"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time", "json"] }
|
||||||
url = { version = "2.5", features = ["serde"] }
|
url = { version = "2.5", features = ["serde"] }
|
||||||
|
|||||||
16
Crawler.toml
Normal file
16
Crawler.toml
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
# Surreal config
|
||||||
|
surreal_url = "localhost:8000"
|
||||||
|
surreal_username = "root"
|
||||||
|
surreal_password = "root"
|
||||||
|
surreal_ns = "test"
|
||||||
|
surreal_db = "v1.12"
|
||||||
|
|
||||||
|
# Minio config
|
||||||
|
s3_bucket = "v1.12"
|
||||||
|
s3_url = "http://localhost:9000"
|
||||||
|
s3_access_key = "jLDPKGuu513VENc8kJwX"
|
||||||
|
s3_secret_key = "4T1nymEzsGYOlKSAb1WX7V3stnQn9a5ZoTQjDfcL"
|
||||||
|
|
||||||
|
# Crawler config
|
||||||
|
crawl_filter = "en.wikipedia.com"
|
||||||
|
budget = 200
|
||||||
@ -50,6 +50,17 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./loki.yaml:/etc/loki/local-config.yaml
|
- ./loki.yaml:/etc/loki/local-config.yaml
|
||||||
|
|
||||||
|
# Metrics collector
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:latest
|
||||||
|
expose:
|
||||||
|
- 9090
|
||||||
|
volumes:
|
||||||
|
- ./prometheus.yaml:/etc/prometheus/prometheus.yml
|
||||||
|
# persist data
|
||||||
|
- prometheus_storage:/prometheus
|
||||||
|
command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
|
||||||
|
|
||||||
# Everything viewer
|
# Everything viewer
|
||||||
grafana:
|
grafana:
|
||||||
image: grafana/grafana:latest
|
image: grafana/grafana:latest
|
||||||
@ -65,6 +76,7 @@ services:
|
|||||||
- 3000:3000
|
- 3000:3000
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
|
prometheus_storage:
|
||||||
grafana_storage:
|
grafana_storage:
|
||||||
alloy_storage:
|
alloy_storage:
|
||||||
surrealdb_storage:
|
surrealdb_storage:
|
||||||
|
|||||||
@ -10,3 +10,15 @@ datasources:
|
|||||||
isDefault: true
|
isDefault: true
|
||||||
version: 1
|
version: 1
|
||||||
editable: false
|
editable: false
|
||||||
|
- name: Prometheus
|
||||||
|
type: prometheus
|
||||||
|
uid: prometheus
|
||||||
|
access: proxy
|
||||||
|
orgId: 1
|
||||||
|
url: http://prometheus:9090
|
||||||
|
basicAuth: false
|
||||||
|
isDefault: false
|
||||||
|
version: 1
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
httpMethod: GET
|
||||||
|
|||||||
16
docker/prometheus.yaml
Normal file
16
docker/prometheus.yaml
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 5s
|
||||||
|
query_log_file: /etc/prometheus/query.log
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: crawler
|
||||||
|
static_configs:
|
||||||
|
# change this your machine's ip, localhost won't work
|
||||||
|
# because localhost refers to the docker container.
|
||||||
|
- targets: ['192.168.8.209:2500']
|
||||||
|
- job_name: loki
|
||||||
|
static_configs:
|
||||||
|
- targets: ['loki:3100']
|
||||||
|
- job_name: prometheus
|
||||||
|
static_configs:
|
||||||
|
- targets: ['localhost:9090']
|
||||||
26
src/db.rs
26
src/db.rs
@ -1,4 +1,5 @@
|
|||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
|
use metrics::counter;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use surrealdb::{
|
use surrealdb::{
|
||||||
engine::remote::ws::{Client, Ws}, error::Db, opt::auth::Root, sql::Thing, Response, Surreal
|
engine::remote::ws::{Client, Ws}, error::Db, opt::auth::Root, sql::Thing, Response, Surreal
|
||||||
@ -8,6 +9,10 @@ use url::Url;
|
|||||||
|
|
||||||
use crate::{Config, Timer};
|
use crate::{Config, Timer};
|
||||||
|
|
||||||
|
const ROUND_TRIP_METRIC: &'static str = "surql_trips";
|
||||||
|
const STORE: &'static str = "surql_store_calls";
|
||||||
|
const LINK: &'static str = "surql_link_calls";
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Clone)]
|
#[derive(Serialize, Deserialize, Clone)]
|
||||||
pub struct Website {
|
pub struct Website {
|
||||||
/// The url that this data is found at
|
/// The url that this data is found at
|
||||||
@ -47,6 +52,7 @@ impl Website {
|
|||||||
|
|
||||||
#[instrument(skip_all)]
|
#[instrument(skip_all)]
|
||||||
pub async fn links_to(&self, other: Vec<Thing>, db: &Surreal<Client>) {
|
pub async fn links_to(&self, other: Vec<Thing>, db: &Surreal<Client>) {
|
||||||
|
|
||||||
let len = other.len();
|
let len = other.len();
|
||||||
if len == 0 {return}
|
if len == 0 {return}
|
||||||
|
|
||||||
@ -57,6 +63,8 @@ impl Website {
|
|||||||
let timer = Timer::start(&msg);
|
let timer = Timer::start(&msg);
|
||||||
// prevent the timer from being dropped instantly.
|
// prevent the timer from being dropped instantly.
|
||||||
let _ = timer;
|
let _ = timer;
|
||||||
|
counter!(ROUND_TRIP_METRIC).increment(1);
|
||||||
|
counter!(LINK).increment(1);
|
||||||
match db
|
match db
|
||||||
.query("COUNT(RELATE (SELECT id FROM website WHERE site = $in) -> links_to -> $out)")
|
.query("COUNT(RELATE (SELECT id FROM website WHERE site = $in) -> links_to -> $out)")
|
||||||
.bind(("in", from))
|
.bind(("in", from))
|
||||||
@ -89,8 +97,12 @@ impl Website {
|
|||||||
|
|
||||||
#[instrument(name = "surql_store", skip_all)]
|
#[instrument(name = "surql_store", skip_all)]
|
||||||
pub async fn store(&self, db: &Surreal<Client>) -> Option<Thing> {
|
pub async fn store(&self, db: &Surreal<Client>) -> Option<Thing> {
|
||||||
|
counter!(STORE).increment(1);
|
||||||
|
let counter = counter!(ROUND_TRIP_METRIC);
|
||||||
let t = Timer::start("Stored link");
|
let t = Timer::start("Stored link");
|
||||||
let _ = t;
|
let _ = t;
|
||||||
|
counter.increment(1);
|
||||||
|
|
||||||
// check if it's been gone thru before
|
// check if it's been gone thru before
|
||||||
let mut response = db
|
let mut response = db
|
||||||
.query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1")
|
.query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1")
|
||||||
@ -105,6 +117,7 @@ impl Website {
|
|||||||
let mut new = self.clone();
|
let mut new = self.clone();
|
||||||
new.crawled = old.crawled | new.crawled;
|
new.crawled = old.crawled | new.crawled;
|
||||||
|
|
||||||
|
counter.increment(1);
|
||||||
// update the record
|
// update the record
|
||||||
match db.upsert((id.tb, id.id.to_string())).content(new).await {
|
match db.upsert((id.tb, id.id.to_string())).content(new).await {
|
||||||
Ok(e) => {
|
Ok(e) => {
|
||||||
@ -130,6 +143,7 @@ impl Website {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
counter.increment(1);
|
||||||
// sites hasn't existed yet
|
// sites hasn't existed yet
|
||||||
match db.create("website").content(self.clone()).await {
|
match db.create("website").content(self.clone()).await {
|
||||||
Ok(e) => {
|
Ok(e) => {
|
||||||
@ -165,23 +179,23 @@ pub struct Record {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip_all, name = "SurrealDB")]
|
#[instrument(skip_all, name = "SurrealDB")]
|
||||||
pub async fn connect(config: &Config<'_>) -> surrealdb::Result<Surreal<Client>> {
|
pub async fn connect(config: &Config) -> surrealdb::Result<Surreal<Client>> {
|
||||||
trace!("Establishing connection to surreal...");
|
trace!("Establishing connection to surreal...");
|
||||||
// Connect to the server
|
// Connect to the server
|
||||||
let db = Surreal::new::<Ws>(config.surreal_url).await?;
|
let db = Surreal::new::<Ws>(&config.surreal_url).await?;
|
||||||
|
|
||||||
trace!("Logging in...");
|
trace!("Logging in...");
|
||||||
// Signin as a namespace, database, or root user
|
// Signin as a namespace, database, or root user
|
||||||
db.signin(Root {
|
db.signin(Root {
|
||||||
username: config.surreal_username,
|
username: &config.surreal_username,
|
||||||
password: config.surreal_password,
|
password: &config.surreal_password,
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Select a specific namespace / database
|
// Select a specific namespace / database
|
||||||
db
|
db
|
||||||
.use_ns(config.surreal_ns)
|
.use_ns(&config.surreal_ns)
|
||||||
.use_db(config.surreal_db)
|
.use_db(&config.surreal_db)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let setup = include_bytes!("setup.surql");
|
let setup = include_bytes!("setup.surql");
|
||||||
|
|||||||
104
src/main.rs
104
src/main.rs
@ -1,32 +1,43 @@
|
|||||||
#![feature(type_alias_impl_trait)]
|
#![feature(ip_from)]
|
||||||
#![feature(const_async_blocks)]
|
|
||||||
|
|
||||||
extern crate html5ever;
|
extern crate html5ever;
|
||||||
|
|
||||||
use std::time::Instant;
|
use std::{fs::File, io::Read, net::{IpAddr, Ipv4Addr}, time::Instant};
|
||||||
|
|
||||||
use db::{connect, Website};
|
use db::{connect, Website};
|
||||||
|
use metrics::{counter, gauge};
|
||||||
|
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||||
use s3::S3;
|
use s3::S3;
|
||||||
|
use serde::Deserialize;
|
||||||
use surrealdb::{engine::remote::ws::Client, Surreal};
|
use surrealdb::{engine::remote::ws::Client, Surreal};
|
||||||
use tokio::{task::JoinSet};
|
use tokio::task::JoinSet;
|
||||||
use tracing::{debug, info, instrument, trace, trace_span, warn};
|
use tracing::{debug, info, instrument, trace, trace_span, warn};
|
||||||
use tracing_subscriber::{fmt::{self, time::LocalTime}, layer::{Filter, SubscriberExt}, EnvFilter, Layer, Registry};
|
use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry};
|
||||||
|
|
||||||
mod db;
|
mod db;
|
||||||
mod parser;
|
mod parser;
|
||||||
mod s3;
|
mod s3;
|
||||||
|
|
||||||
struct Config<'a> {
|
const GET_METRIC: &'static str = "total_gets";
|
||||||
surreal_ns: &'a str,
|
const GET_IN_FLIGHT: &'static str = "gets_in_flight";
|
||||||
surreal_db: &'a str,
|
const SITES_CRAWLED: &'static str = "pages_crawled";
|
||||||
surreal_url: &'a str,
|
const BEING_PROCESSED: &'static str = "pages_being_processed";
|
||||||
surreal_username: &'a str,
|
|
||||||
surreal_password: &'a str,
|
|
||||||
|
|
||||||
s3_url: &'a str,
|
#[derive(Deserialize)]
|
||||||
s3_bucket: &'a str,
|
struct Config {
|
||||||
s3_access_key: &'a str,
|
surreal_ns: String,
|
||||||
s3_secret_key: &'a str,
|
surreal_db: String,
|
||||||
|
surreal_url: String,
|
||||||
|
surreal_username: String,
|
||||||
|
surreal_password: String,
|
||||||
|
|
||||||
|
s3_url: String,
|
||||||
|
s3_bucket: String,
|
||||||
|
s3_access_key: String,
|
||||||
|
s3_secret_key: String,
|
||||||
|
|
||||||
|
crawl_filter: String,
|
||||||
|
budget: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
@ -48,38 +59,33 @@ async fn main() {
|
|||||||
// .with_timer(LocalTime::rfc_3339()) // Loki or alloy does this automatically
|
// .with_timer(LocalTime::rfc_3339()) // Loki or alloy does this automatically
|
||||||
.json()
|
.json()
|
||||||
.with_writer(writer)
|
.with_writer(writer)
|
||||||
|
// .with_filter(EnvFilter::from_default_env())
|
||||||
);
|
);
|
||||||
|
|
||||||
tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber");
|
tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber");
|
||||||
|
|
||||||
// tracing_subscriber::fmt()
|
|
||||||
// .with_env_filter(EnvFilter::from_default_env())
|
let builder = PrometheusBuilder::new();
|
||||||
// .with_line_number(true)
|
builder.with_http_listener(
|
||||||
// .with_thread_ids(true)
|
std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::from_octets([0,0,0,0])), 2500)
|
||||||
// .with_file(true)
|
)
|
||||||
// .with_timer(LocalTime::rfc_3339())
|
.install()
|
||||||
// .init();
|
.expect("failed to install recorder/exporter");
|
||||||
|
|
||||||
debug!("Starting...");
|
debug!("Starting...");
|
||||||
// Would probably take these in as parameters from a cli
|
// Would probably take these in as parameters from a cli
|
||||||
let starting_url = "https://en.wikipedia.org/";
|
let starting_url = "https://en.wikipedia.org/";
|
||||||
// When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
|
// When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
|
||||||
let crawl_filter = "en.wikipedia.org/";
|
// let crawl_filter = "en.wikipedia.org/";
|
||||||
let budget = 50;
|
// let budget = 50;
|
||||||
let mut crawled = 0;
|
let mut crawled = 0;
|
||||||
|
|
||||||
let config = Config {
|
|
||||||
surreal_url: "localhost:8000",
|
|
||||||
surreal_username: "root",
|
|
||||||
surreal_password: "root",
|
|
||||||
surreal_ns: "test",
|
|
||||||
surreal_db: "v1.12",
|
|
||||||
s3_bucket: "v1.12",
|
|
||||||
s3_url: "http://localhost:9000",
|
|
||||||
s3_access_key: "p8gXIZEO2FnWqWBiJYwo",
|
|
||||||
s3_secret_key: "1mRO0EYA2YAQ0xsKrlbkIIz4AT8KNXy6QIQPtxUu",
|
|
||||||
};
|
|
||||||
|
|
||||||
|
let mut file = File::open("./Crawler.toml").expect("Failed to read Crawler.toml");
|
||||||
|
let mut buf = String::new();
|
||||||
|
let _ = file.read_to_string(&mut buf);
|
||||||
|
|
||||||
|
let config: Config = toml::from_str(&buf).expect("Failed to parse Crawler.toml");
|
||||||
|
|
||||||
let db = connect(&config)
|
let db = connect(&config)
|
||||||
.await
|
.await
|
||||||
@ -106,14 +112,14 @@ async fn main() {
|
|||||||
|
|
||||||
let span = trace_span!("Loop");
|
let span = trace_span!("Loop");
|
||||||
let span = span.enter();
|
let span = span.enter();
|
||||||
while crawled < budget {
|
while crawled < config.budget {
|
||||||
let get_num = if budget - crawled < 100 {
|
let get_num = if config.budget - crawled < 100 {
|
||||||
budget - crawled
|
config.budget - crawled
|
||||||
} else {
|
} else {
|
||||||
100
|
100
|
||||||
};
|
};
|
||||||
|
|
||||||
let uncrawled = get_uncrawled_links(&db, get_num, crawl_filter.to_string()).await;
|
let uncrawled = get_uncrawled_links(&db, get_num, config.crawl_filter.clone()).await;
|
||||||
if uncrawled.len() == 0 {
|
if uncrawled.len() == 0 {
|
||||||
info!("Had more budget but finished crawling everything.");
|
info!("Had more budget but finished crawling everything.");
|
||||||
return;
|
return;
|
||||||
@ -126,16 +132,20 @@ async fn main() {
|
|||||||
{
|
{
|
||||||
let mut futures = JoinSet::new();
|
let mut futures = JoinSet::new();
|
||||||
for site in uncrawled {
|
for site in uncrawled {
|
||||||
|
gauge!(BEING_PROCESSED).increment(1);
|
||||||
futures.spawn(get(site, db.clone(), reqwest.clone(), s3.clone()));
|
futures.spawn(get(site, db.clone(), reqwest.clone(), s3.clone()));
|
||||||
// technically the site hasn't be crawled *yet*, but the future
|
|
||||||
// where it is crawled has been set up.
|
|
||||||
crawled += 1;
|
|
||||||
// let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32);
|
// let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32);
|
||||||
// info!("Crawled {crawled} out of {budget} pages. ({percent})");
|
// info!("Crawled {crawled} out of {budget} pages. ({percent})");
|
||||||
}
|
}
|
||||||
debug!("Joining {} futures...", futures.len());
|
debug!("Joining {} futures...", futures.len());
|
||||||
// join all the gets together
|
|
||||||
let _ = futures.join_all().await;
|
let c = counter!(SITES_CRAWLED);
|
||||||
|
// As futures complete runs code in while block
|
||||||
|
while let Some(_) = futures.join_next().await {
|
||||||
|
c.increment(1);
|
||||||
|
gauge!(BEING_PROCESSED).decrement(1);
|
||||||
|
crawled += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
drop(span);
|
drop(span);
|
||||||
@ -154,9 +164,15 @@ async fn get(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client, s
|
|||||||
let request_builder = reqwest.get(site.to_string());
|
let request_builder = reqwest.get(site.to_string());
|
||||||
timer.stop();
|
timer.stop();
|
||||||
|
|
||||||
|
let g = gauge!(GET_IN_FLIGHT);
|
||||||
|
g.increment(1);
|
||||||
let timer = Timer::start("Got page");
|
let timer = Timer::start("Got page");
|
||||||
|
|
||||||
if let Ok(response) = request_builder.send().await {
|
if let Ok(response) = request_builder.send().await {
|
||||||
|
|
||||||
timer.stop();
|
timer.stop();
|
||||||
|
g.decrement(1);
|
||||||
|
counter!(GET_METRIC).increment(1);
|
||||||
debug!("Getting body...");
|
debug!("Getting body...");
|
||||||
|
|
||||||
// Get body
|
// Get body
|
||||||
|
|||||||
@ -97,6 +97,7 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
|
|||||||
let mut links_to = Vec::with_capacity(other_sites.len());
|
let mut links_to = Vec::with_capacity(other_sites.len());
|
||||||
|
|
||||||
for a in other_sites {
|
for a in other_sites {
|
||||||
|
|
||||||
let other = a.store(db).await;
|
let other = a.store(db).await;
|
||||||
if let Some(o) = other {
|
if let Some(o) = other {
|
||||||
links_to.push(o);
|
links_to.push(o);
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
use base64::{alphabet, engine::{self, general_purpose}, Engine};
|
use base64::{alphabet, engine::{self, general_purpose}, Engine};
|
||||||
|
use metrics::counter;
|
||||||
use minio::s3::{
|
use minio::s3::{
|
||||||
args::{BucketExistsArgs, MakeBucketArgs},
|
args::{BucketExistsArgs, MakeBucketArgs},
|
||||||
client::ClientBuilder,
|
client::ClientBuilder,
|
||||||
@ -14,6 +15,8 @@ use crate::{Config, Timer};
|
|||||||
|
|
||||||
const CUSTOM_ENGINE: engine::GeneralPurpose = engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD);
|
const CUSTOM_ENGINE: engine::GeneralPurpose = engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD);
|
||||||
|
|
||||||
|
const ROUND_TRIP_METRIC: &'static str = "s3_trips";
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct S3 {
|
pub struct S3 {
|
||||||
bucket_name: String,
|
bucket_name: String,
|
||||||
@ -22,7 +25,7 @@ pub struct S3 {
|
|||||||
|
|
||||||
impl S3 {
|
impl S3 {
|
||||||
#[instrument(skip_all, name = "S3")]
|
#[instrument(skip_all, name = "S3")]
|
||||||
pub async fn connect(config: &Config<'_>) -> Result<Self, Error> {
|
pub async fn connect(config: &Config) -> Result<Self, Error> {
|
||||||
let base_url = config
|
let base_url = config
|
||||||
.s3_url
|
.s3_url
|
||||||
.parse::<BaseUrl>()
|
.parse::<BaseUrl>()
|
||||||
@ -62,6 +65,7 @@ impl S3 {
|
|||||||
|
|
||||||
#[instrument(name = "s3_store", skip_all)]
|
#[instrument(name = "s3_store", skip_all)]
|
||||||
pub async fn store(&self, data: &str, url: &Url) {
|
pub async fn store(&self, data: &str, url: &Url) {
|
||||||
|
let counter = counter!(ROUND_TRIP_METRIC);
|
||||||
let t = Timer::start("Stored page");
|
let t = Timer::start("Stored page");
|
||||||
let _ = t; // prevent compiler drop
|
let _ = t; // prevent compiler drop
|
||||||
if let Some(domain) = url.domain() {
|
if let Some(domain) = url.domain() {
|
||||||
@ -69,6 +73,7 @@ impl S3 {
|
|||||||
|
|
||||||
trace!("Created filename: {filename} from raw: {}", url.to_string());
|
trace!("Created filename: {filename} from raw: {}", url.to_string());
|
||||||
|
|
||||||
|
counter.increment(1);
|
||||||
let _ = match &self
|
let _ = match &self
|
||||||
.client
|
.client
|
||||||
.put_object_content(&self.bucket_name, &filename, data.to_owned())
|
.put_object_content(&self.bucket_name, &filename, data.to_owned())
|
||||||
@ -83,6 +88,7 @@ impl S3 {
|
|||||||
|
|
||||||
let filename: String = domain.to_owned() + &CUSTOM_ENGINE.encode(url.path());
|
let filename: String = domain.to_owned() + &CUSTOM_ENGINE.encode(url.path());
|
||||||
|
|
||||||
|
counter.increment(1);
|
||||||
let _ = &self
|
let _ = &self
|
||||||
.client
|
.client
|
||||||
.put_object_content(&self.bucket_name, &filename, data.to_owned())
|
.put_object_content(&self.bucket_name, &filename, data.to_owned())
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user