From 2c28d69d55aac73744709096d21410576c2a0f21 Mon Sep 17 00:00:00 2001 From: oliver Date: Tue, 12 Nov 2024 21:03:58 -0700 Subject: [PATCH 01/19] add s3 support --- .gitignore | 1 + .vscode/launch.json | 2 +- Cargo.lock | 254 +++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + compose.yml | 17 ++- schema.surql | 2 - src/db.rs | 29 +++-- src/main.rs | 56 +++++++--- src/s3.rs | 63 +++++++++++ src/setup.surql | 2 + 10 files changed, 398 insertions(+), 30 deletions(-) delete mode 100644 schema.surql create mode 100644 src/s3.rs create mode 100644 src/setup.surql diff --git a/.gitignore b/.gitignore index a968872..301630e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target /.surrealdb +/.minio perf.data flamegraph.svg perf.data.old \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 76fa95c..43deeb5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,7 @@ "request": "launch", "name": "Debug executable 'surreal_spider'", "env": { - "RUST_LOG": "surreal_spider=debug,reqwest=info", + "RUST_LOG": "surreal_spider=trace,reqwest=info", }, "cargo": { "args": [ diff --git a/Cargo.lock b/Cargo.lock index f256eb1..e205787 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -103,6 +103,55 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "any_ascii" version = "0.3.2" @@ -262,6 +311,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -665,6 +725,12 @@ dependencies = [ "inout", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -705,6 +771,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -775,6 +856,20 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.6.0" @@ -791,6 +886,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "deunicode" version = "1.6.0" @@ -895,6 +1001,29 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1279,6 +1408,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "html5ever" version = "0.27.0" @@ -1347,6 +1485,12 @@ version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.1.0" @@ -1366,6 +1510,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1631,6 +1776,12 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -1877,6 +2028,12 @@ dependencies = [ "digest", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.7.4" @@ -1922,6 +2079,46 @@ dependencies = [ "unicase", ] +[[package]] +name = "minio" +version = "0.2.0-alpha" +source = "git+https://github.com/minio/minio-rs.git?rev=c28f576#c28f576cb8f8cf47fb941bb9db62b2cbd6f080c1" +dependencies = [ + "async-recursion", + "async-trait", + "base64 0.22.1", + "byteorder", + "bytes", + "chrono", + "crc", + "dashmap 6.1.0", + "derivative", + "env_logger", + "futures-util", + "hex", + "hmac", + "home", + "http", + "hyper", + "lazy_static", + "log", + "md5", + "multimap", + "os_info", + "percent-encoding", + "rand", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tokio", + "tokio-stream", + "tokio-util", + "urlencoding", + "xmltree", +] + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -1960,6 +2157,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +dependencies = [ + "serde", +] + [[package]] name = "nanoid" version = "0.4.0" @@ -2183,6 +2389,17 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_info" +version = "3.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae99c7fa6dd38c7cafe1ec085e804f8f555a2f8659b0dbe03f1f9963a9b51092" +dependencies = [ + "log", + "serde", + "windows-sys 0.52.0", +] + [[package]] name = "overload" version = "0.1.1" @@ -3437,6 +3654,7 @@ version = "0.1.0" dependencies = [ "html5ever 0.29.0", "markup5ever_rcdom", + "minio", "reqwest", "serde", "surrealdb", @@ -3510,7 +3728,7 @@ dependencies = [ "cedar-policy", "chrono", "ciborium", - "dashmap", + "dashmap 5.5.3", "deunicode", "dmp", "fst", @@ -3840,6 +4058,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.23.1" @@ -4107,6 +4336,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.11.0" @@ -4484,6 +4719,12 @@ dependencies = [ "tap", ] +[[package]] +name = "xml-rs" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af310deaae937e48a26602b730250b4949e125f468f11e6990be3e5304ddd96f" + [[package]] name = "xml5ever" version = "0.20.0" @@ -4495,6 +4736,15 @@ dependencies = [ "markup5ever 0.14.0", ] +[[package]] +name = "xmltree" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b619f8c85654798007fb10afa5125590b43b088c225a25fc2fec100a9fad0fc6" +dependencies = [ + "xml-rs", +] + [[package]] name = "yoke" version = "0.7.4" diff --git a/Cargo.toml b/Cargo.toml index b07be1d..a9c7532 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,8 @@ edition = "2021" [dependencies] html5ever = "0.29.0" markup5ever_rcdom = "0.5.0-unofficial" +# minio = "0.1.0" +minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"} reqwest = "0.12.9" serde = { version = "1.0.214", features = ["derive"] } surrealdb = "2.0.4" diff --git a/compose.yml b/compose.yml index 49b676d..797688f 100644 --- a/compose.yml +++ b/compose.yml @@ -1,5 +1,5 @@ services: - db: + surreal: image: surrealdb/surrealdb:latest-dev ports: - 8000:8000 @@ -14,3 +14,18 @@ services: - --pass - root - rocksdb:/mydata/database.db + minio: + image: quay.io/minio/minio + ports: + - 9000:9000 + - 9001:9001 + environment: + - MINIO_ROOT_USER=root + - MINIO_ROOT_PASSWORD=an8charpassword + volumes: + - ./.minio/:/data + command: + - server + - /data + - --console-address + - ":9001" diff --git a/schema.surql b/schema.surql deleted file mode 100644 index b4a8882..0000000 --- a/schema.surql +++ /dev/null @@ -1,2 +0,0 @@ -DEFINE TABLE website SCHEMALESS; - DEFINE FIELD accessed_at ON TABLE website VALUE time::now(); diff --git a/src/db.rs b/src/db.rs index 1656c4e..b5e1230 100644 --- a/src/db.rs +++ b/src/db.rs @@ -8,12 +8,12 @@ use surrealdb::{ use tracing::{error, instrument, trace, warn}; use url::Url; -use crate::Timer; +use crate::{Config, Timer}; #[derive(Debug, Serialize, Deserialize, Clone)] pub struct Website { /// The url that this data is found at - site: Url, + pub site: Url, /// Wether or not this link has been crawled yet pub crawled: bool, #[serde(skip_serializing)] @@ -39,10 +39,6 @@ impl Website { self.crawled = true } - pub fn mut_url(&mut self) -> &mut Url { - &mut self.site - } - #[instrument(skip_all)] pub async fn links_to(&self, other: Vec, db: &Surreal) { let len = other.len(); @@ -149,19 +145,30 @@ pub struct Record { pub id: Thing, } -pub async fn connect() -> surrealdb::Result> { +#[instrument(skip_all, name = "SurrealDB")] +pub async fn connect(config: &Config<'_>) -> surrealdb::Result> { + trace!("Establishing connection to surreal..."); // Connect to the server - let db = Surreal::new::("127.0.0.1:8000").await?; + let db = Surreal::new::(config.surreal_url).await?; + trace!("Logging in..."); // Signin as a namespace, database, or root user db.signin(Root { - username: "root", - password: "root", + username: config.surreal_username, + password: config.surreal_password, }) .await?; // Select a specific namespace / database - db.use_ns("test").use_db("v1.2").await?; + db + .use_ns(config.surreal_ns) + .use_db(config.surreal_db) + .await?; + + let setup = include_bytes!("setup.surql"); + let file = setup.iter().map(|c| *c as char).collect::(); + + db.query(file).await.expect("Failed to setup surreal tables."); Ok(db) } diff --git a/src/main.rs b/src/main.rs index 751eea9..56dc1b8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,12 +6,27 @@ use html5ever::{ local_name, parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, ParseOpts, }; use rcdom::RcDom; +use s3::S3; use std::time::Instant; use surrealdb::{engine::remote::ws::Client, sql::Thing, Surreal}; use tracing::{debug, info, instrument, trace, trace_span}; use tracing_subscriber::EnvFilter; mod db; +mod s3; + +struct Config<'a> { + surreal_ns: &'a str, + surreal_db: &'a str, + surreal_url: &'a str, + surreal_username: &'a str, + surreal_password: &'a str, + + s3_url: &'a str, + s3_bucket: &'a str, + s3_access_key: &'a str, + s3_secret_key: &'a str, +} #[tokio::main] async fn main() { @@ -21,16 +36,28 @@ async fn main() { .without_time() .init(); debug!("Starting..."); + + let config = Config { + surreal_ns: "test", + surreal_db: "v1.5", + surreal_url: "localhost:8000", + surreal_username: "root", + surreal_password: "root", + s3_url: "http://localhost:9000", + s3_bucket: "v1.5", + s3_access_key: "8tUJn7e1paMFZQr0PKIT", + s3_secret_key: "uSMvYxNOeCejCUgXVqgTfYlUEcmiZY0xcZ91M9E0", + }; // Would probably take these in as parameters from a cli - let url = "https://oliveratkinson.net/"; - // let url = "http://localhost:5500"; - let budget = 1000; + let starting_url = "https://oliveratkinson.net/"; + let budget = 200; let mut crawled = 0; - let db = connect().await.expect("Failed to connect to db, aborting."); - - let client = reqwest::Client::builder() + let s3 = S3::connect(&config).await.expect("Failed to connect to minio, aborting."); + let db = connect(&config).await.expect("Failed to connect to surreal, aborting."); + + let reqwest = reqwest::Client::builder() // .use_rustls_tls() .build() .unwrap(); @@ -40,8 +67,8 @@ async fn main() { let span = trace_span!("Pre-Loop"); let pre_loop_span = span.enter(); // Download the site - let mut site = Website::new(&url, false); - get(&mut site, &db, &client, &mut crawled).await; + let mut site = Website::new(&starting_url, false); + get(&mut site, &db, &reqwest, &s3, &mut crawled).await; drop(pre_loop_span); @@ -65,7 +92,7 @@ async fn main() { let _ = span.enter(); for mut site in uncrawled { - get(&mut site, &db, &client, &mut crawled).await; + get(&mut site, &db, &reqwest, &s3, &mut crawled).await; let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32); info!("Crawled {crawled} out of {budget} pages. ({percent})"); } @@ -80,13 +107,14 @@ async fn main() { async fn get( site: &mut Website, db: &Surreal, - request_client: &reqwest::Client, + reqwest: &reqwest::Client, + s3: &S3, count: &mut usize, ) { trace!("Get: {}", site.to_string()); let timer = Timer::start("Got page"); - if let Ok(response) = request_client.get(site.to_string()).send().await { + if let Ok(response) = reqwest.get(site.to_string()).send().await { timer.stop(); // Get body @@ -98,6 +126,8 @@ async fn get( }, ..Default::default() }; + s3.store(&data, &site.site).await; + // Get DOM let dom = parse_document(RcDom::default(), opts) .from_utf8() @@ -156,10 +186,10 @@ async fn walk( let mut web = site.clone(); // Set url - let url = web.mut_url(); + let mut url = web.site; url.set_fragment(None); // removes #xyz let joined = url.join(&attr.value).unwrap(); - *url = joined; + web.site = joined; // Set other attributes web.crawled = false; diff --git a/src/s3.rs b/src/s3.rs new file mode 100644 index 0000000..ddcc263 --- /dev/null +++ b/src/s3.rs @@ -0,0 +1,63 @@ +use minio::s3::{ + args::{BucketExistsArgs, MakeBucketArgs}, + client::ClientBuilder, + creds::StaticProvider, + error::Error, + http::BaseUrl, + Client, +}; +use tracing::{instrument, trace}; +use url::Url; + +use crate::Config; + +pub struct S3 { + bucket_name: String, + client: Client, +} + +impl S3 { + #[instrument(skip_all, name = "S3")] + pub async fn connect(config: &Config<'_>) -> Result { + let base_url = config.s3_url.parse::().unwrap(); + + let static_provider = + StaticProvider::new(&config.s3_access_key, &config.s3_secret_key, None); + + let client = ClientBuilder::new(base_url) + .provider(Some(Box::new(static_provider))) + .build()?; + + trace!("Checking bucket..."); + let exists = client + .bucket_exists(&BucketExistsArgs::new(&config.s3_bucket).unwrap()) + .await?; + + if !exists { + trace!("Creating bucket..."); + client + .make_bucket(&MakeBucketArgs::new(&config.s3_bucket).unwrap()) + .await?; + } + + trace!("Connection successfull"); + + Ok(Self { + bucket_name: config.s3_bucket.to_owned(), + client: client, + }) + } + + pub async fn store(&self, data: &str, name: &Url) { + if let Some(domain) = name.domain() { + let filename = domain.to_string() + name.path(); + + let _ = &self + .client + .put_object_content(&self.bucket_name, &filename, data.to_owned()) + .send() + .await + .unwrap(); + } + } +} diff --git a/src/setup.surql b/src/setup.surql new file mode 100644 index 0000000..4271c79 --- /dev/null +++ b/src/setup.surql @@ -0,0 +1,2 @@ +DEFINE TABLE IF NOT EXISTS website SCHEMALESS; +DEFINE FIELD IF NOT EXISTS accessed_at ON TABLE website VALUE time::now(); From eaa79b749edc2045e5c10c909a9fdd59723e20d3 Mon Sep 17 00:00:00 2001 From: oliver Date: Tue, 12 Nov 2024 21:19:05 -0700 Subject: [PATCH 02/19] prepare get function for s3 --- src/s3.rs | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/s3.rs b/src/s3.rs index ddcc263..f17b7d1 100644 --- a/src/s3.rs +++ b/src/s3.rs @@ -1,10 +1,5 @@ use minio::s3::{ - args::{BucketExistsArgs, MakeBucketArgs}, - client::ClientBuilder, - creds::StaticProvider, - error::Error, - http::BaseUrl, - Client, + args::{BucketExistsArgs, MakeBucketArgs}, client::ClientBuilder, creds::StaticProvider, error::Error, http::BaseUrl, types::S3Api, Client }; use tracing::{instrument, trace}; use url::Url; @@ -60,4 +55,22 @@ impl S3 { .unwrap(); } } + + pub async fn _get(&self, name: &Url) -> Option { + if let Some(domain) = name.domain() { + let filename = domain.to_string() + name.path(); + + let data = self + .client + .get_object(&self.bucket_name, &filename) + .send() + .await + .unwrap(); + + if let Ok(segments )= data.content.to_segmented_bytes().await { + return Some(segments.to_bytes().iter().map(|c| *c as char).collect::()) + } + } + None + } } From 574a370f3048604ac690b6ff076f7b471fb723a6 Mon Sep 17 00:00:00 2001 From: oliver Date: Tue, 12 Nov 2024 21:24:57 -0700 Subject: [PATCH 03/19] readme updates --- README.md | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a26092e..94913df 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,11 @@ # Surreal Crawler -Mapping with a budget of 1000 (crawl 1000 sites, so many more links are actually discovered), on [my webiste](https://oliveratkinson.net) on 8/26/2024 took 1m9s. +Crawls sites saving all the found links to a surrealdb database. It then proceeds to take batches of 100 uncrawled links untill the crawl budget is reached. It saves the data of each site in a minio database. -This is including the crawl and loading into the database and linking sites. (Locally hosted surreal db instance) -This run created 4299 site links with 23286 links between the sites. (It found my this git site which really bolsters those numbers.) - -## Install / Build - -* You will need rust to compile the crawler [rustup.rs](https://rustup.rs) -* You need python3 (will come installed on most linux distros) and poetry for dependancy management. - * Install `pipx`, `python3` - * Then: `pipx install poetry` - * Then: `poetry install` to install the project dependancies -* You need to install [surrealdb](https://surrealdb.com) - -## Use - -Just run `./crawl.sh {url}` and it will start crawling. You can tweak the budget inside [crawl.sh](https://git.oliveratkinson.net/Oliver/internet_mapper/src/branch/main/crawl.sh) if you want. - -You can also prefix the command with `time` to benchmark the system, such as: `time ./crawl.sh https://discord.com`. +### TODO +- [ ] Domain filtering - prevent the crawler from going on alternate versions of wikipedia. +- [ ] Conditionally save content - based on filename or file contents +- [ ] GUI / TUI ? +- [ ] Better asynchronous getting of the sites. Currently it all happens serially. \ No newline at end of file From 0f8a3d721522b0a7b589c5e62612b2f06e4ad3b4 Mon Sep 17 00:00:00 2001 From: oliver Date: Tue, 12 Nov 2024 23:08:09 -0700 Subject: [PATCH 04/19] using a custom parser now :) --- Cargo.lock | 24 ---------- Cargo.toml | 1 - src/main.rs | 121 ++++++-------------------------------------------- src/parser.rs | 95 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 133 deletions(-) create mode 100644 src/parser.rs diff --git a/Cargo.lock b/Cargo.lock index e205787..0732362 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1987,18 +1987,6 @@ dependencies = [ "tendril", ] -[[package]] -name = "markup5ever_rcdom" -version = "0.5.0-unofficial" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9cb12459c4cab18dcc580159590f404ad78c0a9c5435ace80288ed43abdce31" -dependencies = [ - "html5ever 0.29.0", - "markup5ever 0.14.0", - "tendril", - "xml5ever", -] - [[package]] name = "matchers" version = "0.1.0" @@ -3653,7 +3641,6 @@ name = "surreal_spider" version = "0.1.0" dependencies = [ "html5ever 0.29.0", - "markup5ever_rcdom", "minio", "reqwest", "serde", @@ -4725,17 +4712,6 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af310deaae937e48a26602b730250b4949e125f468f11e6990be3e5304ddd96f" -[[package]] -name = "xml5ever" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2278b4bf33071ba8e30368a59436c65eec8e01c49d5c29b3dfeb0cdc45331383" -dependencies = [ - "log", - "mac", - "markup5ever 0.14.0", -] - [[package]] name = "xmltree" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index a9c7532..cba532f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ edition = "2021" [dependencies] html5ever = "0.29.0" -markup5ever_rcdom = "0.5.0-unofficial" # minio = "0.1.0" minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"} reqwest = "0.12.9" diff --git a/src/main.rs b/src/main.rs index 56dc1b8..f19cd84 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,19 +1,16 @@ extern crate html5ever; -extern crate markup5ever_rcdom as rcdom; + +use std::time::Instant; use db::{connect, Website}; -use html5ever::{ - local_name, parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, ParseOpts, -}; -use rcdom::RcDom; use s3::S3; -use std::time::Instant; -use surrealdb::{engine::remote::ws::Client, sql::Thing, Surreal}; +use surrealdb::{engine::remote::ws::Client, Surreal}; use tracing::{debug, info, instrument, trace, trace_span}; use tracing_subscriber::EnvFilter; mod db; mod s3; +mod parser; struct Config<'a> { surreal_ns: &'a str, @@ -39,19 +36,19 @@ async fn main() { let config = Config { surreal_ns: "test", - surreal_db: "v1.5", + surreal_db: "v1.7", surreal_url: "localhost:8000", surreal_username: "root", surreal_password: "root", s3_url: "http://localhost:9000", - s3_bucket: "v1.5", + s3_bucket: "v1.7", s3_access_key: "8tUJn7e1paMFZQr0PKIT", s3_secret_key: "uSMvYxNOeCejCUgXVqgTfYlUEcmiZY0xcZ91M9E0", }; // Would probably take these in as parameters from a cli let starting_url = "https://oliveratkinson.net/"; - let budget = 200; + let budget = 15; let mut crawled = 0; let s3 = S3::connect(&config).await.expect("Failed to connect to minio, aborting."); @@ -75,11 +72,7 @@ async fn main() { let span = trace_span!("Loop"); let span = span.enter(); while crawled < budget { - let get_num = if budget - crawled < 100 { - budget - crawled - } else { - 100 - }; + let get_num = if budget - crawled < 100 { budget - crawled } else { 100 }; let uncrawled = get_uncrawled_links(&db, get_num).await; if uncrawled.len() == 0 { @@ -93,6 +86,7 @@ async fn main() { for mut site in uncrawled { get(&mut site, &db, &reqwest, &s3, &mut crawled).await; + let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32); info!("Crawled {crawled} out of {budget} pages. ({percent})"); } @@ -103,7 +97,7 @@ async fn main() { } #[instrument(skip_all)] -/// A quick helper function for downloading a url +/// Downloads and crawls and stores a webpage. async fn get( site: &mut Website, db: &Surreal, @@ -113,109 +107,20 @@ async fn get( ) { trace!("Get: {}", site.to_string()); let timer = Timer::start("Got page"); - if let Ok(response) = reqwest.get(site.to_string()).send().await { timer.stop(); // Get body let data = response.text().await.unwrap(); - let opts = ParseOpts { - tree_builder: TreeBuilderOpts { - drop_doctype: true, - ..Default::default() - }, - ..Default::default() - }; + // Store document s3.store(&data, &site.site).await; - - // Get DOM - let dom = parse_document(RcDom::default(), opts) - .from_utf8() - .read_from(&mut data.as_bytes()) - .unwrap(); - - // TODO save the dom to minio if a flag is set - - // Modify record in database - site.set_crawled(); - site.store(db).await; - trace!("Got: {}", site.to_string()); - - // Walk all the children nodes, searching for links to other pages. - let mut buffer = Vec::new(); - let timer = Timer::start("Walked"); - walk(&dom.document, &db, &site, &mut buffer).await; - timer.stop(); - - // Put all the found links into the database. - site.links_to(buffer, &db).await; + // Parse document and store relationships + parser::parse(db, site, data).await; *count += 1; } trace!("Failed to get: {}", site.to_string()); } -/// Walks the givin site, placing it's findings in the database -async fn walk( - node: &rcdom::Handle, - db: &Surreal, - site: &Website, - links_to: &mut Vec, -) { - let span = trace_span!("Walk"); - let span = span.enter(); - // Match each node - node basically means element. - match &node.data { - rcdom::NodeData::Element { name, attrs, .. } => { - for attr in attrs.borrow().clone() { - match name.local { - local_name!("a") - | local_name!("audio") - | local_name!("area") - | local_name!("img") - | local_name!("link") - | local_name!("object") - | local_name!("source") - | local_name!("base") - | local_name!("video") => { - let attribute_name = attr.name.local.to_string(); - if attribute_name == "src" - || attribute_name == "href" - || attribute_name == "data" - { - // Get clone of the current site object - let mut web = site.clone(); - - // Set url - let mut url = web.site; - url.set_fragment(None); // removes #xyz - let joined = url.join(&attr.value).unwrap(); - web.site = joined; - - // Set other attributes - web.crawled = false; - // TODO set element name - // let element_name = name.local.to_string(); - - if let Some(id) = web.store(db).await { - links_to.push(id); - } - } - } - local_name!("button") | local_name!("meta") | local_name!("iframe") => { - // dbg!(attrs); - } - _ => {} - }; - } - } - _ => {} - }; - drop(span); - for child in node.children.borrow().iter() { - Box::pin(walk(child, db, site, links_to)).await; - } -} - /// Returns uncrawled links async fn get_uncrawled_links(db: &Surreal, mut count: usize) -> Vec { if count > 100 { diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000..17143f6 --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,95 @@ +use std::default::Default; +use std::str::FromStr; + +use html5ever::tokenizer::{BufferQueue, TokenizerResult}; +use html5ever::tokenizer::{StartTag, TagToken}; +use html5ever::tokenizer::{Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts}; +use html5ever::{local_name, tendril::*}; +use surrealdb::engine::remote::ws::Client; +use surrealdb::Surreal; + +use crate::db::Website; + +#[derive(Clone)] +struct LinkParser<'a> { + site: &'a Website, +} + +impl TokenSink for LinkParser<'_> { + type Handle = Vec; + + fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult { + match token { + TagToken(tag) => { + if tag.kind == StartTag { + match tag.name { + local_name!("a") + | local_name!("audio") + | local_name!("area") + | local_name!("img") + | local_name!("link") + | local_name!("object") + | local_name!("source") + | local_name!("base") + | local_name!("video") => { + let mut links = Vec::new(); + for attr in &tag.attrs { + let attr_name = attr.name.local.to_string(); + if attr_name == "src" || attr_name == "href" || attr_name == "data" + { + // Get clone of the current site object + let mut web = self.site.clone(); + + // Set url + let mut url = web.site; + url.set_fragment(None); // removes #xyz + let joined = url.join(&attr.value).unwrap(); + web.site = joined; + + web.crawled = false; + + links.push(web); + } + } + + return TokenSinkResult::Script(links); + } + local_name!("button") | local_name!("meta") | local_name!("iframe") => { + // dbg!(attrs); + } + _ => {} + } + } + } + _ => {} + } + TokenSinkResult::Continue + } +} + +pub async fn parse(db: &Surreal, site: &mut Website, data: String) { + + site.set_crawled(); + site.store(db).await; + + let sink = LinkParser { site }; + let chunk = Tendril::from_str(&data).unwrap(); + let mut input = BufferQueue::default(); + input.push_back(chunk.try_reinterpret::().unwrap()); + + let token = Tokenizer::new(sink.clone(), TokenizerOpts::default()); + + let mut links_to = Vec::new(); + while !input.is_empty() { + if let TokenizerResult::Script(s) = token.feed(&mut input) { + for mut web in s { + if let Some(id) = web.store(db).await { + links_to.push(id); + } + } + } + } + sink.site.links_to(links_to, db).await; + assert!(input.is_empty()); + token.end(); +} From c1c8cf07bb153f4d714e5f4f8fb76e08220a3573 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Thu, 12 Dec 2024 11:42:07 -0700 Subject: [PATCH 05/19] unifed settings for testing --- src/main.rs | 18 +++++++++++------- src/parser.rs | 5 +++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index f19cd84..8449efc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,28 +27,30 @@ struct Config<'a> { #[tokio::main] async fn main() { + let total_runtime = Timer::start("Completed"); + tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) .with_line_number(true) - .without_time() + // .without_time() .init(); debug!("Starting..."); let config = Config { - surreal_ns: "test", - surreal_db: "v1.7", surreal_url: "localhost:8000", surreal_username: "root", surreal_password: "root", + surreal_ns: "test", + surreal_db: "custom-engine-v2", + s3_bucket: "custom-engine-v2", s3_url: "http://localhost:9000", - s3_bucket: "v1.7", - s3_access_key: "8tUJn7e1paMFZQr0PKIT", - s3_secret_key: "uSMvYxNOeCejCUgXVqgTfYlUEcmiZY0xcZ91M9E0", + s3_access_key: "0zv7GbLQsw4ZI8TclMps", + s3_secret_key: "5dB7QkGFw7fYbUJ5LpHk2GbWR7Bl710HlRz4NbzB", }; // Would probably take these in as parameters from a cli let starting_url = "https://oliveratkinson.net/"; - let budget = 15; + let budget = 5; let mut crawled = 0; let s3 = S3::connect(&config).await.expect("Failed to connect to minio, aborting."); @@ -94,6 +96,7 @@ async fn main() { drop(span); info!("Done"); + drop(total_runtime); } #[instrument(skip_all)] @@ -117,6 +120,7 @@ async fn get( // Parse document and store relationships parser::parse(db, site, data).await; *count += 1; + return; } trace!("Failed to get: {}", site.to_string()); } diff --git a/src/parser.rs b/src/parser.rs index 17143f6..09d1dff 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -7,8 +7,10 @@ use html5ever::tokenizer::{Token, TokenSink, TokenSinkResult, Tokenizer, Tokeniz use html5ever::{local_name, tendril::*}; use surrealdb::engine::remote::ws::Client; use surrealdb::Surreal; +use tracing::instrument; use crate::db::Website; +use crate::Timer; #[derive(Clone)] struct LinkParser<'a> { @@ -67,6 +69,7 @@ impl TokenSink for LinkParser<'_> { } } +#[instrument(skip_all)] pub async fn parse(db: &Surreal, site: &mut Website, data: String) { site.set_crawled(); @@ -79,6 +82,7 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: String) { let token = Tokenizer::new(sink.clone(), TokenizerOpts::default()); + let t = Timer::start("Stored pages"); let mut links_to = Vec::new(); while !input.is_empty() { if let TokenizerResult::Script(s) = token.feed(&mut input) { @@ -89,6 +93,7 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: String) { } } } + drop(t); sink.site.links_to(links_to, db).await; assert!(input.is_empty()); token.end(); From 22be3b2f613be05c3cf76176dffeaefa8b06a0c0 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Thu, 12 Dec 2024 14:14:38 -0700 Subject: [PATCH 06/19] updating deps --- Cargo.lock | 591 ++++++++++++++++++++++++++++------------------------ Cargo.toml | 14 +- src/main.rs | 17 +- 3 files changed, 338 insertions(+), 284 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0732362..238e64f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,9 +71,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "ammonia" @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arbitrary" -version = "1.3.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" [[package]] name = "argon2" @@ -202,9 +202,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "ascii-canvas" @@ -217,13 +217,14 @@ dependencies = [ [[package]] name = "async-channel" -version = "1.9.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener", + "event-listener-strategy", "futures-core", + "pin-project-lite", ] [[package]] @@ -241,9 +242,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba6d24703c5adc5ba9116901b92ee4e4c0643c01a56c4fd303f3818638d7449" +checksum = "59fd6bd734afb8b6e4d0f84a3e77305ce0a7ccc60d70f6001cb5e1c3f38d8ff1" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -256,25 +257,24 @@ dependencies = [ "futures-timer", "futures-util", "http", - "indexmap 2.6.0", + "indexmap 2.7.0", "mime", "multer", "num-traits", - "once_cell", "pin-project-lite", "regex", "serde", "serde_json", "serde_urlencoded", "static_assertions_next", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "async-graphql-derive" -version = "7.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a94c2d176893486bd37cd1b6defadd999f7357bf5804e92f510c08bcf16c538f" +checksum = "ac38b4dd452d529d6c0248b51df23603f0a875770352e26ae8c346ce6c149b3e" dependencies = [ "Inflector", "async-graphql-parser", @@ -283,15 +283,15 @@ dependencies = [ "proc-macro2", "quote", "strum", - "syn 2.0.85", - "thiserror", + "syn 2.0.90", + "thiserror 1.0.69", ] [[package]] name = "async-graphql-parser" -version = "7.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79272bdbf26af97866e149f05b2b546edb5c00e51b5f916289931ed233e208ad" +checksum = "42d271ddda2f55b13970928abbcbc3423cfc18187c60e8769b48f21a93b7adaa" dependencies = [ "async-graphql-value", "pest", @@ -301,12 +301,12 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5ec94176a12a8cbe985cd73f2e54dc9c702c88c766bdef12f1f3a67cedbee1" +checksum = "aefe909173a037eaf3281b046dc22580b59a38b765d7b8d5116f2ffef098048d" dependencies = [ "bytes", - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_json", ] @@ -319,7 +319,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -341,7 +341,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -358,7 +358,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -483,9 +483,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" dependencies = [ "arrayref", "arrayvec", @@ -515,9 +515,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.5.1" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" +checksum = "2506947f73ad44e344215ccd6403ac2ae18cd8e046e581a441bf8d199f257f03" dependencies = [ "borsh-derive", "cfg_aliases", @@ -525,16 +525,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.5.1" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" +checksum = "c2593a3b8b938bd68373196c9832f516be11fa487ef4ae745eb282e6a56a7244" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.85", - "syn_derive", + "syn 2.0.90", ] [[package]] @@ -567,9 +566,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" [[package]] name = "byteorder" @@ -579,9 +578,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" dependencies = [ "serde", ] @@ -597,9 +596,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "shlex", ] @@ -618,7 +617,7 @@ dependencies = [ "serde", "serde_json", "smol_str", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -641,7 +640,7 @@ dependencies = [ "serde_with", "smol_str", "stacker", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -657,7 +656,7 @@ dependencies = [ "serde_with", "smol_str", "stacker", - "thiserror", + "thiserror 1.0.69", "unicode-security", ] @@ -675,9 +674,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -764,9 +763,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -786,6 +785,25 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -829,7 +847,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -840,7 +858,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -943,7 +961,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -1032,25 +1050,40 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "event-listener" -version = "2.5.3" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" +dependencies = [ + "event-listener", + "pin-project-lite", +] [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fixedbitset" @@ -1166,9 +1199,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" dependencies = [ "fastrand", "futures-core", @@ -1185,7 +1218,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -1263,9 +1296,9 @@ dependencies = [ [[package]] name = "geo-types" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff16065e5720f376fbced200a5ae0f47ace85fd70b7e54269790281353b6d61" +checksum = "b6f47c611187777bbca61ea7aba780213f5f3441fd36294ab333e96cfa791b65" dependencies = [ "approx 0.5.1", "arbitrary", @@ -1304,9 +1337,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", @@ -1314,7 +1347,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.6.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -1361,9 +1394,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heapless" @@ -1428,7 +1461,7 @@ dependencies = [ "markup5ever 0.12.1", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -1442,14 +1475,14 @@ dependencies = [ "markup5ever 0.14.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -1499,9 +1532,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", @@ -1709,7 +1742,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -1752,12 +1785,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.2", "serde", ] @@ -1811,16 +1844,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1867,7 +1901,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" dependencies = [ - "regex-automata 0.4.8", + "regex-automata 0.4.9", ] [[package]] @@ -1887,9 +1921,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.161" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libm" @@ -1916,7 +1950,7 @@ dependencies = [ "ndarray", "num-traits", "rand", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1927,9 +1961,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "lock_api" @@ -2036,7 +2070,7 @@ checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" dependencies = [ "miette-derive", "once_cell", - "thiserror", + "thiserror 1.0.69", "unicode-width", ] @@ -2048,7 +2082,7 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -2118,11 +2152,10 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi", "libc", "wasi", "windows-sys 0.52.0", @@ -2356,7 +2389,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -2379,9 +2412,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.8.2" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae99c7fa6dd38c7cafe1ec085e804f8f555a2f8659b0dbe03f1f9963a9b51092" +checksum = "e5ca711d8b83edbb00b44d504503cd247c9c0bd8b0fa2694f2a1a3d8165379ce" dependencies = [ "log", "serde", @@ -2476,12 +2509,12 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror", + "thiserror 2.0.6", "ucd-trie", ] @@ -2492,7 +2525,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.6.0", + "indexmap 2.7.0", ] [[package]] @@ -2555,7 +2588,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", "unicase", ] @@ -2632,34 +2665,11 @@ dependencies = [ "toml_edit", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2672,9 +2682,9 @@ checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" [[package]] name = "psm" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" dependencies = [ "cc", ] @@ -2713,9 +2723,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", @@ -2724,33 +2734,36 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror", + "thiserror 2.0.6", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", "rustc-hash", "rustls", + "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.6", "tinyvec", "tracing", + "web-time", ] [[package]] name = "quinn-udp" -version = "0.5.6" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" +checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ "cfg_aliases", "libc", @@ -2823,16 +2836,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] -name = "reblessive" -version = "0.4.1" +name = "rayon" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d4f118ca848dfd632a8c0883f9aacd6b58da548eb0629a78cafee3d330938da" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "reblessive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffead9d0a0b45f3e0bc063a244b1779fd53a09d2c2f7282c186a016b1f10a778" [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags", ] @@ -2845,7 +2878,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -2865,7 +2898,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -2876,7 +2909,7 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.9", "regex-syntax 0.8.5", ] @@ -2891,9 +2924,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -2995,7 +3028,7 @@ checksum = "5f0ec466e5d8dca9965eb6871879677bef5590cf7525ad96cae14376efb75073" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -3065,9 +3098,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4b84ba6e838ceb47b41de5194a60244fac43d9fe03b71dbe8c5a201081d6d1" +checksum = "395b0c39c00f9296f3937624c1fa4e0ee44f8c0e4b2c49408179ef381c6c2e6e" dependencies = [ "bytemuck", "byteorder", @@ -3082,9 +3115,9 @@ checksum = "cbf4a6aa5f6d6888f39e980649f3ad6b666acdce1d78e95b8a2cb076e687ae30" [[package]] name = "rstar" -version = "0.12.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "133315eb94c7b1e8d0cb097e5a710d850263372fd028fff18969de708afc7008" +checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb" dependencies = [ "heapless", "num-traits", @@ -3125,9 +3158,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustc_lexer" @@ -3149,22 +3182,22 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "log", "once_cell", @@ -3236,9 +3269,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -3282,9 +3315,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -3307,9 +3340,9 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.214" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] @@ -3325,22 +3358,22 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "itoa", "memchr", "ryu", @@ -3369,7 +3402,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_derive", "serde_json", @@ -3386,7 +3419,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -3449,7 +3482,7 @@ checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ "num-bigint", "num-traits", - "thiserror", + "thiserror 1.0.69", "time", ] @@ -3513,9 +3546,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -3573,7 +3606,7 @@ dependencies = [ "byteorder", "memchr", "serde", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -3627,7 +3660,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -3653,9 +3686,9 @@ dependencies = [ [[package]] name = "surrealdb" -version = "2.0.4" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300749e641e2a5546a142333d91b3537969e9c4cbd1fa2d1d10d92f01624d470" +checksum = "1b69bfa3a8cd087f2d0305be1fed24cb8e18571cb4f116089df7d39f7150f97a" dependencies = [ "arrayvec", "async-channel", @@ -3664,7 +3697,7 @@ dependencies = [ "dmp", "futures", "geo", - "indexmap 2.6.0", + "indexmap 2.7.0", "path-clean", "pharos", "reblessive", @@ -3679,7 +3712,7 @@ dependencies = [ "serde-content", "serde_json", "surrealdb-core", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-tungstenite", "tokio-util", @@ -3694,9 +3727,9 @@ dependencies = [ [[package]] name = "surrealdb-core" -version = "2.0.4" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5795fca60f099754934069dfb3d20824cfec94b4c13c7130d1cd52e0fcadfc42" +checksum = "cfcf8a937ddc5626e845ba715167776008f899795c1274657c2ba1b9f0f4d2bd" dependencies = [ "addr", "ahash 0.8.11", @@ -3742,6 +3775,7 @@ dependencies = [ "quick_cache", "radix_trie", "rand", + "rayon", "reblessive", "regex", "revision", @@ -3759,9 +3793,10 @@ dependencies = [ "sha2", "snap", "storekey", + "strsim", "subtle", "surrealdb-derive", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", "trice", @@ -3769,6 +3804,7 @@ dependencies = [ "unicase", "url", "uuid", + "vart", "wasm-bindgen-futures", "wasmtimer", "ws_stream_wasm", @@ -3797,32 +3833,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "syn_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 2.0.85", -] - [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -3835,7 +3859,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -3867,9 +3891,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -3902,22 +3926,42 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +dependencies = [ + "thiserror-impl 2.0.6", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -3932,9 +3976,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -3953,9 +3997,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -3997,9 +4041,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -4021,7 +4065,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -4036,20 +4080,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ "rustls", - "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -4074,9 +4117,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -4098,7 +4141,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "toml_datetime", "winnow", ] @@ -4111,9 +4154,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -4122,20 +4165,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -4154,9 +4197,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -4203,7 +4246,7 @@ dependencies = [ "rustls", "rustls-pki-types", "sha1", - "thiserror", + "thiserror 1.0.69", "url", "utf-8", ] @@ -4240,9 +4283,9 @@ checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-normalization" @@ -4289,9 +4332,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -4346,6 +4389,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vart" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c92195d375eb94995afddeedfd7f246796eb60b85f727c538e42222c4c9b2d3" + [[package]] name = "vcpkg" version = "0.2.15" @@ -4385,9 +4434,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -4396,36 +4445,36 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4433,22 +4482,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-streams" @@ -4478,9 +4527,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -4498,9 +4547,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -4691,7 +4740,7 @@ dependencies = [ "pharos", "rustc_version", "send_wrapper", - "thiserror", + "thiserror 1.0.69", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -4708,9 +4757,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af310deaae937e48a26602b730250b4949e125f468f11e6990be3e5304ddd96f" +checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" [[package]] name = "xmltree" @@ -4723,9 +4772,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", @@ -4735,13 +4784,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", "synstructure", ] @@ -4763,27 +4812,27 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", "synstructure", ] @@ -4812,5 +4861,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] diff --git a/Cargo.toml b/Cargo.toml index cba532f..72c21f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,13 +4,13 @@ version = "0.1.0" edition = "2021" [dependencies] -html5ever = "0.29.0" +html5ever = "0.29" # minio = "0.1.0" minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"} -reqwest = "0.12.9" -serde = { version = "1.0.214", features = ["derive"] } -surrealdb = "2.0.4" +reqwest = "0.12" +serde = { version = "1.0", features = ["derive"] } +surrealdb = "2.1" tokio = { version="1.41.0", features = ["full"] } -tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } -url = { version = "2.5.3", features = ["serde"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +url = { version = "2.5", features = ["serde"] } diff --git a/src/main.rs b/src/main.rs index 8449efc..e2e3028 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,8 +31,8 @@ async fn main() { tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) - .with_line_number(true) - // .without_time() + .with_line_number(false) + .without_time() .init(); debug!("Starting..."); @@ -49,7 +49,9 @@ async fn main() { }; // Would probably take these in as parameters from a cli - let starting_url = "https://oliveratkinson.net/"; + let starting_url = "https://en.wikipedia.org/"; + // When getting uncrawled pages, name must be LIKE this variable. "" will effectively get ignored. + let crawl_like = "wikipedia"; let budget = 5; let mut crawled = 0; @@ -76,7 +78,7 @@ async fn main() { while crawled < budget { let get_num = if budget - crawled < 100 { budget - crawled } else { 100 }; - let uncrawled = get_uncrawled_links(&db, get_num).await; + let uncrawled = get_uncrawled_links(&db, get_num, crawl_like.to_string()).await; if uncrawled.len() == 0 { info!("Had more budget but finished crawling everything."); return; @@ -126,13 +128,16 @@ async fn get( } /// Returns uncrawled links -async fn get_uncrawled_links(db: &Surreal, mut count: usize) -> Vec { +#[instrument(skip(db))] +async fn get_uncrawled_links(db: &Surreal, mut count: usize, param: String) -> Vec { if count > 100 { count = 100 } + trace!("Getting uncrawled links"); let mut response = db - .query("SELECT * FROM website WHERE crawled = false LIMIT $count") + .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;") + .bind(("format", param)) .bind(("count", count)) .await .expect("Hard-coded query failed..?"); From 215056e493391177cbf496519fc5fdcbd9661317 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Thu, 12 Dec 2024 14:26:49 -0700 Subject: [PATCH 07/19] use contains operator for better output --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index e2e3028..8385a79 100644 --- a/src/main.rs +++ b/src/main.rs @@ -136,7 +136,7 @@ async fn get_uncrawled_links(db: &Surreal, mut count: usize, param: Stri trace!("Getting uncrawled links"); let mut response = db - .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;") + .query("SELECT * FROM website WHERE crawled = false AND site CONTAINS type::string($format) LIMIT $count;") .bind(("format", param)) .bind(("count", count)) .await From 298ad39a790a9648fb7bf04f6f92a0c6769ed9b2 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Thu, 12 Dec 2024 14:59:54 -0700 Subject: [PATCH 08/19] rename --- src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8385a79..6db0563 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,9 +50,9 @@ async fn main() { // Would probably take these in as parameters from a cli let starting_url = "https://en.wikipedia.org/"; - // When getting uncrawled pages, name must be LIKE this variable. "" will effectively get ignored. - let crawl_like = "wikipedia"; - let budget = 5; + // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored. + let crawl_filter = "https://en.wikipedia.org/"; + let budget = 50; let mut crawled = 0; let s3 = S3::connect(&config).await.expect("Failed to connect to minio, aborting."); @@ -78,7 +78,7 @@ async fn main() { while crawled < budget { let get_num = if budget - crawled < 100 { budget - crawled } else { 100 }; - let uncrawled = get_uncrawled_links(&db, get_num, crawl_like.to_string()).await; + let uncrawled = get_uncrawled_links(&db, get_num, crawl_filter.to_string()).await; if uncrawled.len() == 0 { info!("Had more budget but finished crawling everything."); return; From 611a1e923b79b92a072a566e635e89b167214859 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Thu, 12 Dec 2024 15:32:04 -0700 Subject: [PATCH 09/19] starting on the extension --- browser/icon.png | Bin 0 -> 1983 bytes browser/jsconfig.json | 5 +++++ browser/main/main.html | 16 ++++++++++++++++ browser/main/script.js | 5 +++++ browser/main/style.css | 4 ++++ browser/manifest.json | 14 ++++++++++++++ browser/popup/popup.html | 18 ++++++++++++++++++ browser/popup/popup.js | 3 +++ browser/popup/runner.js | 5 +++++ jsconfig.json | 16 ++++++++++++++++ 10 files changed, 86 insertions(+) create mode 100644 browser/icon.png create mode 100644 browser/jsconfig.json create mode 100644 browser/main/main.html create mode 100644 browser/main/script.js create mode 100644 browser/main/style.css create mode 100644 browser/manifest.json create mode 100644 browser/popup/popup.html create mode 100644 browser/popup/popup.js create mode 100644 browser/popup/runner.js create mode 100644 jsconfig.json diff --git a/browser/icon.png b/browser/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..6b41a931d2a06980ea38b1b41dcb279e8701dfa3 GIT binary patch literal 1983 zcmV;w2SE6VP)Px+c}YY;RA>dwS$j}aRTw{acUfR}Sy<%}nhFYPiU=rUQ{zST^OI?a%Z z1Ww^Nq76;jFfyeksPRua)znN`=9pbMk}LA zZR%5!xP+y+0Ger(brfLY*n_-MoBGta3W^H|4GoQ5I>YEvKCoIWnw!#0!@ zARy1*CqnUK4l%99;o?SZ>Z9WHQS2%xCLljQe-`*nGz8pXu~El4S@$36BOCcRecjCi zZYw4L`ORm40?oeG-ir|Rk&TMv+skPH9f}Cpv13OHVka2_ZQ|(YV|~&f8~Kp0b4;JU zat(?Iz>}8Geq8J!LFZxiB{F0qA3B%DNVb6=H>>Q&F0y7((VcjxF^niB^v_n zd*xzx__6N}ZWPeU-216`J@)t8YBJgNnQeT*a zojhR=iyQr$bn5kZuHq$j^6VU!5AduwsRUjH+LhF%V{2<`WmZ;Ji_6hNgA2eTJ|H3@ zBF5oxaH4KxiD;yUalA{X?O;JBD+@L?vZ<4IGo8-+>`OTJ?Rs|X)M9qGIfC754r6wQ zzhv@sY=D>w7`P}BhMPcTV`Jkb@NO6IVA+a$b2=P6UJc}}Xf#-b;GjlkHZ{3wFxrN| z0bluUL|fW|nXNgzTfKWPg0-}UiFSw1BQF4mU0q$HZiF6q`5%nQpNA4m~?xIUTdEB}QKr6%{SozJ0rc zf9jQ&my5~C$$L#E(+D^uR@F`xLoHVYGY0-2N?qevcKG;8q?4Y=??V1B1^F)G)^mOZ z7 z@A9=|_Q%P$z*{5WR+*cYmbSXDAbkp;0S68oI1CY35}N0)H6-A9a8no@Qrj1nBK?)s zPqSmEmw>!L@xZ7nn~01b}opiiO>b=F7F$<3)dcvk*3@+81`e>=!OS$4bf;fH%K0zK5LOBnLqD z>-6&F%RdHS1_dZ7o6kBr1Lfe$@*yn3bdt|-#*#Hwwg5syPX0j00Dl|v#}iViLRf@J zMT#L8yI%sZn&SxAVpjfc6k=}RoTV752mmM-AO({fM2?@^r0^JW#*k0Ht^oX1y8@3y zfOL67RCu6Cm{JVIuK;b#;vV-D0lUqX+&Dh{But9YpG{!dDP*x&MqzdWX^T+r*Y1-L zo}}zj^%P#I?W+(bVN(oj#gFvmFWF`PN9O0}H=YKC$hZSl*G*(+%4e}Vwjn&u*^22b zY1}?GV(8yeT#rY>ZfP4tB518$^61o~Q?>w%d$54Le;QVuJ-MoX3snrl_*?(jW8$P6rC5_*Qo5F*Kn6!{FPPPEw zs|#qUsi}U6iHR-PozN|OHA84?H?z|66n3R%3ML!9qDY%hTRS>B@>Z={#kZi{$aiaQ z-n==)-`_tAof*h=BaI$8?1YdwZa)hQu(2$4%^ZQ$tfZ;m)R>Znjvm&n>X z@G*!#L-0~}jvC&ITmka|^0gZf{d@5RcJgvou3TB|*5B_Y{wo-PowghdT!iZWdjBQ_ zeB%pH?iz6ABxZaM>Uw-^T<`@h$L^efrPv|dgsZUu^>rGD82q;F3qzIw0t4``cDsFT zdV2aPU!ViAJ11n$wr$%c>vX!0fZ>|7f#7u;gW2J)5ZyXeK6xB5RhHx#{$G+-)CsAZ Rx + + + + + + + + + + +

Here is some text

+

old

+ + + \ No newline at end of file diff --git a/browser/main/script.js b/browser/main/script.js new file mode 100644 index 0000000..9fdd6ca --- /dev/null +++ b/browser/main/script.js @@ -0,0 +1,5 @@ +document.getElementById("changeme").innerText = "newer" + +window.fetch("http://127.0.0.1:9001").then((e) => { + console.log(e) +}) diff --git a/browser/main/style.css b/browser/main/style.css new file mode 100644 index 0000000..cab1b1f --- /dev/null +++ b/browser/main/style.css @@ -0,0 +1,4 @@ +html { + background: black; + color: white; +} \ No newline at end of file diff --git a/browser/manifest.json b/browser/manifest.json new file mode 100644 index 0000000..c1d8e0c --- /dev/null +++ b/browser/manifest.json @@ -0,0 +1,14 @@ +{ + "manifest_version": 2, + "name": "Viewer", + "version": "0.1", + "description": "Adds a red border to all webpages matching mozilla.org.", + "permissions": [ + "activeTab" + ], + "browser_action": { + "default_icon": "icon.png", + "default_title": "Click me!", + "default_popup": "popup/popup.html" + } +} \ No newline at end of file diff --git a/browser/popup/popup.html b/browser/popup/popup.html new file mode 100644 index 0000000..dd8094d --- /dev/null +++ b/browser/popup/popup.html @@ -0,0 +1,18 @@ + + + + + + + + +

+ + + + diff --git a/browser/popup/popup.js b/browser/popup/popup.js new file mode 100644 index 0000000..b5d8389 --- /dev/null +++ b/browser/popup/popup.js @@ -0,0 +1,3 @@ +document.getElementById("newtab").addEventListener('click', async function(e) { + await browser.tabs.create({ url: "/main/main.html" }) +}) \ No newline at end of file diff --git a/browser/popup/runner.js b/browser/popup/runner.js new file mode 100644 index 0000000..0eee102 --- /dev/null +++ b/browser/popup/runner.js @@ -0,0 +1,5 @@ +document.body.textContent = ""; + +let header = document.createElement("h1"); +header.textContent = "This page has been eaten"; +document.body.appendChild(header); diff --git a/jsconfig.json b/jsconfig.json new file mode 100644 index 0000000..86f7f90 --- /dev/null +++ b/jsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "ESNext", + "moduleResolution": "Bundler", + "target": "ES2022", + "jsx": "react", + "allowImportingTsExtensions": true, + "strictNullChecks": true, + "strictFunctionTypes": true + }, + "exclude": [ + "node_modules", + "**/node_modules/*" + ], + "typeAcquisition": {"include": ["firefox-webext-browser"]} +} \ No newline at end of file From f42e770a1007f69941ff7fce4a1cb0b1309db479 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Fri, 13 Dec 2024 11:01:35 -0700 Subject: [PATCH 10/19] moved to other repo --- browser/icon.png | Bin 1983 -> 0 bytes browser/jsconfig.json | 5 ----- browser/main/main.html | 16 ---------------- browser/main/script.js | 5 ----- browser/main/style.css | 4 ---- browser/manifest.json | 14 -------------- browser/popup/popup.html | 18 ------------------ browser/popup/popup.js | 3 --- browser/popup/runner.js | 5 ----- 9 files changed, 70 deletions(-) delete mode 100644 browser/icon.png delete mode 100644 browser/jsconfig.json delete mode 100644 browser/main/main.html delete mode 100644 browser/main/script.js delete mode 100644 browser/main/style.css delete mode 100644 browser/manifest.json delete mode 100644 browser/popup/popup.html delete mode 100644 browser/popup/popup.js delete mode 100644 browser/popup/runner.js diff --git a/browser/icon.png b/browser/icon.png deleted file mode 100644 index 6b41a931d2a06980ea38b1b41dcb279e8701dfa3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1983 zcmV;w2SE6VP)Px+c}YY;RA>dwS$j}aRTw{acUfR}Sy<%}nhFYPiU=rUQ{zST^OI?a%Z z1Ww^Nq76;jFfyeksPRua)znN`=9pbMk}LA zZR%5!xP+y+0Ger(brfLY*n_-MoBGta3W^H|4GoQ5I>YEvKCoIWnw!#0!@ zARy1*CqnUK4l%99;o?SZ>Z9WHQS2%xCLljQe-`*nGz8pXu~El4S@$36BOCcRecjCi zZYw4L`ORm40?oeG-ir|Rk&TMv+skPH9f}Cpv13OHVka2_ZQ|(YV|~&f8~Kp0b4;JU zat(?Iz>}8Geq8J!LFZxiB{F0qA3B%DNVb6=H>>Q&F0y7((VcjxF^niB^v_n zd*xzx__6N}ZWPeU-216`J@)t8YBJgNnQeT*a zojhR=iyQr$bn5kZuHq$j^6VU!5AduwsRUjH+LhF%V{2<`WmZ;Ji_6hNgA2eTJ|H3@ zBF5oxaH4KxiD;yUalA{X?O;JBD+@L?vZ<4IGo8-+>`OTJ?Rs|X)M9qGIfC754r6wQ zzhv@sY=D>w7`P}BhMPcTV`Jkb@NO6IVA+a$b2=P6UJc}}Xf#-b;GjlkHZ{3wFxrN| z0bluUL|fW|nXNgzTfKWPg0-}UiFSw1BQF4mU0q$HZiF6q`5%nQpNA4m~?xIUTdEB}QKr6%{SozJ0rc zf9jQ&my5~C$$L#E(+D^uR@F`xLoHVYGY0-2N?qevcKG;8q?4Y=??V1B1^F)G)^mOZ z7 z@A9=|_Q%P$z*{5WR+*cYmbSXDAbkp;0S68oI1CY35}N0)H6-A9a8no@Qrj1nBK?)s zPqSmEmw>!L@xZ7nn~01b}opiiO>b=F7F$<3)dcvk*3@+81`e>=!OS$4bf;fH%K0zK5LOBnLqD z>-6&F%RdHS1_dZ7o6kBr1Lfe$@*yn3bdt|-#*#Hwwg5syPX0j00Dl|v#}iViLRf@J zMT#L8yI%sZn&SxAVpjfc6k=}RoTV752mmM-AO({fM2?@^r0^JW#*k0Ht^oX1y8@3y zfOL67RCu6Cm{JVIuK;b#;vV-D0lUqX+&Dh{But9YpG{!dDP*x&MqzdWX^T+r*Y1-L zo}}zj^%P#I?W+(bVN(oj#gFvmFWF`PN9O0}H=YKC$hZSl*G*(+%4e}Vwjn&u*^22b zY1}?GV(8yeT#rY>ZfP4tB518$^61o~Q?>w%d$54Le;QVuJ-MoX3snrl_*?(jW8$P6rC5_*Qo5F*Kn6!{FPPPEw zs|#qUsi}U6iHR-PozN|OHA84?H?z|66n3R%3ML!9qDY%hTRS>B@>Z={#kZi{$aiaQ z-n==)-`_tAof*h=BaI$8?1YdwZa)hQu(2$4%^ZQ$tfZ;m)R>Znjvm&n>X z@G*!#L-0~}jvC&ITmka|^0gZf{d@5RcJgvou3TB|*5B_Y{wo-PowghdT!iZWdjBQ_ zeB%pH?iz6ABxZaM>Uw-^T<`@h$L^efrPv|dgsZUu^>rGD82q;F3qzIw0t4``cDsFT zdV2aPU!ViAJ11n$wr$%c>vX!0fZ>|7f#7u;gW2J)5ZyXeK6xB5RhHx#{$G+-)CsAZ Rx - - - - - - - - - - -

Here is some text

-

old

- - - \ No newline at end of file diff --git a/browser/main/script.js b/browser/main/script.js deleted file mode 100644 index 9fdd6ca..0000000 --- a/browser/main/script.js +++ /dev/null @@ -1,5 +0,0 @@ -document.getElementById("changeme").innerText = "newer" - -window.fetch("http://127.0.0.1:9001").then((e) => { - console.log(e) -}) diff --git a/browser/main/style.css b/browser/main/style.css deleted file mode 100644 index cab1b1f..0000000 --- a/browser/main/style.css +++ /dev/null @@ -1,4 +0,0 @@ -html { - background: black; - color: white; -} \ No newline at end of file diff --git a/browser/manifest.json b/browser/manifest.json deleted file mode 100644 index c1d8e0c..0000000 --- a/browser/manifest.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "manifest_version": 2, - "name": "Viewer", - "version": "0.1", - "description": "Adds a red border to all webpages matching mozilla.org.", - "permissions": [ - "activeTab" - ], - "browser_action": { - "default_icon": "icon.png", - "default_title": "Click me!", - "default_popup": "popup/popup.html" - } -} \ No newline at end of file diff --git a/browser/popup/popup.html b/browser/popup/popup.html deleted file mode 100644 index dd8094d..0000000 --- a/browser/popup/popup.html +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - -

- - - - diff --git a/browser/popup/popup.js b/browser/popup/popup.js deleted file mode 100644 index b5d8389..0000000 --- a/browser/popup/popup.js +++ /dev/null @@ -1,3 +0,0 @@ -document.getElementById("newtab").addEventListener('click', async function(e) { - await browser.tabs.create({ url: "/main/main.html" }) -}) \ No newline at end of file diff --git a/browser/popup/runner.js b/browser/popup/runner.js deleted file mode 100644 index 0eee102..0000000 --- a/browser/popup/runner.js +++ /dev/null @@ -1,5 +0,0 @@ -document.body.textContent = ""; - -let header = document.createElement("h1"); -header.textContent = "This page has been eaten"; -document.body.appendChild(header); From 82929fd0fcd7f169d94bc1320d6acb948115f6b6 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Fri, 13 Dec 2024 13:28:24 -0700 Subject: [PATCH 11/19] updating for base64 --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 10 +++++----- src/s3.rs | 9 +++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 238e64f..cbb76f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3673,6 +3673,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" name = "surreal_spider" version = "0.1.0" dependencies = [ + "base64 0.22.1", "html5ever 0.29.0", "minio", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 72c21f7..ed51242 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] +base64 = "0.22.1" html5ever = "0.29" # minio = "0.1.0" minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"} diff --git a/src/main.rs b/src/main.rs index 6db0563..7ae82fd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,17 +41,17 @@ async fn main() { surreal_username: "root", surreal_password: "root", surreal_ns: "test", - surreal_db: "custom-engine-v2", - s3_bucket: "custom-engine-v2", + surreal_db: "b64v1", + s3_bucket: "b64v1", s3_url: "http://localhost:9000", - s3_access_key: "0zv7GbLQsw4ZI8TclMps", - s3_secret_key: "5dB7QkGFw7fYbUJ5LpHk2GbWR7Bl710HlRz4NbzB", + s3_access_key: "8UO76z8wCs9DnpxSbQUY", + s3_secret_key: "xwKVMpf2jzgprsdo85Dvo74UmO84y0aRrAUorYY5", }; // Would probably take these in as parameters from a cli let starting_url = "https://en.wikipedia.org/"; // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored. - let crawl_filter = "https://en.wikipedia.org/"; + let crawl_filter = "wikipedia.org/"; let budget = 50; let mut crawled = 0; diff --git a/src/s3.rs b/src/s3.rs index f17b7d1..6ed33cb 100644 --- a/src/s3.rs +++ b/src/s3.rs @@ -43,9 +43,14 @@ impl S3 { }) } + #[instrument(skip_all)] pub async fn store(&self, data: &str, name: &Url) { - if let Some(domain) = name.domain() { - let filename = domain.to_string() + name.path(); + if let Some(domain) = name.to_string().split('#').collect::>().get(0) { + use base64::prelude::*; + // FIXME can still get unsupported characters, _ I think + let filename = BASE64_URL_SAFE.encode(domain); + + trace!("Filename: {filename} from {domain}"); let _ = &self .client From b7540a4680ef650ed13154a2132ad5292e2d6978 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Tue, 18 Mar 2025 10:53:06 -0600 Subject: [PATCH 12/19] checkpoint - onto profiling --- Cargo.lock | 1210 +++++++++++++++++++++++++++++++------------------ Cargo.toml | 6 +- src/db.rs | 24 +- src/main.rs | 100 ++-- src/parser.rs | 73 +-- src/s3.rs | 80 ++-- 6 files changed, 941 insertions(+), 552 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbb76f9..cbcede6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,7 +42,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -54,10 +54,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -144,11 +144,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", + "once_cell", "windows-sys 0.59.0", ] @@ -227,6 +228,19 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0cf008e5e1a9e9e22a7d3c9a4992e21a350290069e36d8fb72304ed17e8f2d2" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-executor" version = "1.13.1" @@ -242,9 +256,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.13" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fd6bd734afb8b6e4d0f84a3e77305ce0a7ccc60d70f6001cb5e1c3f38d8ff1" +checksum = "bfff2b17d272a5e3e201feda444e2c24b011fa722951268d1bd8b9b5bc6dc449" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -257,7 +271,7 @@ dependencies = [ "futures-timer", "futures-util", "http", - "indexmap 2.7.0", + "indexmap 2.8.0", "mime", "multer", "num-traits", @@ -272,9 +286,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.13" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac38b4dd452d529d6c0248b51df23603f0a875770352e26ae8c346ce6c149b3e" +checksum = "d8e5d0c6697def2f79ccbd972fb106b633173a6066e430b480e1ff9376a7561a" dependencies = [ "Inflector", "async-graphql-parser", @@ -283,15 +297,15 @@ dependencies = [ "proc-macro2", "quote", "strum", - "syn 2.0.90", + "syn 2.0.100", "thiserror 1.0.69", ] [[package]] name = "async-graphql-parser" -version = "7.0.13" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d271ddda2f55b13970928abbcbc3423cfc18187c60e8769b48f21a93b7adaa" +checksum = "8531ee6d292c26df31c18c565ff22371e7bdfffe7f5e62b69537db0b8fd554dc" dependencies = [ "async-graphql-value", "pest", @@ -301,12 +315,12 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.13" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aefe909173a037eaf3281b046dc22580b59a38b765d7b8d5116f2ffef098048d" +checksum = "741110dda927420a28fbc1c310543d3416f789a6ba96859c2c265843a0a96887" dependencies = [ "bytes", - "indexmap 2.7.0", + "indexmap 2.8.0", "serde", "serde_json", ] @@ -319,7 +333,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -341,7 +355,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -352,13 +366,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -396,7 +410,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -413,9 +427,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.6.0" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" [[package]] name = "bcrypt" @@ -425,7 +439,7 @@ checksum = "e65938ed058ef47d92cf8b346cc76ef48984572ade631927e9937b5ffc7662c7" dependencies = [ "base64 0.22.1", "blowfish", - "getrandom", + "getrandom 0.2.15", "subtle", "zeroize", ] @@ -456,9 +470,9 @@ checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" [[package]] name = "bitvec" @@ -483,9 +497,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.5" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +checksum = "675f87afced0413c9bb02843499dbbd3882a237645883f71a2b59644a6d2f753" dependencies = [ "arrayref", "arrayvec", @@ -515,9 +529,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.5.3" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2506947f73ad44e344215ccd6403ac2ae18cd8e046e581a441bf8d199f257f03" +checksum = "b2b74d67a0fc0af8e9823b79fd1c43a0900e5a8f0e0f4cc9210796bf3a820126" dependencies = [ "borsh-derive", "cfg_aliases", @@ -525,22 +539,22 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.5.3" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2593a3b8b938bd68373196c9832f516be11fa487ef4ae745eb282e6a56a7244" +checksum = "2d37ed1b2c9b78421218a0b4f6d8349132d6ec2cfeba1cfb0118b0a8e268df9e" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytecheck" @@ -566,9 +580,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.20.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" +checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" [[package]] name = "byteorder" @@ -578,9 +592,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] @@ -596,9 +610,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.3" +version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ "shlex", ] @@ -674,9 +688,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ "android-tzdata", "iana-time-zone", @@ -684,7 +698,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets", + "windows-link", ] [[package]] @@ -763,9 +777,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -786,10 +800,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] -name = "crossbeam-deque" -version = "0.8.5" +name = "crc32fast" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -806,15 +829,15 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-common" @@ -847,7 +870,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -858,7 +881,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -890,15 +913,15 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", "serde", @@ -917,9 +940,9 @@ dependencies = [ [[package]] name = "deunicode" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" +checksum = "dc55fe0d1f6c107595572ec8b107c0999bb1a2e0b75e37429a4fb0d6474a0e7d" [[package]] name = "digest" @@ -961,7 +984,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -991,9 +1014,9 @@ dependencies = [ [[package]] name = "either" -version = "1.13.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "ena" @@ -1021,9 +1044,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "env_filter" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" dependencies = [ "log", "regex", @@ -1031,22 +1054,22 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.5" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" dependencies = [ "anstream", "anstyle", "env_filter", - "humantime", + "jiff", "log", ] [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -1060,9 +1083,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", @@ -1091,6 +1114,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "float_next_after" version = "1.0.0" @@ -1103,6 +1136,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1199,9 +1238,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ "fastrand", "futures-core", @@ -1218,7 +1257,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -1296,9 +1335,9 @@ dependencies = [ [[package]] name = "geo-types" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f47c611187777bbca61ea7aba780213f5f3441fd36294ab333e96cfa791b65" +checksum = "3bd1157f0f936bf0cd68dec91e8f7c311afe60295574d62b70d4861a1bfdf2d9" dependencies = [ "approx 0.5.1", "arbitrary", @@ -1325,7 +1364,21 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", "wasm-bindgen", ] @@ -1337,9 +1390,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "h2" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" dependencies = [ "atomic-waker", "bytes", @@ -1347,7 +1400,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.7.0", + "indexmap 2.8.0", "slab", "tokio", "tokio-util", @@ -1356,9 +1409,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" dependencies = [ "cfg-if", "crunchy", @@ -1387,16 +1440,17 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash 0.8.11", - "allocator-api2", -] [[package]] name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heapless" @@ -1443,11 +1497,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1461,28 +1515,26 @@ dependencies = [ "markup5ever 0.12.1", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] name = "html5ever" -version = "0.29.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e15626aaf9c351bc696217cbe29cb9b5e86c43f8a46b5e2f5c6c5cf7cb904ce" +checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c" dependencies = [ "log", "mac", - "markup5ever 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.90", + "markup5ever 0.14.1", + "match_token", ] [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -1501,12 +1553,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", + "futures-core", "http", "http-body", "pin-project-lite", @@ -1514,9 +1566,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -1526,15 +1578,15 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "hyper" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", @@ -1553,9 +1605,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http", @@ -1615,7 +1667,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -1742,7 +1794,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -1785,9 +1837,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1796,18 +1848,18 @@ dependencies = [ [[package]] name = "inout" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ "generic-array", ] [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is_terminal_polyfill" @@ -1844,15 +1896,39 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jiff" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -1860,11 +1936,11 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "9.3.0" +version = "9.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "js-sys", "pem", "ring", @@ -1921,9 +1997,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.168" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libm" @@ -1949,21 +2025,21 @@ checksum = "56e7562b41c8876d3367897067013bb2884cc78e6893f092ecd26b305176ac82" dependencies = [ "ndarray", "num-traits", - "rand", + "rand 0.8.5", "thiserror 1.0.69", ] [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" [[package]] name = "litemap" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" [[package]] name = "lock_api" @@ -1977,9 +2053,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "mac" @@ -2009,9 +2085,9 @@ dependencies = [ [[package]] name = "markup5ever" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c88c6129bd24319e62a0359cb6b958fa7e8be6e19bb1663bc396b90883aca5" +checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" dependencies = [ "log", "phf", @@ -2021,6 +2097,17 @@ dependencies = [ "tendril", ] +[[package]] +name = "match_token" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "matchers" version = "0.1.0" @@ -2082,7 +2169,7 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -2128,7 +2215,7 @@ dependencies = [ "multimap", "os_info", "percent-encoding", - "rand", + "rand 0.8.5", "regex", "reqwest", "serde", @@ -2143,9 +2230,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", ] @@ -2157,7 +2244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -2193,14 +2280,14 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8" dependencies = [ - "rand", + "rand 0.8.5", ] [[package]] name = "native-tls" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" dependencies = [ "libc", "log", @@ -2239,7 +2326,7 @@ dependencies = [ "noisy_float", "num-integer", "num-traits", - "rand", + "rand 0.8.5", ] [[package]] @@ -2266,6 +2353,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2331,10 +2427,19 @@ dependencies = [ ] [[package]] -name = "object" -version = "0.36.5" +name = "num_threads" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -2362,15 +2467,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" [[package]] name = "openssl" -version = "0.10.68" +version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ "bitflags", "cfg-if", @@ -2389,20 +2494,20 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.104" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", @@ -2412,9 +2517,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ca711d8b83edbb00b44d504503cd247c9c0bd8b0fa2694f2a1a3d8165379ce" +checksum = "2a604e53c24761286860eba4e2c8b23a0161526476b1de520139d69cdb85a6b5" dependencies = [ "log", "serde", @@ -2453,7 +2558,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2463,7 +2568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -2493,9 +2598,9 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" dependencies = [ "base64 0.22.1", "serde", @@ -2514,7 +2619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.6", + "thiserror 2.0.12", "ucd-trie", ] @@ -2525,7 +2630,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.7.0", + "indexmap 2.8.0", ] [[package]] @@ -2540,72 +2645,53 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_macros", - "phf_shared 0.11.2", + "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator 0.11.2", - "phf_shared 0.11.2", + "phf_generator", + "phf_shared", ] [[package]] name = "phf_generator" -version = "0.10.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared 0.10.0", - "rand", -] - -[[package]] -name = "phf_generator" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" -dependencies = [ - "phf_shared 0.11.2", - "rand", + "phf_shared", + "rand 0.8.5", ] [[package]] name = "phf_macros" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ - "phf_generator 0.11.2", - "phf_shared 0.11.2", + "phf_generator", + "phf_shared", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", "unicase", ] [[package]] name = "phf_shared" -version = "0.10.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", "unicase", @@ -2619,9 +2705,9 @@ checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2631,9 +2717,24 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] [[package]] name = "powerfmt" @@ -2643,11 +2744,11 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy", + "zerocopy 0.8.23", ] [[package]] @@ -2658,18 +2759,18 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "proc-macro-crate" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] @@ -2682,9 +2783,9 @@ checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" [[package]] name = "psm" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88" dependencies = [ "cc", ] @@ -2723,37 +2824,39 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" dependencies = [ "bytes", + "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash", "rustls", "socket2", - "thiserror 2.0.6", + "thiserror 2.0.12", "tokio", "tracing", + "web-time", ] [[package]] name = "quinn-proto" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" dependencies = [ "bytes", - "getrandom", - "rand", + "getrandom 0.3.2", + "rand 0.9.0", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.6", + "thiserror 2.0.12", "tinyvec", "tracing", "web-time", @@ -2761,9 +2864,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" dependencies = [ "cfg_aliases", "libc", @@ -2775,13 +2878,19 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radium" version = "0.7.0" @@ -2806,8 +2915,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.23", ] [[package]] @@ -2817,7 +2937,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -2826,7 +2956,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", ] [[package]] @@ -2863,9 +3002,9 @@ checksum = "ffead9d0a0b45f3e0bc063a244b1779fd53a09d2c2f7282c186a016b1f10a778" [[package]] name = "redox_syscall" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ "bitflags", ] @@ -2876,29 +3015,29 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror 1.0.69", ] [[package]] name = "ref-cast" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -2956,10 +3095,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "989e327e510263980e231de548a33e63d34962d29ae61b467389a1a09627a254" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "encoding_rs", @@ -2995,6 +3135,7 @@ dependencies = [ "tokio-native-tls", "tokio-rustls", "tokio-util", + "tower", "tower-service", "url", "wasm-bindgen", @@ -3007,9 +3148,9 @@ dependencies = [ [[package]] name = "revision" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f53179a035f881adad8c4d58a2c599c6b4a8325b989c68d178d7a34d1b1e4c" +checksum = "54b8ee532f15b2f0811eb1a50adf10d036e14a6cdae8d99893e7f3b921cb227d" dependencies = [ "chrono", "geo", @@ -3022,26 +3163,25 @@ dependencies = [ [[package]] name = "revision-derive" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0ec466e5d8dca9965eb6871879677bef5590cf7525ad96cae14376efb75073" +checksum = "d3415e1bc838c36f9a0a2ac60c0fa0851c72297685e66592c44870d82834dfa2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] name = "ring" -version = "0.17.8" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] @@ -3098,9 +3238,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.8" +version = "0.10.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395b0c39c00f9296f3937624c1fa4e0ee44f8c0e4b2c49408179ef381c6c2e6e" +checksum = "a652edd001c53df0b3f96a36a8dc93fce6866988efc16808235653c6bcac8bf2" dependencies = [ "bytemuck", "byteorder", @@ -3136,15 +3276,15 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.36.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" +checksum = "5c24af6e7ac43c88a8a458d1139d0246fdce2f6cd2f1ac6cb51eb88b29c978af" dependencies = [ "arrayvec", "borsh", "bytes", "num-traits", - "rand", + "rand 0.8.5", "rkyv", "serde", "serde_json", @@ -3158,9 +3298,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_lexer" @@ -3182,9 +3322,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.42" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" dependencies = [ "bitflags", "errno", @@ -3195,9 +3335,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ "log", "once_cell", @@ -3219,18 +3359,18 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" dependencies = [ "web-time", ] [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f" dependencies = [ "ring", "rustls-pki-types", @@ -3239,15 +3379,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "salsa20" @@ -3315,9 +3455,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -3325,9 +3465,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" dependencies = [ "serde", ] @@ -3340,40 +3480,40 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.216" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde-content" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e255eaf9f3814135df4f959c9f404ebb2e67238bae0ed412da10518d0629e7c9" +checksum = "3753ca04f350fa92d00b6146a3555e63c55388c9ef2e11e09bce2ff1c0b509c6" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.8.0", "itoa", "memchr", "ryu", @@ -3394,15 +3534,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.7.0", + "indexmap 2.8.0", "serde", "serde_derive", "serde_json", @@ -3412,14 +3552,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -3476,21 +3616,21 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "simple_asn1" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 1.0.69", + "thiserror 2.0.12", "time", ] [[package]] name = "siphasher" -version = "0.3.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" @@ -3503,9 +3643,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "smol_str" @@ -3556,11 +3696,11 @@ dependencies = [ [[package]] name = "spade" -version = "2.12.1" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f5ef1f863aca7d1d7dda7ccfc36a0a4279bd6d3c375176e5e0712e25cb4889" +checksum = "1ece03ff43cd2a9b57ebf776ea5e78bd30b3b4185a619f041079f4109f385034" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", "num-traits", "robust", "smallvec", @@ -3580,9 +3720,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +checksum = "d9156ebd5870ef293bfb43f91c7a74528d363ec0d424afe24160ed5a4343d08a" dependencies = [ "cc", "cfg-if", @@ -3611,26 +3751,25 @@ dependencies = [ [[package]] name = "string_cache" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +checksum = "938d512196766101d333398efde81bc1f37b00cb42c2f8350e5df639f040bbbe" dependencies = [ "new_debug_unreachable", - "once_cell", "parking_lot", - "phf_shared 0.10.0", + "phf_shared", "precomputed-hash", "serde", ] [[package]] name = "string_cache_codegen" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", + "phf_generator", + "phf_shared", "proc-macro2", "quote", ] @@ -3660,7 +3799,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -3674,7 +3813,7 @@ name = "surreal_spider" version = "0.1.0" dependencies = [ "base64 0.22.1", - "html5ever 0.29.0", + "html5ever 0.29.1", "minio", "reqwest", "serde", @@ -3687,9 +3826,9 @@ dependencies = [ [[package]] name = "surrealdb" -version = "2.1.3" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b69bfa3a8cd087f2d0305be1fed24cb8e18571cb4f116089df7d39f7150f97a" +checksum = "aa1aa87197bad9dd12c93350533a8da09bae064a411f445c97ca0e64faabc304" dependencies = [ "arrayvec", "async-channel", @@ -3698,7 +3837,7 @@ dependencies = [ "dmp", "futures", "geo", - "indexmap 2.7.0", + "indexmap 2.8.0", "path-clean", "pharos", "reblessive", @@ -3728,9 +3867,9 @@ dependencies = [ [[package]] name = "surrealdb-core" -version = "2.1.3" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcf8a937ddc5626e845ba715167776008f899795c1274657c2ba1b9f0f4d2bd" +checksum = "4ceb9f421e07af67b06c57ea7a8b08c3b4e4677de483435ca6d69bd00600a571" dependencies = [ "addr", "ahash 0.8.11", @@ -3758,6 +3897,7 @@ dependencies = [ "geo", "geo-types", "hex", + "http", "ipnet", "jsonwebtoken", "lexicmp", @@ -3769,13 +3909,14 @@ dependencies = [ "num-traits", "num_cpus", "object_store", + "parking_lot", "pbkdf2", "pharos", "phf", "pin-project-lite", "quick_cache", "radix_trie", - "rand", + "rand 0.8.5", "rayon", "reblessive", "regex", @@ -3796,7 +3937,7 @@ dependencies = [ "storekey", "strsim", "subtle", - "surrealdb-derive", + "sysinfo", "thiserror 1.0.69", "tokio", "tracing", @@ -3811,16 +3952,6 @@ dependencies = [ "ws_stream_wasm", ] -[[package]] -name = "surrealdb-derive" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacdb4c58b9ebef0291310afcd63af0012d85610d361f3785952c61b6f1dddf4" -dependencies = [ - "quote", - "syn 1.0.109", -] - [[package]] name = "syn" version = "1.0.109" @@ -3834,9 +3965,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -3860,7 +3991,21 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", +] + +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", ] [[package]] @@ -3892,12 +4037,12 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600" dependencies = [ - "cfg-if", "fastrand", + "getrandom 0.3.2", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3936,11 +4081,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.12", ] [[package]] @@ -3951,18 +4096,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -3977,13 +4122,15 @@ dependencies = [ [[package]] name = "time" -version = "0.3.37" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +checksum = "9d9c75b47bdff86fa3334a3db91356b8d7d86a9b839dab7d0bdc5c3d3a077618" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", @@ -3992,15 +4139,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.19" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +checksum = "29aa485584182073ed57fd5004aa09c371f021325014694e432313345865fd04" dependencies = [ "num-conv", "time-core", @@ -4027,9 +4174,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -4042,9 +4189,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" dependencies = [ "backtrace", "bytes", @@ -4060,13 +4207,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -4081,9 +4228,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ "rustls", "tokio", @@ -4118,9 +4265,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", @@ -4138,15 +4285,36 @@ checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" -version = "0.22.22" +version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.8.0", "toml_datetime", "winnow", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4172,7 +4340,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] [[package]] @@ -4209,6 +4377,7 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", + "time", "tracing", "tracing-core", "tracing-log", @@ -4243,7 +4412,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.8.5", "rustls", "rustls-pki-types", "sha1", @@ -4254,9 +4423,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "ucd-trie" @@ -4266,27 +4435,26 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "ulid" -version = "1.1.3" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f903f293d11f31c0c29e4148f6dc0d033a7f80cebc0282bea147611667d289" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "getrandom", - "rand", + "rand 0.9.0", "serde", "web-time", ] [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-normalization" @@ -4375,26 +4543,27 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.11.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom", + "getrandom 0.3.2", + "js-sys", "serde", "wasm-bindgen", ] [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vart" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c92195d375eb94995afddeedfd7f246796eb60b85f727c538e42222c4c9b2d3" +checksum = "87782b74f898179396e93c0efabb38de0d58d50bbd47eae00c71b3a1144dbbae" [[package]] name = "vcpkg" @@ -4434,35 +4603,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "wasm-bindgen" -version = "0.2.99" +name = "wasi" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -4473,9 +4652,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4483,22 +4662,25 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" @@ -4528,9 +4710,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -4548,9 +4730,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.7" +version = "0.26.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" dependencies = [ "rustls-pki-types", ] @@ -4586,43 +4768,101 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core 0.57.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] -name = "windows-registry" -version = "0.2.0" +name = "windows-core" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" dependencies = [ - "windows-result", + "windows-implement", + "windows-interface", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-registry" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +dependencies = [ + "windows-result 0.3.2", "windows-strings", - "windows-targets", + "windows-targets 0.53.0", ] [[package]] name = "windows-result" -version = "0.2.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", ] [[package]] name = "windows-strings" -version = "0.1.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-result", - "windows-targets", + "windows-link", ] [[package]] @@ -4631,7 +4871,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4640,7 +4880,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4649,14 +4889,30 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -4665,42 +4921,84 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -4708,14 +5006,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winnow" -version = "0.6.20" +name = "windows_x86_64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "winnow" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + [[package]] name = "write16" version = "1.0.0" @@ -4758,9 +5071,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" +checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" [[package]] name = "xmltree" @@ -4791,7 +5104,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", "synstructure", ] @@ -4801,8 +5114,16 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" +dependencies = [ + "zerocopy-derive 0.8.23", ] [[package]] @@ -4813,27 +5134,38 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", ] [[package]] name = "zerofrom" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", "synstructure", ] @@ -4862,5 +5194,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.100", ] diff --git a/Cargo.toml b/Cargo.toml index ed51242..2dcb455 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,10 @@ base64 = "0.22.1" html5ever = "0.29" # minio = "0.1.0" minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"} -reqwest = "0.12" +reqwest = { version = "0.12", features = ["gzip"] } serde = { version = "1.0", features = ["derive"] } -surrealdb = "2.1" +surrealdb = "2.2" tokio = { version="1.41.0", features = ["full"] } tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time"] } url = { version = "2.5", features = ["serde"] } diff --git a/src/db.rs b/src/db.rs index b5e1230..e8c19f4 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,9 +1,6 @@ use serde::{Deserialize, Serialize}; use surrealdb::{ - engine::remote::ws::{Client, Ws}, - opt::auth::Root, - sql::Thing, - Response, Surreal, + engine::remote::ws::{Client, Ws}, error::Db, opt::auth::Root, sql::Thing, Response, Surreal }; use tracing::{error, instrument, trace, warn}; use url::Url; @@ -82,15 +79,15 @@ impl Website { } #[instrument(skip_all)] - pub async fn store(&mut self, db: &Surreal) -> Option { + pub async fn store(&self, db: &Surreal) -> Option { // check if it's been gone thru before let mut response = db .query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1") .bind(("site", self.site.to_string())) .await - .unwrap(); + .expect("Failed to check surreal for duplicates!"); - if let Some(old) = response.take::>(0).unwrap() { + if let Some(old) = response.take::>(0).expect("Failed to read reponse from surreal for duplicates.") { // site exists already if let Some(id) = old.id { // make sure to preserve the "crawled status" @@ -106,7 +103,18 @@ impl Website { } } Err(e) => { - error!("{}", e); + match e { + surrealdb::Error::Db(error) => { + match error { + Db::QueryCancelled => todo!(), + Db::QueryNotExecuted => todo!(), + Db::QueryNotExecutedDetail { message } => todo!(), + _=>{}, + } + }, + _=>{}, + } + // error!("{}", e); } }; } diff --git a/src/main.rs b/src/main.rs index 7ae82fd..a87301d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,6 @@ +#![feature(type_alias_impl_trait)] +#![feature(const_async_blocks)] + extern crate html5ever; use std::time::Instant; @@ -5,12 +8,13 @@ use std::time::Instant; use db::{connect, Website}; use s3::S3; use surrealdb::{engine::remote::ws::Client, Surreal}; +use tokio::task::JoinSet; use tracing::{debug, info, instrument, trace, trace_span}; -use tracing_subscriber::EnvFilter; +use tracing_subscriber::{fmt::time::LocalTime, EnvFilter}; mod db; -mod s3; mod parser; +mod s3; struct Config<'a> { surreal_ns: &'a str, @@ -31,52 +35,62 @@ async fn main() { tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) - .with_line_number(false) - .without_time() + .with_line_number(true) + .with_file(true) + .with_timer(LocalTime::rfc_3339()) .init(); debug!("Starting..."); - + // Would probably take these in as parameters from a cli + let starting_url = "https://en.wikipedia.org/"; + // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored. + let crawl_filter = "en.wikipedia.org/"; + let budget = 50; + let mut crawled = 0; + let config = Config { surreal_url: "localhost:8000", surreal_username: "root", surreal_password: "root", surreal_ns: "test", - surreal_db: "b64v1", - s3_bucket: "b64v1", + surreal_db: "v1.11.2", + s3_bucket: "v1.11.2", s3_url: "http://localhost:9000", s3_access_key: "8UO76z8wCs9DnpxSbQUY", s3_secret_key: "xwKVMpf2jzgprsdo85Dvo74UmO84y0aRrAUorYY5", }; + - // Would probably take these in as parameters from a cli - let starting_url = "https://en.wikipedia.org/"; - // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored. - let crawl_filter = "wikipedia.org/"; - let budget = 50; - let mut crawled = 0; + let db = connect(&config) + .await + .expect("Failed to connect to surreal, aborting."); + let s3 = S3::connect(&config) + .await + .expect("Failed to connect to minio, aborting."); - let s3 = S3::connect(&config).await.expect("Failed to connect to minio, aborting."); - let db = connect(&config).await.expect("Failed to connect to surreal, aborting."); - let reqwest = reqwest::Client::builder() // .use_rustls_tls() + .gzip(true) .build() - .unwrap(); + .expect("Failed to build reqwest client."); // Kick off the whole machine - This Website object doesn't matter, it's just to allow for // get() to work. let span = trace_span!("Pre-Loop"); let pre_loop_span = span.enter(); // Download the site - let mut site = Website::new(&starting_url, false); - get(&mut site, &db, &reqwest, &s3, &mut crawled).await; + let site = Website::new(&starting_url, false); + get(site, db.clone(), reqwest.clone(), s3.clone()).await; drop(pre_loop_span); let span = trace_span!("Loop"); let span = span.enter(); while crawled < budget { - let get_num = if budget - crawled < 100 { budget - crawled } else { 100 }; + let get_num = if budget - crawled < 100 { + budget - crawled + } else { + 100 + }; let uncrawled = get_uncrawled_links(&db, get_num, crawl_filter.to_string()).await; if uncrawled.len() == 0 { @@ -88,11 +102,19 @@ async fn main() { let span = trace_span!("Crawling"); let _ = span.enter(); - for mut site in uncrawled { - get(&mut site, &db, &reqwest, &s3, &mut crawled).await; - - let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32); - info!("Crawled {crawled} out of {budget} pages. ({percent})"); + { + let mut futures = JoinSet::new(); + for site in uncrawled { + futures.spawn(get(site, db.clone(), reqwest.clone(), s3.clone())); + // technically the site hasn't be crawled *yet*, but the future + // where it is crawled has been set up. + crawled += 1; + // let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32); + // info!("Crawled {crawled} out of {budget} pages. ({percent})"); + } + debug!("Joining {} futures...", futures.len()); + // join all the gets together + let _ = futures.join_all().await; } } drop(span); @@ -103,25 +125,19 @@ async fn main() { #[instrument(skip_all)] /// Downloads and crawls and stores a webpage. -async fn get( - site: &mut Website, - db: &Surreal, - reqwest: &reqwest::Client, - s3: &S3, - count: &mut usize, -) { +/// It is acceptable to clone `db`, `reqwest`, and `s3` because they all use `Arc`s internally. - Noted by Oliver +async fn get(mut site: Website, db: Surreal, reqwest: reqwest::Client, s3: S3) { trace!("Get: {}", site.to_string()); let timer = Timer::start("Got page"); if let Ok(response) = reqwest.get(site.to_string()).send().await { timer.stop(); // Get body - let data = response.text().await.unwrap(); + let data = response.text().await.expect("Failed to read http response's body!"); // Store document - s3.store(&data, &site.site).await; + s3.store(&data, &site.site).await; // Parse document and store relationships - parser::parse(db, site, data).await; - *count += 1; + parser::parse(&db, &mut site, &data).await; return; } trace!("Failed to get: {}", site.to_string()); @@ -129,15 +145,19 @@ async fn get( /// Returns uncrawled links #[instrument(skip(db))] -async fn get_uncrawled_links(db: &Surreal, mut count: usize, param: String) -> Vec { +async fn get_uncrawled_links( + db: &Surreal, + mut count: usize, + filter: String, +) -> Vec { if count > 100 { count = 100 } - trace!("Getting uncrawled links"); + debug!("Getting uncrawled links"); let mut response = db .query("SELECT * FROM website WHERE crawled = false AND site CONTAINS type::string($format) LIMIT $count;") - .bind(("format", param)) + .bind(("format", filter)) .bind(("count", count)) .await .expect("Hard-coded query failed..?"); @@ -162,7 +182,7 @@ impl<'a> Timer<'a> { pub fn stop(&self) -> f64 { let dif = self.start.elapsed().as_micros(); let ms = dif as f64 / 1000.; - debug!("{}", format!("{} in {:.3}ms", self.msg, ms)); + trace!("{}", format!("{} in {:.3}ms", self.msg, ms)); ms } } diff --git a/src/parser.rs b/src/parser.rs index 09d1dff..3357059 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -12,12 +12,7 @@ use tracing::instrument; use crate::db::Website; use crate::Timer; -#[derive(Clone)] -struct LinkParser<'a> { - site: &'a Website, -} - -impl TokenSink for LinkParser<'_> { +impl TokenSink for Website { type Handle = Vec; fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult { @@ -40,20 +35,20 @@ impl TokenSink for LinkParser<'_> { if attr_name == "src" || attr_name == "href" || attr_name == "data" { // Get clone of the current site object - let mut web = self.site.clone(); + let mut web = self.clone(); // Set url let mut url = web.site; url.set_fragment(None); // removes #xyz - let joined = url.join(&attr.value).unwrap(); + let joined = url.join(&attr.value).expect("Failed to join url during parsing!"); web.site = joined; web.crawled = false; - + links.push(web); } } - + return TokenSinkResult::Script(links); } local_name!("button") | local_name!("meta") | local_name!("iframe") => { @@ -70,31 +65,49 @@ impl TokenSink for LinkParser<'_> { } #[instrument(skip_all)] -pub async fn parse(db: &Surreal, site: &mut Website, data: String) { - +pub async fn parse(db: &Surreal, site: &mut Website, data: &str) { + // update self in db site.set_crawled(); site.store(db).await; - - let sink = LinkParser { site }; - let chunk = Tendril::from_str(&data).unwrap(); - let mut input = BufferQueue::default(); - input.push_back(chunk.try_reinterpret::().unwrap()); - let token = Tokenizer::new(sink.clone(), TokenizerOpts::default()); - - let t = Timer::start("Stored pages"); - let mut links_to = Vec::new(); - while !input.is_empty() { - if let TokenizerResult::Script(s) = token.feed(&mut input) { - for mut web in s { - if let Some(id) = web.store(db).await { - links_to.push(id); + // prep work + let mut other_sites = Vec::new(); + { // using blocks to prevent compiler's async worries + let _t = Timer::start("Parsed page"); + + // change data into something that can be tokenized + let chunk = Tendril::from_str(&data).expect("Failed to parse string into Tendril!"); + // create buffer of tokens and push our input into it + let mut token_buffer = BufferQueue::default(); + token_buffer.push_back(chunk.try_reinterpret::().expect("Failed to reinterprt chunk!")); + // create the tokenizer + let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default()); + + // go thru buffer + while let TokenizerResult::Script(sites) = tokenizer.feed(&mut token_buffer) { + other_sites.push(sites); + } + + assert!(token_buffer.is_empty()); + tokenizer.end(); + } + + { + let t = Timer::start("Stored pages"); + let mut links_to = Vec::new(); + + // this is a 2d vec accidentally + for a in other_sites { + for b in a { + // TODO this can become a JoinSet later + let other = b.store(db).await; + if let Some(o) = other { + links_to.push(o); } } } + + site.links_to(links_to, db).await; + drop(t); } - drop(t); - sink.site.links_to(links_to, db).await; - assert!(input.is_empty()); - token.end(); } diff --git a/src/s3.rs b/src/s3.rs index 6ed33cb..a54d84c 100644 --- a/src/s3.rs +++ b/src/s3.rs @@ -1,11 +1,20 @@ +use base64::{alphabet, engine::{self, general_purpose}, Engine}; use minio::s3::{ - args::{BucketExistsArgs, MakeBucketArgs}, client::ClientBuilder, creds::StaticProvider, error::Error, http::BaseUrl, types::S3Api, Client + args::{BucketExistsArgs, MakeBucketArgs}, + client::ClientBuilder, + creds::StaticProvider, + error::Error, + http::BaseUrl, + Client, }; -use tracing::{instrument, trace}; +use tracing::{instrument, trace, warn}; use url::Url; use crate::Config; + +const CUSTOM_ENGINE: engine::GeneralPurpose = engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD); +#[derive(Clone)] pub struct S3 { bucket_name: String, client: Client, @@ -14,7 +23,10 @@ pub struct S3 { impl S3 { #[instrument(skip_all, name = "S3")] pub async fn connect(config: &Config<'_>) -> Result { - let base_url = config.s3_url.parse::().unwrap(); + let base_url = config + .s3_url + .parse::() + .expect("Failed to parse url into BaseUrl"); let static_provider = StaticProvider::new(&config.s3_access_key, &config.s3_secret_key, None); @@ -25,17 +37,22 @@ impl S3 { trace!("Checking bucket..."); let exists = client - .bucket_exists(&BucketExistsArgs::new(&config.s3_bucket).unwrap()) + .bucket_exists( + &BucketExistsArgs::new(&config.s3_bucket) + .expect("Failed to check if bucket exists"), + ) .await?; if !exists { trace!("Creating bucket..."); client - .make_bucket(&MakeBucketArgs::new(&config.s3_bucket).unwrap()) + .make_bucket( + &MakeBucketArgs::new(&config.s3_bucket).expect("Failed to create bucket!"), + ) .await?; } - trace!("Connection successfull"); + trace!("Connection successful"); Ok(Self { bucket_name: config.s3_bucket.to_owned(), @@ -44,38 +61,37 @@ impl S3 { } #[instrument(skip_all)] - pub async fn store(&self, data: &str, name: &Url) { - if let Some(domain) = name.to_string().split('#').collect::>().get(0) { - use base64::prelude::*; - // FIXME can still get unsupported characters, _ I think - let filename = BASE64_URL_SAFE.encode(domain); + pub async fn store(&self, data: &str, url: &Url) { + if let Some(domain) = url.domain() { + let filename = domain.to_owned() + url.path(); - trace!("Filename: {filename} from {domain}"); + trace!("Created filename: {filename} from raw: {}", url.to_string()); - let _ = &self + let _ = match &self .client .put_object_content(&self.bucket_name, &filename, data.to_owned()) .send() - .await - .unwrap(); - } - } - - pub async fn _get(&self, name: &Url) -> Option { - if let Some(domain) = name.domain() { - let filename = domain.to_string() + name.path(); + .await { + Ok(_) => {}, + Err(err) => { + match err { + Error::InvalidObjectName(_) => { - let data = self - .client - .get_object(&self.bucket_name, &filename) - .send() - .await - .unwrap(); - - if let Ok(segments )= data.content.to_segmented_bytes().await { - return Some(segments.to_bytes().iter().map(|c| *c as char).collect::()) - } + warn!("Tried storing invalid object name, retrying with Base64 encoding. Last try."); + + let filename: String = domain.to_owned() + &CUSTOM_ENGINE.encode(url.path()); + + let _ = &self + .client + .put_object_content(&self.bucket_name, &filename, data.to_owned()) + .send() + .await + .unwrap(); + }, + _ => {}, + } + }, + }; } - None } } From bd0b9462450fc78f791225565b1035a7458722b9 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Tue, 18 Mar 2025 15:02:32 -0600 Subject: [PATCH 13/19] fixed tracing --- Cargo.toml | 2 +- src/db.rs | 13 ++++++++++++- src/main.rs | 21 +++++++++++++++++---- src/parser.rs | 2 -- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2dcb455..b04036c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ base64 = "0.22.1" html5ever = "0.29" # minio = "0.1.0" minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"} -reqwest = { version = "0.12", features = ["gzip"] } +reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } surrealdb = "2.2" tokio = { version="1.41.0", features = ["full"] } diff --git a/src/db.rs b/src/db.rs index e8c19f4..02602dc 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,3 +1,4 @@ +use std::fmt::Debug; use serde::{Deserialize, Serialize}; use surrealdb::{ engine::remote::ws::{Client, Ws}, error::Db, opt::auth::Root, sql::Thing, Response, Surreal @@ -7,7 +8,7 @@ use url::Url; use crate::{Config, Timer}; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone)] pub struct Website { /// The url that this data is found at pub site: Url, @@ -17,6 +18,14 @@ pub struct Website { id: Option, } +// manual impl to make tracing look nicer +impl Debug for Website { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let site = (self.site.domain().unwrap_or("n/a")).to_string() + self.site.path(); + f.debug_struct("Website").field("site", &site).finish() + } +} + impl Website { /// Creates a blank site (assumes that url param is site's root) pub fn new(url: &str, crawled: bool) -> Self { @@ -80,6 +89,8 @@ impl Website { #[instrument(skip_all)] pub async fn store(&self, db: &Surreal) -> Option { + let t = Timer::start("Stored page"); + let _ = t; // check if it's been gone thru before let mut response = db .query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1") diff --git a/src/main.rs b/src/main.rs index a87301d..a60f093 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ use db::{connect, Website}; use s3::S3; use surrealdb::{engine::remote::ws::Client, Surreal}; use tokio::task::JoinSet; -use tracing::{debug, info, instrument, trace, trace_span}; +use tracing::{debug, info, instrument, trace, trace_span, warn}; use tracing_subscriber::{fmt::time::LocalTime, EnvFilter}; mod db; @@ -36,6 +36,7 @@ async fn main() { tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) .with_line_number(true) + .with_thread_ids(true) .with_file(true) .with_timer(LocalTime::rfc_3339()) .init(); @@ -123,14 +124,20 @@ async fn main() { drop(total_runtime); } -#[instrument(skip_all)] +#[instrument(skip (db, s3, reqwest))] /// Downloads and crawls and stores a webpage. /// It is acceptable to clone `db`, `reqwest`, and `s3` because they all use `Arc`s internally. - Noted by Oliver async fn get(mut site: Website, db: Surreal, reqwest: reqwest::Client, s3: S3) { trace!("Get: {}", site.to_string()); + + let timer = Timer::start("Built request"); + let request_builder = reqwest.get(site.to_string()); + timer.stop(); + let timer = Timer::start("Got page"); - if let Ok(response) = reqwest.get(site.to_string()).send().await { + if let Ok(response) = request_builder.send().await { timer.stop(); + debug!("Getting body..."); // Get body let data = response.text().await.expect("Failed to read http response's body!"); @@ -182,7 +189,13 @@ impl<'a> Timer<'a> { pub fn stop(&self) -> f64 { let dif = self.start.elapsed().as_micros(); let ms = dif as f64 / 1000.; - trace!("{}", format!("{} in {:.3}ms", self.msg, ms)); + + if ms > 200. { + warn!("{}", format!("{} in {:.3}ms", self.msg, ms)); + } else { + trace!("{}", format!("{} in {:.3}ms", self.msg, ms)); + } + ms } } diff --git a/src/parser.rs b/src/parser.rs index 3357059..68786b0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -93,7 +93,6 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: &str) { } { - let t = Timer::start("Stored pages"); let mut links_to = Vec::new(); // this is a 2d vec accidentally @@ -108,6 +107,5 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: &str) { } site.links_to(links_to, db).await; - drop(t); } } From 3b4e6a40cefcb4a4580a0d67c65cba8dfacf5af9 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Tue, 18 Mar 2025 15:07:50 -0600 Subject: [PATCH 14/19] minimize vec resizing --- src/parser.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 68786b0..4c4a1ca 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -71,7 +71,7 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: &str) { site.store(db).await; // prep work - let mut other_sites = Vec::new(); + let mut other_sites: Vec = Vec::new(); { // using blocks to prevent compiler's async worries let _t = Timer::start("Parsed page"); @@ -84,8 +84,9 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: &str) { let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default()); // go thru buffer - while let TokenizerResult::Script(sites) = tokenizer.feed(&mut token_buffer) { - other_sites.push(sites); + while let TokenizerResult::Script(mut sites) = tokenizer.feed(&mut token_buffer) { + other_sites.append(&mut sites); + // other_sites.push(sites); } assert!(token_buffer.is_empty()); @@ -93,16 +94,12 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: &str) { } { - let mut links_to = Vec::new(); + let mut links_to = Vec::with_capacity(other_sites.len()); - // this is a 2d vec accidentally for a in other_sites { - for b in a { - // TODO this can become a JoinSet later - let other = b.store(db).await; - if let Some(o) = other { - links_to.push(o); - } + let other = a.store(db).await; + if let Some(o) = other { + links_to.push(o); } } From f2a3e836a05b9b727132033a037404e3ee4d1ef8 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Tue, 18 Mar 2025 15:08:29 -0600 Subject: [PATCH 15/19] spelling and clippy --- src/db.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/db.rs b/src/db.rs index 02602dc..dd4f81f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -98,7 +98,7 @@ impl Website { .await .expect("Failed to check surreal for duplicates!"); - if let Some(old) = response.take::>(0).expect("Failed to read reponse from surreal for duplicates.") { + if let Some(old) = response.take::>(0).expect("Failed to read response from surreal for duplicates.") { // site exists already if let Some(id) = old.id { // make sure to preserve the "crawled status" @@ -119,7 +119,7 @@ impl Website { match error { Db::QueryCancelled => todo!(), Db::QueryNotExecuted => todo!(), - Db::QueryNotExecutedDetail { message } => todo!(), + Db::QueryNotExecutedDetail { message: _ } => todo!(), _=>{}, } }, From d11e7dd27cfc85fd917a02e5c78d3a2cd852cc62 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Tue, 18 Mar 2025 15:25:40 -0600 Subject: [PATCH 16/19] the biggest 1 line improvement ever --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index a60f093..aaffb91 100644 --- a/src/main.rs +++ b/src/main.rs @@ -163,7 +163,7 @@ async fn get_uncrawled_links( debug!("Getting uncrawled links"); let mut response = db - .query("SELECT * FROM website WHERE crawled = false AND site CONTAINS type::string($format) LIMIT $count;") + .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;") .bind(("format", filter)) .bind(("count", count)) .await From e3e4175f513d8d6e19075d7cd2ec47f55f000e25 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Tue, 18 Mar 2025 15:25:56 -0600 Subject: [PATCH 17/19] logging improvements --- src/db.rs | 4 ++-- src/main.rs | 4 ++-- src/s3.rs | 6 ++++-- src/setup.surql | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/db.rs b/src/db.rs index dd4f81f..9b14d89 100644 --- a/src/db.rs +++ b/src/db.rs @@ -87,9 +87,9 @@ impl Website { } } - #[instrument(skip_all)] + #[instrument(name = "surql_store", skip_all)] pub async fn store(&self, db: &Surreal) -> Option { - let t = Timer::start("Stored page"); + let t = Timer::start("Stored link"); let _ = t; // check if it's been gone thru before let mut response = db diff --git a/src/main.rs b/src/main.rs index aaffb91..2475cb5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,8 +53,8 @@ async fn main() { surreal_username: "root", surreal_password: "root", surreal_ns: "test", - surreal_db: "v1.11.2", - s3_bucket: "v1.11.2", + surreal_db: "v1.12", + s3_bucket: "v1.12", s3_url: "http://localhost:9000", s3_access_key: "8UO76z8wCs9DnpxSbQUY", s3_secret_key: "xwKVMpf2jzgprsdo85Dvo74UmO84y0aRrAUorYY5", diff --git a/src/s3.rs b/src/s3.rs index a54d84c..4439df2 100644 --- a/src/s3.rs +++ b/src/s3.rs @@ -10,7 +10,7 @@ use minio::s3::{ use tracing::{instrument, trace, warn}; use url::Url; -use crate::Config; +use crate::{Config, Timer}; const CUSTOM_ENGINE: engine::GeneralPurpose = engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD); @@ -60,8 +60,10 @@ impl S3 { }) } - #[instrument(skip_all)] + #[instrument(name = "s3_store", skip_all)] pub async fn store(&self, data: &str, url: &Url) { + let t = Timer::start("Stored page"); + let _ = t; // prevent compiler drop if let Some(domain) = url.domain() { let filename = domain.to_owned() + url.path(); diff --git a/src/setup.surql b/src/setup.surql index 4271c79..61c020c 100644 --- a/src/setup.surql +++ b/src/setup.surql @@ -1,2 +1,3 @@ DEFINE TABLE IF NOT EXISTS website SCHEMALESS; DEFINE FIELD IF NOT EXISTS accessed_at ON TABLE website VALUE time::now(); +DEFINE INDEX IF NOT EXISTS idx ON TABLE website COLUMNS site UNIQUE; From de80418c0011f352d9215343e90549654b3e9968 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Tue, 18 Mar 2025 16:09:46 -0600 Subject: [PATCH 18/19] better logging --- .gitignore | 3 +- Cargo.lock | 13 ++ Cargo.toml | 2 +- compose.yml | 31 ----- docker/alloy.conf | 14 ++ docker/compose.yml | 71 ++++++++++ docker/grafana.yaml | 12 ++ docker/load-me-into-grafana.json | 223 +++++++++++++++++++++++++++++++ docker/loki.yaml | 62 +++++++++ src/main.rs | 44 ++++-- 10 files changed, 430 insertions(+), 45 deletions(-) delete mode 100644 compose.yml create mode 100644 docker/alloy.conf create mode 100644 docker/compose.yml create mode 100644 docker/grafana.yaml create mode 100644 docker/load-me-into-grafana.json create mode 100644 docker/loki.yaml diff --git a/.gitignore b/.gitignore index 301630e..dbaa546 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /.minio perf.data flamegraph.svg -perf.data.old \ No newline at end of file +perf.data.old +/docker/logs/* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index cbcede6..8c6a73f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4364,6 +4364,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.19" @@ -4374,6 +4384,8 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", @@ -4381,6 +4393,7 @@ dependencies = [ "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b04036c..8499bea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,5 +13,5 @@ serde = { version = "1.0", features = ["derive"] } surrealdb = "2.2" tokio = { version="1.41.0", features = ["full"] } tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time", "json"] } url = { version = "2.5", features = ["serde"] } diff --git a/compose.yml b/compose.yml deleted file mode 100644 index 797688f..0000000 --- a/compose.yml +++ /dev/null @@ -1,31 +0,0 @@ -services: - surreal: - image: surrealdb/surrealdb:latest-dev - ports: - - 8000:8000 - volumes: - - ./.surrealdb/:/mydata - command: - - start - - --log - - debug - - --user - - root - - --pass - - root - - rocksdb:/mydata/database.db - minio: - image: quay.io/minio/minio - ports: - - 9000:9000 - - 9001:9001 - environment: - - MINIO_ROOT_USER=root - - MINIO_ROOT_PASSWORD=an8charpassword - volumes: - - ./.minio/:/data - command: - - server - - /data - - --console-address - - ":9001" diff --git a/docker/alloy.conf b/docker/alloy.conf new file mode 100644 index 0000000..35babb3 --- /dev/null +++ b/docker/alloy.conf @@ -0,0 +1,14 @@ +local.file_match "tmplogs" { + path_targets = [{"__path__" = "/tmp/alloy-logs/*.log"}] +} + +loki.source.file "local_files" { + targets = local.file_match.tmplogs.targets + forward_to = [loki.write.local_loki.receiver] +} + +loki.write "local_loki" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} \ No newline at end of file diff --git a/docker/compose.yml b/docker/compose.yml new file mode 100644 index 0000000..48c1a90 --- /dev/null +++ b/docker/compose.yml @@ -0,0 +1,71 @@ +services: + surreal: + image: surrealdb/surrealdb:latest-dev + ports: + - 8000:8000 + volumes: + - surrealdb_storage:/mydata + command: + - start + - --log + - debug + - --user + - root + - --pass + - root + - rocksdb:/mydata/database.db + minio: + image: quay.io/minio/minio + ports: + - 9000:9000 + - 9001:9001 + environment: + - MINIO_ROOT_USER=root + - MINIO_ROOT_PASSWORD=an8charpassword + volumes: + - minio_storage:/data + command: + - server + - /data + - --console-address + - ":9001" + + alloy: + image: grafana/alloy:latest + ports: + - 12345:12345 + volumes: + # if you change this, you also need to change it in the alloy config file + - ./logs/:/tmp/alloy-logs + - ./alloy.conf:/etc/alloy/config.alloy + - alloy_storage:/var/lib/alloy + command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy + + #logs + loki: + image: grafana/loki:latest + ports: + - 3100:3100 + command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./loki.yaml:/etc/loki/local-config.yaml + + # Everything viewer + grafana: + image: grafana/grafana:latest + volumes: + - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - grafana_storage:/var/lib/grafana + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor + ports: + - 3000:3000 + +volumes: + grafana_storage: + alloy_storage: + surrealdb_storage: + minio_storage: \ No newline at end of file diff --git a/docker/grafana.yaml b/docker/grafana.yaml new file mode 100644 index 0000000..be79737 --- /dev/null +++ b/docker/grafana.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: true + version: 1 + editable: false diff --git a/docker/load-me-into-grafana.json b/docker/load-me-into-grafana.json new file mode 100644 index 0000000..198e006 --- /dev/null +++ b/docker/load-me-into-grafana.json @@ -0,0 +1,223 @@ +{ + "__inputs": [ + { + "name": "DS_LOKI", + "label": "Loki", + "description": "", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.3.1" + }, + { + "type": "panel", + "id": "logs", + "name": "Logs", + "version": "" + }, + { + "type": "datasource", + "id": "loki", + "name": "Loki", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `ERROR` | line_format \"{{.fields_message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Errors", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `WARN` | line_format \"{{.fields_message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Warnings", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `DEBUG` | line_format \"{{.fields_message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Debug", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `TRACE` | line_format \"{{.fields_message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Trace", + "type": "logs" + } + ], + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "New dashboard", + "uid": "ceg90x34pqgowd", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/loki.yaml b/docker/loki.yaml new file mode 100644 index 0000000..819d266 --- /dev/null +++ b/docker/loki.yaml @@ -0,0 +1,62 @@ +# this is mostly the default config from grafana's website + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +frontend: + encoding: protobuf + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false diff --git a/src/main.rs b/src/main.rs index 2475cb5..bd97892 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,9 +8,9 @@ use std::time::Instant; use db::{connect, Website}; use s3::S3; use surrealdb::{engine::remote::ws::Client, Surreal}; -use tokio::task::JoinSet; +use tokio::{task::JoinSet}; use tracing::{debug, info, instrument, trace, trace_span, warn}; -use tracing_subscriber::{fmt::time::LocalTime, EnvFilter}; +use tracing_subscriber::{fmt::{self, time::LocalTime}, layer::{Filter, SubscriberExt}, EnvFilter, Layer, Registry}; mod db; mod parser; @@ -33,13 +33,33 @@ struct Config<'a> { async fn main() { let total_runtime = Timer::start("Completed"); - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .with_line_number(true) - .with_thread_ids(true) - .with_file(true) - .with_timer(LocalTime::rfc_3339()) - .init(); + let writer = std::fs::OpenOptions::new() + .append(true) + .create(true) + .open("./docker/logs/tracing.log") + .expect("Couldn't make log file!"); + + let registry = Registry::default() + .with( + fmt::layer() + .with_line_number(true) + .with_thread_ids(true) + .with_file(true) + // .with_timer(LocalTime::rfc_3339()) // Loki or alloy does this automatically + .json() + .with_writer(writer) + ); + + tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber"); + + // tracing_subscriber::fmt() + // .with_env_filter(EnvFilter::from_default_env()) + // .with_line_number(true) + // .with_thread_ids(true) + // .with_file(true) + // .with_timer(LocalTime::rfc_3339()) + // .init(); + debug!("Starting..."); // Would probably take these in as parameters from a cli let starting_url = "https://en.wikipedia.org/"; @@ -56,8 +76,8 @@ async fn main() { surreal_db: "v1.12", s3_bucket: "v1.12", s3_url: "http://localhost:9000", - s3_access_key: "8UO76z8wCs9DnpxSbQUY", - s3_secret_key: "xwKVMpf2jzgprsdo85Dvo74UmO84y0aRrAUorYY5", + s3_access_key: "p8gXIZEO2FnWqWBiJYwo", + s3_secret_key: "1mRO0EYA2YAQ0xsKrlbkIIz4AT8KNXy6QIQPtxUu", }; @@ -66,7 +86,7 @@ async fn main() { .expect("Failed to connect to surreal, aborting."); let s3 = S3::connect(&config) .await - .expect("Failed to connect to minio, aborting."); + .expect("Failed to connect to minio, aborting.\n\nThis probably means you need to login to the minio console and get a new access key!\n\n(Probably here) http://localhost:9001/access-keys/new-account\n\n"); let reqwest = reqwest::Client::builder() // .use_rustls_tls() From 9aa34b3eee0ddbb3bda5c2c98e3603d1681aea86 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 19 Mar 2025 04:59:50 +0000 Subject: [PATCH 19/19] epic metrics --- Cargo.lock | 350 ++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 3 + Crawler.toml | 16 ++ docker/compose.yml | 14 +- docker/grafana.yaml | 12 ++ docker/prometheus.yaml | 16 ++ src/db.rs | 26 ++- src/main.rs | 108 +++++++------ src/parser.rs | 1 + src/s3.rs | 8 +- 10 files changed, 493 insertions(+), 61 deletions(-) create mode 100644 Crawler.toml create mode 100644 docker/prometheus.yaml diff --git a/Cargo.lock b/Cargo.lock index 8c6a73f..dd31000 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -398,6 +398,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-lc-rs" +version = "1.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -453,6 +476,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.11.0", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.100", + "which", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -614,6 +660,8 @@ version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -674,6 +722,15 @@ dependencies = [ "unicode-security", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -738,6 +795,26 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -769,6 +846,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1002,6 +1089,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "earcutr" version = "0.4.3" @@ -1166,6 +1259,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fst" version = "0.4.7" @@ -1388,6 +1487,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "h2" version = "0.4.8" @@ -1614,6 +1719,7 @@ dependencies = [ "hyper", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1924,6 +2030,15 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -1986,6 +2101,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lexicmp" version = "0.1.0" @@ -2001,6 +2122,16 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.11" @@ -2029,6 +2160,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.9.3" @@ -2149,6 +2286,53 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "metrics" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7deb012b3b2767169ff203fadb4c6b0b82b947512e5eb9e0b78c2e186ad9e3" +dependencies = [ + "ahash 0.8.11", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" +dependencies = [ + "base64 0.22.1", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "indexmap 2.8.0", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd4884b1dd24f7d6628274a2f5ae22465c337c5ba065ec9b6edccddf8acc673" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.2", + "metrics", + "quanta", + "rand 0.8.5", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "miette" version = "5.10.0" @@ -2188,6 +2372,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minio" version = "0.2.0-alpha" @@ -2295,7 +2485,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -2353,6 +2543,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntapi" version = "0.4.1" @@ -2757,6 +2957,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "prettyplease" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +dependencies = [ + "proc-macro2", + "syn 2.0.100", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -2810,6 +3020,21 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quanta" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick_cache" version = "0.5.2" @@ -2833,7 +3058,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "socket2", "thiserror 2.0.12", @@ -2852,7 +3077,7 @@ dependencies = [ "getrandom 0.3.2", "rand 0.9.0", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", @@ -2968,6 +3193,24 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" +dependencies = [ + "bitflags", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -3296,6 +3539,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -3320,6 +3569,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.0.3" @@ -3329,7 +3591,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.9.3", "windows-sys 0.59.0", ] @@ -3339,6 +3601,7 @@ version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -3348,6 +3611,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -3372,6 +3647,7 @@ version = "0.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3447,7 +3723,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -3520,6 +3809,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3632,6 +3930,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +[[package]] +name = "sketches-ddsketch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" + [[package]] name = "slab" version = "0.4.9" @@ -3814,11 +4118,14 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "html5ever 0.29.1", + "metrics", + "metrics-exporter-prometheus", "minio", "reqwest", "serde", "surrealdb", "tokio", + "toml", "tracing", "tracing-subscriber", "url", @@ -4015,7 +4322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4044,7 +4351,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix", + "rustix 1.0.3", "windows-sys 0.59.0", ] @@ -4277,11 +4584,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -4290,6 +4612,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap 2.8.0", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] @@ -4750,6 +5074,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 8499bea..6a1d88e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,12 +6,15 @@ edition = "2021" [dependencies] base64 = "0.22.1" html5ever = "0.29" +metrics = "0.24.1" +metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]} # minio = "0.1.0" minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"} reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } surrealdb = "2.2" tokio = { version="1.41.0", features = ["full"] } +toml = "0.8.20" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time", "json"] } url = { version = "2.5", features = ["serde"] } diff --git a/Crawler.toml b/Crawler.toml new file mode 100644 index 0000000..29625a8 --- /dev/null +++ b/Crawler.toml @@ -0,0 +1,16 @@ +# Surreal config +surreal_url = "localhost:8000" +surreal_username = "root" +surreal_password = "root" +surreal_ns = "test" +surreal_db = "v1.12" + +# Minio config +s3_bucket = "v1.12" +s3_url = "http://localhost:9000" +s3_access_key = "jLDPKGuu513VENc8kJwX" +s3_secret_key = "4T1nymEzsGYOlKSAb1WX7V3stnQn9a5ZoTQjDfcL" + +# Crawler config +crawl_filter = "en.wikipedia.com" +budget = 200 \ No newline at end of file diff --git a/docker/compose.yml b/docker/compose.yml index 48c1a90..1f3faf3 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -50,6 +50,17 @@ services: volumes: - ./loki.yaml:/etc/loki/local-config.yaml + # Metrics collector + prometheus: + image: prom/prometheus:latest + expose: + - 9090 + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + # persist data + - prometheus_storage:/prometheus + command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml + # Everything viewer grafana: image: grafana/grafana:latest @@ -63,8 +74,9 @@ services: - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor ports: - 3000:3000 - + volumes: + prometheus_storage: grafana_storage: alloy_storage: surrealdb_storage: diff --git a/docker/grafana.yaml b/docker/grafana.yaml index be79737..ac68607 100644 --- a/docker/grafana.yaml +++ b/docker/grafana.yaml @@ -10,3 +10,15 @@ datasources: isDefault: true version: 1 editable: false +- name: Prometheus + type: prometheus + uid: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: false + version: 1 + editable: false + jsonData: + httpMethod: GET diff --git a/docker/prometheus.yaml b/docker/prometheus.yaml new file mode 100644 index 0000000..b8f5992 --- /dev/null +++ b/docker/prometheus.yaml @@ -0,0 +1,16 @@ +global: + scrape_interval: 5s + query_log_file: /etc/prometheus/query.log + +scrape_configs: + - job_name: crawler + static_configs: + # change this your machine's ip, localhost won't work + # because localhost refers to the docker container. + - targets: ['192.168.8.209:2500'] + - job_name: loki + static_configs: + - targets: ['loki:3100'] + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/src/db.rs b/src/db.rs index 9b14d89..bb0edac 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,4 +1,5 @@ use std::fmt::Debug; +use metrics::counter; use serde::{Deserialize, Serialize}; use surrealdb::{ engine::remote::ws::{Client, Ws}, error::Db, opt::auth::Root, sql::Thing, Response, Surreal @@ -8,6 +9,10 @@ use url::Url; use crate::{Config, Timer}; +const ROUND_TRIP_METRIC: &'static str = "surql_trips"; +const STORE: &'static str = "surql_store_calls"; +const LINK: &'static str = "surql_link_calls"; + #[derive(Serialize, Deserialize, Clone)] pub struct Website { /// The url that this data is found at @@ -47,6 +52,7 @@ impl Website { #[instrument(skip_all)] pub async fn links_to(&self, other: Vec, db: &Surreal) { + let len = other.len(); if len == 0 {return} @@ -57,6 +63,8 @@ impl Website { let timer = Timer::start(&msg); // prevent the timer from being dropped instantly. let _ = timer; + counter!(ROUND_TRIP_METRIC).increment(1); + counter!(LINK).increment(1); match db .query("COUNT(RELATE (SELECT id FROM website WHERE site = $in) -> links_to -> $out)") .bind(("in", from)) @@ -89,8 +97,12 @@ impl Website { #[instrument(name = "surql_store", skip_all)] pub async fn store(&self, db: &Surreal) -> Option { + counter!(STORE).increment(1); + let counter = counter!(ROUND_TRIP_METRIC); let t = Timer::start("Stored link"); let _ = t; + counter.increment(1); + // check if it's been gone thru before let mut response = db .query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1") @@ -105,6 +117,7 @@ impl Website { let mut new = self.clone(); new.crawled = old.crawled | new.crawled; + counter.increment(1); // update the record match db.upsert((id.tb, id.id.to_string())).content(new).await { Ok(e) => { @@ -130,6 +143,7 @@ impl Website { }; } } else { + counter.increment(1); // sites hasn't existed yet match db.create("website").content(self.clone()).await { Ok(e) => { @@ -165,23 +179,23 @@ pub struct Record { } #[instrument(skip_all, name = "SurrealDB")] -pub async fn connect(config: &Config<'_>) -> surrealdb::Result> { +pub async fn connect(config: &Config) -> surrealdb::Result> { trace!("Establishing connection to surreal..."); // Connect to the server - let db = Surreal::new::(config.surreal_url).await?; + let db = Surreal::new::(&config.surreal_url).await?; trace!("Logging in..."); // Signin as a namespace, database, or root user db.signin(Root { - username: config.surreal_username, - password: config.surreal_password, + username: &config.surreal_username, + password: &config.surreal_password, }) .await?; // Select a specific namespace / database db - .use_ns(config.surreal_ns) - .use_db(config.surreal_db) + .use_ns(&config.surreal_ns) + .use_db(&config.surreal_db) .await?; let setup = include_bytes!("setup.surql"); diff --git a/src/main.rs b/src/main.rs index bd97892..42f65ae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,32 +1,43 @@ -#![feature(type_alias_impl_trait)] -#![feature(const_async_blocks)] +#![feature(ip_from)] extern crate html5ever; -use std::time::Instant; +use std::{fs::File, io::Read, net::{IpAddr, Ipv4Addr}, time::Instant}; use db::{connect, Website}; +use metrics::{counter, gauge}; +use metrics_exporter_prometheus::PrometheusBuilder; use s3::S3; +use serde::Deserialize; use surrealdb::{engine::remote::ws::Client, Surreal}; -use tokio::{task::JoinSet}; +use tokio::task::JoinSet; use tracing::{debug, info, instrument, trace, trace_span, warn}; -use tracing_subscriber::{fmt::{self, time::LocalTime}, layer::{Filter, SubscriberExt}, EnvFilter, Layer, Registry}; +use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry}; mod db; mod parser; mod s3; -struct Config<'a> { - surreal_ns: &'a str, - surreal_db: &'a str, - surreal_url: &'a str, - surreal_username: &'a str, - surreal_password: &'a str, +const GET_METRIC: &'static str = "total_gets"; +const GET_IN_FLIGHT: &'static str = "gets_in_flight"; +const SITES_CRAWLED: &'static str = "pages_crawled"; +const BEING_PROCESSED: &'static str = "pages_being_processed"; - s3_url: &'a str, - s3_bucket: &'a str, - s3_access_key: &'a str, - s3_secret_key: &'a str, +#[derive(Deserialize)] +struct Config { + surreal_ns: String, + surreal_db: String, + surreal_url: String, + surreal_username: String, + surreal_password: String, + + s3_url: String, + s3_bucket: String, + s3_access_key: String, + s3_secret_key: String, + + crawl_filter: String, + budget: usize, } #[tokio::main] @@ -48,38 +59,33 @@ async fn main() { // .with_timer(LocalTime::rfc_3339()) // Loki or alloy does this automatically .json() .with_writer(writer) + // .with_filter(EnvFilter::from_default_env()) ); tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber"); - // tracing_subscriber::fmt() - // .with_env_filter(EnvFilter::from_default_env()) - // .with_line_number(true) - // .with_thread_ids(true) - // .with_file(true) - // .with_timer(LocalTime::rfc_3339()) - // .init(); + + let builder = PrometheusBuilder::new(); + builder.with_http_listener( + std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::from_octets([0,0,0,0])), 2500) + ) + .install() + .expect("failed to install recorder/exporter"); debug!("Starting..."); - // Would probably take these in as parameters from a cli + // Would probably take these in as parameters from a cli let starting_url = "https://en.wikipedia.org/"; // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored. - let crawl_filter = "en.wikipedia.org/"; - let budget = 50; + // let crawl_filter = "en.wikipedia.org/"; + // let budget = 50; let mut crawled = 0; - let config = Config { - surreal_url: "localhost:8000", - surreal_username: "root", - surreal_password: "root", - surreal_ns: "test", - surreal_db: "v1.12", - s3_bucket: "v1.12", - s3_url: "http://localhost:9000", - s3_access_key: "p8gXIZEO2FnWqWBiJYwo", - s3_secret_key: "1mRO0EYA2YAQ0xsKrlbkIIz4AT8KNXy6QIQPtxUu", - }; - + + let mut file = File::open("./Crawler.toml").expect("Failed to read Crawler.toml"); + let mut buf = String::new(); + let _ = file.read_to_string(&mut buf); + + let config: Config = toml::from_str(&buf).expect("Failed to parse Crawler.toml"); let db = connect(&config) .await @@ -106,14 +112,14 @@ async fn main() { let span = trace_span!("Loop"); let span = span.enter(); - while crawled < budget { - let get_num = if budget - crawled < 100 { - budget - crawled + while crawled < config.budget { + let get_num = if config.budget - crawled < 100 { + config.budget - crawled } else { 100 }; - let uncrawled = get_uncrawled_links(&db, get_num, crawl_filter.to_string()).await; + let uncrawled = get_uncrawled_links(&db, get_num, config.crawl_filter.clone()).await; if uncrawled.len() == 0 { info!("Had more budget but finished crawling everything."); return; @@ -126,16 +132,20 @@ async fn main() { { let mut futures = JoinSet::new(); for site in uncrawled { + gauge!(BEING_PROCESSED).increment(1); futures.spawn(get(site, db.clone(), reqwest.clone(), s3.clone())); - // technically the site hasn't be crawled *yet*, but the future - // where it is crawled has been set up. - crawled += 1; // let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32); // info!("Crawled {crawled} out of {budget} pages. ({percent})"); } debug!("Joining {} futures...", futures.len()); - // join all the gets together - let _ = futures.join_all().await; + + let c = counter!(SITES_CRAWLED); + // As futures complete runs code in while block + while let Some(_) = futures.join_next().await { + c.increment(1); + gauge!(BEING_PROCESSED).decrement(1); + crawled += 1; + } } } drop(span); @@ -154,9 +164,15 @@ async fn get(mut site: Website, db: Surreal, reqwest: reqwest::Client, s let request_builder = reqwest.get(site.to_string()); timer.stop(); + let g = gauge!(GET_IN_FLIGHT); + g.increment(1); let timer = Timer::start("Got page"); + if let Ok(response) = request_builder.send().await { + timer.stop(); + g.decrement(1); + counter!(GET_METRIC).increment(1); debug!("Getting body..."); // Get body diff --git a/src/parser.rs b/src/parser.rs index 4c4a1ca..0e5dc38 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -97,6 +97,7 @@ pub async fn parse(db: &Surreal, site: &mut Website, data: &str) { let mut links_to = Vec::with_capacity(other_sites.len()); for a in other_sites { + let other = a.store(db).await; if let Some(o) = other { links_to.push(o); diff --git a/src/s3.rs b/src/s3.rs index 4439df2..99bd91a 100644 --- a/src/s3.rs +++ b/src/s3.rs @@ -1,4 +1,5 @@ use base64::{alphabet, engine::{self, general_purpose}, Engine}; +use metrics::counter; use minio::s3::{ args::{BucketExistsArgs, MakeBucketArgs}, client::ClientBuilder, @@ -14,6 +15,8 @@ use crate::{Config, Timer}; const CUSTOM_ENGINE: engine::GeneralPurpose = engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD); +const ROUND_TRIP_METRIC: &'static str = "s3_trips"; + #[derive(Clone)] pub struct S3 { bucket_name: String, @@ -22,7 +25,7 @@ pub struct S3 { impl S3 { #[instrument(skip_all, name = "S3")] - pub async fn connect(config: &Config<'_>) -> Result { + pub async fn connect(config: &Config) -> Result { let base_url = config .s3_url .parse::() @@ -62,6 +65,7 @@ impl S3 { #[instrument(name = "s3_store", skip_all)] pub async fn store(&self, data: &str, url: &Url) { + let counter = counter!(ROUND_TRIP_METRIC); let t = Timer::start("Stored page"); let _ = t; // prevent compiler drop if let Some(domain) = url.domain() { @@ -69,6 +73,7 @@ impl S3 { trace!("Created filename: {filename} from raw: {}", url.to_string()); + counter.increment(1); let _ = match &self .client .put_object_content(&self.bucket_name, &filename, data.to_owned()) @@ -83,6 +88,7 @@ impl S3 { let filename: String = domain.to_owned() + &CUSTOM_ENGINE.encode(url.path()); + counter.increment(1); let _ = &self .client .put_object_content(&self.bucket_name, &filename, data.to_owned())