use std::cell::LazyCell; use std::sync::{Arc, RwLock}; use serde::{Deserialize, Serialize}; use spider::hashbrown::HashMap; use surrealdb::engine::remote::ws::{Ws, Client}; use surrealdb::opt::auth::Root; use surrealdb::sql::Thing; use surrealdb::Surreal; use tokio::spawn; use tokio::sync::broadcast::{self, Sender, Receiver}; #[derive(Debug, Deserialize)] struct Record { #[allow(dead_code)] id: Thing, } #[derive(Debug, Serialize)] struct Website<'a> { url: &'a str, crawled: bool } #[tokio::main] async fn main() -> surrealdb::Result<()> { let db_handle = tokio::spawn(async move { let db = Surreal::new::("127.0.0.1:8000").await.unwrap(); db.signin(Root { username: "root", password: "root", }) .await.unwrap(); db.use_ns("test").use_db("test").await.unwrap(); let mut sub = PIPE.subscribe(); loop { match sub.recv().await { Ok((from, to)) => { // let f: Vec = db.create("website").content(Website {url: &from, crawled: true}).await.unwrap(); // let t: Vec = db.create("website").content(Website {url: &to, crawled: false}).await.unwrap(); // println!("{:?} {:?}", f , t); }, Err(e) => { eprintln!("Error: {}", e); }, } } }); let mut site = spider::website::Website::new("https://oliveratkinson.net") .with_budget(Some(HashMap::from([ ("*", 25), ]))) .with_tld(true) .with_on_link_find_callback(Some(|from, to| { let from = from.as_ref().to_string(); let to = to.as_ref().to_string(); match PIPE.send((from.clone(), to.clone())) { Ok(_) => {}, Err(e) => { eprintln!("{:?}", e); }, }; println!("{from} -> {to}"); })) .build() .unwrap(); site.crawl().await; let _ = db_handle.await; Ok(()) } const PIPE: LazyCell> = LazyCell::new(|| { broadcast::channel(100).0 });