79 lines
2.2 KiB
Rust
79 lines
2.2 KiB
Rust
|
use std::cell::LazyCell;
|
||
|
|
use std::sync::{Arc, RwLock};
|
||
|
|
|
||
|
|
use serde::{Deserialize, Serialize};
|
||
|
|
use spider::hashbrown::HashMap;
|
||
|
|
use surrealdb::engine::remote::ws::{Ws, Client};
|
||
|
|
use surrealdb::opt::auth::Root;
|
||
|
|
use surrealdb::sql::Thing;
|
||
|
|
use surrealdb::Surreal;
|
||
|
|
use tokio::spawn;
|
||
|
|
use tokio::sync::broadcast::{self, Sender, Receiver};
|
||
|
|
|
||
|
|
#[derive(Debug, Deserialize)]
|
||
|
|
struct Record {
|
||
|
|
#[allow(dead_code)]
|
||
|
|
id: Thing,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Serialize)]
|
||
|
|
struct Website<'a> {
|
||
|
|
url: &'a str,
|
||
|
|
crawled: bool
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::main]
|
||
|
|
async fn main() -> surrealdb::Result<()> {
|
||
|
|
|
||
|
|
let db_handle = tokio::spawn(async move {
|
||
|
|
let db = Surreal::new::<Ws>("127.0.0.1:8000").await.unwrap();
|
||
|
|
db.signin(Root {
|
||
|
|
username: "root",
|
||
|
|
password: "root",
|
||
|
|
})
|
||
|
|
.await.unwrap();
|
||
|
|
db.use_ns("test").use_db("test").await.unwrap();
|
||
|
|
|
||
|
|
let mut sub = PIPE.subscribe();
|
||
|
|
loop {
|
||
|
|
match sub.recv().await {
|
||
|
|
Ok((from, to)) => {
|
||
|
|
// let f: Vec<Record> = db.create("website").content(Website {url: &from, crawled: true}).await.unwrap();
|
||
|
|
// let t: Vec<Record> = db.create("website").content(Website {url: &to, crawled: false}).await.unwrap();
|
||
|
|
// println!("{:?} {:?}", f , t);
|
||
|
|
},
|
||
|
|
Err(e) => {
|
||
|
|
eprintln!("Error: {}", e);
|
||
|
|
},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
let mut site = spider::website::Website::new("https://oliveratkinson.net")
|
||
|
|
.with_budget(Some(HashMap::from([
|
||
|
|
("*", 25),
|
||
|
|
])))
|
||
|
|
.with_tld(true)
|
||
|
|
.with_on_link_find_callback(Some(|from, to| {
|
||
|
|
let from = from.as_ref().to_string();
|
||
|
|
let to = to.as_ref().to_string();
|
||
|
|
match PIPE.send((from.clone(), to.clone())) {
|
||
|
|
Ok(_) => {},
|
||
|
|
Err(e) => {
|
||
|
|
eprintln!("{:?}", e);
|
||
|
|
},
|
||
|
|
};
|
||
|
|
println!("{from} -> {to}");
|
||
|
|
}))
|
||
|
|
.build()
|
||
|
|
.unwrap();
|
||
|
|
|
||
|
|
site.crawl().await;
|
||
|
|
let _ = db_handle.await;
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
const PIPE: LazyCell<Sender<(String, String)>> = LazyCell::new(|| {
|
||
|
|
broadcast::channel(100).0
|
||
|
|
});
|