Oliver/internet_mapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unifed settings for testing

Browse Source
This commit is contained in:
Oliver Atkinson 2024-12-12 11:41:39 -07:00
parent fd71a8bc13
commit e4bf320ecd
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main.rs
View File

@ -30,10 +30,11 @@ struct Config<'a> {
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
let total_runtime = Timer::start("Completed");
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_env_filter(EnvFilter::from_default_env()) .with_env_filter(EnvFilter::from_default_env())
.with_line_number(true) .with_line_number(true)
.without_time() // .without_time()
.init(); .init();
debug!("Starting..."); debug!("Starting...");
@ -42,16 +43,15 @@ async fn main() {
surreal_username: "root", surreal_username: "root",
surreal_password: "root", surreal_password: "root",
surreal_ns: "test", surreal_ns: "test",
surreal_db: "v1.9", surreal_db: "spider",
s3_bucket: "v1.9", s3_bucket: "spider",
s3_url: "http://localhost:9000", s3_url: "http://localhost:9000",
s3_access_key: "0zv7GbLQsw4ZI8TclMps", s3_access_key: "0zv7GbLQsw4ZI8TclMps",
s3_secret_key: "5dB7QkGFw7fYbUJ5LpHk2GbWR7Bl710HlRz4NbzB", s3_secret_key: "5dB7QkGFw7fYbUJ5LpHk2GbWR7Bl710HlRz4NbzB",
}; };
// Would probably take these in as parameters from a cli // Would probably take these in as parameters from a cli
// let starting_url = "https://oliveratkinson.net/"; let starting_url = "https://oliveratkinson.net/";
let starting_url = "https://en.wikipedia.org/wiki/Main_Page";
let s3 = S3::connect(&config) let s3 = S3::connect(&config)
.await .await
@ -61,7 +61,7 @@ async fn main() {
.expect("Failed to connect to surreal, aborting."); .expect("Failed to connect to surreal, aborting.");
let mut site = spider::website::Website::new(&starting_url) let mut site = spider::website::Website::new(&starting_url)
.with_limit(4) .with_limit(5)
.with_depth(0) .with_depth(0)
.build() .build()
.unwrap(); .unwrap();
@ -110,6 +110,7 @@ async fn main() {
subscriber.await.unwrap(); subscriber.await.unwrap();
info!("Done"); info!("Done");
drop(total_runtime);
} }
/// Returns uncrawled links /// Returns uncrawled links