unifed settings for testing

This commit is contained in:
Oliver Atkinson 2024-12-12 11:41:39 -07:00
parent fd71a8bc13
commit e4bf320ecd

View File

@ -30,10 +30,11 @@ struct Config<'a> {
#[tokio::main]
async fn main() {
let total_runtime = Timer::start("Completed");
tracing_subscriber::fmt()
.with_env_filter(EnvFilter::from_default_env())
.with_line_number(true)
.without_time()
// .without_time()
.init();
debug!("Starting...");
@ -42,16 +43,15 @@ async fn main() {
surreal_username: "root",
surreal_password: "root",
surreal_ns: "test",
surreal_db: "v1.9",
s3_bucket: "v1.9",
surreal_db: "spider",
s3_bucket: "spider",
s3_url: "http://localhost:9000",
s3_access_key: "0zv7GbLQsw4ZI8TclMps",
s3_secret_key: "5dB7QkGFw7fYbUJ5LpHk2GbWR7Bl710HlRz4NbzB",
};
// Would probably take these in as parameters from a cli
// let starting_url = "https://oliveratkinson.net/";
let starting_url = "https://en.wikipedia.org/wiki/Main_Page";
let starting_url = "https://oliveratkinson.net/";
let s3 = S3::connect(&config)
.await
@ -61,7 +61,7 @@ async fn main() {
.expect("Failed to connect to surreal, aborting.");
let mut site = spider::website::Website::new(&starting_url)
.with_limit(4)
.with_limit(5)
.with_depth(0)
.build()
.unwrap();
@ -110,6 +110,7 @@ async fn main() {
subscriber.await.unwrap();
info!("Done");
drop(total_runtime);
}
/// Returns uncrawled links