unifed settings for testing
This commit is contained in:
parent
fd71a8bc13
commit
e4bf320ecd
13
src/main.rs
13
src/main.rs
@ -30,10 +30,11 @@ struct Config<'a> {
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let total_runtime = Timer::start("Completed");
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(EnvFilter::from_default_env())
|
||||
.with_line_number(true)
|
||||
.without_time()
|
||||
// .without_time()
|
||||
.init();
|
||||
debug!("Starting...");
|
||||
|
||||
@ -42,16 +43,15 @@ async fn main() {
|
||||
surreal_username: "root",
|
||||
surreal_password: "root",
|
||||
surreal_ns: "test",
|
||||
surreal_db: "v1.9",
|
||||
s3_bucket: "v1.9",
|
||||
surreal_db: "spider",
|
||||
s3_bucket: "spider",
|
||||
s3_url: "http://localhost:9000",
|
||||
s3_access_key: "0zv7GbLQsw4ZI8TclMps",
|
||||
s3_secret_key: "5dB7QkGFw7fYbUJ5LpHk2GbWR7Bl710HlRz4NbzB",
|
||||
};
|
||||
|
||||
// Would probably take these in as parameters from a cli
|
||||
// let starting_url = "https://oliveratkinson.net/";
|
||||
let starting_url = "https://en.wikipedia.org/wiki/Main_Page";
|
||||
let starting_url = "https://oliveratkinson.net/";
|
||||
|
||||
let s3 = S3::connect(&config)
|
||||
.await
|
||||
@ -61,7 +61,7 @@ async fn main() {
|
||||
.expect("Failed to connect to surreal, aborting.");
|
||||
|
||||
let mut site = spider::website::Website::new(&starting_url)
|
||||
.with_limit(4)
|
||||
.with_limit(5)
|
||||
.with_depth(0)
|
||||
.build()
|
||||
.unwrap();
|
||||
@ -110,6 +110,7 @@ async fn main() {
|
||||
subscriber.await.unwrap();
|
||||
|
||||
info!("Done");
|
||||
drop(total_runtime);
|
||||
}
|
||||
|
||||
/// Returns uncrawled links
|
||||
|
Loading…
Reference in New Issue
Block a user