unifed settings for testing
This commit is contained in:
		
							
								
								
									
										13
									
								
								src/main.rs
									
									
									
									
									
								
							
							
						
						
									
										13
									
								
								src/main.rs
									
									
									
									
									
								
							@@ -30,10 +30,11 @@ struct Config<'a> {
 | 
			
		||||
 | 
			
		||||
#[tokio::main]
 | 
			
		||||
async fn main() {
 | 
			
		||||
    let total_runtime = Timer::start("Completed");
 | 
			
		||||
    tracing_subscriber::fmt()
 | 
			
		||||
        .with_env_filter(EnvFilter::from_default_env())
 | 
			
		||||
        .with_line_number(true)
 | 
			
		||||
        .without_time()
 | 
			
		||||
        // .without_time()
 | 
			
		||||
        .init();
 | 
			
		||||
    debug!("Starting...");
 | 
			
		||||
 | 
			
		||||
@@ -42,16 +43,15 @@ async fn main() {
 | 
			
		||||
        surreal_username: "root",
 | 
			
		||||
        surreal_password: "root",
 | 
			
		||||
        surreal_ns: "test",
 | 
			
		||||
        surreal_db: "v1.9",
 | 
			
		||||
        s3_bucket: "v1.9",
 | 
			
		||||
        surreal_db: "spider",
 | 
			
		||||
        s3_bucket: "spider",
 | 
			
		||||
        s3_url: "http://localhost:9000",
 | 
			
		||||
        s3_access_key: "0zv7GbLQsw4ZI8TclMps",
 | 
			
		||||
        s3_secret_key: "5dB7QkGFw7fYbUJ5LpHk2GbWR7Bl710HlRz4NbzB",
 | 
			
		||||
    };
 | 
			
		||||
 | 
			
		||||
    // Would probably take these in as parameters from a cli
 | 
			
		||||
    // let starting_url = "https://oliveratkinson.net/";
 | 
			
		||||
    let starting_url = "https://en.wikipedia.org/wiki/Main_Page";
 | 
			
		||||
    let starting_url = "https://oliveratkinson.net/";
 | 
			
		||||
 | 
			
		||||
    let s3 = S3::connect(&config)
 | 
			
		||||
        .await
 | 
			
		||||
@@ -61,7 +61,7 @@ async fn main() {
 | 
			
		||||
        .expect("Failed to connect to surreal, aborting.");
 | 
			
		||||
 | 
			
		||||
    let mut site = spider::website::Website::new(&starting_url)
 | 
			
		||||
        .with_limit(4)
 | 
			
		||||
        .with_limit(5)
 | 
			
		||||
        .with_depth(0)
 | 
			
		||||
        .build()
 | 
			
		||||
        .unwrap();
 | 
			
		||||
@@ -110,6 +110,7 @@ async fn main() {
 | 
			
		||||
    subscriber.await.unwrap();
 | 
			
		||||
 | 
			
		||||
    info!("Done");
 | 
			
		||||
    drop(total_runtime);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
/// Returns uncrawled links
 | 
			
		||||
 
 | 
			
		||||
		Reference in New Issue
	
	Block a user