logging updates

src/db.rs

@@ -43,7 +43,6 @@ impl Website {
     }
 
     pub fn set_crawled(&mut self) {
-        trace!("Set crawled to true");
         self.crawled = true
     }
 

src/main.rs

@@ -20,7 +20,7 @@ use metrics_exporter_prometheus::PrometheusBuilder;
 use serde::Deserialize;
 use surrealdb::{engine::remote::ws::Client, Surreal};
 use tokio::{io::AsyncWriteExt, task::JoinSet};
-use tracing::{debug, error, info, instrument, level_filters::LevelFilter, trace, trace_span};
+use tracing::{debug, error, info, instrument, level_filters::LevelFilter, trace, trace_span, warn};
 use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry};
 
 mod db;
@@ -174,8 +174,11 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
     // Send the http request (get)
     if let Ok(response) = request_builder.send().await {
         // Get body from response
+        let headers = response.headers();
+        let ct = headers.get("Content-Type");
 
-        let path = filesystem::as_path(&site.site);
+        if let Some(ct) = ct {
+            let path = filesystem::as_path(&site.site, ct);
 
             // make sure that the file is good to go
             if let Some(mut file) = filesystem::init(&path).await {
@@ -227,6 +230,9 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
             } else {
                 error!("File failed to cooperate: {:?}", path);
             }
+        } else {
+            warn!("Server did not respond with Content-Type header: {}", site.site.to_string());
+        }
     } else {
         error!("Failed to get: {}", &site.site);
     }

src/parser.rs

@@ -127,16 +127,15 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
                         }
                         let url = origin.clone() + link;
 
-                        trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
-
                         if let Ok(url) = Url::parse(&url) {
-                            trace!("Saved relative url `{}` AS: `{}`", link, url);
+                            trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
                             Some(url)
                         } else {
                             error!(
-                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`",
+                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`. Failed url was: {}",
                                 link,
-                                parent.to_string()
+                                parent.to_string(),
+                                url
                             );
                             None
                         }