better logging

src/filesystem.rs

@@ -2,7 +2,7 @@ use std::{io::ErrorKind, path::PathBuf};
 
 use reqwest::header::HeaderValue;
 use tokio::fs;
-use tracing::{error, trace, warn};
+use tracing::{error, event, trace, warn, Level};
 use url::Url;
 
 pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
@@ -18,7 +18,7 @@ pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
         };
 
         if let Some((ttype, subtype)) = ttype.split_once('/') {
-            trace!("Found Content-Type to be: {ttype}/{subtype} for {}", url.to_string());
+            trace!(url = url.to_string(), main_type = ttype, sub_type = subtype, "Found Content-Type to be: {ttype}/{subtype}");
             // If the Content-Type header is "*/html" (most likely "text/html") and the path's
             // extension is anything but html:
             if subtype=="html" && !url_path.extension().is_some_and(|f| f=="html" || f=="htm" ) {
@@ -29,7 +29,7 @@ pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
     } else {
         warn!("Header: {:?} couldn't be parsed into a string!", content_type);
     }
-    trace!("Final path for {} is: {:?}", url, url_path);
+    trace!(url = url.to_string(), path = &*url_path.to_string_lossy(), "Converted URL into path");
 
     url_path
 }

src/main.rs

@@ -200,7 +200,7 @@ async fn process_single_thread(config: Arc<Config>, db: Surreal<Client>, reqwest
 async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client) {
 
     // METRICS
-    trace!("Process: {}", &site.site);
+    trace!(url = &site.site.as_str(), "Process: {}", &site.site);
     BEING_PROCESSED.add(1, &[]);
     // let mut process_span = TRACER.start("Process");
 
@@ -274,8 +274,13 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
             }
             let _ = writer.flush().await;
             // rename the temp file into the real file name
-            if let Err(err) = tokio::fs::rename(tmp_path, path).await {
-                error!("{}", err);
+            if let Err(err) = tokio::fs::rename(&tmp_path, &path).await {
+                error!(
+                    from = &*tmp_path.to_string_lossy(),
+                    to = &*path.to_string_lossy(),
+                    "Error renaming file: {}",
+                    err
+                );
             }
 
             // stream_span.end();
@@ -302,6 +307,8 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
 
                 // parsing_span.end();
                 BEING_PARSED.add(-1, &[]);
+            } else {
+                trace!("Did not parse: {}", site.site.as_str());
             }
 
             // update self in db

src/parser.rs

@@ -12,7 +12,6 @@ use crate::db::Website;
 impl TokenSink for Website {
     type Handle = Vec<Website>;
 
-    #[instrument(skip(token, _line_number))]
     fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<Self::Handle> {
         match token {
             TagToken(tag) => {
@@ -33,13 +32,13 @@ impl TokenSink for Website {
                                 let attr_name = attr.name.local.to_string();
                                 if attr_name == "src" || attr_name == "href" || attr_name == "data"
                                 {
-                                    trace!("Found `{}` in html `{}` tag", &attr.value, tag.name);
+                                    trace!(url = self.site.as_str(),"Found `{}` in html `{}` tag", &attr.value, tag.name);
                                     let url = try_get_url(&self.site, &attr.value);
 
                                     if let Some(mut parsed) = url {
                                         parsed.set_query(None);
                                         parsed.set_fragment(None);
-                                        trace!("Final cleaned URL: `{}`", parsed.to_string());
+                                        trace!(url = self.site.as_str(), "Final cleaned URL: `{}`", parsed.to_string());
                                         let web = Website::new(&parsed.to_string(), false);
                                         links.push(web);
                                     }
@@ -60,10 +59,10 @@ impl TokenSink for Website {
     }
 }
 
-#[instrument(skip_all)]
+#[instrument(skip(data))]
 /// Parses the passed site and returns all the sites it links to.
 pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
-    trace!("Parsing {}", site.site.to_string());
+    trace!(url = site.site.as_str(), "Parsing {}", site.site.to_string());
     // prep work
     let mut other_sites: Vec<Website> = Vec::new();
 
@@ -88,7 +87,7 @@ pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
         assert!(token_buffer.is_empty());
         tokenizer.end();
     } else {
-        warn!("Tendril failed to parse on: {}", site.site.to_string());
+        warn!(url = site.site.as_str(), "Tendril failed to parse on: {}", site.site.to_string());
     }
 
     other_sites
@@ -100,7 +99,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
         Ok(ok) => Some(ok),
         Err(e) => {
             if link.starts_with('#') {
-                trace!("Rejecting # url");
+                trace!(url = parent.as_str(), "Rejecting # url");
                 None
             } else if link.starts_with("//") {
                 // if a url starts with "//" is assumed that it will adopt
@@ -128,7 +127,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
                         let url = origin.clone() + link;
 
                         if let Ok(url) = Url::parse(&url) {
-                            trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
+                            trace!(url = parent.as_str(), "Built `{url}` from `{origin} + `{}`", link.to_string());
                             Some(url)
                         } else {
                             error!(