From 76e78cc74511d57c8784bbf7e72d6944c9378cd1 Mon Sep 17 00:00:00 2001 From: Rushmore75 Date: Wed, 16 Jul 2025 16:02:16 -0600 Subject: [PATCH] better logging --- src/filesystem.rs | 6 +++--- src/main.rs | 13 ++++++++++--- src/parser.rs | 15 +++++++-------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/filesystem.rs b/src/filesystem.rs index 9977c16..84a3a01 100644 --- a/src/filesystem.rs +++ b/src/filesystem.rs @@ -2,7 +2,7 @@ use std::{io::ErrorKind, path::PathBuf}; use reqwest::header::HeaderValue; use tokio::fs; -use tracing::{error, trace, warn}; +use tracing::{error, event, trace, warn, Level}; use url::Url; pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf { @@ -18,7 +18,7 @@ pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf { }; if let Some((ttype, subtype)) = ttype.split_once('/') { - trace!("Found Content-Type to be: {ttype}/{subtype} for {}", url.to_string()); + trace!(url = url.to_string(), main_type = ttype, sub_type = subtype, "Found Content-Type to be: {ttype}/{subtype}"); // If the Content-Type header is "*/html" (most likely "text/html") and the path's // extension is anything but html: if subtype=="html" && !url_path.extension().is_some_and(|f| f=="html" || f=="htm" ) { @@ -29,7 +29,7 @@ pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf { } else { warn!("Header: {:?} couldn't be parsed into a string!", content_type); } - trace!("Final path for {} is: {:?}", url, url_path); + trace!(url = url.to_string(), path = &*url_path.to_string_lossy(), "Converted URL into path"); url_path } diff --git a/src/main.rs b/src/main.rs index c2550b8..b4575bf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -200,7 +200,7 @@ async fn process_single_thread(config: Arc, db: Surreal, reqwest async fn process(mut site: Website, db: Surreal, reqwest: reqwest::Client) { // METRICS - trace!("Process: {}", &site.site); + trace!(url = &site.site.as_str(), "Process: {}", &site.site); BEING_PROCESSED.add(1, &[]); // let mut process_span = TRACER.start("Process"); @@ -274,8 +274,13 @@ async fn process(mut site: Website, db: Surreal, reqwest: reqwest::Clien } let _ = writer.flush().await; // rename the temp file into the real file name - if let Err(err) = tokio::fs::rename(tmp_path, path).await { - error!("{}", err); + if let Err(err) = tokio::fs::rename(&tmp_path, &path).await { + error!( + from = &*tmp_path.to_string_lossy(), + to = &*path.to_string_lossy(), + "Error renaming file: {}", + err + ); } // stream_span.end(); @@ -302,6 +307,8 @@ async fn process(mut site: Website, db: Surreal, reqwest: reqwest::Clien // parsing_span.end(); BEING_PARSED.add(-1, &[]); + } else { + trace!("Did not parse: {}", site.site.as_str()); } // update self in db diff --git a/src/parser.rs b/src/parser.rs index 48db2d2..dc04373 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -12,7 +12,6 @@ use crate::db::Website; impl TokenSink for Website { type Handle = Vec; - #[instrument(skip(token, _line_number))] fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult { match token { TagToken(tag) => { @@ -33,13 +32,13 @@ impl TokenSink for Website { let attr_name = attr.name.local.to_string(); if attr_name == "src" || attr_name == "href" || attr_name == "data" { - trace!("Found `{}` in html `{}` tag", &attr.value, tag.name); + trace!(url = self.site.as_str(),"Found `{}` in html `{}` tag", &attr.value, tag.name); let url = try_get_url(&self.site, &attr.value); if let Some(mut parsed) = url { parsed.set_query(None); parsed.set_fragment(None); - trace!("Final cleaned URL: `{}`", parsed.to_string()); + trace!(url = self.site.as_str(), "Final cleaned URL: `{}`", parsed.to_string()); let web = Website::new(&parsed.to_string(), false); links.push(web); } @@ -60,10 +59,10 @@ impl TokenSink for Website { } } -#[instrument(skip_all)] +#[instrument(skip(data))] /// Parses the passed site and returns all the sites it links to. pub async fn parse(site: &Website, data: &[u8]) -> Vec { - trace!("Parsing {}", site.site.to_string()); + trace!(url = site.site.as_str(), "Parsing {}", site.site.to_string()); // prep work let mut other_sites: Vec = Vec::new(); @@ -88,7 +87,7 @@ pub async fn parse(site: &Website, data: &[u8]) -> Vec { assert!(token_buffer.is_empty()); tokenizer.end(); } else { - warn!("Tendril failed to parse on: {}", site.site.to_string()); + warn!(url = site.site.as_str(), "Tendril failed to parse on: {}", site.site.to_string()); } other_sites @@ -100,7 +99,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option { Ok(ok) => Some(ok), Err(e) => { if link.starts_with('#') { - trace!("Rejecting # url"); + trace!(url = parent.as_str(), "Rejecting # url"); None } else if link.starts_with("//") { // if a url starts with "//" is assumed that it will adopt @@ -128,7 +127,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option { let url = origin.clone() + link; if let Ok(url) = Url::parse(&url) { - trace!("Built `{url}` from `{origin} + `{}`", link.to_string()); + trace!(url = parent.as_str(), "Built `{url}` from `{origin} + `{}`", link.to_string()); Some(url) } else { error!(