better logging

This commit is contained in:
Rushmore75 2025-07-16 16:02:16 -06:00
parent b4038b76dd
commit 76e78cc745
3 changed files with 20 additions and 14 deletions

View File

@ -2,7 +2,7 @@ use std::{io::ErrorKind, path::PathBuf};
use reqwest::header::HeaderValue;
use tokio::fs;
use tracing::{error, trace, warn};
use tracing::{error, event, trace, warn, Level};
use url::Url;
pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
@ -18,7 +18,7 @@ pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
};
if let Some((ttype, subtype)) = ttype.split_once('/') {
trace!("Found Content-Type to be: {ttype}/{subtype} for {}", url.to_string());
trace!(url = url.to_string(), main_type = ttype, sub_type = subtype, "Found Content-Type to be: {ttype}/{subtype}");
// If the Content-Type header is "*/html" (most likely "text/html") and the path's
// extension is anything but html:
if subtype=="html" && !url_path.extension().is_some_and(|f| f=="html" || f=="htm" ) {
@ -29,7 +29,7 @@ pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
} else {
warn!("Header: {:?} couldn't be parsed into a string!", content_type);
}
trace!("Final path for {} is: {:?}", url, url_path);
trace!(url = url.to_string(), path = &*url_path.to_string_lossy(), "Converted URL into path");
url_path
}

View File

@ -200,7 +200,7 @@ async fn process_single_thread(config: Arc<Config>, db: Surreal<Client>, reqwest
async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client) {
// METRICS
trace!("Process: {}", &site.site);
trace!(url = &site.site.as_str(), "Process: {}", &site.site);
BEING_PROCESSED.add(1, &[]);
// let mut process_span = TRACER.start("Process");
@ -274,8 +274,13 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
}
let _ = writer.flush().await;
// rename the temp file into the real file name
if let Err(err) = tokio::fs::rename(tmp_path, path).await {
error!("{}", err);
if let Err(err) = tokio::fs::rename(&tmp_path, &path).await {
error!(
from = &*tmp_path.to_string_lossy(),
to = &*path.to_string_lossy(),
"Error renaming file: {}",
err
);
}
// stream_span.end();
@ -302,6 +307,8 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
// parsing_span.end();
BEING_PARSED.add(-1, &[]);
} else {
trace!("Did not parse: {}", site.site.as_str());
}
// update self in db

View File

@ -12,7 +12,6 @@ use crate::db::Website;
impl TokenSink for Website {
type Handle = Vec<Website>;
#[instrument(skip(token, _line_number))]
fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<Self::Handle> {
match token {
TagToken(tag) => {
@ -33,13 +32,13 @@ impl TokenSink for Website {
let attr_name = attr.name.local.to_string();
if attr_name == "src" || attr_name == "href" || attr_name == "data"
{
trace!("Found `{}` in html `{}` tag", &attr.value, tag.name);
trace!(url = self.site.as_str(),"Found `{}` in html `{}` tag", &attr.value, tag.name);
let url = try_get_url(&self.site, &attr.value);
if let Some(mut parsed) = url {
parsed.set_query(None);
parsed.set_fragment(None);
trace!("Final cleaned URL: `{}`", parsed.to_string());
trace!(url = self.site.as_str(), "Final cleaned URL: `{}`", parsed.to_string());
let web = Website::new(&parsed.to_string(), false);
links.push(web);
}
@ -60,10 +59,10 @@ impl TokenSink for Website {
}
}
#[instrument(skip_all)]
#[instrument(skip(data))]
/// Parses the passed site and returns all the sites it links to.
pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
trace!("Parsing {}", site.site.to_string());
trace!(url = site.site.as_str(), "Parsing {}", site.site.to_string());
// prep work
let mut other_sites: Vec<Website> = Vec::new();
@ -88,7 +87,7 @@ pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
assert!(token_buffer.is_empty());
tokenizer.end();
} else {
warn!("Tendril failed to parse on: {}", site.site.to_string());
warn!(url = site.site.as_str(), "Tendril failed to parse on: {}", site.site.to_string());
}
other_sites
@ -100,7 +99,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
Ok(ok) => Some(ok),
Err(e) => {
if link.starts_with('#') {
trace!("Rejecting # url");
trace!(url = parent.as_str(), "Rejecting # url");
None
} else if link.starts_with("//") {
// if a url starts with "//" is assumed that it will adopt
@ -128,7 +127,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
let url = origin.clone() + link;
if let Ok(url) = Url::parse(&url) {
trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
trace!(url = parent.as_str(), "Built `{url}` from `{origin} + `{}`", link.to_string());
Some(url)
} else {
error!(