better logging
This commit is contained in:
parent
b4038b76dd
commit
76e78cc745
@ -2,7 +2,7 @@ use std::{io::ErrorKind, path::PathBuf};
|
||||
|
||||
use reqwest::header::HeaderValue;
|
||||
use tokio::fs;
|
||||
use tracing::{error, trace, warn};
|
||||
use tracing::{error, event, trace, warn, Level};
|
||||
use url::Url;
|
||||
|
||||
pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
|
||||
@ -18,7 +18,7 @@ pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
|
||||
};
|
||||
|
||||
if let Some((ttype, subtype)) = ttype.split_once('/') {
|
||||
trace!("Found Content-Type to be: {ttype}/{subtype} for {}", url.to_string());
|
||||
trace!(url = url.to_string(), main_type = ttype, sub_type = subtype, "Found Content-Type to be: {ttype}/{subtype}");
|
||||
// If the Content-Type header is "*/html" (most likely "text/html") and the path's
|
||||
// extension is anything but html:
|
||||
if subtype=="html" && !url_path.extension().is_some_and(|f| f=="html" || f=="htm" ) {
|
||||
@ -29,7 +29,7 @@ pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
|
||||
} else {
|
||||
warn!("Header: {:?} couldn't be parsed into a string!", content_type);
|
||||
}
|
||||
trace!("Final path for {} is: {:?}", url, url_path);
|
||||
trace!(url = url.to_string(), path = &*url_path.to_string_lossy(), "Converted URL into path");
|
||||
|
||||
url_path
|
||||
}
|
||||
|
13
src/main.rs
13
src/main.rs
@ -200,7 +200,7 @@ async fn process_single_thread(config: Arc<Config>, db: Surreal<Client>, reqwest
|
||||
async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client) {
|
||||
|
||||
// METRICS
|
||||
trace!("Process: {}", &site.site);
|
||||
trace!(url = &site.site.as_str(), "Process: {}", &site.site);
|
||||
BEING_PROCESSED.add(1, &[]);
|
||||
// let mut process_span = TRACER.start("Process");
|
||||
|
||||
@ -274,8 +274,13 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
|
||||
}
|
||||
let _ = writer.flush().await;
|
||||
// rename the temp file into the real file name
|
||||
if let Err(err) = tokio::fs::rename(tmp_path, path).await {
|
||||
error!("{}", err);
|
||||
if let Err(err) = tokio::fs::rename(&tmp_path, &path).await {
|
||||
error!(
|
||||
from = &*tmp_path.to_string_lossy(),
|
||||
to = &*path.to_string_lossy(),
|
||||
"Error renaming file: {}",
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
// stream_span.end();
|
||||
@ -302,6 +307,8 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
|
||||
|
||||
// parsing_span.end();
|
||||
BEING_PARSED.add(-1, &[]);
|
||||
} else {
|
||||
trace!("Did not parse: {}", site.site.as_str());
|
||||
}
|
||||
|
||||
// update self in db
|
||||
|
@ -12,7 +12,6 @@ use crate::db::Website;
|
||||
impl TokenSink for Website {
|
||||
type Handle = Vec<Website>;
|
||||
|
||||
#[instrument(skip(token, _line_number))]
|
||||
fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<Self::Handle> {
|
||||
match token {
|
||||
TagToken(tag) => {
|
||||
@ -33,13 +32,13 @@ impl TokenSink for Website {
|
||||
let attr_name = attr.name.local.to_string();
|
||||
if attr_name == "src" || attr_name == "href" || attr_name == "data"
|
||||
{
|
||||
trace!("Found `{}` in html `{}` tag", &attr.value, tag.name);
|
||||
trace!(url = self.site.as_str(),"Found `{}` in html `{}` tag", &attr.value, tag.name);
|
||||
let url = try_get_url(&self.site, &attr.value);
|
||||
|
||||
if let Some(mut parsed) = url {
|
||||
parsed.set_query(None);
|
||||
parsed.set_fragment(None);
|
||||
trace!("Final cleaned URL: `{}`", parsed.to_string());
|
||||
trace!(url = self.site.as_str(), "Final cleaned URL: `{}`", parsed.to_string());
|
||||
let web = Website::new(&parsed.to_string(), false);
|
||||
links.push(web);
|
||||
}
|
||||
@ -60,10 +59,10 @@ impl TokenSink for Website {
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
#[instrument(skip(data))]
|
||||
/// Parses the passed site and returns all the sites it links to.
|
||||
pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
|
||||
trace!("Parsing {}", site.site.to_string());
|
||||
trace!(url = site.site.as_str(), "Parsing {}", site.site.to_string());
|
||||
// prep work
|
||||
let mut other_sites: Vec<Website> = Vec::new();
|
||||
|
||||
@ -88,7 +87,7 @@ pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
|
||||
assert!(token_buffer.is_empty());
|
||||
tokenizer.end();
|
||||
} else {
|
||||
warn!("Tendril failed to parse on: {}", site.site.to_string());
|
||||
warn!(url = site.site.as_str(), "Tendril failed to parse on: {}", site.site.to_string());
|
||||
}
|
||||
|
||||
other_sites
|
||||
@ -100,7 +99,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
|
||||
Ok(ok) => Some(ok),
|
||||
Err(e) => {
|
||||
if link.starts_with('#') {
|
||||
trace!("Rejecting # url");
|
||||
trace!(url = parent.as_str(), "Rejecting # url");
|
||||
None
|
||||
} else if link.starts_with("//") {
|
||||
// if a url starts with "//" is assumed that it will adopt
|
||||
@ -128,7 +127,7 @@ fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
|
||||
let url = origin.clone() + link;
|
||||
|
||||
if let Ok(url) = Url::parse(&url) {
|
||||
trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
|
||||
trace!(url = parent.as_str(), "Built `{url}` from `{origin} + `{}`", link.to_string());
|
||||
Some(url)
|
||||
} else {
|
||||
error!(
|
||||
|
Loading…
x
Reference in New Issue
Block a user