Oliver/internet_mapper
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

working filesystem storage

Browse Source
This commit is contained in:
Rushmore75
2025-03-21 11:42:43 -06:00
parent 808790a7c3
commit b750d88d48
5 changed files with 63 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/filesystem.rs
View File

@@ -1,7 +1,7 @@
use std::{ffi::OsStr, path::PathBuf};
use tokio::fs;
use tracing::{error, instrument, trace, warn};
use tracing::{debug, error, info, instrument, trace, warn};
use url::Url;
#[instrument(skip(data))]
@@ -21,6 +21,8 @@ pub async fn store(data: &str, url: &Url) {
(url_path.clone(), "index.html".into())
};
info!("Writing at: {:?} {:?}", basepath, filename);
// create the folders
if let Err(err) = fs::create_dir_all(&basepath).await {
error!("Dir creation: {err} {:?}", basepath);

90
src/parser.rs
View File

@@ -19,8 +19,7 @@ impl TokenSink for Website {
TagToken(tag) => {
if tag.kind == StartTag {
match tag.name {
// this should be all the html
// elements that have links
// this should be all the html elements that have links
local_name!("a")
| local_name!("audio")
| local_name!("area")
@@ -35,37 +34,9 @@ impl TokenSink for Website {
let attr_name = attr.name.local.to_string();
if attr_name == "src" || attr_name == "href" || attr_name == "data"
{
let url: Option<Url> = match Url::parse(&attr.value) {
Ok(ok) => {
trace!("Found `{}` in the html on `{}` tag", ok.to_string(), tag.name);
Some(ok)
},
Err(e) => {
if attr.value.starts_with('#') {
trace!("Rejecting # url");
None
} else {
match e {
url::ParseError::RelativeUrlWithoutBase => {
let origin = self.site.origin().ascii_serialization();
let url = origin.clone() + &attr.value;
trace!("Built `{url}` from `{origin} + {}`", &attr.value.to_string());
if let Ok(url) = Url::parse(&url) {
trace!("Saved relative url `{}` AS: `{}`", &attr.value, url);
Some(url)
} else {
error!("Failed to reconstruct a url from relative url: `{}` on site: `{}`", &attr.value, self.site.to_string());
None
}
},
_ => {
error!("MISC error: {:?} {:?}", e, &attr.value);
None
},
}
}
},
};
trace!("Found `{}` in html `{}` tag", &attr.value, tag.name);
let url = try_get_url(&self.site, &attr.value);
if let Some(mut parsed) = url {
parsed.set_query(None);
parsed.set_fragment(None);
@@ -119,3 +90,56 @@ pub async fn parse(site: &Website, data: &str) -> Vec<Website> {
other_sites
}
#[instrument]
fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
match Url::parse(link) {
Ok(ok) => Some(ok),
Err(e) => {
if link.starts_with('#') {
trace!("Rejecting # url");
None
} else if link.starts_with("//") {
// if a url starts with "//" is assumed that it will adopt
// the same scheme as it's parent
// https://stackoverflow.com/questions/9646407/two-forward-slashes-in-a-url-src-href-attribute
let scheme = parent.scheme();
match Url::parse(&format!("{scheme}://{}", link)) {
Ok(url) => Some(url),
Err(err) => {
error!("Failed parsing realative scheme url: {}", err);
None
}
}
} else {
// # This is some sort of realative url, gonna try patching it up into an absolute
// url
match e {
url::ParseError::RelativeUrlWithoutBase => {
// Is: scheme://host:port
let origin = parent.origin().ascii_serialization();
let url = origin.clone() + link;
trace!("Built `{url}` from `{origin} + {}`", link.to_string());
if let Ok(url) = Url::parse(&url) {
trace!("Saved relative url `{}` AS: `{}`", link, url);
Some(url)
} else {
error!(
"Failed to reconstruct a url from relative url: `{}` on site: `{}`",
link,
parent.to_string()
);
None
}
}
_ => {
error!("MISC error: {:?} {:?}", e, link);
None
}
}
}
}
}
}