78 lines
2.5 KiB
Rust
78 lines
2.5 KiB
Rust
use std::{ffi::OsStr, path::PathBuf};
|
|
|
|
use tokio::fs;
|
|
use tracing::{debug, error, instrument, trace, warn};
|
|
use url::Url;
|
|
|
|
#[instrument(skip(data))]
|
|
pub async fn store(data: &str, url: &Url) {
|
|
// extract data from url to save it accurately
|
|
let url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
|
|
|
|
// if it's a file
|
|
let (basepath, filename) = if url_path.extension().filter(valid_file_extension).is_some() {
|
|
// get everything up till the file
|
|
let basepath = url_path.ancestors().skip(1).take(1).collect::<PathBuf>();
|
|
// get the file name
|
|
let filename = url_path.file_name().expect("This should exist").to_string_lossy();
|
|
trace!("Save path: {:?} and base path: {:?}", &url_path, &basepath);
|
|
(basepath, filename.to_string())
|
|
} else {
|
|
(url_path.clone(), "index.html".into())
|
|
};
|
|
|
|
debug!("Writing at: {:?} {:?}", basepath, filename);
|
|
|
|
// create the folders
|
|
if let Err(err) = fs::create_dir_all(&basepath).await {
|
|
error!("Dir creation: {err} {:?}", basepath);
|
|
} else {
|
|
// FIXME I don't think this handles index.html files well...
|
|
// TODO this should probably append .html to non-described files
|
|
// create the file if that was successful
|
|
if let Err(err) = fs::write(&basepath.join(filename), data).await {
|
|
error!("File creation: {err} {:?}", url_path);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn valid_file_extension(take: &&OsStr) -> bool {
|
|
let los = take.to_string_lossy();
|
|
let all = los.split('.');
|
|
match all.last() {
|
|
Some(s) => {
|
|
match s.to_lowercase().as_str() {
|
|
"html" => true,
|
|
"css" => true,
|
|
"js" => true,
|
|
"ts" => true,
|
|
"otf" => true, // font
|
|
|
|
"png" => true,
|
|
"svg" => true,
|
|
"jpg" => true,
|
|
"jpeg" => true,
|
|
"mp4" => true,
|
|
"mp3" => true,
|
|
"webp" => true,
|
|
|
|
"pdf" => true,
|
|
"json" => true,
|
|
"xml" => true,
|
|
|
|
// IGNORE
|
|
// TODO Should this be a list of all domains?
|
|
"org" => false,
|
|
"com" => false,
|
|
"net" => false,
|
|
|
|
_ => {
|
|
warn!("Might be forgetting a file extension: {s}");
|
|
false
|
|
}
|
|
}
|
|
},
|
|
None => false,
|
|
}
|
|
}
|