use std::{ffi::OsStr, path::PathBuf}; use tokio::fs; use tracing::{debug, error, instrument, trace, warn}; use url::Url; #[instrument(skip(data))] pub async fn store(data: &str, url: &Url) { // extract data from url to save it accurately let url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path()); // if it's a file let (basepath, filename) = if url_path.extension().filter(valid_file_extension).is_some() { // get everything up till the file let basepath = url_path.ancestors().skip(1).take(1).collect::(); // get the file name let filename = url_path.file_name().expect("This should exist").to_string_lossy(); trace!("Save path: {:?} and base path: {:?}", &url_path, &basepath); (basepath, filename.to_string()) } else { (url_path.clone(), "index.html".into()) }; debug!("Writing at: {:?} {:?}", basepath, filename); // create the folders if let Err(err) = fs::create_dir_all(&basepath).await { error!("Dir creation: {err} {:?}", basepath); } else { // FIXME I don't think this handles index.html files well... // TODO this should probably append .html to non-described files // create the file if that was successful if let Err(err) = fs::write(&basepath.join(filename), data).await { error!("File creation: {err} {:?}", url_path); } } } fn valid_file_extension(take: &&OsStr) -> bool { let los = take.to_string_lossy(); let all = los.split('.'); match all.last() { Some(s) => { match s.to_lowercase().as_str() { "html" => true, "css" => true, "js" => true, "ts" => true, "otf" => true, // font "png" => true, "svg" => true, "jpg" => true, "jpeg" => true, "mp4" => true, "mp3" => true, "webp" => true, "pdf" => true, "json" => true, "xml" => true, // IGNORE // TODO Should this be a list of all domains? "org" => false, "com" => false, "net" => false, _ => { warn!("Might be forgetting a file extension: {s}"); false } } }, None => false, } }