src/filesystem.rs

@@ -1,24 +1,36 @@
 use std::path::PathBuf;
 
+use reqwest::header::HeaderValue;
 use tokio::fs;
 use tracing::{error, instrument, trace};
 use url::Url;
 
 #[instrument(skip(data))]
 pub async fn store(data: &str, url: &Url) {
-    let path = PathBuf::from("./downloaded".to_string() + url.path());
-    let basepath = path.ancestors().skip(1).take(1).collect::<PathBuf>();
-    trace!("Save path: {:?} and base path: {:?}", &path, &basepath);
-    if let Err(err) = fs::create_dir_all(&basepath).await {
-        let ex = path.ancestors().fold(String::new(), |mut s, item| {
-            s += ", ";
-            s += &item.to_string_lossy().to_string();
-            s
-        });
-        error!("Dir creation: {err} {:?} {ex}", basepath);
+    // extract data from url to save it accurately
+    let url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
+
+    // if it's a file
+    let (basepath, filename) = if url_path.extension().is_some() {
+        // get everything up till the file
+        let basepath = url_path.ancestors().skip(1).take(1).collect::<PathBuf>();
+        // get the file name
+        let filename = url_path.file_name().expect("This should exist").to_string_lossy();
+        trace!("Save path: {:?} and base path: {:?}", &url_path, &basepath);
+        (basepath, filename.to_string())
     } else {
-        if let Err(err) = fs::write(&path, data).await {
-            error!("File creation: {err} {:?}", path);
+        (url_path.clone(), "index.html".into())
+    };
+
+    // create the folders
+    if let Err(err) = fs::create_dir_all(&basepath).await {
+        error!("Dir creation: {err} {:?}", basepath);
+    } else {
+        // FIXME I don't think this handles index.html files well...
+        // TODO this should probably append .html to non-described files
+        // create the file if that was successful
+        if let Err(err) = fs::write(&basepath.join(filename), data).await {
+            error!("File creation: {err} {:?}", url_path);
         }
     }
 }

src/main.rs

@@ -7,6 +7,7 @@ use std::{
 };
 
 use db::{connect, Website};
+use filesystem::FileType;
 use metrics::{counter, gauge};
 use metrics_exporter_prometheus::PrometheusBuilder;
 use serde::Deserialize;
@@ -167,6 +168,7 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
             .text()
             .await
             .expect("Failed to read http response's body!");
+
         // Store document
         filesystem::store(&data, &site.site).await;