src/filesystem.rs

@@ -6,30 +6,19 @@ use url::Url;
 
 #[instrument(skip(data))]
 pub async fn store(data: &str, url: &Url) {
-    // extract data from url to save it accurately
+    let path = PathBuf::from("./downloaded".to_string() + url.path());
-    let url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
+    let basepath = path.ancestors().skip(1).take(1).collect::<PathBuf>();
-
+    trace!("Save path: {:?} and base path: {:?}", &path, &basepath);
-    // if it's a file
-    let (basepath, filename) = if url_path.extension().is_some() {
-        // get everything up till the file
-        let basepath = url_path.ancestors().skip(1).take(1).collect::<PathBuf>();
-        // get the file name
-        let filename = url_path.file_name().expect("This should exist").to_string_lossy();
-        trace!("Save path: {:?} and base path: {:?}", &url_path, &basepath);
-        (basepath, filename.to_string())
-    } else {
-        (url_path.clone(), "index.html".into())
-    };
-
-    // create the folders
     if let Err(err) = fs::create_dir_all(&basepath).await {
-        error!("Dir creation: {err} {:?}", basepath);
+        let ex = path.ancestors().fold(String::new(), |mut s, item| {
+            s += ", ";
+            s += &item.to_string_lossy().to_string();
+            s
+        });
+        error!("Dir creation: {err} {:?} {ex}", basepath);
     } else {
-        // FIXME I don't think this handles index.html files well...
+        if let Err(err) = fs::write(&path, data).await {
-        // TODO this should probably append .html to non-described files
+            error!("File creation: {err} {:?}", path);
-        // create the file if that was successful
-        if let Err(err) = fs::write(&basepath.join(filename), data).await {
-            error!("File creation: {err} {:?}", url_path);
         }
     }
 }

src/main.rs

@@ -167,7 +167,6 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
             .text()
             .await
             .expect("Failed to read http response's body!");
-
         // Store document
         filesystem::store(&data, &site.site).await;