updating for base64

src/main.rs

@@ -41,17 +41,17 @@ async fn main() {
         surreal_username: "root",
         surreal_password: "root",
         surreal_ns: "test",
-        surreal_db: "custom-engine-v2",
-        s3_bucket: "custom-engine-v2",
+        surreal_db: "b64v1",
+        s3_bucket: "b64v1",
         s3_url: "http://localhost:9000",
-        s3_access_key: "0zv7GbLQsw4ZI8TclMps",
-        s3_secret_key: "5dB7QkGFw7fYbUJ5LpHk2GbWR7Bl710HlRz4NbzB",
+        s3_access_key: "8UO76z8wCs9DnpxSbQUY",
+        s3_secret_key: "xwKVMpf2jzgprsdo85Dvo74UmO84y0aRrAUorYY5",
     };
 
     // Would probably take these in as parameters from a cli
     let starting_url = "https://en.wikipedia.org/";
     // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
-    let crawl_filter = "https://en.wikipedia.org/";
+    let crawl_filter = "wikipedia.org/";
     let budget = 50;
     let mut crawled = 0;
 

src/s3.rs

@@ -43,9 +43,14 @@ impl S3 {
         })
     }
 
+    #[instrument(skip_all)]
     pub async fn store(&self, data: &str, name: &Url) {
-        if let Some(domain) = name.domain() {
-            let filename = domain.to_string() + name.path();
+        if let Some(domain) = name.to_string().split('#').collect::<Vec<&str>>().get(0) {
+            use base64::prelude::*;
+            // FIXME can still get unsupported characters, _ I think
+            let filename = BASE64_URL_SAFE.encode(domain);
+
+            trace!("Filename: {filename} from {domain}");
 
             let _ = &self
                 .client