changes

Reviewed-on: #6

.gitignore

@@ -3,4 +3,6 @@
 /.minio
 perf.data
 flamegraph.svg
-perf.data.old
+perf.data.old
+/docker/logs/*
+/downloaded

.vscode/launch.json

@@ -7,18 +7,15 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug executable 'surreal_spider'",
+            "name": "Debug executable 'internet_mapper'",
-            "env": {
-                "RUST_LOG": "surreal_spider=trace,reqwest=info",
-            },
             "cargo": {
                 "args": [
                     "build",
-                    "--bin=surreal_spider",
+                    "--bin=internet_mapper",
-                    "--package=surreal_spider"
+                    "--package=internet_mapper"
                 ],
                 "filter": {
-                    "name": "surreal_spider",
+                    "name": "internet_mapper",
                     "kind": "bin"
                 }
             },
@@ -28,16 +25,16 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug unit tests in executable 'surreal_spider'",
+            "name": "Debug unit tests in executable 'internet_mapper'",
             "cargo": {
                 "args": [
                     "test",
                     "--no-run",
-                    "--bin=surreal_spider",
+                    "--bin=internet_mapper",
-                    "--package=surreal_spider"
+                    "--package=internet_mapper"
                 ],
                 "filter": {
-                    "name": "surreal_spider",
+                    "name": "internet_mapper",
                     "kind": "bin"
                 }
             },

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "cSpell.words": [
+        "creds",
+        "reqwest",
+        "rustls",
+        "surql",
+    ]
+}

Cargo.lock

@@ -398,6 +398,29 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
+[[package]]
+name = "aws-lc-rs"
+version = "1.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01"
+dependencies = [
+ "aws-lc-sys",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f"
+dependencies = [
+ "bindgen",
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.74"
@@ -453,6 +476,29 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.69.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "clang-sys",
+ "itertools 0.11.0",
+ "lazy_static",
+ "lazycell",
+ "log",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash 1.1.0",
+ "shlex",
+ "syn 2.0.100",
+ "which",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.5.3"
@@ -614,6 +660,8 @@ version = "1.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
 dependencies = [
+ "jobserver",
+ "libc",
  "shlex",
 ]
 
@@ -674,6 +722,15 @@ dependencies = [
  "unicode-security",
 ]
 
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -738,6 +795,26 @@ dependencies = [
  "inout",
 ]
 
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "cmake"
+version = "0.1.54"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "colorchoice"
 version = "1.0.3"
@@ -769,6 +846,16 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "core-foundation"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.7"
@@ -1002,6 +1089,12 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
 
+[[package]]
+name = "dunce"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
+
 [[package]]
 name = "earcutr"
 version = "0.4.3"
@@ -1166,6 +1259,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
 [[package]]
 name = "fst"
 version = "0.4.7"
@@ -1388,6 +1487,12 @@ version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
+[[package]]
+name = "glob"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
+
 [[package]]
 name = "h2"
 version = "0.4.8"
@@ -1614,6 +1719,7 @@ dependencies = [
  "hyper",
  "hyper-util",
  "rustls",
+ "rustls-native-certs",
  "rustls-pki-types",
  "tokio",
  "tokio-rustls",
@@ -1855,6 +1961,26 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "internet_mapper"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "futures-util",
+ "html5ever 0.29.1",
+ "metrics",
+ "metrics-exporter-prometheus",
+ "minio",
+ "reqwest",
+ "serde",
+ "surrealdb",
+ "tokio",
+ "toml",
+ "tracing",
+ "tracing-subscriber",
+ "url",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.11.0"
@@ -1924,6 +2050,15 @@ dependencies = [
  "syn 2.0.100",
 ]
 
+[[package]]
+name = "jobserver"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.77"
@@ -1986,6 +2121,12 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
 [[package]]
 name = "lexicmp"
 version = "0.1.0"
@@ -2001,6 +2142,16 @@ version = "0.2.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
+[[package]]
+name = "libloading"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
+dependencies = [
+ "cfg-if",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "libm"
 version = "0.2.11"
@@ -2029,6 +2180,12 @@ dependencies = [
  "thiserror 1.0.69",
 ]
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.9.3"
@@ -2149,6 +2306,53 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "metrics"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a7deb012b3b2767169ff203fadb4c6b0b82b947512e5eb9e0b78c2e186ad9e3"
+dependencies = [
+ "ahash 0.8.11",
+ "portable-atomic",
+]
+
+[[package]]
+name = "metrics-exporter-prometheus"
+version = "0.16.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034"
+dependencies = [
+ "base64 0.22.1",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "indexmap 2.8.0",
+ "ipnet",
+ "metrics",
+ "metrics-util",
+ "quanta",
+ "thiserror 1.0.69",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbd4884b1dd24f7d6628274a2f5ae22465c337c5ba065ec9b6edccddf8acc673"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.15.2",
+ "metrics",
+ "quanta",
+ "rand 0.8.5",
+ "rand_xoshiro",
+ "sketches-ddsketch",
+]
+
 [[package]]
 name = "miette"
 version = "5.10.0"
@@ -2188,6 +2392,12 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "minio"
 version = "0.2.0-alpha"
@@ -2295,7 +2505,7 @@ dependencies = [
  "openssl-probe",
  "openssl-sys",
  "schannel",
- "security-framework",
+ "security-framework 2.11.1",
  "security-framework-sys",
  "tempfile",
 ]
@@ -2353,6 +2563,16 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "ntapi"
 version = "0.4.1"
@@ -2757,6 +2977,16 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 
+[[package]]
+name = "prettyplease"
+version = "0.2.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb"
+dependencies = [
+ "proc-macro2",
+ "syn 2.0.100",
+]
+
 [[package]]
 name = "proc-macro-crate"
 version = "3.3.0"
@@ -2810,6 +3040,21 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "quanta"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "once_cell",
+ "raw-cpuid",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "web-sys",
+ "winapi",
+]
+
 [[package]]
 name = "quick_cache"
 version = "0.5.2"
@@ -2833,7 +3078,7 @@ dependencies = [
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
- "rustc-hash",
+ "rustc-hash 2.1.1",
  "rustls",
  "socket2",
  "thiserror 2.0.12",
@@ -2852,7 +3097,7 @@ dependencies = [
  "getrandom 0.3.2",
  "rand 0.9.0",
  "ring",
- "rustc-hash",
+ "rustc-hash 2.1.1",
  "rustls",
  "rustls-pki-types",
  "slab",
@@ -2968,6 +3213,24 @@ dependencies = [
  "getrandom 0.3.2",
 ]
 
+[[package]]
+name = "rand_xoshiro"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
+dependencies = [
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "raw-cpuid"
+version = "11.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "rawpointer"
 version = "0.2.1"
@@ -3296,6 +3559,12 @@ version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.1"
@@ -3320,6 +3589,19 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustix"
+version = "0.38.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.4.15",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "rustix"
 version = "1.0.3"
@@ -3329,7 +3611,7 @@ dependencies = [
  "bitflags",
  "errno",
  "libc",
- "linux-raw-sys",
+ "linux-raw-sys 0.9.3",
  "windows-sys 0.59.0",
 ]
 
@@ -3339,6 +3621,7 @@ version = "0.23.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c"
 dependencies = [
+ "aws-lc-rs",
  "log",
  "once_cell",
  "ring",
@@ -3348,6 +3631,18 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3"
+dependencies = [
+ "openssl-probe",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework 3.2.0",
+]
+
 [[package]]
 name = "rustls-pemfile"
 version = "2.2.0"
@@ -3372,6 +3667,7 @@ version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f"
 dependencies = [
+ "aws-lc-rs",
  "ring",
  "rustls-pki-types",
  "untrusted",
@@ -3447,7 +3743,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
 dependencies = [
  "bitflags",
- "core-foundation",
+ "core-foundation 0.9.4",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
+dependencies = [
+ "bitflags",
+ "core-foundation 0.10.0",
  "core-foundation-sys",
  "libc",
  "security-framework-sys",
@@ -3520,6 +3829,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_urlencoded"
 version = "0.7.1"
@@ -3632,6 +3950,12 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
 
+[[package]]
+name = "sketches-ddsketch"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a"
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -3808,22 +4132,6 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
-[[package]]
-name = "surreal_spider"
-version = "0.1.0"
-dependencies = [
- "base64 0.22.1",
- "html5ever 0.29.1",
- "minio",
- "reqwest",
- "serde",
- "surrealdb",
- "tokio",
- "tracing",
- "tracing-subscriber",
- "url",
-]
-
 [[package]]
 name = "surrealdb"
 version = "2.2.1"
@@ -4015,7 +4323,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
 dependencies = [
  "bitflags",
- "core-foundation",
+ "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
 
@@ -4044,7 +4352,7 @@ dependencies = [
  "fastrand",
  "getrandom 0.3.2",
  "once_cell",
- "rustix",
+ "rustix 1.0.3",
  "windows-sys 0.59.0",
 ]
 
@@ -4277,11 +4585,26 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "toml"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
 [[package]]
 name = "toml_datetime"
 version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "toml_edit"
@@ -4290,6 +4613,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
 dependencies = [
  "indexmap 2.8.0",
+ "serde",
+ "serde_spanned",
  "toml_datetime",
  "winnow",
 ]
@@ -4364,6 +4689,16 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-serde"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.19"
@@ -4374,6 +4709,8 @@ dependencies = [
  "nu-ansi-term",
  "once_cell",
  "regex",
+ "serde",
+ "serde_json",
  "sharded-slab",
  "smallvec",
  "thread_local",
@@ -4381,6 +4718,7 @@ dependencies = [
  "tracing",
  "tracing-core",
  "tracing-log",
+ "tracing-serde",
 ]
 
 [[package]]
@@ -4737,6 +5075,18 @@ dependencies = [
  "rustls-pki-types",
 ]
 
+[[package]]
+name = "which"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
+dependencies = [
+ "either",
+ "home",
+ "once_cell",
+ "rustix 0.38.44",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"

Cargo.toml

@@ -1,17 +1,21 @@
 [package]
-name = "surreal_spider"
+name = "internet_mapper"
 version = "0.1.0"
 edition = "2021"
 
 [dependencies]
 base64 = "0.22.1"
+futures-util = "0.3.31"
 html5ever = "0.29"
+metrics = "0.24.1"
+metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]}
 # minio = "0.1.0"
 minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"}
-reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] }
+reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls", "stream"] }
 serde = { version = "1.0", features = ["derive"] }
 surrealdb = "2.2"
 tokio = { version="1.41.0", features = ["full"] }
+toml = "0.8.20"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time", "json"] }
 url = { version = "2.5", features = ["serde"] }

Crawler.toml

@@ -0,0 +1,12 @@
+# Surreal config
+surreal_url = "localhost:8000"
+surreal_username = "root"
+surreal_password = "root"
+surreal_ns = "test"
+surreal_db = "v1.21.1"
+
+# Crawler config
+crawl_filter = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI" 
+start_url = "https://ftpgeoinfo.msl.mt.gov/Data/Spatial/MSDI"
+budget = 10000
+batch_size = 50

README.md

@@ -2,10 +2,54 @@
 
 Crawls sites saving all the found links to a surrealdb database. It then proceeds to take batches of 100 uncrawled links untill the crawl budget is reached. It saves the data of each site in a minio database.
 
+## How to use
+
+1. Clone the repo and `cd` into it.
+2. Build the repo with `cargo build -r`
+3. Start the docker conatiners
+	1. cd into the docker folder `cd docker`
+	2. Bring up the docker containers `docker compose up -d`
+4. From the project's root, edit the `Crawler.toml` file to your liking.
+5. Run with `./target/release/internet_mapper`
+
+You can view stats of the project at `http://<your-ip>:3000/dashboards`
+
+```bash
+# Untested script but probably works
+git clone https://git.oliveratkinson.net/Oliver/internet_mapper.git
+cd internet_mapper
+
+cargo build -r
+
+cd docker
+docker compose up -d
+cd ..
+
+$EDITOR Crawler.toml
+
+./target/release/internet_mapper
+
+```
 
 ### TODO
 
-- [ ] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
+- [x] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
 - [ ] Conditionally save content - based on filename or file contents
-- [ ] GUI / TUI ?
+- [x] GUI / TUI ? - Graphana
-- [ ] Better asynchronous getting of the sites. Currently it all happens serially.
+- [x] Better asynchronous getting of the sites. Currently it all happens serially.
+- [x] Allow for storing asynchronously - dropping the "links to" logic fixes this need
+- [x] Control crawler via config file (no recompliation needed)
+
+3/17/25: Took >1hr to crawl 100 pages
+
+3/19/25: Took 20min to crawl 1000 pages
+This ment we stored 1000 pages, 142,997 urls, and 1,425,798 links between the two.
+
+3/20/25: Took 5min to crawl 1000 pages
+
+3/21/25: Took 3min to crawl 1000 pages
+
+# About
+
+![Screenshot](/pngs/graphana.png)
+

compose.yml

@@ -1,31 +0,0 @@
-services:
-  surreal:
-    image: surrealdb/surrealdb:latest-dev
-    ports:
-    - 8000:8000
-    volumes:
-      - ./.surrealdb/:/mydata
-    command:
-      - start
-      - --log
-      - debug
-      - --user
-      - root
-      - --pass
-      - root
-      - rocksdb:/mydata/database.db
-  minio:
-    image: quay.io/minio/minio
-    ports:
-      - 9000:9000
-      - 9001:9001
-    environment:
-      - MINIO_ROOT_USER=root
-      - MINIO_ROOT_PASSWORD=an8charpassword
-    volumes:
-      - ./.minio/:/data
-    command:
-      - server
-      - /data
-      - --console-address
-      - ":9001"

docker/alloy.conf

@@ -0,0 +1,14 @@
+local.file_match "tmplogs" {
+    path_targets = [{"__path__" = "/tmp/alloy-logs/*.log"}]
+}
+
+loki.source.file "local_files" {
+    targets    = local.file_match.tmplogs.targets
+    forward_to = [loki.write.local_loki.receiver]
+}
+
+loki.write "local_loki" {
+    endpoint {
+        url = "http://loki:3100/loki/api/v1/push"
+    }
+}

docker/compose.yml

@@ -0,0 +1,68 @@
+services:
+  surreal:
+    image: surrealdb/surrealdb:latest-dev
+    ports:
+    - 8000:8000
+    volumes:
+      - surrealdb_storage:/mydata
+    command:
+      - start
+      - --log
+      - debug
+      - --user
+      - root
+      - --pass
+      - root
+      - rocksdb:/mydata/database.db
+
+  alloy:
+    image: grafana/alloy:latest
+    ports:
+      - 12345:12345
+    volumes:
+      # if you change this, you also need to change it in the alloy config file
+      - ./logs/:/tmp/alloy-logs
+      - ./alloy.conf:/etc/alloy/config.alloy
+      - alloy_storage:/var/lib/alloy
+    command:  run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+
+  #logs
+  loki:
+    image: grafana/loki:latest
+    ports:
+      - 3100:3100
+    command: -config.file=/etc/loki/local-config.yaml
+    volumes:
+      - ./loki.yaml:/etc/loki/local-config.yaml
+
+  # Metrics collector
+  prometheus:
+    image: prom/prometheus:latest
+    expose:
+      - 9090
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+      # persist data
+      - prometheus_storage:/prometheus
+    command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
+
+  # Everything viewer
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - ./dashboards:/var/lib/grafana/dashboards
+      - grafana_storage:/var/lib/grafana
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor
+    ports:
+      - 3000:3000
+  
+volumes:
+  prometheus_storage:
+  grafana_storage:
+  alloy_storage:
+  surrealdb_storage:

docker/dashboards/crawler-dashboard.json

@@ -0,0 +1,648 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "Prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    },
+    {
+      "name": "DS_LOKI",
+      "label": "Loki",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "loki",
+      "pluginName": "Loki"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "11.3.1"
+    },
+    {
+      "type": "panel",
+      "id": "logs",
+      "name": "Logs",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "loki",
+      "name": "Loki",
+      "version": "1.0.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": 300000,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_trips",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "Trips to Surreal",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "s3_trips",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "Trips to S3",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "pages_crawled",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "total crawled",
+          "range": true,
+          "refId": "C",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "pages_being_processed",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "Pages being processed",
+          "range": true,
+          "refId": "E",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "gets_in_flight",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "D",
+          "useBackend": false
+        }
+      ],
+      "title": "Crawler stats",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": 300000,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 9,
+        "x": 8,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_trips",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "Trips to Surreal",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_link_calls",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "link calls",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_store_calls",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "store calls",
+          "range": true,
+          "refId": "C",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "pages_being_processed",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "Pages being processed",
+          "range": true,
+          "refId": "E",
+          "useBackend": false
+        }
+      ],
+      "title": "Surreal stats",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "This is across all threads, so this isn't wall clock time",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 7,
+        "x": 17,
+        "y": 0
+      },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_lock_waiting_ms",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Time spend waiting on lock",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 18,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "id": 1,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `ERROR` | line_format \"{{.threadId}} {{.filename_extracted}}:{{.line_number}}  {{.fields_message}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Errors",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 26
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `DEBUG` | line_format \"{{.fields_message}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Debug",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 16,
+        "w": 12,
+        "x": 12,
+        "y": 26
+      },
+      "id": 4,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `TRACE` | line_format \"{{.fields_message}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Trace",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 34
+      },
+      "id": 3,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `WARN` | line_format \"{{.fields_message}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Warnings",
+      "type": "logs"
+    }
+  ],
+  "schemaVersion": 40,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "datasource": {
+          "type": "loki",
+          "uid": "P8E80F9AEF21F6940"
+        },
+        "filters": [],
+        "name": "Filters",
+        "type": "adhoc"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Crawler",
+  "uid": "ceg90x34pqgowd",
+  "version": 21,
+  "weekStart": ""
+}

docker/grafana.yaml

@@ -0,0 +1,24 @@
+apiVersion: 1
+
+datasources:
+- name: Loki
+  type: loki
+  access: proxy 
+  orgId: 1
+  url: http://loki:3100
+  basicAuth: false
+  isDefault: true 
+  version: 1
+  editable: false
+- name: Prometheus
+  type: prometheus
+  uid: prometheus
+  access: proxy
+  orgId: 1
+  url: http://prometheus:9090
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
+  jsonData:
+    httpMethod: GET

docker/loki.yaml

@@ -0,0 +1,62 @@
+# this is mostly the default config from grafana's website
+
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  metric_aggregation:
+    loki_address: localhost:3100
+
+frontend:
+  encoding: protobuf
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+analytics:
+  reporting_enabled: false

docker/prometheus.yaml

@@ -0,0 +1,17 @@
+global:
+  scrape_interval: 5s 
+  query_log_file: /etc/prometheus/query.log
+
+scrape_configs:
+  - job_name: crawler
+    static_configs:
+    # change this your machine's ip, localhost won't work
+    # because localhost refers to the docker container.
+      - targets: ['192.168.1.200:2500']
+        #- targets: ['192.168.8.209:2500']
+  - job_name: loki
+    static_configs:
+      - targets: ['loki:3100']
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']

src/db.rs

@@ -1,28 +1,37 @@
+use metrics::counter;
 use std::fmt::Debug;
 use serde::{Deserialize, Serialize};
 use surrealdb::{
-    engine::remote::ws::{Client, Ws}, error::Db, opt::auth::Root, sql::Thing, Response, Surreal
+    engine::remote::ws::{Client, Ws},
+    opt::auth::Root,
+    sql::Thing,
+    Surreal,
 };
-use tracing::{error, instrument, trace, warn};
+use tracing::{error, instrument, trace};
 use url::Url;
 
-use crate::{Config, Timer};
+use crate::Config;
 
-#[derive(Serialize, Deserialize, Clone)]
+const STORE: &str = "surql_store_calls";
+
+#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
 pub struct Website {
     /// The url that this data is found at
     pub site: Url,
     /// Wether or not this link has been crawled yet
     pub crawled: bool,
-    #[serde(skip_serializing)]
+    /// 200, 404, etc
-    id: Option<Thing>,
+    pub status_code: u16,
 }
 
 // manual impl to make tracing look nicer
 impl Debug for Website {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let site = (self.site.domain().unwrap_or("n/a")).to_string() + self.site.path();
+        f.debug_struct("Website")
-        f.debug_struct("Website").field("site", &site).finish()
+            .field("host", &self.site.host())
+            .field("path", &self.site.path())
+            .field("status_code", &self.status_code)
+            .finish()
     }
 }
 
@@ -34,121 +43,42 @@ impl Website {
             Err(_) => todo!(),
         };
         Self {
-            id: None,
             crawled,
             site,
+            status_code: 0,
         }
     }
 
-    pub fn set_crawled(&mut self) {
+    // Insert ever item in the vec into surreal, crawled state will be preserved as TRUE
-        trace!("Set crawled to true");
+    // if already in the database as such or incoming data is TRUE.
-        self.crawled = true
+    #[instrument(skip(db))]
-    }
+    pub async fn store_all(all: Vec<Self>, db: &Surreal<Client>) -> Vec<Thing> {
+        counter!(STORE).increment(1);
+        let mut things = Vec::with_capacity(all.len());
 
-    #[instrument(skip_all)]
+        // FIXME failes *sometimes* because "Resource Busy"
-    pub async fn links_to(&self, other: Vec<Thing>, db: &Surreal<Client>) {
-        let len = other.len();
-        if len == 0 {return}
-
-        let from = self.site.to_string();
-        // let to = other.site.to_string();
-        trace!("Linking {from} to {} other pages.", other.len());
-        let msg = format!("Linked {len} pages");
-        let timer = Timer::start(&msg);
-        // prevent the timer from being dropped instantly.
-        let _ = timer;
         match db
-            .query("COUNT(RELATE (SELECT id FROM website WHERE site = $in) -> links_to -> $out)")
+            .query(
-            .bind(("in", from))
+                "INSERT INTO website $array
-            .bind(("out", other))
+                    ON DUPLICATE KEY UPDATE
+                        accessed_at = time::now(),
+                        status_code = $input.status_code,
+                        crawled = crawled OR $input.crawled
+                    RETURN VALUE id;
+                 ",
+            )
+            .bind(("array", all))
             .await
         {
-            Ok(mut e) => {
+            Ok(mut id) => match id.take::<Vec<Thing>>(0) {
-                // The relate could technically "fail" (not relate anything), this just means that
+                Ok(mut x) => things.append(&mut x),
-                // the query was ok.
+                Err(err) => error!("{:?}", err),
-                let _: Response = e;
-                if let Ok(vec) = e.take(0) {
-                    let _: Vec<usize> = vec;
-                    if let Some(num) = vec.get(0) {
-                        if *num == len {
-                            trace!("Link OK");
-                            return;
-                        } else {
-                            warn!("Didn't link all the records. {num}/{len}");
-                            return;
-                        }
-                    }
-                }
-                warn!("Linking request succeeded but couldn't verify the results.");
             },
-            Err(e) => {
+            Err(err) => {
-                error!("{}", e.to_string());
+                error!("{:?}", err);
-            },
-        }
-    }
-
-    #[instrument(skip_all)]
-    pub async fn store(&self, db: &Surreal<Client>) -> Option<Thing> {
-        let t = Timer::start("Stored page");
-        let _ = t;
-        // check if it's been gone thru before
-        let mut response = db
-            .query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1")
-            .bind(("site", self.site.to_string()))
-            .await
-            .expect("Failed to check surreal for duplicates!");
-
-        if let Some(old) = response.take::<Option<Website>>(0).expect("Failed to read reponse from surreal for duplicates.") {
-            // site exists already
-            if let Some(id) = old.id {
-                // make sure to preserve the "crawled status"
-                let mut new = self.clone();
-                new.crawled = old.crawled | new.crawled;
-
-                // update the record
-                match db.upsert((id.tb, id.id.to_string())).content(new).await {
-                    Ok(e) => {
-                        if let Some(a) = e {
-                            let _: Record = a;
-                            return Some(a.id);
-                        }
-                    }
-                    Err(e) => {
-                        match e {
-                            surrealdb::Error::Db(error) => {
-                                match error {
-                                    Db::QueryCancelled => todo!(),
-                                    Db::QueryNotExecuted => todo!(),
-                                    Db::QueryNotExecutedDetail { message } => todo!(),
-                                   _=>{},
-                                }
-                            },
-                            _=>{},
-                        }
-                        // error!("{}", e);
-                    }
-                };
             }
-        } else {
-            // sites hasn't existed yet
-            match db.create("website").content(self.clone()).await {
-                Ok(e) => {
-                    let _: Option<Record> = e;
-                    if let Some(a) = e {
-                        let _: Record = a;
-                        return Some(a.id);
-                    }
-                }
-                Err(a) => error!("{:?}", a),
-            };
         }
-        None
+        things
-    }
-}
-
-impl ToString for Website {
-    fn to_string(&self) -> String {
-        self.site.to_string()
     }
 }
 
@@ -165,29 +95,31 @@ pub struct Record {
 }
 
 #[instrument(skip_all, name = "SurrealDB")]
-pub async fn connect(config: &Config<'_>) -> surrealdb::Result<Surreal<Client>> {
+pub async fn connect(config: &Config) -> surrealdb::Result<Surreal<Client>> {
     trace!("Establishing connection to surreal...");
     // Connect to the server
-    let db = Surreal::new::<Ws>(config.surreal_url).await?;
+    let db = Surreal::new::<Ws>(&config.surreal_url).await?;
 
     trace!("Logging in...");
     // Signin as a namespace, database, or root user
     db.signin(Root {
-        username: config.surreal_username,
+        username: &config.surreal_username,
-        password: config.surreal_password,
+        password: &config.surreal_password,
     })
     .await?;
 
     // Select a specific namespace / database
-    db
+    db.use_ns(&config.surreal_ns)
-        .use_ns(config.surreal_ns)
+        .use_db(&config.surreal_db)
-        .use_db(config.surreal_db)
         .await?;
-    
+
     let setup = include_bytes!("setup.surql");
     let file = setup.iter().map(|c| *c as char).collect::<String>();
 
-    db.query(file).await.expect("Failed to setup surreal tables.");
+    db.query(file)
+        .await
+        .expect("Failed to setup surreal tables.");
 
     Ok(db)
 }
+

src/filesystem.rs

@@ -0,0 +1,66 @@
+use std::{io::ErrorKind, path::PathBuf};
+
+use reqwest::header::HeaderValue;
+use tokio::fs;
+use tracing::{error, trace, warn};
+use url::Url;
+
+pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
+    // extract data from url to save it accurately
+    let mut url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
+
+    if let Ok(header) = content_type.to_str() {
+        // text/html; charset=UTF-8; option=value
+        let ttype = if let Some((t, _)) = header.split_once(';') {
+            t
+        } else {
+            header
+        };
+
+        if let Some((ttype, subtype)) = ttype.split_once('/') {
+            trace!("Found Content-Type to be: {ttype}/{subtype} for {}", url.to_string());
+            // If the Content-Type header is "*/html" (most likely "text/html") and the path's
+            // extension is anything but html:
+            if subtype=="html" && !url_path.extension().is_some_and(|f| f=="html" || f=="htm" ) {
+                // time to slap a index.html to the end of that path there!
+                url_path = url_path.join("index.html");
+            }
+        }
+    } else {
+        warn!("Header: {:?} couldn't be parsed into a string!", content_type);
+    }
+    trace!("Final path for {} is: {:?}", url, url_path);
+
+    url_path
+}
+
+pub async fn init(filename: &PathBuf) -> Option<fs::File> {
+    let file = async || tokio::fs::OpenOptions::new()
+        .append(true)
+        .create(true)
+        .open(&filename).await;
+
+    match file().await {
+        Ok(ok) => Some(ok),
+        Err(err) => {
+            // the file/folder isn't found
+            if err.kind() == ErrorKind::NotFound {
+                if let Some(parent ) = &filename.parent() {
+                    // create the folders
+                    if let Err(err) = fs::create_dir_all(&parent).await {
+                        error!("Dir creation: {err} {:?}", filename);
+                        eprintln!("{}", err)
+                    } else if let Ok(ok) = file().await {
+                        return Some(ok);
+                    }
+                } else {
+                    error!("Couldn't get file's parents: {:?}", &filename);
+                }
+            } else {
+                error!("File open error: {err} {:?}", filename);
+            }
+            // we don't care about other errors, we can't/shouldn't fix them
+            None
+        }
+    }
+}

src/main.rs

@@ -1,72 +1,102 @@
-#![feature(type_alias_impl_trait)]
+#![feature(ip_from)]
-#![feature(const_async_blocks)]
+#![warn(clippy::expect_used)]
+#![deny(clippy::unwrap_used)]
 
 extern crate html5ever;
 
-use std::time::Instant;
+use futures_util::StreamExt;
+
+use std::{
+    collections::HashSet,
+    fs::File,
+    io::Read,
+    net::{IpAddr, Ipv4Addr},
+};
 
 use db::{connect, Website};
-use s3::S3;
+use metrics::{counter, gauge};
+use metrics_exporter_prometheus::PrometheusBuilder;
+use serde::Deserialize;
 use surrealdb::{engine::remote::ws::Client, Surreal};
-use tokio::task::JoinSet;
+use tokio::{io::{AsyncWriteExt, BufWriter}, task::JoinSet};
-use tracing::{debug, info, instrument, trace, trace_span, warn};
+use tracing::{debug, debug_span, error, info, instrument, level_filters::LevelFilter, trace, trace_span, warn};
-use tracing_subscriber::{fmt::time::LocalTime, EnvFilter};
+use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry};
 
 mod db;
+mod filesystem;
 mod parser;
-mod s3;
 
-struct Config<'a> {
+const GET_METRIC: &str = "total_gets";
-    surreal_ns: &'a str,
+const GET_IN_FLIGHT: &str = "gets_in_flight";
-    surreal_db: &'a str,
+const SITES_CRAWLED: &str = "pages_crawled";
-    surreal_url: &'a str,
+const BEING_PROCESSED: &str = "pages_being_processed";
-    surreal_username: &'a str,
-    surreal_password: &'a str,
 
-    s3_url: &'a str,
+#[derive(Deserialize)]
-    s3_bucket: &'a str,
+struct Config {
-    s3_access_key: &'a str,
+    surreal_ns: String,
-    s3_secret_key: &'a str,
+    surreal_db: String,
+    surreal_url: String,
+    surreal_username: String,
+    surreal_password: String,
+
+    crawl_filter: String,
+    start_url: String,
+    budget: usize,
+    batch_size: usize,
 }
 
 #[tokio::main]
 async fn main() {
-    let total_runtime = Timer::start("Completed");
+    println!("Logs and metrics are provided to the Grafana dashboard");
+
+    let writer = std::fs::OpenOptions::new()
+        .append(true)
+        .create(true)
+        .open("./docker/logs/tracing.log")
+        .expect("Couldn't make log file!");
+
+    let filter = EnvFilter::builder()
+        .with_default_directive(LevelFilter::DEBUG.into())
+        .from_env_lossy();
+
+    let registry = Registry::default().with(
+        fmt::layer()
+            .with_line_number(true)
+            .with_thread_ids(true)
+            .with_file(true)
+            .json()
+            .with_writer(writer)
+            .with_filter(filter)
+    );
+
+    tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber");
+
+    let builder = PrometheusBuilder::new();
+    builder
+        .with_http_listener(std::net::SocketAddr::new(
+            IpAddr::V4(Ipv4Addr::from_octets([0, 0, 0, 0])),
+            2500,
+        ))
+        .install()
+        .expect("failed to install recorder/exporter");
+
+    info!("Starting...");
 
-    tracing_subscriber::fmt()
-        .with_env_filter(EnvFilter::from_default_env())
-        .with_line_number(true)
-        .with_thread_ids(true)
-        .with_file(true)
-        .with_timer(LocalTime::rfc_3339())
-        .init();
-    debug!("Starting...");
-   // Would probably take these in as parameters from a cli
-    let starting_url = "https://en.wikipedia.org/";
     // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
-    let crawl_filter = "en.wikipedia.org/";
+    // let crawl_filter = "en.wikipedia.org/";
-    let budget = 50;
+    // let budget = 50;
     let mut crawled = 0;
 
-    let config = Config {
+    let mut file = File::open("./Crawler.toml").expect("Failed to read Crawler.toml");
-        surreal_url: "localhost:8000",
+    let mut buf = String::new();
-        surreal_username: "root",
+    let _ = file.read_to_string(&mut buf);
-        surreal_password: "root",
+
-        surreal_ns: "test",
+    let config: Config = toml::from_str(&buf).expect("Failed to parse Crawler.toml");
-        surreal_db: "v1.11.2",
+    let starting_url = &config.start_url;
-        s3_bucket: "v1.11.2",
-        s3_url: "http://localhost:9000",
-        s3_access_key: "8UO76z8wCs9DnpxSbQUY",
-        s3_secret_key: "xwKVMpf2jzgprsdo85Dvo74UmO84y0aRrAUorYY5",
-    };
-   
 
     let db = connect(&config)
         .await
         .expect("Failed to connect to surreal, aborting.");
-    let s3 = S3::connect(&config)
-        .await
-        .expect("Failed to connect to minio, aborting.");
 
     let reqwest = reqwest::Client::builder()
         // .use_rustls_tls()
@@ -79,91 +109,176 @@ async fn main() {
     let span = trace_span!("Pre-Loop");
     let pre_loop_span = span.enter();
     // Download the site
-    let site = Website::new(&starting_url, false);
+    let site = Website::new(starting_url, false);
-    get(site, db.clone(), reqwest.clone(), s3.clone()).await;
+    process(site, db.clone(), reqwest.clone()).await;
 
     drop(pre_loop_span);
 
     let span = trace_span!("Loop");
     let span = span.enter();
-    while crawled < budget {
+    while crawled < config.budget {
-        let get_num = if budget - crawled < 100 {
+        let uncrawled =
-            budget - crawled
+            get_uncrawled_links(&db, config.budget - crawled, config.crawl_filter.clone(), &config).await;
-        } else {
+        if uncrawled.is_empty() {
-            100
-        };
-
-        let uncrawled = get_uncrawled_links(&db, get_num, crawl_filter.to_string()).await;
-        if uncrawled.len() == 0 {
             info!("Had more budget but finished crawling everything.");
             return;
         }
-        debug!("Crawling {} pages...", uncrawled.len());
-
-        let span = trace_span!("Crawling");
-        let _ = span.enter();
 
         {
             let mut futures = JoinSet::new();
             for site in uncrawled {
-                futures.spawn(get(site, db.clone(), reqwest.clone(), s3.clone()));
+                gauge!(BEING_PROCESSED).increment(1);
-                // technically the site hasn't be crawled *yet*, but the future
+                futures.spawn(process(site, db.clone(), reqwest.clone()));
-                // where it is crawled has been set up.
-                crawled += 1;
                 // let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32);
                 // info!("Crawled {crawled} out of {budget} pages. ({percent})");
             }
-            debug!("Joining {} futures...", futures.len());
+
-            // join all the gets together
+            let c = counter!(SITES_CRAWLED);
-            let _ = futures.join_all().await;
+            // As futures complete runs code in while block
+            while futures.join_next().await.is_some() {
+                c.increment(1);
+                gauge!(BEING_PROCESSED).decrement(1);
+                crawled += 1;
+            }
         }
     }
     drop(span);
 
+    if let Ok(mut ok) = db
+        .query("count(select id from website where crawled = true)")
+        .await
+    {
+        let res = ok.take::<Option<usize>>(0);
+        if let Ok(Some(n)) = res {
+            info!("Total crawled pages now equals {n}");
+        }
+    }
+
     info!("Done");
-    drop(total_runtime);
 }
 
-#[instrument(skip (db, s3, reqwest))]
+#[instrument(skip(db, reqwest))]
 /// Downloads and crawls and stores a webpage.
-/// It is acceptable to clone `db`, `reqwest`, and `s3` because they all use `Arc`s internally. - Noted by Oliver 
+/// It is acceptable to clone `db`, `reqwest`, and `s3` because they all use `Arc`s internally. - Noted by Oliver
-async fn get(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client, s3: S3) {
+async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client) {
-    trace!("Get: {}", site.to_string());
+    // METRICS
+    trace!("Process: {}", &site.site);
+    // Build the request
+    let request_builder = reqwest.get(site.site.to_string());
 
-    let timer = Timer::start("Built request");
+    // METRICS
-    let request_builder = reqwest.get(site.to_string());
+    let g = gauge!(GET_IN_FLIGHT);
-    timer.stop();
+    g.increment(1);
 
-    let timer = Timer::start("Got page");
+    // Send the http request (get)
     if let Ok(response) = request_builder.send().await {
-        timer.stop();
+        let headers = response.headers();
-        debug!("Getting body...");
+        let code = response.status();
 
-        // Get body
+        #[allow(non_snake_case)]
-        let data = response.text().await.expect("Failed to read http response's body!");
+        let CT = headers.get("Content-Type");
-        // Store document
+        let ct = headers.get("content-type");
-        s3.store(&data, &site.site).await;
+
-        // Parse document and store relationships
+        let ct = match (CT,ct) {
-        parser::parse(&db, &mut site, &data).await;
+            (None, None) => {
-        return;
+                warn!("Server did not respond with Content-Type header. Url: {} Headers: ({:?})", site.site.to_string(), headers);
+                return
+            },
+            (None, Some(a)) => a,
+            (Some(a), None) => a,
+            (Some(a), Some(_)) => a,
+        };
+
+        // create filepath (handles / -> /index.html)
+        let path = filesystem::as_path(&site.site, ct);
+
+        // make sure that the file is good to go
+        if let Some(file) = filesystem::init(&path).await {
+            // Get body from response
+            // stream the response onto the disk
+            let mut stream = response.bytes_stream();
+
+            let should_parse = path.to_string_lossy().ends_with(".html");
+            let mut writer = BufWriter::new(file);
+            let mut buf: Vec<u8> = Vec::new();
+
+            // Write file to disk
+            info!("Writing at: {:?}", path);
+            while let Some(data) = stream.next().await {
+                match data {
+                    Ok(data) => {
+                        let _ = writer.write_all(&data).await;
+                        // If we are going to parse this file later, we will save it
+                        // into memory as well as the disk.
+                        // We do this because the data here might be incomplete
+                        if should_parse {
+                            data.iter().for_each(|f| buf.push(*f));
+                        }
+                    },
+                    Err(err) => {
+                        eprintln!("{}", err)
+                    },
+                }
+            }
+            let _ = writer.flush();
+
+
+            // (If needed) Parse the file
+            if should_parse {
+                let span = debug_span!("Should Parse");
+                let enter = span.enter();
+
+                // Parse document and get relationships
+                let sites = parser::parse(&site, &buf).await;
+                // De-duplicate this list
+                let prev_len = sites.len();
+                let set = sites.into_iter().fold(HashSet::new(), |mut set, item| {
+                    set.insert(item);
+                    set
+                });
+                let de_dupe_sites: Vec<Website> = set.into_iter().collect();
+                let diff = prev_len - de_dupe_sites.len();
+                trace!("Saved {diff} from being entered into the db by de-duping");
+                // Store all the other sites so that we can link to them.
+                let _ = Website::store_all(de_dupe_sites, &db).await;
+
+                drop(enter);
+            }
+
+            // METRICS
+            g.decrement(1);
+            counter!(GET_METRIC).increment(1);
+
+            // update self in db
+            site.crawled = true;
+            site.status_code = code.as_u16();
+            Website::store_all(vec![site.clone()], &db).await;
+        } else {
+            error!("File failed to cooperate: {:?}", path);
+        }
+
+        trace!("Done processing: {}", &site.site);
+    } else {
+        error!("Failed to get: {}", &site.site);
     }
-    trace!("Failed to get: {}", site.to_string());
 }
 
 /// Returns uncrawled links
-#[instrument(skip(db))]
+#[instrument(skip(db, config))]
 async fn get_uncrawled_links(
     db: &Surreal<Client>,
     mut count: usize,
     filter: String,
+    config: &Config,
 ) -> Vec<Website> {
-    if count > 100 {
+    if count > config.batch_size {
-        count = 100
+        count = config.batch_size;
     }
-    debug!("Getting uncrawled links");
+
+    debug!("Getting {} uncrawled links", count);
 
     let mut response = db
-        .query("SELECT * FROM website WHERE crawled = false AND site CONTAINS type::string($format) LIMIT $count;")
+        .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;")
         .bind(("format", filter))
         .bind(("count", count))
         .await
@@ -172,36 +287,3 @@ async fn get_uncrawled_links(
         .take(0)
         .expect("Returned websites couldn't be parsed")
 }
-
-pub struct Timer<'a> {
-    start: Instant,
-    msg: &'a str,
-}
-
-impl<'a> Timer<'a> {
-    #[inline]
-    pub fn start(msg: &'a str) -> Self {
-        Self {
-            start: Instant::now(),
-            msg,
-        }
-    }
-    pub fn stop(&self) -> f64 {
-        let dif = self.start.elapsed().as_micros();
-        let ms = dif as f64 / 1000.;
-
-        if ms > 200. {
-            warn!("{}", format!("{} in {:.3}ms", self.msg, ms));
-        } else {
-            trace!("{}", format!("{} in {:.3}ms", self.msg, ms));
-        }
-
-        ms
-    }
-}
-
-impl Drop for Timer<'_> {
-    fn drop(&mut self) {
-        self.stop();
-    }
-}

src/parser.rs

@@ -1,25 +1,24 @@
 use std::default::Default;
-use std::str::FromStr;
 
 use html5ever::tokenizer::{BufferQueue, TokenizerResult};
 use html5ever::tokenizer::{StartTag, TagToken};
 use html5ever::tokenizer::{Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts};
 use html5ever::{local_name, tendril::*};
-use surrealdb::engine::remote::ws::Client;
+use tracing::{debug, error, instrument, trace, warn};
-use surrealdb::Surreal;
+use url::Url;
-use tracing::instrument;
 
 use crate::db::Website;
-use crate::Timer;
 
 impl TokenSink for Website {
     type Handle = Vec<Website>;
 
+    #[instrument(skip(token, _line_number))]
     fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<Self::Handle> {
         match token {
             TagToken(tag) => {
                 if tag.kind == StartTag {
                     match tag.name {
+                        // this should be all the html elements that have links
                         local_name!("a")
                         | local_name!("audio")
                         | local_name!("area")
@@ -34,21 +33,18 @@ impl TokenSink for Website {
                                 let attr_name = attr.name.local.to_string();
                                 if attr_name == "src" || attr_name == "href" || attr_name == "data"
                                 {
-                                    // Get clone of the current site object
+                                    trace!("Found `{}` in html `{}` tag", &attr.value, tag.name);
-                                    let mut web = self.clone();
+                                    let url = try_get_url(&self.site, &attr.value);
 
-                                    // Set url
+                                    if let Some(mut parsed) = url {
-                                    let mut url = web.site;
+                                        parsed.set_query(None);
-                                    url.set_fragment(None); // removes #xyz
+                                        parsed.set_fragment(None);
-                                    let joined = url.join(&attr.value).expect("Failed to join url during parsing!");
+                                        trace!("Final cleaned URL: `{}`", parsed.to_string());
-                                    web.site = joined;
+                                        let web = Website::new(&parsed.to_string(), false);
-
+                                        links.push(web);
-                                    web.crawled = false;
+                                    }
-
-                                    links.push(web);
                                 }
                             }
-
                             return TokenSinkResult::Script(links);
                         }
                         local_name!("button") | local_name!("meta") | local_name!("iframe") => {
@@ -65,47 +61,91 @@ impl TokenSink for Website {
 }
 
 #[instrument(skip_all)]
-pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
+/// Parses the passed site and returns all the sites it links to.
-    // update self in db
+pub async fn parse(site: &Website, data: &[u8]) -> Vec<Website> {
-    site.set_crawled();
+    debug!("Parsing {}", site.site.to_string());
-    site.store(db).await;
-
     // prep work
-    let mut other_sites = Vec::new();
+    let mut other_sites: Vec<Website> = Vec::new();
-    { // using blocks to prevent compiler's async worries
-        let _t = Timer::start("Parsed page");
 
-        // change data into something that can be tokenized
+    // change data into something that can be tokenized
-        let chunk = Tendril::from_str(&data).expect("Failed to parse string into Tendril!");
+    let s: Result<Tendril<fmt::UTF8>, ()> = Tendril::try_from_byte_slice(data);
+    if let Ok(chunk) = s {
         // create buffer of tokens and push our input into it
-        let mut token_buffer = BufferQueue::default();
+        let token_buffer = BufferQueue::default();
-        token_buffer.push_back(chunk.try_reinterpret::<fmt::UTF8>().expect("Failed to reinterprt chunk!"));
+        token_buffer.push_back(
+            chunk
+                .try_reinterpret::<fmt::UTF8>()
+                .expect("Failed to reinterpret chunk!"),
+        );
         // create the tokenizer
         let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
 
         // go thru buffer
-        while let TokenizerResult::Script(sites) = tokenizer.feed(&mut token_buffer) {
+        while let TokenizerResult::Script(mut sites) = tokenizer.feed(&token_buffer) {
-            other_sites.push(sites);
+            other_sites.append(&mut sites);
+            // other_sites.push(sites);
         }
-
         assert!(token_buffer.is_empty());
         tokenizer.end();
+    } else {
+        warn!("Tendril failed to parse on: {}", site.site.to_string());
     }
 
-    {
+    other_sites
-        let mut links_to = Vec::new();
+}
 
-        // this is a 2d vec accidentally
+#[instrument]
-        for a in other_sites {
+fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
-            for b in a {
+    match Url::parse(link) {
-                // TODO this can become a JoinSet later
+        Ok(ok) => Some(ok),
-                let other = b.store(db).await;
+        Err(e) => {
-                if let Some(o) = other {
+            if link.starts_with('#') {
-                    links_to.push(o);
+                trace!("Rejecting # url");
+                None
+            } else if link.starts_with("//") {
+                // if a url starts with "//" is assumed that it will adopt
+                // the same scheme as it's parent
+                // https://stackoverflow.com/questions/9646407/two-forward-slashes-in-a-url-src-href-attribute
+                let scheme = parent.scheme();
+
+                match Url::parse(&format!("{scheme}://{}", link)) {
+                    Ok(url) => Some(url),
+                    Err(err) => {
+                        error!("Failed parsing relative scheme url: {}", err);
+                        None
+                    }
+                }
+            } else {
+                // # This is some sort of realative url, gonna try patching it up into an absolute
+                // url
+                match e {
+                    url::ParseError::RelativeUrlWithoutBase => {
+                        // Is: scheme://host:port
+                        let mut origin = parent.origin().ascii_serialization();
+                        if !origin.ends_with('/') && !link.starts_with('/') {
+                            origin += "/";
+                        }
+                        let url = origin.clone() + link;
+
+                        if let Ok(url) = Url::parse(&url) {
+                            trace!("Built `{url}` from `{origin} + `{}`", link.to_string());
+                            Some(url)
+                        } else {
+                            error!(
+                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`. Failed url was: {}",
+                                link,
+                                parent.to_string(),
+                                url
+                            );
+                            None
+                        }
+                    }
+                    _ => {
+                        error!("MISC error: {:?} {:?}", e, link);
+                        None
+                    }
                 }
             }
         }
-
-        site.links_to(links_to, db).await;
     }
 }

src/s3.rs

@@ -1,97 +0,0 @@
-use base64::{alphabet, engine::{self, general_purpose}, Engine};
-use minio::s3::{
-    args::{BucketExistsArgs, MakeBucketArgs},
-    client::ClientBuilder,
-    creds::StaticProvider,
-    error::Error,
-    http::BaseUrl,
-    Client,
-};
-use tracing::{instrument, trace, warn};
-use url::Url;
-
-use crate::Config;
-        
-const CUSTOM_ENGINE: engine::GeneralPurpose = engine::GeneralPurpose::new(&alphabet::URL_SAFE, general_purpose::NO_PAD);
-
-#[derive(Clone)]
-pub struct S3 {
-    bucket_name: String,
-    client: Client,
-}
-
-impl S3 {
-    #[instrument(skip_all, name = "S3")]
-    pub async fn connect(config: &Config<'_>) -> Result<Self, Error> {
-        let base_url = config
-            .s3_url
-            .parse::<BaseUrl>()
-            .expect("Failed to parse url into BaseUrl");
-
-        let static_provider =
-            StaticProvider::new(&config.s3_access_key, &config.s3_secret_key, None);
-
-        let client = ClientBuilder::new(base_url)
-            .provider(Some(Box::new(static_provider)))
-            .build()?;
-
-        trace!("Checking bucket...");
-        let exists = client
-            .bucket_exists(
-                &BucketExistsArgs::new(&config.s3_bucket)
-                    .expect("Failed to check if bucket exists"),
-            )
-            .await?;
-
-        if !exists {
-            trace!("Creating bucket...");
-            client
-                .make_bucket(
-                    &MakeBucketArgs::new(&config.s3_bucket).expect("Failed to create bucket!"),
-                )
-                .await?;
-        }
-
-        trace!("Connection successful");
-
-        Ok(Self {
-            bucket_name: config.s3_bucket.to_owned(),
-            client: client,
-        })
-    }
-
-    #[instrument(skip_all)]
-    pub async fn store(&self, data: &str, url: &Url) {
-        if let Some(domain) = url.domain() {
-            let filename = domain.to_owned() + url.path();
-
-            trace!("Created filename: {filename} from raw: {}", url.to_string());
-
-            let _ = match &self
-                .client
-                .put_object_content(&self.bucket_name, &filename, data.to_owned())
-                .send()
-                .await {
-                    Ok(_) => {},
-                    Err(err) => {
-                        match err {
-                            Error::InvalidObjectName(_) => {
-
-                                warn!("Tried storing invalid object name, retrying with Base64 encoding. Last try.");
-                                
-                                let filename: String = domain.to_owned() + &CUSTOM_ENGINE.encode(url.path());
-
-                                let _ = &self
-                                    .client
-                                    .put_object_content(&self.bucket_name, &filename, data.to_owned())
-                                    .send()
-                                    .await
-                                    .unwrap();
-                            },
-                            _ => {},
-                       }
-                    },
-                };
-        }
-    }
-}

src/setup.surql

@@ -1,2 +1,9 @@
 DEFINE TABLE IF NOT EXISTS website SCHEMALESS;
+
+DEFINE FIELD IF NOT EXISTS site ON TABLE website TYPE string;
+DEFINE INDEX IF NOT EXISTS idx ON TABLE website COLUMNS site UNIQUE;
+
+DEFINE FIELD IF NOT EXISTS crawled ON TABLE website TYPE bool;
+
 DEFINE FIELD IF NOT EXISTS accessed_at ON TABLE website VALUE time::now();
+DEFINE FIELD IF NOT EXISTS first_accessed_at ON TABLE website VALUE time::now();