checkpoint

add speed improvements
:)
2025-07-10 18:46:25 -06:00 · 2025-03-21 12:14:29 -06:00 · 2025-03-21 12:11:05 -06:00 · 2025-03-21 11:42:43 -06:00 · 2025-03-21 07:11:51 +00:00 · 2025-03-21 06:48:39 +00:00
33 changed files with 3350 additions and 1236 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,8 @@
 /target
+/.surrealdb
+/.minio
+perf.data
+flamegraph.svg
+perf.data.old
+/docker/logs/*
+/downloaded
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -8,6 +8,9 @@
            "type": "lldb",
            "request": "launch",
            "name": "Debug executable 'surreal_spider'",
+            "env": {
+                "RUST_LOG": "surreal_spider=trace,reqwest=info",
+            },
            "cargo": {
                "args": [
                    "build",
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -0,0 +1,8 @@
+{
+    "cSpell.words": [
+        "creds",
+        "reqwest",
+        "rustls",
+        "surql"
+    ]
+}
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,7 +1,20 @@
 [package]
-name = "surreal_spider"
+name = "internet_mapper"
 version = "0.1.0"
 edition = "2021"

 [dependencies]
-spider = { features = [], git="https://github.com/Rushmore75/spider.git", rev="ff91646973ad04ff423010f36206f550e37c4278" }
+base64 = "0.22.1"
+html5ever = "0.29"
+metrics = "0.24.1"
+metrics-exporter-prometheus = { version = "0.16.2", features=["http-listener"]}
+# minio = "0.1.0"
+minio = {git="https://github.com/minio/minio-rs.git", rev = "c28f576"}
+reqwest = { version = "0.12", features = ["gzip", "default", "rustls-tls"] }
+rusqlite = { version = "0.34.0", features = ["bundled"] }
+serde = { version = "1.0", features = ["derive"] }
+tokio = { version="1.41.0", features = ["full"] }
+toml = "0.8.20"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time", "json"] }
+url = { version = "2.5", features = ["serde"] }
--- a/Crawler.toml
+++ b/Crawler.toml
@@ -0,0 +1,10 @@
+# Surreal config
+surreal_url = "localhost:8000"
+surreal_username = "root"
+surreal_password = "root"
+surreal_ns = "test"
+surreal_db = "v1.19.2"
+
+# Crawler config
+crawl_filter = "en.wikipedia.com" 
+budget = 1000
--- a/README.md
+++ b/README.md
@@ -1,23 +1,25 @@
 # Surreal Crawler

-Mapping with a budget of 1000 (crawl 1000 sites, so many more links are actually discovered), on [my webiste](https://oliveratkinson.net) on 8/26/2024 took 1m9s.
+Crawls sites saving all the found links to a surrealdb database. It then proceeds to take batches of 100 uncrawled links untill the crawl budget is reached. It saves the data of each site in a minio database.

-This is including the crawl and loading into the database and linking sites. (Locally hosted surreal db instance)
+### TODO

-This run created 4299 site links with 23286 links between the sites. (It found my this git site which really bolsters those numbers.)
+- [ ] Domain filtering - prevent the crawler from going on alternate versions of wikipedia.
+- [ ] Conditionally save content - based on filename or file contents
+- [x] GUI / TUI ? - Graphana
+- [x] Better asynchronous getting of the sites. Currently it all happens serially.
+- [ ] Allow for storing asynchronously

-## Install / Build
+3/17/25: Took >1hr to crawl 100 pages

-* You will need rust to compile the crawler [rustup.rs](https://rustup.rs)
-* You need python3 (will come installed on most linux distros) and poetry for dependancy management.
-    * Install `pipx`, `python3`
-    * Then: `pipx install poetry`
-    * Then: `poetry install` to install the project dependancies
-* You need to install [surrealdb](https://surrealdb.com)
+3/19/25: Took 20min to crawl 1000 pages
+This ment we stored 1000 pages, 142,997 urls, and 1,425,798 links between the two.

-## Use
+3/20/25: Took 5min to crawl 1000 pages

-Just run `./crawl.sh {url}` and it will start crawling. You can tweak the budget inside [crawl.sh](https://git.oliveratkinson.net/Oliver/internet_mapper/src/branch/main/crawl.sh) if you want.
+3/21/25: Took 3min to crawl 1000 pages

-You can also prefix the command with `time` to benchmark the system, such as: `time ./crawl.sh https://discord.com`.
+# About
+
+![Screenshot](/pngs/graphana.png)

--- a/compose.yml
+++ b/compose.yml
@@ -1,13 +0,0 @@
-services:
-  db:
-    image: surrealdb/surrealdb:latest-dev
-    ports:
-    - 8000:8000
-    command:
-      - start
-      - --log
-      - debug
-      - --user
-      - root
-      - --pass
-      - root
--- a/crawl.sh
+++ b/crawl.sh
@@ -1,2 +0,0 @@
-#!/bin/bash
-cargo run -- $1 1000 | python3 driver.py
--- a/demo_site/audio.mp3
+++ b/demo_site/audio.mp3
--- a/demo_site/index.html
+++ b/demo_site/index.html
@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Page Title</title>
+    <script src='main.js'></script>
+	<base href="/" target="_blank">
+	<link rel="stylesheet" href="style.css">
+</head>
+<body>
+    <a href="/">Link text</a>
+	<img src="/picture.png"></img>
+	<audio src="/audio.mp3" controls>Audio is unsupported</audio>
+	<video src="/video.mp4" controls>Video is unsupported</video>
+	<audio controls>
+		<source src="audio.mp3"></source>
+	</audio>
+	<map>
+		<area href="/"></area>
+	</map>
+	<object data="/video.mp4"></object>
+	<iframe src="/" title="The same Site"></iframe>
+	<meta>
+</body>
+</html>
--- a/demo_site/picture.png
+++ b/demo_site/picture.png
--- a/demo_site/script.js
+++ b/demo_site/script.js
@@ -0,0 +1 @@
+alert("alert")
--- a/demo_site/style.css
+++ b/demo_site/style.css
@@ -0,0 +1,3 @@
+html {
+	background-color: white;
+}
--- a/demo_site/video.mp4
+++ b/demo_site/video.mp4
--- a/docker/alloy.conf
+++ b/docker/alloy.conf
@@ -0,0 +1,14 @@
+local.file_match "tmplogs" {
+    path_targets = [{"__path__" = "/tmp/alloy-logs/*.log"}]
+}
+
+loki.source.file "local_files" {
+    targets    = local.file_match.tmplogs.targets
+    forward_to = [loki.write.local_loki.receiver]
+}
+
+loki.write "local_loki" {
+    endpoint {
+        url = "http://loki:3100/loki/api/v1/push"
+    }
+}
--- a/docker/compose.yml
+++ b/docker/compose.yml
@@ -0,0 +1,69 @@
+services:
+  surreal:
+    image: surrealdb/surrealdb:latest-dev
+    ports:
+    - 8000:8000
+    volumes:
+      - surrealdb_storage:/mydata
+    command:
+      - start
+      - --log
+      - debug
+      - --user
+      - root
+      - --pass
+      - root
+      - rocksdb:/mydata/database.db
+
+  alloy:
+    image: grafana/alloy:latest
+    ports:
+      - 12345:12345
+    volumes:
+      # if you change this, you also need to change it in the alloy config file
+      - ./logs/:/tmp/alloy-logs
+      - ./alloy.conf:/etc/alloy/config.alloy
+      - alloy_storage:/var/lib/alloy
+    command:  run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+
+  #logs
+  loki:
+    image: grafana/loki:latest
+    ports:
+      - 3100:3100
+    command: -config.file=/etc/loki/local-config.yaml
+    volumes:
+      - ./loki.yaml:/etc/loki/local-config.yaml
+
+  # Metrics collector
+  prometheus:
+    image: prom/prometheus:latest
+    expose:
+      - 9090
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+      # persist data
+      - prometheus_storage:/prometheus
+    command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
+
+  # Everything viewer
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - ./dashboards:/var/lib/grafana/dashboards
+      - grafana_storage:/var/lib/grafana
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor
+    ports:
+      - 3000:3000
+  
+volumes:
+  prometheus_storage:
+  grafana_storage:
+  alloy_storage:
+  surrealdb_storage:
+  minio_storage:
--- a/docker/dashboards/crawler-dashboard.json
+++ b/docker/dashboards/crawler-dashboard.json
@@ -0,0 +1,648 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "Prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    },
+    {
+      "name": "DS_LOKI",
+      "label": "Loki",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "loki",
+      "pluginName": "Loki"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "11.3.1"
+    },
+    {
+      "type": "panel",
+      "id": "logs",
+      "name": "Logs",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "loki",
+      "name": "Loki",
+      "version": "1.0.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": 300000,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_trips",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "Trips to Surreal",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "s3_trips",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "Trips to S3",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "pages_crawled",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "total crawled",
+          "range": true,
+          "refId": "C",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "pages_being_processed",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "Pages being processed",
+          "range": true,
+          "refId": "E",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "gets_in_flight",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "D",
+          "useBackend": false
+        }
+      ],
+      "title": "Crawler stats",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": 300000,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 9,
+        "x": 8,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_trips",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "Trips to Surreal",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_link_calls",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "link calls",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_store_calls",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "store calls",
+          "range": true,
+          "refId": "C",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "pages_being_processed",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "Pages being processed",
+          "range": true,
+          "refId": "E",
+          "useBackend": false
+        }
+      ],
+      "title": "Surreal stats",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "This is across all threads, so this isn't wall clock time",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 7,
+        "x": 17,
+        "y": 0
+      },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "surql_lock_waiting_ms",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Time spend waiting on lock",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 18,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "id": 1,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `ERROR` | line_format \"{{.threadId}} {{.filename_extracted}}:{{.line_number}}  {{.fields_message}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Errors",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 26
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `DEBUG` | line_format \"{{.fields_message}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Debug",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 16,
+        "w": 12,
+        "x": 12,
+        "y": 26
+      },
+      "id": 4,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `TRACE` | line_format \"{{.fields_message}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Trace",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 34
+      },
+      "id": 3,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `WARN` | line_format \"{{.fields_message}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Warnings",
+      "type": "logs"
+    }
+  ],
+  "schemaVersion": 40,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "datasource": {
+          "type": "loki",
+          "uid": "P8E80F9AEF21F6940"
+        },
+        "filters": [],
+        "name": "Filters",
+        "type": "adhoc"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Crawler",
+  "uid": "ceg90x34pqgowd",
+  "version": 21,
+  "weekStart": ""
+}
--- a/docker/grafana.yaml
+++ b/docker/grafana.yaml
@@ -0,0 +1,24 @@
+apiVersion: 1
+
+datasources:
+- name: Loki
+  type: loki
+  access: proxy 
+  orgId: 1
+  url: http://loki:3100
+  basicAuth: false
+  isDefault: true 
+  version: 1
+  editable: false
+- name: Prometheus
+  type: prometheus
+  uid: prometheus
+  access: proxy
+  orgId: 1
+  url: http://prometheus:9090
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
+  jsonData:
+    httpMethod: GET
--- a/docker/loki.yaml
+++ b/docker/loki.yaml
@@ -0,0 +1,62 @@
+# this is mostly the default config from grafana's website
+
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  metric_aggregation:
+    loki_address: localhost:3100
+
+frontend:
+  encoding: protobuf
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+analytics:
+  reporting_enabled: false
--- a/docker/prometheus.yaml
+++ b/docker/prometheus.yaml
@@ -0,0 +1,17 @@
+global:
+  scrape_interval: 5s 
+  query_log_file: /etc/prometheus/query.log
+
+scrape_configs:
+  - job_name: crawler
+    static_configs:
+    # change this your machine's ip, localhost won't work
+    # because localhost refers to the docker container.
+      - targets: ['172.20.239.48:2500']
+        #- targets: ['192.168.8.209:2500']
+  - job_name: loki
+    static_configs:
+      - targets: ['loki:3100']
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
--- a/driver.py
+++ b/driver.py
@@ -1,49 +0,0 @@
-from surrealdb import Surreal
-import fileinput
-import asyncio
-
-
-async def main():
-    async with Surreal("ws://localhost:8000/rpc") as db:
-
-        await db.signin({"user": "root", "pass": "root"})
-        await db.use("test", "test")
-
-        for line in fileinput.input():
-        # Testing
-        # for line in ["https://oliveratkinson.net;->;http://google.com"]:
-
-            urls = line.split(";->;")
-            ffrom = urls[0].strip()
-            to= urls[1].strip()
-
-            to_id = None
-            from_id = None
-
-            if (ffrom == to):
-                print('Site has self-reference, ignoring')
-                continue
-
-            # FROM
-            result = await db.query(f"SELECT id FROM website WHERE url='{ffrom}'")
-            if (len(result[0]['result']) > 0):
-                from_id = result[0]['result'][0]['id']
-                await db.query(f"UPDATE {from_id} MERGE {{ crawled: True }}")
-            else:
-                from_response = await db.create("website", { "url":ffrom, "crawled":True, } )
-                from_id = from_response[0]["id"]
-
-            # TO
-            result = await db.query(f"SELECT id FROM website WHERE url='{to}'")
-            if (len(result[0]['result']) < 1):
-                # Object doesn't exists yet
-                to_response = await db.create("website", { "url":to })
-                to_id = to_response[0]["id"]
-            else:
-                to_id = result[0]['result'][0]['id']
-
-            await db.query(f"RELATE {from_id} -> links_to -> {to_id}");
-
-if __name__ == "__main__":
-    asyncio.run(main())
-
--- a/jsconfig.json
+++ b/jsconfig.json
@@ -0,0 +1,16 @@
+{
+    "compilerOptions": {
+        "module": "ESNext",
+        "moduleResolution": "Bundler",
+        "target": "ES2022",
+        "jsx": "react",
+        "allowImportingTsExtensions": true,
+        "strictNullChecks": true,
+        "strictFunctionTypes": true
+    },
+    "exclude": [
+        "node_modules",
+        "**/node_modules/*"
+    ],
+    "typeAcquisition": {"include": ["firefox-webext-browser"]}
+}
--- a/pngs/graphana.png
+++ b/pngs/graphana.png
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,370 +0,0 @@
-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
-
-[[package]]
-name = "annotated-types"
-version = "0.7.0"
-description = "Reusable constraint types to use with typing.Annotated"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
-    {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
-]
-
-[[package]]
-name = "anyio"
-version = "4.4.0"
-description = "High level compatibility layer for multiple asynchronous event loop implementations"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"},
-    {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"},
-]
-
-[package.dependencies]
-exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""}
-idna = ">=2.8"
-sniffio = ">=1.1"
-typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""}
-
-[package.extras]
-doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
-test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
-trio = ["trio (>=0.23)"]
-
-[[package]]
-name = "certifi"
-version = "2024.7.4"
-description = "Python package for providing Mozilla's CA Bundle."
-optional = false
-python-versions = ">=3.6"
-files = [
-    {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"},
-    {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"},
-]
-
-[[package]]
-name = "exceptiongroup"
-version = "1.2.2"
-description = "Backport of PEP 654 (exception groups)"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"},
-    {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"},
-]
-
-[package.extras]
-test = ["pytest (>=6)"]
-
-[[package]]
-name = "h11"
-version = "0.14.0"
-description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
-    {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
-]
-
-[[package]]
-name = "httpcore"
-version = "1.0.5"
-description = "A minimal low-level HTTP client."
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"},
-    {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"},
-]
-
-[package.dependencies]
-certifi = "*"
-h11 = ">=0.13,<0.15"
-
-[package.extras]
-asyncio = ["anyio (>=4.0,<5.0)"]
-http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
-trio = ["trio (>=0.22.0,<0.26.0)"]
-
-[[package]]
-name = "httpx"
-version = "0.27.0"
-description = "The next generation HTTP client."
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"},
-    {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"},
-]
-
-[package.dependencies]
-anyio = "*"
-certifi = "*"
-httpcore = "==1.*"
-idna = "*"
-sniffio = "*"
-
-[package.extras]
-brotli = ["brotli", "brotlicffi"]
-cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
-http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
-
-[[package]]
-name = "idna"
-version = "3.8"
-description = "Internationalized Domain Names in Applications (IDNA)"
-optional = false
-python-versions = ">=3.6"
-files = [
-    {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"},
-    {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"},
-]
-
-[[package]]
-name = "pydantic"
-version = "2.8.2"
-description = "Data validation using Python type hints"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"},
-    {file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"},
-]
-
-[package.dependencies]
-annotated-types = ">=0.4.0"
-pydantic-core = "2.20.1"
-typing-extensions = [
-    {version = ">=4.12.2", markers = "python_version >= \"3.13\""},
-    {version = ">=4.6.1", markers = "python_version < \"3.13\""},
-]
-
-[package.extras]
-email = ["email-validator (>=2.0.0)"]
-
-[[package]]
-name = "pydantic-core"
-version = "2.20.1"
-description = "Core functionality for Pydantic validation and serialization"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "pydantic_core-2.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3acae97ffd19bf091c72df4d726d552c473f3576409b2a7ca36b2f535ffff4a3"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41f4c96227a67a013e7de5ff8f20fb496ce573893b7f4f2707d065907bffdbd6"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f239eb799a2081495ea659d8d4a43a8f42cd1fe9ff2e7e436295c38a10c286a"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53e431da3fc53360db73eedf6f7124d1076e1b4ee4276b36fb25514544ceb4a3"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1f62b2413c3a0e846c3b838b2ecd6c7a19ec6793b2a522745b0869e37ab5bc1"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d41e6daee2813ecceea8eda38062d69e280b39df793f5a942fa515b8ed67953"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d482efec8b7dc6bfaedc0f166b2ce349df0011f5d2f1f25537ced4cfc34fd98"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e93e1a4b4b33daed65d781a57a522ff153dcf748dee70b40c7258c5861e1768a"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7c4ea22b6739b162c9ecaaa41d718dfad48a244909fe7ef4b54c0b530effc5a"},
-    {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4f2790949cf385d985a31984907fecb3896999329103df4e4983a4a41e13e840"},
-    {file = "pydantic_core-2.20.1-cp310-none-win32.whl", hash = "sha256:5e999ba8dd90e93d57410c5e67ebb67ffcaadcea0ad973240fdfd3a135506250"},
-    {file = "pydantic_core-2.20.1-cp310-none-win_amd64.whl", hash = "sha256:512ecfbefef6dac7bc5eaaf46177b2de58cdf7acac8793fe033b24ece0b9566c"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"},
-    {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"},
-    {file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"},
-    {file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4f55095ad087474999ee28d3398bae183a66be4823f753cd7d67dd0153427c9"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9aa05d09ecf4c75157197f27cdc9cfaeb7c5f15021c6373932bf3e124af029f"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e97fdf088d4b31ff4ba35db26d9cc472ac7ef4a2ff2badeabf8d727b3377fc52"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc633a9fe1eb87e250b5c57d389cf28998e4292336926b0b6cdaee353f89a237"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d573faf8eb7e6b1cbbcb4f5b247c60ca8be39fe2c674495df0eb4318303137fe"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26dc97754b57d2fd00ac2b24dfa341abffc380b823211994c4efac7f13b9e90e"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:33499e85e739a4b60c9dac710c20a08dc73cb3240c9a0e22325e671b27b70d24"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bebb4d6715c814597f85297c332297c6ce81e29436125ca59d1159b07f423eb1"},
-    {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:516d9227919612425c8ef1c9b869bbbee249bc91912c8aaffb66116c0b447ebd"},
-    {file = "pydantic_core-2.20.1-cp312-none-win32.whl", hash = "sha256:469f29f9093c9d834432034d33f5fe45699e664f12a13bf38c04967ce233d688"},
-    {file = "pydantic_core-2.20.1-cp312-none-win_amd64.whl", hash = "sha256:035ede2e16da7281041f0e626459bcae33ed998cca6a0a007a5ebb73414ac72d"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0827505a5c87e8aa285dc31e9ec7f4a17c81a813d45f70b1d9164e03a813a686"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19c0fa39fa154e7e0b7f82f88ef85faa2a4c23cc65aae2f5aea625e3c13c735a"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa223cd1e36b642092c326d694d8bf59b71ddddc94cdb752bbbb1c5c91d833b"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c336a6d235522a62fef872c6295a42ecb0c4e1d0f1a3e500fe949415761b8a19"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7eb6a0587eded33aeefea9f916899d42b1799b7b14b8f8ff2753c0ac1741edac"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70c8daf4faca8da5a6d655f9af86faf6ec2e1768f4b8b9d0226c02f3d6209703"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fa4c9bf273ca41f940bceb86922a7667cd5bf90e95dbb157cbb8441008482c"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:11b71d67b4725e7e2a9f6e9c0ac1239bbc0c48cce3dc59f98635efc57d6dac83"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:270755f15174fb983890c49881e93f8f1b80f0b5e3a3cc1394a255706cabd203"},
-    {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c81131869240e3e568916ef4c307f8b99583efaa60a8112ef27a366eefba8ef0"},
-    {file = "pydantic_core-2.20.1-cp313-none-win32.whl", hash = "sha256:b91ced227c41aa29c672814f50dbb05ec93536abf8f43cd14ec9521ea09afe4e"},
-    {file = "pydantic_core-2.20.1-cp313-none-win_amd64.whl", hash = "sha256:65db0f2eefcaad1a3950f498aabb4875c8890438bc80b19362cf633b87a8ab20"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4745f4ac52cc6686390c40eaa01d48b18997cb130833154801a442323cc78f91"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8ad4c766d3f33ba8fd692f9aa297c9058970530a32c728a2c4bfd2616d3358b"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41e81317dd6a0127cabce83c0c9c3fbecceae981c8391e6f1dec88a77c8a569a"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04024d270cf63f586ad41fff13fde4311c4fc13ea74676962c876d9577bcc78f"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaad4ff2de1c3823fddf82f41121bdf453d922e9a238642b1dedb33c4e4f98ad"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26ab812fa0c845df815e506be30337e2df27e88399b985d0bb4e3ecfe72df31c"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c5ebac750d9d5f2706654c638c041635c385596caf68f81342011ddfa1e5598"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2aafc5a503855ea5885559eae883978c9b6d8c8993d67766ee73d82e841300dd"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4868f6bd7c9d98904b748a2653031fc9c2f85b6237009d475b1008bfaeb0a5aa"},
-    {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa2f457b4af386254372dfa78a2eda2563680d982422641a85f271c859df1987"},
-    {file = "pydantic_core-2.20.1-cp38-none-win32.whl", hash = "sha256:225b67a1f6d602de0ce7f6c1c3ae89a4aa25d3de9be857999e9124f15dab486a"},
-    {file = "pydantic_core-2.20.1-cp38-none-win_amd64.whl", hash = "sha256:6b507132dcfc0dea440cce23ee2182c0ce7aba7054576efc65634f080dbe9434"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b03f7941783b4c4a26051846dea594628b38f6940a2fdc0df00b221aed39314c"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1eedfeb6089ed3fad42e81a67755846ad4dcc14d73698c120a82e4ccf0f1f9f6"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:635fee4e041ab9c479e31edda27fcf966ea9614fff1317e280d99eb3e5ab6fe2"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77bf3ac639c1ff567ae3b47f8d4cc3dc20f9966a2a6dd2311dcc055d3d04fb8a"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ed1b0132f24beeec5a78b67d9388656d03e6a7c837394f99257e2d55b461611"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6514f963b023aeee506678a1cf821fe31159b925c4b76fe2afa94cc70b3222b"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10d4204d8ca33146e761c79f83cc861df20e7ae9f6487ca290a97702daf56006"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d036c7187b9422ae5b262badb87a20a49eb6c5238b2004e96d4da1231badef1"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ebfef07dbe1d93efb94b4700f2d278494e9162565a54f124c404a5656d7ff09"},
-    {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6b9d9bb600328a1ce523ab4f454859e9d439150abb0906c5a1983c146580ebab"},
-    {file = "pydantic_core-2.20.1-cp39-none-win32.whl", hash = "sha256:784c1214cb6dd1e3b15dd8b91b9a53852aed16671cc3fbe4786f4f1db07089e2"},
-    {file = "pydantic_core-2.20.1-cp39-none-win_amd64.whl", hash = "sha256:d2fe69c5434391727efa54b47a1e7986bb0186e72a41b203df8f5b0a19a4f669"},
-    {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"},
-    {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"},
-    {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"},
-    {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"},
-    {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"},
-    {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"},
-    {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"},
-    {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"},
-    {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"},
-    {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"},
-    {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"},
-    {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"},
-    {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"},
-    {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"},
-    {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"},
-    {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"},
-    {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"},
-]
-
-[package.dependencies]
-typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
-
-[[package]]
-name = "sniffio"
-version = "1.3.1"
-description = "Sniff out which async library your code is running under"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"},
-    {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
-]
-
-[[package]]
-name = "surrealdb"
-version = "0.3.2"
-description = "The official SurrealDB library for Python."
-optional = false
-python-versions = ">=3.8,<4.0"
-files = [
-    {file = "surrealdb-0.3.2-py3-none-any.whl", hash = "sha256:30424971be9698f0fce61dc8387576d19302e0bb0ae708d311e79a261ead7701"},
-    {file = "surrealdb-0.3.2.tar.gz", hash = "sha256:8ab570b8d4299f35443f0d4fa636947cf1a18f67fd88acc00e5e5a36356e82cd"},
-]
-
-[package.dependencies]
-httpx = ">=0.23.0"
-pydantic = ">=2.1.0,<3.0.0"
-websockets = ">=10.4,<11.0"
-
-[[package]]
-name = "typing-extensions"
-version = "4.12.2"
-description = "Backported and Experimental Type Hints for Python 3.8+"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
-    {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
-]
-
-[[package]]
-name = "websockets"
-version = "10.4"
-description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "websockets-10.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d58804e996d7d2307173d56c297cf7bc132c52df27a3efaac5e8d43e36c21c48"},
-    {file = "websockets-10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc0b82d728fe21a0d03e65f81980abbbcb13b5387f733a1a870672c5be26edab"},
-    {file = "websockets-10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ba089c499e1f4155d2a3c2a05d2878a3428cf321c848f2b5a45ce55f0d7d310c"},
-    {file = "websockets-10.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33d69ca7612f0ddff3316b0c7b33ca180d464ecac2d115805c044bf0a3b0d032"},
-    {file = "websockets-10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62e627f6b6d4aed919a2052efc408da7a545c606268d5ab5bfab4432734b82b4"},
-    {file = "websockets-10.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ea7b82bfcae927eeffc55d2ffa31665dc7fec7b8dc654506b8e5a518eb4d50"},
-    {file = "websockets-10.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e0cb5cc6ece6ffa75baccfd5c02cffe776f3f5c8bf486811f9d3ea3453676ce8"},
-    {file = "websockets-10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae5e95cfb53ab1da62185e23b3130e11d64431179debac6dc3c6acf08760e9b1"},
-    {file = "websockets-10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7c584f366f46ba667cfa66020344886cf47088e79c9b9d39c84ce9ea98aaa331"},
-    {file = "websockets-10.4-cp310-cp310-win32.whl", hash = "sha256:b029fb2032ae4724d8ae8d4f6b363f2cc39e4c7b12454df8df7f0f563ed3e61a"},
-    {file = "websockets-10.4-cp310-cp310-win_amd64.whl", hash = "sha256:8dc96f64ae43dde92530775e9cb169979f414dcf5cff670455d81a6823b42089"},
-    {file = "websockets-10.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:47a2964021f2110116cc1125b3e6d87ab5ad16dea161949e7244ec583b905bb4"},
-    {file = "websockets-10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e789376b52c295c4946403bd0efecf27ab98f05319df4583d3c48e43c7342c2f"},
-    {file = "websockets-10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d3f0b61c45c3fa9a349cf484962c559a8a1d80dae6977276df8fd1fa5e3cb8c"},
-    {file = "websockets-10.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f55b5905705725af31ccef50e55391621532cd64fbf0bc6f4bac935f0fccec46"},
-    {file = "websockets-10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00c870522cdb69cd625b93f002961ffb0c095394f06ba8c48f17eef7c1541f96"},
-    {file = "websockets-10.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f38706e0b15d3c20ef6259fd4bc1700cd133b06c3c1bb108ffe3f8947be15fa"},
-    {file = "websockets-10.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f2c38d588887a609191d30e902df2a32711f708abfd85d318ca9b367258cfd0c"},
-    {file = "websockets-10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fe10ddc59b304cb19a1bdf5bd0a7719cbbc9fbdd57ac80ed436b709fcf889106"},
-    {file = "websockets-10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:90fcf8929836d4a0e964d799a58823547df5a5e9afa83081761630553be731f9"},
-    {file = "websockets-10.4-cp311-cp311-win32.whl", hash = "sha256:b9968694c5f467bf67ef97ae7ad4d56d14be2751000c1207d31bf3bb8860bae8"},
-    {file = "websockets-10.4-cp311-cp311-win_amd64.whl", hash = "sha256:a7a240d7a74bf8d5cb3bfe6be7f21697a28ec4b1a437607bae08ac7acf5b4882"},
-    {file = "websockets-10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:74de2b894b47f1d21cbd0b37a5e2b2392ad95d17ae983e64727e18eb281fe7cb"},
-    {file = "websockets-10.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3a686ecb4aa0d64ae60c9c9f1a7d5d46cab9bfb5d91a2d303d00e2cd4c4c5cc"},
-    {file = "websockets-10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d15c968ea7a65211e084f523151dbf8ae44634de03c801b8bd070b74e85033"},
-    {file = "websockets-10.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00213676a2e46b6ebf6045bc11d0f529d9120baa6f58d122b4021ad92adabd41"},
-    {file = "websockets-10.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:e23173580d740bf8822fd0379e4bf30aa1d5a92a4f252d34e893070c081050df"},
-    {file = "websockets-10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:dd500e0a5e11969cdd3320935ca2ff1e936f2358f9c2e61f100a1660933320ea"},
-    {file = "websockets-10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4239b6027e3d66a89446908ff3027d2737afc1a375f8fd3eea630a4842ec9a0c"},
-    {file = "websockets-10.4-cp37-cp37m-win32.whl", hash = "sha256:8a5cc00546e0a701da4639aa0bbcb0ae2bb678c87f46da01ac2d789e1f2d2038"},
-    {file = "websockets-10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:a9f9a735deaf9a0cadc2d8c50d1a5bcdbae8b6e539c6e08237bc4082d7c13f28"},
-    {file = "websockets-10.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5c1289596042fad2cdceb05e1ebf7aadf9995c928e0da2b7a4e99494953b1b94"},
-    {file = "websockets-10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0cff816f51fb33c26d6e2b16b5c7d48eaa31dae5488ace6aae468b361f422b63"},
-    {file = "websockets-10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dd9becd5fe29773d140d68d607d66a38f60e31b86df75332703757ee645b6faf"},
-    {file = "websockets-10.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45ec8e75b7dbc9539cbfafa570742fe4f676eb8b0d3694b67dabe2f2ceed8aa6"},
-    {file = "websockets-10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f72e5cd0f18f262f5da20efa9e241699e0cf3a766317a17392550c9ad7b37d8"},
-    {file = "websockets-10.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185929b4808b36a79c65b7865783b87b6841e852ef5407a2fb0c03381092fa3b"},
-    {file = "websockets-10.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7d27a7e34c313b3a7f91adcd05134315002aaf8540d7b4f90336beafaea6217c"},
-    {file = "websockets-10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:884be66c76a444c59f801ac13f40c76f176f1bfa815ef5b8ed44321e74f1600b"},
-    {file = "websockets-10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:931c039af54fc195fe6ad536fde4b0de04da9d5916e78e55405436348cfb0e56"},
-    {file = "websockets-10.4-cp38-cp38-win32.whl", hash = "sha256:db3c336f9eda2532ec0fd8ea49fef7a8df8f6c804cdf4f39e5c5c0d4a4ad9a7a"},
-    {file = "websockets-10.4-cp38-cp38-win_amd64.whl", hash = "sha256:48c08473563323f9c9debac781ecf66f94ad5a3680a38fe84dee5388cf5acaf6"},
-    {file = "websockets-10.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:40e826de3085721dabc7cf9bfd41682dadc02286d8cf149b3ad05bff89311e4f"},
-    {file = "websockets-10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:56029457f219ade1f2fc12a6504ea61e14ee227a815531f9738e41203a429112"},
-    {file = "websockets-10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f5fc088b7a32f244c519a048c170f14cf2251b849ef0e20cbbb0fdf0fdaf556f"},
-    {file = "websockets-10.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc8709c00704194213d45e455adc106ff9e87658297f72d544220e32029cd3d"},
-    {file = "websockets-10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0154f7691e4fe6c2b2bc275b5701e8b158dae92a1ab229e2b940efe11905dff4"},
-    {file = "websockets-10.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c6d2264f485f0b53adf22697ac11e261ce84805c232ed5dbe6b1bcb84b00ff0"},
-    {file = "websockets-10.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9bc42e8402dc5e9905fb8b9649f57efcb2056693b7e88faa8fb029256ba9c68c"},
-    {file = "websockets-10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:edc344de4dac1d89300a053ac973299e82d3db56330f3494905643bb68801269"},
-    {file = "websockets-10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:84bc2a7d075f32f6ed98652db3a680a17a4edb21ca7f80fe42e38753a58ee02b"},
-    {file = "websockets-10.4-cp39-cp39-win32.whl", hash = "sha256:c94ae4faf2d09f7c81847c63843f84fe47bf6253c9d60b20f25edfd30fb12588"},
-    {file = "websockets-10.4-cp39-cp39-win_amd64.whl", hash = "sha256:bbccd847aa0c3a69b5f691a84d2341a4f8a629c6922558f2a70611305f902d74"},
-    {file = "websockets-10.4-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:82ff5e1cae4e855147fd57a2863376ed7454134c2bf49ec604dfe71e446e2193"},
-    {file = "websockets-10.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d210abe51b5da0ffdbf7b43eed0cfdff8a55a1ab17abbec4301c9ff077dd0342"},
-    {file = "websockets-10.4-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:942de28af58f352a6f588bc72490ae0f4ccd6dfc2bd3de5945b882a078e4e179"},
-    {file = "websockets-10.4-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9b27d6c1c6cd53dc93614967e9ce00ae7f864a2d9f99fe5ed86706e1ecbf485"},
-    {file = "websockets-10.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3d3cac3e32b2c8414f4f87c1b2ab686fa6284a980ba283617404377cd448f631"},
-    {file = "websockets-10.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:da39dd03d130162deb63da51f6e66ed73032ae62e74aaccc4236e30edccddbb0"},
-    {file = "websockets-10.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389f8dbb5c489e305fb113ca1b6bdcdaa130923f77485db5b189de343a179393"},
-    {file = "websockets-10.4-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09a1814bb15eff7069e51fed0826df0bc0702652b5cb8f87697d469d79c23576"},
-    {file = "websockets-10.4-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff64a1d38d156d429404aaa84b27305e957fd10c30e5880d1765c9480bea490f"},
-    {file = "websockets-10.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b343f521b047493dc4022dd338fc6db9d9282658862756b4f6fd0e996c1380e1"},
-    {file = "websockets-10.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:932af322458da7e4e35df32f050389e13d3d96b09d274b22a7aa1808f292fee4"},
-    {file = "websockets-10.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a4162139374a49eb18ef5b2f4da1dd95c994588f5033d64e0bbfda4b6b6fcf"},
-    {file = "websockets-10.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c57e4c1349fbe0e446c9fa7b19ed2f8a4417233b6984277cce392819123142d3"},
-    {file = "websockets-10.4-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b627c266f295de9dea86bd1112ed3d5fafb69a348af30a2422e16590a8ecba13"},
-    {file = "websockets-10.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:05a7233089f8bd355e8cbe127c2e8ca0b4ea55467861906b80d2ebc7db4d6b72"},
-    {file = "websockets-10.4.tar.gz", hash = "sha256:eef610b23933c54d5d921c92578ae5f89813438fded840c2e9809d378dc765d3"},
-]
-
-[metadata]
-lock-version = "2.0"
-python-versions = "^3.10"
-content-hash = "e5d2be363dd4d62589502cb214d66ac9f21992ad2fb62a09a50e243000600518"
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,15 +0,0 @@
-[tool.poetry]
-name = "surreal_spider"
-version = "0.1.0"
-description = ""
-authors = ["oliver <oliveratkinsoneng@gmail.com>"]
-
-[tool.poetry.dependencies]
-python = "^3.10"
-surrealdb = "^0.3.2"
-
-[tool.poetry.dev-dependencies]
-
-[build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
--- a/schema.surql
+++ b/schema.surql
@@ -1,15 +0,0 @@
-DELETE website;
-DELETE links_to;
-DELETE url_index;
-
-DEFINE TABLE website SCHEMAFULL;
-    DEFINE FIELD accessed_at ON TABLE website VALUE time::now();
-    DEFINE FIELD crawled     ON TABLE website TYPE bool DEFAULT false;
-    DEFINE FIELD url         ON TABLE website TYPE string;
-    DEFINE FIELD domain      ON TABLE website VALUE parse::url::domain($this.url) ASSERT !type::is::none($value);
-    DEFINE FIELD path        ON TABLE website VALUE parse::url::path($this.url) ASSERT !type::is::none($value);
-    DEFINE INDEX url_index   ON TABLE website COLUMNS url UNIQUE;
-
-DEFINE TABLE links_to SCHEMAFULL TYPE RELATION FROM website TO website;
-    DEFINE FIELD discovered_at ON TABLE links_to TYPE datetime DEFAULT time::now();
-
--- a/src/db.rs
+++ b/src/db.rs
@@ -0,0 +1,99 @@
+use metrics::counter;
+use rusqlite::Connection;
+use std::fmt::Debug;
+use serde::{Deserialize, Serialize};
+use tracing::{error, instrument, trace};
+use url::Url;
+
+use crate::Config;
+
+const STORE: &str = "surql_store_calls";
+
+#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
+pub struct Website {
+    /// The url that this data is found at
+    pub site: Url,
+    /// Wether or not this link has been crawled yet
+    pub crawled: bool,
+}
+
+// manual impl to make tracing look nicer
+impl Debug for Website {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Website").field("site", &self.site).finish()
+    }
+}
+
+impl Website {
+    /// Creates a blank site (assumes that url param is site's root)
+    pub fn new(url: &str, crawled: bool) -> Self {
+        let site = match Url::parse(url) {
+            Ok(a) => a,
+            Err(_) => todo!(),
+        };
+        Self {
+            crawled,
+            site
+        }
+    }
+
+    pub fn set_crawled(&mut self) {
+        trace!("Set crawled to true");
+        self.crawled = true
+    }
+
+    // Insert ever item in the vec into surreal, crawled state will be preserved as TRUE
+    // if already in the database as such or incoming data is TRUE.
+    pub async fn store_all(all: Vec<Self>, db: &Connection) {
+        counter!(STORE).increment(1);
+        let mut things = Vec::with_capacity(all.len());
+
+        rusqlite::ParamsFromIter;
+
+        db.execute("",
+            params![]
+        );
+
+        match db
+            .query(
+                "INSERT INTO website $array
+                    ON DUPLICATE KEY UPDATE
+                        accessed_at = time::now(),
+                        crawled = crawled OR $input.crawled
+                    RETURN VALUE id;
+                 ",
+            )
+            .bind(("array", all))
+            .await
+        {
+            Ok(mut id) => match id.take::<Vec<Thing>>(0) {
+                Ok(mut x) => things.append(&mut x),
+                Err(err) => error!("{:?}", err),
+            },
+            Err(err) => {
+                error!("{:?}", err);
+            }
+        }
+        things
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct Email {
+    pub email: String,
+    pub on: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Record {
+    #[allow(dead_code)]
+    pub id: Thing,
+}
+
+#[instrument(skip_all, name = "sqlite_connect")]
+pub async fn connect(config: &Config) -> Result<Connection, rusqlite::Error> {
+    trace!("Establishing connection to sqlite...");
+    // Connect to the server
+    Connection::open("./squeelite.db")
+}
+
--- a/src/filesystem.rs
+++ b/src/filesystem.rs
@@ -0,0 +1,70 @@
+use std::{ffi::OsStr, path::PathBuf};
+
+use tokio::fs;
+use tracing::{debug, error, instrument, trace, warn};
+use url::Url;
+
+#[instrument(skip(data))]
+pub async fn store(data: &str, url: &Url) {
+    // extract data from url to save it accurately
+    let url_path = PathBuf::from("./downloaded/".to_string() + url.domain().unwrap_or("UnknownDomain") + url.path());
+
+    // if it's a file
+    let (basepath, filename) = if url_path.extension().filter(valid_file_extension).is_some() {
+        // get everything up till the file
+        let basepath = url_path.ancestors().skip(1).take(1).collect::<PathBuf>();
+        // get the file name
+        let filename = url_path.file_name().expect("This should exist").to_string_lossy();
+        trace!("Save path: {:?} and base path: {:?}", &url_path, &basepath);
+        (basepath, filename.to_string())
+    } else {
+        (url_path.clone(), "index.html".into())
+    };
+
+    debug!("Writing at: {:?} {:?}", basepath, filename);
+
+    // create the folders
+    if let Err(err) = fs::create_dir_all(&basepath).await {
+        error!("Dir creation: {err} {:?}", basepath);
+    } else {
+        // FIXME I don't think this handles index.html files well...
+        // TODO this should probably append .html to non-described files
+        // create the file if that was successful
+        if let Err(err) = fs::write(&basepath.join(filename), data).await {
+            error!("File creation: {err} {:?}", url_path);
+        }
+    }
+}
+
+fn valid_file_extension(take: &&OsStr) -> bool {
+    let los = take.to_string_lossy();
+    let all = los.split('.');
+    match all.last() {
+        Some(s) => {
+            match s.to_lowercase().as_str() {
+                "html" => true,
+                "css" => true,
+                "js" => true,
+                "ts" => true,
+                "otf" => true, // font
+
+                "png" => true,
+                "svg" => true,
+                "jpg" => true,
+                "jpeg" => true,
+                "mp4" => true,
+                "mp3" => true,
+                "webp" => true,
+
+                "pdf" => true,
+                "json" => true,
+                "xml" => true,
+                _ => {
+                    warn!("Might be forgetting a file extension: {s}");
+                    false
+                }
+            }
+        },
+        None => false,
+    }
+}
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,35 +1,221 @@
-use spider::{hashbrown::HashMap, tokio};
+#![feature(ip_from)]
+
+extern crate html5ever;
+
+use std::{
+    collections::HashSet, fs::File, io::Read, net::{IpAddr, Ipv4Addr}
+};
+
+use db::{connect, Website};
+use metrics::{counter, gauge};
+use metrics_exporter_prometheus::PrometheusBuilder;
+use serde::Deserialize;
+use surrealdb::{engine::remote::ws::Client, Surreal};
+use tokio::task::JoinSet;
+use tracing::{debug, error, info, instrument, level_filters::LevelFilter, trace, trace_span};
+use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer, Registry};
+
+mod db;
+mod parser;
+mod filesystem;
+
+const GET_METRIC: &str = "total_gets";
+const GET_IN_FLIGHT: &str = "gets_in_flight";
+const SITES_CRAWLED: &str = "pages_crawled";
+const BEING_PROCESSED: &str = "pages_being_processed";
+
+#[derive(Deserialize)]
+struct Config {
+    surreal_ns: String,
+    surreal_db: String,
+    surreal_url: String,
+    surreal_username: String,
+    surreal_password: String,
+
+    crawl_filter: String,
+    budget: usize,
+}

 #[tokio::main]
 async fn main() {
+    let writer = std::fs::OpenOptions::new()
+        .append(true)
+        .create(true)
+        .open("./docker/logs/tracing.log")
+        .expect("Couldn't make log file!");

-    let args = std::env::args().collect::<Vec<String>>();
+    let filter = EnvFilter::builder()
+        .with_default_directive(LevelFilter::DEBUG.into())
+        .from_env_lossy();

-    let url = &args[1];
-    let budget = &args[2];
-    let budget = match budget.parse::<u32>() {
-        Ok(x) => x,
-        Err(_) => panic!("Second arg must be a int"),
-    };
+    let registry = Registry::default().with(
+        fmt::layer()
+            .with_line_number(true)
+            .with_thread_ids(true)
+            .with_file(true)
+            .json()
+            .with_writer(writer)
+            .with_filter(filter)
+    );

+    tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber");

-    let mut site = spider::website::Website::new(url)
-        .with_budget(Some(HashMap::from([
-            ("*", budget),
-        ])))
-        .with_tld(true)
-        .with_on_link_find_callback(Some(|from, to| {
-            let from = from.as_ref().to_string();
-            let to = to.as_ref().to_string();
+    let builder = PrometheusBuilder::new();
+    builder
+        .with_http_listener(std::net::SocketAddr::new(
+            IpAddr::V4(Ipv4Addr::from_octets([0, 0, 0, 0])),
+            2500,
+        ))
+        .install()
+        .expect("failed to install recorder/exporter");

-            let from = from.trim();
-            let to= to.trim();
+    info!("Starting...");
+    // Would probably take these in as parameters from a cli
+    let starting_url = "https://en.wikipedia.org/";
+    // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
+    // let crawl_filter = "en.wikipedia.org/";
+    // let budget = 50;
+    let mut crawled = 0;

-            println!("{from};->;{to}"); 
-        }))
+    let mut file = File::open("./Crawler.toml").expect("Failed to read Crawler.toml");
+    let mut buf = String::new();
+    let _ = file.read_to_string(&mut buf);
+
+    let config: Config = toml::from_str(&buf).expect("Failed to parse Crawler.toml");
+
+    let db = connect(&config)
+        .await
+        .expect("Failed to connect to surreal, aborting.");
+
+    let reqwest = reqwest::Client::builder()
+        // .use_rustls_tls()
+        .gzip(true)
        .build()
-        .unwrap();
+        .expect("Failed to build reqwest client.");

-    site.crawl().await;
+    // Kick off the whole machine - This Website object doesn't matter, it's just to allow for
+    // get() to work.
+    let span = trace_span!("Pre-Loop");
+    let pre_loop_span = span.enter();
+    // Download the site
+    let site = Website::new(starting_url, false);
+    process(site, db.clone(), reqwest.clone()).await;
+
+    drop(pre_loop_span);
+
+    let span = trace_span!("Loop");
+    let span = span.enter();
+    while crawled < config.budget {
+        let get_num = if config.budget - crawled < 100 {
+            config.budget - crawled
+        } else {
+            100
+        };
+
+        let uncrawled = get_uncrawled_links(&db, get_num, config.crawl_filter.clone()).await;
+        if uncrawled.is_empty() {
+            info!("Had more budget but finished crawling everything.");
+            return;
+        }
+
+        {
+            let mut futures = JoinSet::new();
+            for site in uncrawled {
+                gauge!(BEING_PROCESSED).increment(1);
+                futures.spawn(process(site, db.clone(), reqwest.clone()));
+                // let percent = format!("{:.2}%", (crawled as f32 / budget as f32) * 100f32);
+                // info!("Crawled {crawled} out of {budget} pages. ({percent})");
+            }
+
+            let c = counter!(SITES_CRAWLED);
+            // As futures complete runs code in while block
+            while futures.join_next().await.is_some() {
+                c.increment(1);
+                gauge!(BEING_PROCESSED).decrement(1);
+                crawled += 1;
+            }
+        }
+    }
+    drop(span);
+
+    info!("Done");
+}
+
+#[instrument(skip(db, reqwest))]
+/// Downloads and crawls and stores a webpage.
+/// It is acceptable to clone `db`, `reqwest`, and `s3` because they all use `Arc`s internally. - Noted by Oliver
+async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Client) {
+    
+    // METRICS
+    trace!("Process: {}", &site.site);
+    // Build the request
+    let request_builder = reqwest.get(site.site.to_string());
+
+    // METRICS
+    let g = gauge!(GET_IN_FLIGHT);
+    g.increment(1);
+
+    // Send the http request (get)
+    if let Ok(response) = request_builder.send().await {
+
+        // METRICS
+        g.decrement(1);
+        counter!(GET_METRIC).increment(1);
+
+        // Get body from response
+        let data = response
+            .text()
+            .await
+            .expect("Failed to read http response's body!");
+
+        // Store document
+        filesystem::store(&data, &site.site).await;
+
+        // Parse document and get relationships
+        let sites = parser::parse(&site, &data).await;
+        
+        // update self in db
+        site.set_crawled();
+        Website::store_all(vec![site], &db).await;
+
+        // De-duplicate this list
+        let prev_len = sites.len();
+        let set = sites.into_iter().fold(HashSet::new(), |mut set,item| {
+            set.insert(item);
+            set
+        });
+        let de_dupe_sites: Vec<Website> = set.into_iter().collect();
+        let diff = prev_len - de_dupe_sites.len();
+        trace!("Saved {diff} from being entered into the db by de-duping");
+
+        // Store all the other sites so that we can link to them.
+        let _ = Website::store_all(de_dupe_sites, &db).await;
+
+    } else {
+        error!("Failed to get: {}", &site.site);
+    }
+}
+
+/// Returns uncrawled links
+#[instrument(skip(db))]
+async fn get_uncrawled_links(
+    db: &Surreal<Client>,
+    mut count: usize,
+    filter: String,
+) -> Vec<Website> {
+    if count > 100 {
+        count = 100
+    }
+    debug!("Getting uncrawled links");
+
+    let mut response = db
+        .query("SELECT * FROM website WHERE crawled = false AND site ~ type::string($format) LIMIT $count;")
+        .bind(("format", filter))
+        .bind(("count", count))
+        .await
+        .expect("Hard-coded query failed..?");
+    response
+        .take(0)
+        .expect("Returned websites couldn't be parsed")
 }

--- a/src/parser.rs
+++ b/src/parser.rs
@@ -0,0 +1,145 @@
+use std::default::Default;
+use std::str::FromStr;
+
+use html5ever::tokenizer::{BufferQueue, TokenizerResult};
+use html5ever::tokenizer::{StartTag, TagToken};
+use html5ever::tokenizer::{Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts};
+use html5ever::{local_name, tendril::*};
+use tracing::{debug, error, instrument, trace, warn};
+use url::Url;
+
+use crate::db::Website;
+
+impl TokenSink for Website {
+    type Handle = Vec<Website>;
+
+    #[instrument(skip(token, _line_number))]
+    fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<Self::Handle> {
+        match token {
+            TagToken(tag) => {
+                if tag.kind == StartTag {
+                    match tag.name {
+                        // this should be all the html elements that have links
+                        local_name!("a")
+                        | local_name!("audio")
+                        | local_name!("area")
+                        | local_name!("img")
+                        | local_name!("link")
+                        | local_name!("object")
+                        | local_name!("source")
+                        | local_name!("base")
+                        | local_name!("video") => {
+                            let mut links = Vec::new();
+                            for attr in &tag.attrs {
+                                let attr_name = attr.name.local.to_string();
+                                if attr_name == "src" || attr_name == "href" || attr_name == "data"
+                                {
+                                    trace!("Found `{}` in html `{}` tag", &attr.value, tag.name);
+                                    let url = try_get_url(&self.site, &attr.value);
+
+                                    if let Some(mut parsed) = url {
+                                        parsed.set_query(None);
+                                        parsed.set_fragment(None);
+                                        debug!("Final cleaned URL: `{}`", parsed.to_string());
+                                        let web = Website::new(&parsed.to_string(), false);
+                                        links.push(web);
+                                    }
+                                }
+                            }
+                            return TokenSinkResult::Script(links);
+                        }
+                        local_name!("button") | local_name!("meta") | local_name!("iframe") => {
+                            // dbg!(attrs);
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            _ => {}
+        }
+        TokenSinkResult::Continue
+    }
+}
+
+#[instrument(skip_all)]
+/// Parses the passed site and returns all the sites it links to.
+pub async fn parse(site: &Website, data: &str) -> Vec<Website> {
+    // prep work
+    let mut other_sites: Vec<Website> = Vec::new();
+
+    // change data into something that can be tokenized
+    let chunk = Tendril::from_str(data).expect("Failed to parse string into Tendril!");
+    // create buffer of tokens and push our input into it
+    let token_buffer = BufferQueue::default();
+    token_buffer.push_back(
+        chunk
+            .try_reinterpret::<fmt::UTF8>()
+            .expect("Failed to reinterpret chunk!"),
+    );
+    // create the tokenizer
+    let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
+
+    // go thru buffer
+    while let TokenizerResult::Script(mut sites) = tokenizer.feed(&token_buffer) {
+        other_sites.append(&mut sites);
+        // other_sites.push(sites);
+    }
+    assert!(token_buffer.is_empty());
+    tokenizer.end();
+
+    other_sites
+}
+
+#[instrument]
+fn try_get_url(parent: &Url, link: &str) -> Option<Url> {
+    match Url::parse(link) {
+        Ok(ok) => Some(ok),
+        Err(e) => {
+            if link.starts_with('#') {
+                trace!("Rejecting # url");
+                None
+            } else if link.starts_with("//") {
+                // if a url starts with "//" is assumed that it will adopt
+                // the same scheme as it's parent
+                // https://stackoverflow.com/questions/9646407/two-forward-slashes-in-a-url-src-href-attribute
+                let scheme = parent.scheme();
+
+                match Url::parse(&format!("{scheme}://{}", link)) {
+                    Ok(url) => Some(url),
+                    Err(err) => {
+                        error!("Failed parsing realative scheme url: {}", err);
+                        None
+                    }
+                }
+            } else {
+                // # This is some sort of realative url, gonna try patching it up into an absolute
+                // url
+                match e {
+                    url::ParseError::RelativeUrlWithoutBase => {
+                        // Is: scheme://host:port
+                        let origin = parent.origin().ascii_serialization();
+                        let url = origin.clone() + link;
+
+                        trace!("Built `{url}` from `{origin} + {}`", link.to_string());
+
+                        if let Ok(url) = Url::parse(&url) {
+                            trace!("Saved relative url `{}` AS: `{}`", link, url);
+                            Some(url)
+                        } else {
+                            error!(
+                                "Failed to reconstruct a url from relative url: `{}` on site: `{}`",
+                                link,
+                                parent.to_string()
+                            );
+                            None
+                        }
+                    }
+                    _ => {
+                        error!("MISC error: {:?} {:?}", e, link);
+                        None
+                    }
+                }
+            }
+        }
+    }
+}
--- a/src/setup.surql
+++ b/src/setup.surql
@@ -0,0 +1,9 @@
+DEFINE TABLE IF NOT EXISTS website SCHEMALESS;
+
+DEFINE FIELD IF NOT EXISTS site ON TABLE website TYPE string;
+DEFINE INDEX IF NOT EXISTS idx ON TABLE website COLUMNS site UNIQUE;
+
+DEFINE FIELD IF NOT EXISTS crawled ON TABLE website TYPE bool;
+
+DEFINE FIELD IF NOT EXISTS accessed_at ON TABLE website VALUE time::now();
+DEFINE FIELD IF NOT EXISTS first_accessed_at ON TABLE website VALUE time::now();
--- a/1
+++ b/1
@@ -1 +0,0 @@
-[{"result":[{"accessed_at":"2024-08-25T20:07:25.969525156Z","crawled":false,"domain":"google.com","id":"website:fd46b0cr5f5y3d57eje8","path":"/","url":"https://google.com"}],"status":"OK","time":"205.7
Author	SHA1	Message	Date
Oliver	4989a59ddf	checkpoint	2025-07-10 18:46:25 -06:00
Rushmore75	6fc71c7a78	add speed improvements	2025-03-21 12:14:29 -06:00
Rushmore75	96a3ca092a	:)	2025-03-21 12:11:05 -06:00
Rushmore75	b750d88d48	working filesystem storage	2025-03-21 11:42:43 -06:00
Oliver	808790a7c3	file patch;	2025-03-21 07:11:51 +00:00
Oliver	2de01b2a0e	remove removed code	2025-03-21 06:48:39 +00:00
Oliver	be0fd5505b	i think the files work better	2025-03-21 06:48:17 +00:00
Oliver	a23429104c	dead code removal	2025-03-21 06:03:34 +00:00
Oliver	66581cc453	getting there	2025-03-21 05:59:40 +00:00
Rushmore75	7df19a480f	updates	2025-03-20 15:11:01 -06:00
Rushmore75	b9c1f0b492	readme updates	2025-03-19 15:05:32 -06:00
Rushmore75	71b7b2d7bc	it works and it is awesome	2025-03-19 15:04:00 -06:00
Rushmore75	bac3cd9d1d	add most recent long run	2025-03-19 15:03:49 -06:00
Rushmore75	1f6a0acce3	shutup spellchecker	2025-03-19 15:03:39 -06:00
Rushmore75	53dbf53ab9	newest settings	2025-03-19 15:03:24 -06:00
Rushmore75	0477bb26e4	viz improvements	2025-03-19 15:03:11 -06:00
Rushmore75	6409baaffb	Reducted trips to surreal by x500	2025-03-19 12:41:08 -06:00
Oliver	135a7e4957	Merge pull request 'multithreading' (#2 ) from multithreading into main Reviewed-on: #2	2025-03-19 05:00:59 +00:00
Oliver	9aa34b3eee	epic metrics	2025-03-19 04:59:50 +00:00
Rushmore75	de80418c00	better logging	2025-03-18 16:09:46 -06:00
Rushmore75	e3e4175f51	logging improvements	2025-03-18 15:25:56 -06:00
Rushmore75	d11e7dd27c	the biggest 1 line improvement ever	2025-03-18 15:25:40 -06:00
Rushmore75	f2a3e836a0	spelling and clippy	2025-03-18 15:08:29 -06:00
Rushmore75	3b4e6a40ce	minimize vec resizing	2025-03-18 15:07:50 -06:00
Rushmore75	bd0b946245	fixed tracing	2025-03-18 15:02:32 -06:00
Rushmore75	b7540a4680	checkpoint - onto profiling	2025-03-18 10:53:06 -06:00
Oliver Atkinson	82929fd0fc	updating for base64	2024-12-13 13:28:24 -07:00
Oliver Atkinson	f42e770a10	moved to other repo	2024-12-13 11:01:35 -07:00
Oliver Atkinson	611a1e923b	starting on the extension	2024-12-12 15:32:04 -07:00
Oliver Atkinson	298ad39a79	rename	2024-12-12 14:59:54 -07:00
Oliver Atkinson	215056e493	use contains operator for better output	2024-12-12 14:26:49 -07:00
Oliver Atkinson	22be3b2f61	updating deps	2024-12-12 14:14:38 -07:00
Oliver Atkinson	c1c8cf07bb	unifed settings for testing	2024-12-12 11:42:07 -07:00
oliver	0f8a3d7215	using a custom parser now :)	2024-11-12 23:08:09 -07:00
oliver	574a370f30	readme updates	2024-11-12 21:24:57 -07:00
oliver	eaa79b749e	prepare get function for s3	2024-11-12 21:19:05 -07:00
oliver	2c28d69d55	add s3 support	2024-11-12 21:03:58 -07:00
oliver	d28d18de08	formatting and timer changes, consolidated functions	2024-11-12 18:40:10 -07:00
Oliver	8a5ac61b26	Merge pull request 'custom_engine' (#1 ) from custom_engine into main Reviewed-on: #1	2024-11-13 00:53:31 +00:00
oliver	d9d4c56142	fits with compose.yml	2024-11-12 17:51:13 -07:00
oliver	f87a43c3a9	added demo site for testing elements	2024-11-12 17:50:59 -07:00
oliver	7cac880f8e	remove un-used function	2024-11-12 17:50:28 -07:00
oliver	720adaa552	added support for nearly all html tags that can have a link	2024-11-12 17:50:06 -07:00
oliver	7c32600694	this is really all that's needed	2024-11-12 17:49:45 -07:00
oliver	399510c599	use reqwest client for epic speedup	2024-11-10 20:37:00 -07:00
oliver	ec66c4e765	remove unused import	2024-11-10 20:36:39 -07:00
oliver	a9628ee5e4	working, now onto speeding it up	2024-11-10 20:24:04 -07:00
oliver	5404d5c3e8	it works :party:	2024-11-09 23:30:57 -07:00
oliver	fd971bafbf	it works now	2024-11-09 15:28:10 -07:00
oliver	c3997b0bb7	works more, but still not all the way	2024-11-09 11:30:32 -07:00
Oliver Atkinson	7826c4cec6	jank-ish fix but it sure does work make the root record (for links https://example.com/) have a record id of the url, thus preventing duplication when using upsert	2024-10-31 15:32:37 -06:00
Oliver Atkinson	3a46dd937b	updates	2024-10-31 15:09:48 -06:00
Oliver Atkinson	fbca067b1f	clean up walk()	2024-10-31 14:10:14 -06:00
Oliver Atkinson	9324160e74	crawling 🕷️	2024-10-07 11:14:56 -06:00
Oliver Atkinson	974bccc457	no longer using spider, just wiritng my own crawler	2024-10-04 13:52:34 -06:00
				`@@ -1 +0,0 @@`
				`[{"result":[{"accessed_at":"2024-08-25T20:07:25.969525156Z","crawled":false,"domain":"google.com","id":"website:fd46b0cr5f5y3d57eje8","path":"/","url":"https://google.com"}],"status":"OK","time":"205.7`