custom_engine #1
4
.gitignore
vendored
4
.gitignore
vendored
@ -1 +1,5 @@
|
|||||||
/target
|
/target
|
||||||
|
/.surrealdb
|
||||||
|
perf.data
|
||||||
|
flamegraph.svg
|
||||||
|
perf.data.old
|
||||||
3
.vscode/launch.json
vendored
3
.vscode/launch.json
vendored
@ -8,6 +8,9 @@
|
|||||||
"type": "lldb",
|
"type": "lldb",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"name": "Debug executable 'surreal_spider'",
|
"name": "Debug executable 'surreal_spider'",
|
||||||
|
"env": {
|
||||||
|
"RUST_LOG": "surreal_spider=debug,reqwest=info",
|
||||||
|
},
|
||||||
"cargo": {
|
"cargo": {
|
||||||
"args": [
|
"args": [
|
||||||
"build",
|
"build",
|
||||||
|
|||||||
3375
Cargo.lock
generated
3375
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
10
Cargo.toml
10
Cargo.toml
@ -4,4 +4,12 @@ version = "0.1.0"
|
|||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
spider = { features = [], git="https://github.com/Rushmore75/spider.git", rev="ff91646973ad04ff423010f36206f550e37c4278" }
|
html5ever = "0.29.0"
|
||||||
|
markup5ever_rcdom = "0.5.0-unofficial"
|
||||||
|
reqwest = "0.12.9"
|
||||||
|
serde = { version = "1.0.214", features = ["derive"] }
|
||||||
|
surrealdb = "2.0.4"
|
||||||
|
tokio = { version="1.41.0", features = ["full"] }
|
||||||
|
tracing = "0.1.40"
|
||||||
|
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
|
||||||
|
url = { version = "2.5.3", features = ["serde"] }
|
||||||
|
|||||||
@ -3,6 +3,8 @@ services:
|
|||||||
image: surrealdb/surrealdb:latest-dev
|
image: surrealdb/surrealdb:latest-dev
|
||||||
ports:
|
ports:
|
||||||
- 8000:8000
|
- 8000:8000
|
||||||
|
volumes:
|
||||||
|
- ./.surrealdb/:/mydata
|
||||||
command:
|
command:
|
||||||
- start
|
- start
|
||||||
- --log
|
- --log
|
||||||
@ -11,3 +13,4 @@ services:
|
|||||||
- root
|
- root
|
||||||
- --pass
|
- --pass
|
||||||
- root
|
- root
|
||||||
|
- rocksdb:/mydata/database.db
|
||||||
|
|||||||
BIN
demo_site/audio.mp3
Normal file
BIN
demo_site/audio.mp3
Normal file
Binary file not shown.
24
demo_site/index.html
Normal file
24
demo_site/index.html
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Page Title</title>
|
||||||
|
<script src='main.js'></script>
|
||||||
|
<base href="/" target="_blank">
|
||||||
|
<link rel="stylesheet" href="style.css">
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<a href="/">Link text</a>
|
||||||
|
<img src="/picture.png"></img>
|
||||||
|
<audio src="/audio.mp3" controls>Audio is unsupported</audio>
|
||||||
|
<video src="/video.mp4" controls>Video is unsupported</video>
|
||||||
|
<audio controls>
|
||||||
|
<source src="audio.mp3"></source>
|
||||||
|
</audio>
|
||||||
|
<map>
|
||||||
|
<area href="/"></area>
|
||||||
|
</map>
|
||||||
|
<object data="/video.mp4"></object>
|
||||||
|
<iframe src="/" title="The same Site"></iframe>
|
||||||
|
<meta>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
BIN
demo_site/picture.png
Normal file
BIN
demo_site/picture.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 111 KiB |
1
demo_site/script.js
Normal file
1
demo_site/script.js
Normal file
@ -0,0 +1 @@
|
|||||||
|
alert("alert")
|
||||||
3
demo_site/style.css
Normal file
3
demo_site/style.css
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
html {
|
||||||
|
background-color: white;
|
||||||
|
}
|
||||||
BIN
demo_site/video.mp4
Normal file
BIN
demo_site/video.mp4
Normal file
Binary file not shown.
49
driver.py
49
driver.py
@ -1,49 +0,0 @@
|
|||||||
from surrealdb import Surreal
|
|
||||||
import fileinput
|
|
||||||
import asyncio
|
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
|
||||||
async with Surreal("ws://localhost:8000/rpc") as db:
|
|
||||||
|
|
||||||
await db.signin({"user": "root", "pass": "root"})
|
|
||||||
await db.use("test", "test")
|
|
||||||
|
|
||||||
for line in fileinput.input():
|
|
||||||
# Testing
|
|
||||||
# for line in ["https://oliveratkinson.net;->;http://google.com"]:
|
|
||||||
|
|
||||||
urls = line.split(";->;")
|
|
||||||
ffrom = urls[0].strip()
|
|
||||||
to= urls[1].strip()
|
|
||||||
|
|
||||||
to_id = None
|
|
||||||
from_id = None
|
|
||||||
|
|
||||||
if (ffrom == to):
|
|
||||||
print('Site has self-reference, ignoring')
|
|
||||||
continue
|
|
||||||
|
|
||||||
# FROM
|
|
||||||
result = await db.query(f"SELECT id FROM website WHERE url='{ffrom}'")
|
|
||||||
if (len(result[0]['result']) > 0):
|
|
||||||
from_id = result[0]['result'][0]['id']
|
|
||||||
await db.query(f"UPDATE {from_id} MERGE {{ crawled: True }}")
|
|
||||||
else:
|
|
||||||
from_response = await db.create("website", { "url":ffrom, "crawled":True, } )
|
|
||||||
from_id = from_response[0]["id"]
|
|
||||||
|
|
||||||
# TO
|
|
||||||
result = await db.query(f"SELECT id FROM website WHERE url='{to}'")
|
|
||||||
if (len(result[0]['result']) < 1):
|
|
||||||
# Object doesn't exists yet
|
|
||||||
to_response = await db.create("website", { "url":to })
|
|
||||||
to_id = to_response[0]["id"]
|
|
||||||
else:
|
|
||||||
to_id = result[0]['result'][0]['id']
|
|
||||||
|
|
||||||
await db.query(f"RELATE {from_id} -> links_to -> {to_id}");
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(main())
|
|
||||||
|
|
||||||
370
poetry.lock
generated
370
poetry.lock
generated
@ -1,370 +0,0 @@
|
|||||||
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "annotated-types"
|
|
||||||
version = "0.7.0"
|
|
||||||
description = "Reusable constraint types to use with typing.Annotated"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.8"
|
|
||||||
files = [
|
|
||||||
{file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
|
|
||||||
{file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "anyio"
|
|
||||||
version = "4.4.0"
|
|
||||||
description = "High level compatibility layer for multiple asynchronous event loop implementations"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.8"
|
|
||||||
files = [
|
|
||||||
{file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"},
|
|
||||||
{file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.dependencies]
|
|
||||||
exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""}
|
|
||||||
idna = ">=2.8"
|
|
||||||
sniffio = ">=1.1"
|
|
||||||
typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""}
|
|
||||||
|
|
||||||
[package.extras]
|
|
||||||
doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
|
|
||||||
test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
|
|
||||||
trio = ["trio (>=0.23)"]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "certifi"
|
|
||||||
version = "2024.7.4"
|
|
||||||
description = "Python package for providing Mozilla's CA Bundle."
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.6"
|
|
||||||
files = [
|
|
||||||
{file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"},
|
|
||||||
{file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "exceptiongroup"
|
|
||||||
version = "1.2.2"
|
|
||||||
description = "Backport of PEP 654 (exception groups)"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.7"
|
|
||||||
files = [
|
|
||||||
{file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"},
|
|
||||||
{file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.extras]
|
|
||||||
test = ["pytest (>=6)"]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "h11"
|
|
||||||
version = "0.14.0"
|
|
||||||
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.7"
|
|
||||||
files = [
|
|
||||||
{file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
|
|
||||||
{file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "httpcore"
|
|
||||||
version = "1.0.5"
|
|
||||||
description = "A minimal low-level HTTP client."
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.8"
|
|
||||||
files = [
|
|
||||||
{file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"},
|
|
||||||
{file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.dependencies]
|
|
||||||
certifi = "*"
|
|
||||||
h11 = ">=0.13,<0.15"
|
|
||||||
|
|
||||||
[package.extras]
|
|
||||||
asyncio = ["anyio (>=4.0,<5.0)"]
|
|
||||||
http2 = ["h2 (>=3,<5)"]
|
|
||||||
socks = ["socksio (==1.*)"]
|
|
||||||
trio = ["trio (>=0.22.0,<0.26.0)"]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "httpx"
|
|
||||||
version = "0.27.0"
|
|
||||||
description = "The next generation HTTP client."
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.8"
|
|
||||||
files = [
|
|
||||||
{file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"},
|
|
||||||
{file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.dependencies]
|
|
||||||
anyio = "*"
|
|
||||||
certifi = "*"
|
|
||||||
httpcore = "==1.*"
|
|
||||||
idna = "*"
|
|
||||||
sniffio = "*"
|
|
||||||
|
|
||||||
[package.extras]
|
|
||||||
brotli = ["brotli", "brotlicffi"]
|
|
||||||
cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
|
|
||||||
http2 = ["h2 (>=3,<5)"]
|
|
||||||
socks = ["socksio (==1.*)"]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "idna"
|
|
||||||
version = "3.8"
|
|
||||||
description = "Internationalized Domain Names in Applications (IDNA)"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.6"
|
|
||||||
files = [
|
|
||||||
{file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"},
|
|
||||||
{file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pydantic"
|
|
||||||
version = "2.8.2"
|
|
||||||
description = "Data validation using Python type hints"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.8"
|
|
||||||
files = [
|
|
||||||
{file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"},
|
|
||||||
{file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.dependencies]
|
|
||||||
annotated-types = ">=0.4.0"
|
|
||||||
pydantic-core = "2.20.1"
|
|
||||||
typing-extensions = [
|
|
||||||
{version = ">=4.12.2", markers = "python_version >= \"3.13\""},
|
|
||||||
{version = ">=4.6.1", markers = "python_version < \"3.13\""},
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.extras]
|
|
||||||
email = ["email-validator (>=2.0.0)"]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pydantic-core"
|
|
||||||
version = "2.20.1"
|
|
||||||
description = "Core functionality for Pydantic validation and serialization"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.8"
|
|
||||||
files = [
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3acae97ffd19bf091c72df4d726d552c473f3576409b2a7ca36b2f535ffff4a3"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41f4c96227a67a013e7de5ff8f20fb496ce573893b7f4f2707d065907bffdbd6"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f239eb799a2081495ea659d8d4a43a8f42cd1fe9ff2e7e436295c38a10c286a"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53e431da3fc53360db73eedf6f7124d1076e1b4ee4276b36fb25514544ceb4a3"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1f62b2413c3a0e846c3b838b2ecd6c7a19ec6793b2a522745b0869e37ab5bc1"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d41e6daee2813ecceea8eda38062d69e280b39df793f5a942fa515b8ed67953"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d482efec8b7dc6bfaedc0f166b2ce349df0011f5d2f1f25537ced4cfc34fd98"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e93e1a4b4b33daed65d781a57a522ff153dcf748dee70b40c7258c5861e1768a"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7c4ea22b6739b162c9ecaaa41d718dfad48a244909fe7ef4b54c0b530effc5a"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4f2790949cf385d985a31984907fecb3896999329103df4e4983a4a41e13e840"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-none-win32.whl", hash = "sha256:5e999ba8dd90e93d57410c5e67ebb67ffcaadcea0ad973240fdfd3a135506250"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp310-none-win_amd64.whl", hash = "sha256:512ecfbefef6dac7bc5eaaf46177b2de58cdf7acac8793fe033b24ece0b9566c"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4f55095ad087474999ee28d3398bae183a66be4823f753cd7d67dd0153427c9"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9aa05d09ecf4c75157197f27cdc9cfaeb7c5f15021c6373932bf3e124af029f"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e97fdf088d4b31ff4ba35db26d9cc472ac7ef4a2ff2badeabf8d727b3377fc52"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc633a9fe1eb87e250b5c57d389cf28998e4292336926b0b6cdaee353f89a237"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d573faf8eb7e6b1cbbcb4f5b247c60ca8be39fe2c674495df0eb4318303137fe"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26dc97754b57d2fd00ac2b24dfa341abffc380b823211994c4efac7f13b9e90e"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:33499e85e739a4b60c9dac710c20a08dc73cb3240c9a0e22325e671b27b70d24"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bebb4d6715c814597f85297c332297c6ce81e29436125ca59d1159b07f423eb1"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:516d9227919612425c8ef1c9b869bbbee249bc91912c8aaffb66116c0b447ebd"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-none-win32.whl", hash = "sha256:469f29f9093c9d834432034d33f5fe45699e664f12a13bf38c04967ce233d688"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp312-none-win_amd64.whl", hash = "sha256:035ede2e16da7281041f0e626459bcae33ed998cca6a0a007a5ebb73414ac72d"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0827505a5c87e8aa285dc31e9ec7f4a17c81a813d45f70b1d9164e03a813a686"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19c0fa39fa154e7e0b7f82f88ef85faa2a4c23cc65aae2f5aea625e3c13c735a"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa223cd1e36b642092c326d694d8bf59b71ddddc94cdb752bbbb1c5c91d833b"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c336a6d235522a62fef872c6295a42ecb0c4e1d0f1a3e500fe949415761b8a19"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7eb6a0587eded33aeefea9f916899d42b1799b7b14b8f8ff2753c0ac1741edac"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70c8daf4faca8da5a6d655f9af86faf6ec2e1768f4b8b9d0226c02f3d6209703"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fa4c9bf273ca41f940bceb86922a7667cd5bf90e95dbb157cbb8441008482c"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:11b71d67b4725e7e2a9f6e9c0ac1239bbc0c48cce3dc59f98635efc57d6dac83"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:270755f15174fb983890c49881e93f8f1b80f0b5e3a3cc1394a255706cabd203"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c81131869240e3e568916ef4c307f8b99583efaa60a8112ef27a366eefba8ef0"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-none-win32.whl", hash = "sha256:b91ced227c41aa29c672814f50dbb05ec93536abf8f43cd14ec9521ea09afe4e"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp313-none-win_amd64.whl", hash = "sha256:65db0f2eefcaad1a3950f498aabb4875c8890438bc80b19362cf633b87a8ab20"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4745f4ac52cc6686390c40eaa01d48b18997cb130833154801a442323cc78f91"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8ad4c766d3f33ba8fd692f9aa297c9058970530a32c728a2c4bfd2616d3358b"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41e81317dd6a0127cabce83c0c9c3fbecceae981c8391e6f1dec88a77c8a569a"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04024d270cf63f586ad41fff13fde4311c4fc13ea74676962c876d9577bcc78f"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaad4ff2de1c3823fddf82f41121bdf453d922e9a238642b1dedb33c4e4f98ad"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26ab812fa0c845df815e506be30337e2df27e88399b985d0bb4e3ecfe72df31c"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c5ebac750d9d5f2706654c638c041635c385596caf68f81342011ddfa1e5598"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2aafc5a503855ea5885559eae883978c9b6d8c8993d67766ee73d82e841300dd"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4868f6bd7c9d98904b748a2653031fc9c2f85b6237009d475b1008bfaeb0a5aa"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa2f457b4af386254372dfa78a2eda2563680d982422641a85f271c859df1987"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-none-win32.whl", hash = "sha256:225b67a1f6d602de0ce7f6c1c3ae89a4aa25d3de9be857999e9124f15dab486a"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp38-none-win_amd64.whl", hash = "sha256:6b507132dcfc0dea440cce23ee2182c0ce7aba7054576efc65634f080dbe9434"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b03f7941783b4c4a26051846dea594628b38f6940a2fdc0df00b221aed39314c"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1eedfeb6089ed3fad42e81a67755846ad4dcc14d73698c120a82e4ccf0f1f9f6"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:635fee4e041ab9c479e31edda27fcf966ea9614fff1317e280d99eb3e5ab6fe2"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77bf3ac639c1ff567ae3b47f8d4cc3dc20f9966a2a6dd2311dcc055d3d04fb8a"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ed1b0132f24beeec5a78b67d9388656d03e6a7c837394f99257e2d55b461611"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6514f963b023aeee506678a1cf821fe31159b925c4b76fe2afa94cc70b3222b"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10d4204d8ca33146e761c79f83cc861df20e7ae9f6487ca290a97702daf56006"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d036c7187b9422ae5b262badb87a20a49eb6c5238b2004e96d4da1231badef1"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ebfef07dbe1d93efb94b4700f2d278494e9162565a54f124c404a5656d7ff09"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6b9d9bb600328a1ce523ab4f454859e9d439150abb0906c5a1983c146580ebab"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-none-win32.whl", hash = "sha256:784c1214cb6dd1e3b15dd8b91b9a53852aed16671cc3fbe4786f4f1db07089e2"},
|
|
||||||
{file = "pydantic_core-2.20.1-cp39-none-win_amd64.whl", hash = "sha256:d2fe69c5434391727efa54b47a1e7986bb0186e72a41b203df8f5b0a19a4f669"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"},
|
|
||||||
{file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"},
|
|
||||||
{file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.dependencies]
|
|
||||||
typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "sniffio"
|
|
||||||
version = "1.3.1"
|
|
||||||
description = "Sniff out which async library your code is running under"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.7"
|
|
||||||
files = [
|
|
||||||
{file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"},
|
|
||||||
{file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "surrealdb"
|
|
||||||
version = "0.3.2"
|
|
||||||
description = "The official SurrealDB library for Python."
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.8,<4.0"
|
|
||||||
files = [
|
|
||||||
{file = "surrealdb-0.3.2-py3-none-any.whl", hash = "sha256:30424971be9698f0fce61dc8387576d19302e0bb0ae708d311e79a261ead7701"},
|
|
||||||
{file = "surrealdb-0.3.2.tar.gz", hash = "sha256:8ab570b8d4299f35443f0d4fa636947cf1a18f67fd88acc00e5e5a36356e82cd"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.dependencies]
|
|
||||||
httpx = ">=0.23.0"
|
|
||||||
pydantic = ">=2.1.0,<3.0.0"
|
|
||||||
websockets = ">=10.4,<11.0"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "typing-extensions"
|
|
||||||
version = "4.12.2"
|
|
||||||
description = "Backported and Experimental Type Hints for Python 3.8+"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.8"
|
|
||||||
files = [
|
|
||||||
{file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
|
|
||||||
{file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "websockets"
|
|
||||||
version = "10.4"
|
|
||||||
description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.7"
|
|
||||||
files = [
|
|
||||||
{file = "websockets-10.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d58804e996d7d2307173d56c297cf7bc132c52df27a3efaac5e8d43e36c21c48"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc0b82d728fe21a0d03e65f81980abbbcb13b5387f733a1a870672c5be26edab"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ba089c499e1f4155d2a3c2a05d2878a3428cf321c848f2b5a45ce55f0d7d310c"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33d69ca7612f0ddff3316b0c7b33ca180d464ecac2d115805c044bf0a3b0d032"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62e627f6b6d4aed919a2052efc408da7a545c606268d5ab5bfab4432734b82b4"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ea7b82bfcae927eeffc55d2ffa31665dc7fec7b8dc654506b8e5a518eb4d50"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e0cb5cc6ece6ffa75baccfd5c02cffe776f3f5c8bf486811f9d3ea3453676ce8"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae5e95cfb53ab1da62185e23b3130e11d64431179debac6dc3c6acf08760e9b1"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7c584f366f46ba667cfa66020344886cf47088e79c9b9d39c84ce9ea98aaa331"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-win32.whl", hash = "sha256:b029fb2032ae4724d8ae8d4f6b363f2cc39e4c7b12454df8df7f0f563ed3e61a"},
|
|
||||||
{file = "websockets-10.4-cp310-cp310-win_amd64.whl", hash = "sha256:8dc96f64ae43dde92530775e9cb169979f414dcf5cff670455d81a6823b42089"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:47a2964021f2110116cc1125b3e6d87ab5ad16dea161949e7244ec583b905bb4"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e789376b52c295c4946403bd0efecf27ab98f05319df4583d3c48e43c7342c2f"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d3f0b61c45c3fa9a349cf484962c559a8a1d80dae6977276df8fd1fa5e3cb8c"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f55b5905705725af31ccef50e55391621532cd64fbf0bc6f4bac935f0fccec46"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00c870522cdb69cd625b93f002961ffb0c095394f06ba8c48f17eef7c1541f96"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f38706e0b15d3c20ef6259fd4bc1700cd133b06c3c1bb108ffe3f8947be15fa"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f2c38d588887a609191d30e902df2a32711f708abfd85d318ca9b367258cfd0c"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fe10ddc59b304cb19a1bdf5bd0a7719cbbc9fbdd57ac80ed436b709fcf889106"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:90fcf8929836d4a0e964d799a58823547df5a5e9afa83081761630553be731f9"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-win32.whl", hash = "sha256:b9968694c5f467bf67ef97ae7ad4d56d14be2751000c1207d31bf3bb8860bae8"},
|
|
||||||
{file = "websockets-10.4-cp311-cp311-win_amd64.whl", hash = "sha256:a7a240d7a74bf8d5cb3bfe6be7f21697a28ec4b1a437607bae08ac7acf5b4882"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:74de2b894b47f1d21cbd0b37a5e2b2392ad95d17ae983e64727e18eb281fe7cb"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3a686ecb4aa0d64ae60c9c9f1a7d5d46cab9bfb5d91a2d303d00e2cd4c4c5cc"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d15c968ea7a65211e084f523151dbf8ae44634de03c801b8bd070b74e85033"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00213676a2e46b6ebf6045bc11d0f529d9120baa6f58d122b4021ad92adabd41"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:e23173580d740bf8822fd0379e4bf30aa1d5a92a4f252d34e893070c081050df"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:dd500e0a5e11969cdd3320935ca2ff1e936f2358f9c2e61f100a1660933320ea"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4239b6027e3d66a89446908ff3027d2737afc1a375f8fd3eea630a4842ec9a0c"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-win32.whl", hash = "sha256:8a5cc00546e0a701da4639aa0bbcb0ae2bb678c87f46da01ac2d789e1f2d2038"},
|
|
||||||
{file = "websockets-10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:a9f9a735deaf9a0cadc2d8c50d1a5bcdbae8b6e539c6e08237bc4082d7c13f28"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5c1289596042fad2cdceb05e1ebf7aadf9995c928e0da2b7a4e99494953b1b94"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0cff816f51fb33c26d6e2b16b5c7d48eaa31dae5488ace6aae468b361f422b63"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dd9becd5fe29773d140d68d607d66a38f60e31b86df75332703757ee645b6faf"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45ec8e75b7dbc9539cbfafa570742fe4f676eb8b0d3694b67dabe2f2ceed8aa6"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f72e5cd0f18f262f5da20efa9e241699e0cf3a766317a17392550c9ad7b37d8"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185929b4808b36a79c65b7865783b87b6841e852ef5407a2fb0c03381092fa3b"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7d27a7e34c313b3a7f91adcd05134315002aaf8540d7b4f90336beafaea6217c"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:884be66c76a444c59f801ac13f40c76f176f1bfa815ef5b8ed44321e74f1600b"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:931c039af54fc195fe6ad536fde4b0de04da9d5916e78e55405436348cfb0e56"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-win32.whl", hash = "sha256:db3c336f9eda2532ec0fd8ea49fef7a8df8f6c804cdf4f39e5c5c0d4a4ad9a7a"},
|
|
||||||
{file = "websockets-10.4-cp38-cp38-win_amd64.whl", hash = "sha256:48c08473563323f9c9debac781ecf66f94ad5a3680a38fe84dee5388cf5acaf6"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:40e826de3085721dabc7cf9bfd41682dadc02286d8cf149b3ad05bff89311e4f"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:56029457f219ade1f2fc12a6504ea61e14ee227a815531f9738e41203a429112"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f5fc088b7a32f244c519a048c170f14cf2251b849ef0e20cbbb0fdf0fdaf556f"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc8709c00704194213d45e455adc106ff9e87658297f72d544220e32029cd3d"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0154f7691e4fe6c2b2bc275b5701e8b158dae92a1ab229e2b940efe11905dff4"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c6d2264f485f0b53adf22697ac11e261ce84805c232ed5dbe6b1bcb84b00ff0"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9bc42e8402dc5e9905fb8b9649f57efcb2056693b7e88faa8fb029256ba9c68c"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:edc344de4dac1d89300a053ac973299e82d3db56330f3494905643bb68801269"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:84bc2a7d075f32f6ed98652db3a680a17a4edb21ca7f80fe42e38753a58ee02b"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-win32.whl", hash = "sha256:c94ae4faf2d09f7c81847c63843f84fe47bf6253c9d60b20f25edfd30fb12588"},
|
|
||||||
{file = "websockets-10.4-cp39-cp39-win_amd64.whl", hash = "sha256:bbccd847aa0c3a69b5f691a84d2341a4f8a629c6922558f2a70611305f902d74"},
|
|
||||||
{file = "websockets-10.4-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:82ff5e1cae4e855147fd57a2863376ed7454134c2bf49ec604dfe71e446e2193"},
|
|
||||||
{file = "websockets-10.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d210abe51b5da0ffdbf7b43eed0cfdff8a55a1ab17abbec4301c9ff077dd0342"},
|
|
||||||
{file = "websockets-10.4-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:942de28af58f352a6f588bc72490ae0f4ccd6dfc2bd3de5945b882a078e4e179"},
|
|
||||||
{file = "websockets-10.4-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9b27d6c1c6cd53dc93614967e9ce00ae7f864a2d9f99fe5ed86706e1ecbf485"},
|
|
||||||
{file = "websockets-10.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3d3cac3e32b2c8414f4f87c1b2ab686fa6284a980ba283617404377cd448f631"},
|
|
||||||
{file = "websockets-10.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:da39dd03d130162deb63da51f6e66ed73032ae62e74aaccc4236e30edccddbb0"},
|
|
||||||
{file = "websockets-10.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389f8dbb5c489e305fb113ca1b6bdcdaa130923f77485db5b189de343a179393"},
|
|
||||||
{file = "websockets-10.4-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09a1814bb15eff7069e51fed0826df0bc0702652b5cb8f87697d469d79c23576"},
|
|
||||||
{file = "websockets-10.4-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff64a1d38d156d429404aaa84b27305e957fd10c30e5880d1765c9480bea490f"},
|
|
||||||
{file = "websockets-10.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b343f521b047493dc4022dd338fc6db9d9282658862756b4f6fd0e996c1380e1"},
|
|
||||||
{file = "websockets-10.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:932af322458da7e4e35df32f050389e13d3d96b09d274b22a7aa1808f292fee4"},
|
|
||||||
{file = "websockets-10.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a4162139374a49eb18ef5b2f4da1dd95c994588f5033d64e0bbfda4b6b6fcf"},
|
|
||||||
{file = "websockets-10.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c57e4c1349fbe0e446c9fa7b19ed2f8a4417233b6984277cce392819123142d3"},
|
|
||||||
{file = "websockets-10.4-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b627c266f295de9dea86bd1112ed3d5fafb69a348af30a2422e16590a8ecba13"},
|
|
||||||
{file = "websockets-10.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:05a7233089f8bd355e8cbe127c2e8ca0b4ea55467861906b80d2ebc7db4d6b72"},
|
|
||||||
{file = "websockets-10.4.tar.gz", hash = "sha256:eef610b23933c54d5d921c92578ae5f89813438fded840c2e9809d378dc765d3"},
|
|
||||||
]
|
|
||||||
|
|
||||||
[metadata]
|
|
||||||
lock-version = "2.0"
|
|
||||||
python-versions = "^3.10"
|
|
||||||
content-hash = "e5d2be363dd4d62589502cb214d66ac9f21992ad2fb62a09a50e243000600518"
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
[tool.poetry]
|
|
||||||
name = "surreal_spider"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = ""
|
|
||||||
authors = ["oliver <oliveratkinsoneng@gmail.com>"]
|
|
||||||
|
|
||||||
[tool.poetry.dependencies]
|
|
||||||
python = "^3.10"
|
|
||||||
surrealdb = "^0.3.2"
|
|
||||||
|
|
||||||
[tool.poetry.dev-dependencies]
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["poetry-core>=1.0.0"]
|
|
||||||
build-backend = "poetry.core.masonry.api"
|
|
||||||
15
schema.surql
15
schema.surql
@ -1,15 +1,2 @@
|
|||||||
DELETE website;
|
DEFINE TABLE website SCHEMALESS;
|
||||||
DELETE links_to;
|
|
||||||
DELETE url_index;
|
|
||||||
|
|
||||||
DEFINE TABLE website SCHEMAFULL;
|
|
||||||
DEFINE FIELD accessed_at ON TABLE website VALUE time::now();
|
DEFINE FIELD accessed_at ON TABLE website VALUE time::now();
|
||||||
DEFINE FIELD crawled ON TABLE website TYPE bool DEFAULT false;
|
|
||||||
DEFINE FIELD url ON TABLE website TYPE string;
|
|
||||||
DEFINE FIELD domain ON TABLE website VALUE parse::url::domain($this.url) ASSERT !type::is::none($value);
|
|
||||||
DEFINE FIELD path ON TABLE website VALUE parse::url::path($this.url) ASSERT !type::is::none($value);
|
|
||||||
DEFINE INDEX url_index ON TABLE website COLUMNS url UNIQUE;
|
|
||||||
|
|
||||||
DEFINE TABLE links_to SCHEMAFULL TYPE RELATION FROM website TO website;
|
|
||||||
DEFINE FIELD discovered_at ON TABLE links_to TYPE datetime DEFAULT time::now();
|
|
||||||
|
|
||||||
|
|||||||
167
src/db.rs
Normal file
167
src/db.rs
Normal file
@ -0,0 +1,167 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use surrealdb::{
|
||||||
|
engine::remote::ws::{Client, Ws},
|
||||||
|
opt::auth::Root,
|
||||||
|
sql::Thing,
|
||||||
|
Response, Surreal,
|
||||||
|
};
|
||||||
|
use tracing::{error, instrument, trace, warn};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
|
use crate::Timer;
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||||
|
pub struct Website {
|
||||||
|
/// The url that this data is found at
|
||||||
|
site: Url,
|
||||||
|
/// Wether or not this link has been crawled yet
|
||||||
|
pub crawled: bool,
|
||||||
|
#[serde(skip_serializing)]
|
||||||
|
id: Option<Thing>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Website {
|
||||||
|
/// Creates a blank site (assumes that url param is site's root)
|
||||||
|
pub fn new(url: &str, crawled: bool) -> Self {
|
||||||
|
let site = match Url::parse(url) {
|
||||||
|
Ok(a) => a,
|
||||||
|
Err(_) => todo!(),
|
||||||
|
};
|
||||||
|
Self {
|
||||||
|
id: None,
|
||||||
|
crawled,
|
||||||
|
site,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_crawled(&mut self) {
|
||||||
|
trace!("Set crawled to true");
|
||||||
|
self.crawled = true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn mut_url(&mut self) -> &mut Url {
|
||||||
|
&mut self.site
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all)]
|
||||||
|
pub async fn links_to(&self, other: Vec<Thing>, db: &Surreal<Client>) {
|
||||||
|
let len = other.len();
|
||||||
|
if len == 0 {return}
|
||||||
|
|
||||||
|
let from = self.site.to_string();
|
||||||
|
// let to = other.site.to_string();
|
||||||
|
trace!("Linking {from} to {} other pages.", other.len());
|
||||||
|
let msg = format!("Linked {len} pages");
|
||||||
|
let timer = Timer::start(&msg);
|
||||||
|
// prevent the timer from being dropped instantly.
|
||||||
|
let _ = timer;
|
||||||
|
match db
|
||||||
|
.query("COUNT(RELATE (SELECT id FROM website WHERE site = $in) -> links_to -> $out)")
|
||||||
|
.bind(("in", from))
|
||||||
|
.bind(("out", other))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(mut e) => {
|
||||||
|
// The relate could technically "fail" (not relate anything), this just means that
|
||||||
|
// the query was ok.
|
||||||
|
let _: Response = e;
|
||||||
|
if let Ok(vec) = e.take(0) {
|
||||||
|
let _: Vec<usize> = vec;
|
||||||
|
if let Some(num) = vec.get(0) {
|
||||||
|
if *num == len {
|
||||||
|
trace!("Link OK");
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
warn!("Didn't link all the records. {num}/{len}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
warn!("Linking request succeeded but couldn't verify the results.");
|
||||||
|
},
|
||||||
|
Err(e) => {
|
||||||
|
error!("{}", e.to_string());
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all)]
|
||||||
|
pub async fn store(&mut self, db: &Surreal<Client>) -> Option<Thing> {
|
||||||
|
// check if it's been gone thru before
|
||||||
|
let mut response = db
|
||||||
|
.query("SELECT * FROM ONLY website WHERE site = $site LIMIT 1")
|
||||||
|
.bind(("site", self.site.to_string()))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
if let Some(old) = response.take::<Option<Website>>(0).unwrap() {
|
||||||
|
// site exists already
|
||||||
|
if let Some(id) = old.id {
|
||||||
|
// make sure to preserve the "crawled status"
|
||||||
|
let mut new = self.clone();
|
||||||
|
new.crawled = old.crawled | new.crawled;
|
||||||
|
|
||||||
|
// update the record
|
||||||
|
match db.upsert((id.tb, id.id.to_string())).content(new).await {
|
||||||
|
Ok(e) => {
|
||||||
|
if let Some(a) = e {
|
||||||
|
let _: Record = a;
|
||||||
|
return Some(a.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("{}", e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// sites hasn't existed yet
|
||||||
|
match db.create("website").content(self.clone()).await {
|
||||||
|
Ok(e) => {
|
||||||
|
let _: Option<Record> = e;
|
||||||
|
if let Some(a) = e {
|
||||||
|
let _: Record = a;
|
||||||
|
return Some(a.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(a) => error!("{:?}", a),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ToString for Website {
|
||||||
|
fn to_string(&self) -> String {
|
||||||
|
self.site.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct Email {
|
||||||
|
pub email: String,
|
||||||
|
pub on: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct Record {
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub id: Thing,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn connect() -> surrealdb::Result<Surreal<Client>> {
|
||||||
|
// Connect to the server
|
||||||
|
let db = Surreal::new::<Ws>("127.0.0.1:8000").await?;
|
||||||
|
|
||||||
|
// Signin as a namespace, database, or root user
|
||||||
|
db.signin(Root {
|
||||||
|
username: "root",
|
||||||
|
password: "root",
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
// Select a specific namespace / database
|
||||||
|
db.use_ns("test").use_db("v1.2").await?;
|
||||||
|
|
||||||
|
Ok(db)
|
||||||
|
}
|
||||||
223
src/main.rs
223
src/main.rs
@ -1,35 +1,210 @@
|
|||||||
use spider::{hashbrown::HashMap, tokio};
|
extern crate markup5ever_rcdom as rcdom;
|
||||||
|
extern crate html5ever;
|
||||||
|
|
||||||
|
use std::{path::is_separator, rc::Rc, time::Instant};
|
||||||
|
use db::{connect, Website};
|
||||||
|
use html5ever::{local_name, parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, ParseOpts};
|
||||||
|
use rcdom::{Node, RcDom};
|
||||||
|
use surrealdb::{engine::remote::ws::Client, sql::Thing, Surreal};
|
||||||
|
use tracing::{debug, info, instrument, trace, trace_span, warn};
|
||||||
|
use tracing_subscriber::EnvFilter;
|
||||||
|
|
||||||
|
mod db;
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() {
|
async fn main() {
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(EnvFilter::from_default_env())
|
||||||
|
.with_line_number(true)
|
||||||
|
.without_time()
|
||||||
|
.init();
|
||||||
|
debug!("Starting...");
|
||||||
|
|
||||||
let args = std::env::args().collect::<Vec<String>>();
|
// Would probably take these in as parameters from a cli
|
||||||
|
// let url = "https://oliveratkinson.net/";
|
||||||
|
let url = "http://localhost:5500";
|
||||||
|
let budget = 50;
|
||||||
|
let mut crawled = 0;
|
||||||
|
|
||||||
let url = &args[1];
|
let db = connect().await.expect("Failed to connect to db, aborting.");
|
||||||
let budget = &args[2];
|
|
||||||
let budget = match budget.parse::<u32>() {
|
|
||||||
Ok(x) => x,
|
|
||||||
Err(_) => panic!("Second arg must be a int"),
|
|
||||||
};
|
|
||||||
|
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
let mut site = spider::website::Website::new(url)
|
// .use_rustls_tls()
|
||||||
.with_budget(Some(HashMap::from([
|
|
||||||
("*", budget),
|
|
||||||
])))
|
|
||||||
.with_tld(true)
|
|
||||||
.with_on_link_find_callback(Some(|from, to| {
|
|
||||||
let from = from.as_ref().to_string();
|
|
||||||
let to = to.as_ref().to_string();
|
|
||||||
|
|
||||||
let from = from.trim();
|
|
||||||
let to= to.trim();
|
|
||||||
|
|
||||||
println!("{from};->;{to}");
|
|
||||||
}))
|
|
||||||
.build()
|
.build()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
site.crawl().await;
|
// Kick off the whole machine - This Website object doesn't matter, it's just to allow for
|
||||||
|
// get() to work.
|
||||||
|
let span = trace_span!("Pre-Loop");
|
||||||
|
let pre_loop_span = span.enter();
|
||||||
|
// Download the site
|
||||||
|
let mut site = Website::new(&url, false);
|
||||||
|
let dom = get(&mut site, &db, &client).await.expect("Inital page returned None.");
|
||||||
|
|
||||||
|
crawl_wrapper(&dom, &db, &site, &mut crawled).await;
|
||||||
|
|
||||||
|
drop(pre_loop_span);
|
||||||
|
|
||||||
|
let span = trace_span!("Loop");
|
||||||
|
let span = span.enter();
|
||||||
|
while crawled < budget {
|
||||||
|
let get_num = if budget - crawled < 100 {
|
||||||
|
budget - crawled
|
||||||
|
} else {100};
|
||||||
|
|
||||||
|
let uncrawled = get_uncrawled_links(&db, get_num).await;
|
||||||
|
if uncrawled.len() == 0 {
|
||||||
|
info!("Had more budget but finished crawling everything.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
debug!("Crawling {} pages...", uncrawled.len());
|
||||||
|
|
||||||
|
let span = trace_span!("Crawling");
|
||||||
|
let _ = span.enter();
|
||||||
|
|
||||||
|
for mut site in uncrawled {
|
||||||
|
if let Some(dom) = get(&mut site, &db, &client).await {
|
||||||
|
crawl_wrapper(&dom, &db, &site, &mut crawled).await;
|
||||||
|
let percent = format!("{:.2}%", (crawled as f32/budget as f32) * 100f32);
|
||||||
|
info!("Crawled {crawled} out of {budget} pages. ({percent})");
|
||||||
|
} else {
|
||||||
|
warn!("Failed to get {}", site.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(span);
|
||||||
|
|
||||||
|
info!("Done");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn crawl_wrapper(dom: &Rc<Node>, db: &Surreal<Client>, site: &Website, count: &mut usize) {
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
let timer= Timer::start("Walked");
|
||||||
|
walk(&dom, &db, &site, &mut buffer).await;
|
||||||
|
drop(timer);
|
||||||
|
site.links_to(buffer, &db).await;
|
||||||
|
*count += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all)]
|
||||||
|
/// A quick helper function for downloading a url
|
||||||
|
async fn get(site: &mut Website, db: &Surreal<Client>, getter: &reqwest::Client) -> Option<Rc<Node>> {
|
||||||
|
trace!("Get: {}", site.to_string());
|
||||||
|
let timer = Timer::start("Got page");
|
||||||
|
|
||||||
|
if let Ok(response) = getter.get(site.to_string()).send().await {
|
||||||
|
drop(timer);
|
||||||
|
|
||||||
|
let data = response.text().await.unwrap();
|
||||||
|
let opts = ParseOpts {
|
||||||
|
tree_builder: TreeBuilderOpts {
|
||||||
|
drop_doctype: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let dom = parse_document(RcDom::default(), opts)
|
||||||
|
.from_utf8()
|
||||||
|
.read_from(&mut data.as_bytes())
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// TODO save the dom to minio if a flag is set
|
||||||
|
|
||||||
|
site.set_crawled();
|
||||||
|
site.store(db).await;
|
||||||
|
trace!("Got: {}", site.to_string());
|
||||||
|
return Some(dom.document);
|
||||||
|
}
|
||||||
|
trace!("Failed to get: {}", site.to_string());
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Walks the givin site, placing it's findings in the database
|
||||||
|
async fn walk(node: &rcdom::Handle, db: &Surreal<Client> , site: &Website, links_to: &mut Vec<Thing>) {
|
||||||
|
let span = trace_span!("Walk");
|
||||||
|
let span = span.enter();
|
||||||
|
// Match each node - node basically means element.
|
||||||
|
match &node.data {
|
||||||
|
rcdom::NodeData::Element { name, attrs, template_contents, mathml_annotation_xml_integration_point } => {
|
||||||
|
for attr in attrs.borrow().clone() {
|
||||||
|
match name.local {
|
||||||
|
local_name!("a") |
|
||||||
|
local_name!("audio") |
|
||||||
|
local_name!("area") |
|
||||||
|
local_name!("img") |
|
||||||
|
local_name!("link") |
|
||||||
|
local_name!("object") |
|
||||||
|
local_name!("source") |
|
||||||
|
local_name!("base") |
|
||||||
|
local_name!("video") => {
|
||||||
|
let attribute_name = attr.name.local.to_string();
|
||||||
|
if attribute_name == "src" || attribute_name == "href" || attribute_name == "data" {
|
||||||
|
// Get clone of the current site object
|
||||||
|
let mut web = site.clone();
|
||||||
|
|
||||||
|
// Set url
|
||||||
|
let url = web.mut_url();
|
||||||
|
url.set_fragment(None); // removes #xyz
|
||||||
|
let joined = url.join(&attr.value).unwrap();
|
||||||
|
*url = joined;
|
||||||
|
|
||||||
|
// Set other attributes
|
||||||
|
web.crawled = false;
|
||||||
|
// TODO set element name
|
||||||
|
// let element_name = name.local.to_string();
|
||||||
|
|
||||||
|
if let Some(id) = web.store(db).await {
|
||||||
|
links_to.push(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
local_name!("button") |
|
||||||
|
local_name!("meta") |
|
||||||
|
local_name!("iframe") => {
|
||||||
|
// dbg!(attrs);
|
||||||
|
}
|
||||||
|
_ => {/**/}
|
||||||
|
};
|
||||||
|
};
|
||||||
|
},
|
||||||
|
_ => {},
|
||||||
|
};
|
||||||
|
drop(span);
|
||||||
|
for child in node.children.borrow().iter() {
|
||||||
|
Box::pin(walk(child, db, site, links_to)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns uncrawled links
|
||||||
|
async fn get_uncrawled_links(db: &Surreal<Client>, mut count: usize) -> Vec<Website> {
|
||||||
|
if count > 100 {
|
||||||
|
count = 100
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut response = db
|
||||||
|
.query("SELECT * FROM website WHERE crawled = false LIMIT $count")
|
||||||
|
.bind(("count", count))
|
||||||
|
.await
|
||||||
|
.expect("Hard-coded query failed..?");
|
||||||
|
response.take(0).expect("Returned websites couldn't be parsed")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct Timer<'a> {
|
||||||
|
start: Instant,
|
||||||
|
msg: &'a str,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> Timer<'a> {
|
||||||
|
#[inline]
|
||||||
|
pub fn start(msg: &'a str) -> Self {
|
||||||
|
Self { start: Instant::now(), msg }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> Drop for Timer<'a> {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let dif = self.start.elapsed().as_micros();
|
||||||
|
debug!("{}", format!("{} in {:.3}ms", self.msg, dif as f64/1000.));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user