50 lines
1.6 KiB
Python
50 lines
1.6 KiB
Python
from surrealdb import Surreal
|
|
import fileinput
|
|
import asyncio
|
|
|
|
|
|
async def main():
|
|
async with Surreal("ws://localhost:8000/rpc") as db:
|
|
|
|
await db.signin({"user": "root", "pass": "root"})
|
|
await db.use("test", "test")
|
|
|
|
for line in fileinput.input():
|
|
# Testing
|
|
# for line in ["https://oliveratkinson.net;->;http://google.com"]:
|
|
|
|
urls = line.split(";->;")
|
|
ffrom = urls[0].strip()
|
|
to= urls[1].strip()
|
|
|
|
to_id = None
|
|
from_id = None
|
|
|
|
if (ffrom == to):
|
|
print('Site has self-reference, ignoring')
|
|
continue
|
|
|
|
# FROM
|
|
result = await db.query(f"SELECT id FROM website WHERE url='{ffrom}'")
|
|
if (len(result[0]['result']) > 0):
|
|
from_id = result[0]['result'][0]['id']
|
|
await db.query(f"UPDATE {from_id} MERGE {{ crawled: True }}")
|
|
else:
|
|
from_response = await db.create("website", { "url":ffrom, "crawled":True, } )
|
|
from_id = from_response[0]["id"]
|
|
|
|
# TO
|
|
result = await db.query(f"SELECT id FROM website WHERE url='{to}'")
|
|
if (len(result[0]['result']) < 1):
|
|
# Object doesn't exists yet
|
|
to_response = await db.create("website", { "url":to })
|
|
to_id = to_response[0]["id"]
|
|
else:
|
|
to_id = result[0]['result'][0]['id']
|
|
|
|
await db.query(f"RELATE {from_id} -> links_to -> {to_id}");
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|
|
|