Oliver/internet_mapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

it helps when you don't reference [0] for two separate elements...

Browse Source
This commit is contained in:
oliver 2024-08-26 00:55:36 -06:00
parent edf9bfc1f5
commit 67d89ff9eb
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawl.sh
View File

@ -1,2 +1,2 @@
#!/bin/bash #!/bin/bash
cargo run -- https://discord.com 5 | python3 driver.py cargo run -- $1 1000 | python3 driver.py

8
driver.py
View File

@ -14,13 +14,15 @@ async def main():
# for line in ["https://oliveratkinson.net;->;http://google.com"]: # for line in ["https://oliveratkinson.net;->;http://google.com"]:
urls = line.split(";->;") urls = line.split(";->;")
ffrom = urls[0] ffrom = urls[0].strip()
to= urls[0] to= urls[1].strip()
to_id = None to_id = None
from_id = None from_id = None
print(ffrom, "->", to) if (ffrom == to):
print('Site has self-reference, ignoring')
continue
# FROM # FROM
result = await db.query(f"SELECT id FROM website WHERE url='{ffrom}'") result = await db.query(f"SELECT id FROM website WHERE url='{ffrom}'")