Oliver/internet_mapper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Oliver:399510c599df5acf19363b4e52c4315c2765f559
Branches Tags
Oliver:main
Oliver:status_codes
Oliver:no-lock
Oliver:spider-rs
...
compare: Oliver:d9d4c5614234b64dfefcad06fde00be6ab7d7f7b
Branches Tags
Oliver:status_codes
Oliver:main
Oliver:no-lock
Oliver:spider-rs

5 Commits
399510c599 ... d9d4c56142

Author SHA1 Message Date
oliver
d9d4c56142 fits with compose.yml 2024-11-12 17:51:13 -07:00
oliver
f87a43c3a9 added demo site for testing elements 2024-11-12 17:50:59 -07:00
oliver
7cac880f8e remove un-used function 2024-11-12 17:50:28 -07:00
oliver
720adaa552 added support for nearly all html tags that can have a link 2024-11-12 17:50:06 -07:00
oliver
7c32600694 this is really all that's needed 2024-11-12 17:49:45 -07:00
11 changed files with 82 additions and 55 deletions
Show Stats Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
/target
/.surrealdb
perf.data
flamegraph.svg
perf.data.old

BIN
demo_site/audio.mp3 Normal file
View File

Binary file not shown.

24
demo_site/index.html Normal file
View File

@ -0,0 +1,24 @@
<!DOCTYPE html>
<html>
<head>
<title>Page Title</title>
<script src='main.js'></script>
<base href="/" target="_blank">
<link rel="stylesheet" href="style.css">
</head>
<body>
<a href="/">Link text</a>
<img src="/picture.png"></img>
<audio src="/audio.mp3" controls>Audio is unsupported</audio>
<video src="/video.mp4" controls>Video is unsupported</video>
<audio controls>
<source src="audio.mp3"></source>
</audio>
<map>
<area href="/"></area>
</map>
<object data="/video.mp4"></object>
<iframe src="/" title="The same Site"></iframe>
<meta>
</body>
</html>

BIN
demo_site/picture.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 111 KiB

1
demo_site/script.js Normal file
View File

@ -0,0 +1 @@
alert("alert")

3
demo_site/style.css Normal file
View File

@ -0,0 +1,3 @@
html {
background-color: white;
}

BIN
demo_site/video.mp4 Normal file
View File

Binary file not shown.

21
schema.surql
View File

@ -1,21 +1,2 @@
DELETE website;
DELETE links_to;
DELETE url_index;
DEFINE TABLE website SCHEMAFULL;
DEFINE FIELD url ON TABLE website TYPE string;
DEFINE FIELD crawled ON TABLE website TYPE bool DEFAULT false;
DEFINE FIELD domain ON TABLE website VALUE parse::url::domain($this.url) ASSERT !type::is::none($value);
DEFINE FIELD path ON TABLE website VALUE parse::url::path($this.url) ASSERT !type::is::none($value);
DEFINE TABLE website SCHEMALESS;
DEFINE FIELD accessed_at ON TABLE website VALUE time::now();
DEFINE INDEX url_index ON TABLE website COLUMNS url UNIQUE;
DEFINE TABLE email SCHEMAFULL;
DEFINE FIELD email ON TABLE email TYPE string;
DEFINE TABLE found_on SCHEMAFULL TYPE RELATION FROM email TO website;
DEFINE FIELD discovered_at ON TABLE found_on TYPE datetime DEFAULT time::now();
DEFINE TABLE links_to SCHEMAFULL TYPE RELATION FROM website TO website;
DEFINE FIELD discovered_at ON TABLE links_to TYPE datetime DEFAULT time::now();

12
src/db.rs
View File

@ -15,7 +15,7 @@ pub struct Website {
/// The url that this data is found at
site: Url,
/// Wether or not this link has been crawled yet
crawled: bool,
pub crawled: bool,
#[serde(skip_serializing)]
id: Option<Thing>,
}
@ -39,18 +39,10 @@ impl Website {
self.crawled = true
}
pub fn crawled(&mut self) -> &mut bool {
&mut self.crawled
}
pub fn mut_url(&mut self) -> &mut Url {
&mut self.site
}
pub fn domain_str(&self) -> &str {
self.site.as_str()
}
#[instrument(skip_all)]
pub async fn links_to(&self, other: Vec<Thing>, db: &Surreal<Client>) {
let len = other.len();
@ -105,9 +97,11 @@ impl Website {
if let Some(old) = response.take::<Option<Website>>(0).unwrap() {
// site exists already
if let Some(id) = old.id {
// make sure to preserve the "crawled status"
let mut new = self.clone();
new.crawled = old.crawled | new.crawled;
// update the record
match db.upsert((id.tb, id.id.to_string())).content(new).await {
Ok(e) => {
if let Some(a) = e {

70
src/main.rs
View File

@ -1,9 +1,9 @@
extern crate markup5ever_rcdom as rcdom;
extern crate html5ever;
use std::{rc::Rc, time::Instant};
use std::{path::is_separator, rc::Rc, time::Instant};
use db::{connect, Website};
use html5ever::{parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, ParseOpts};
use html5ever::{local_name, parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, ParseOpts};
use rcdom::{Node, RcDom};
use surrealdb::{engine::remote::ws::Client, sql::Thing, Surreal};
use tracing::{debug, info, instrument, trace, trace_span, warn};
@ -21,7 +21,8 @@ async fn main() {
debug!("Starting...");
// Would probably take these in as parameters from a cli
let url = "https://oliveratkinson.net/";
// let url = "https://oliveratkinson.net/";
let url = "http://localhost:5500";
let budget = 50;
let mut crawled = 0;
@ -36,9 +37,12 @@ async fn main() {
// get() to work.
let span = trace_span!("Pre-Loop");
let pre_loop_span = span.enter();
// Download the site
let mut site = Website::new(&url, false);
let dom = get(&mut site, &db, &client).await.expect("Inital page returned None.");
crawl_wrapper(&dom, &db, &site, &mut crawled).await;
drop(pre_loop_span);
let span = trace_span!("Loop");
@ -49,6 +53,10 @@ async fn main() {
} else {100};
let uncrawled = get_uncrawled_links(&db, get_num).await;
if uncrawled.len() == 0 {
info!("Had more budget but finished crawling everything.");
return;
}
debug!("Crawling {} pages...", uncrawled.len());
let span = trace_span!("Crawling");
@ -101,6 +109,8 @@ async fn get(site: &mut Website, db: &Surreal<Client>, getter: &reqwest::Client)
.read_from(&mut data.as_bytes())
.unwrap();
// TODO save the dom to minio if a flag is set
site.set_crawled();
site.store(db).await;
trace!("Got: {}", site.to_string());
@ -114,34 +124,48 @@ async fn get(site: &mut Website, db: &Surreal<Client>, getter: &reqwest::Client)
async fn walk(node: &rcdom::Handle, db: &Surreal<Client> , site: &Website, links_to: &mut Vec<Thing>) {
let span = trace_span!("Walk");
let span = span.enter();
// Match each node - node basically means element.
match &node.data {
rcdom::NodeData::Element { name, attrs, template_contents, mathml_annotation_xml_integration_point } => {
for attr in attrs.borrow().clone() {
if name.local.to_string() == "a" {
if attr.value.starts_with("mailto") {
trace!("Is mailto");
// mailto link, lol
let _created: Option<db::Record> = db.create("email").content(db::Email {
email: attr.value.to_string(),
on: site.domain_str().to_owned(),
}).await.unwrap();
} else {
let mut web = site.clone();
let url = web.mut_url();
match name.local {
local_name!("a") |
local_name!("audio") |
local_name!("area") |
local_name!("img") |
local_name!("link") |
local_name!("object") |
local_name!("source") |
local_name!("base") |
local_name!("video") => {
let attribute_name = attr.name.local.to_string();
if attribute_name == "src" || attribute_name == "href" || attribute_name == "data" {
// Get clone of the current site object
let mut web = site.clone();
// TODO remove #xyz
let joined = url.join(&attr.value).unwrap();
*url = joined;
// Set url
let url = web.mut_url();
url.set_fragment(None); // removes #xyz
let joined = url.join(&attr.value).unwrap();
*url = joined;
let crawled = web.crawled();
*crawled = false;
// Set other attributes
web.crawled = false;
// TODO set element name
// let element_name = name.local.to_string();
if let Some(id) = web.store(db).await {
links_to.push(id);
if let Some(id) = web.store(db).await {
links_to.push(id);
}
}
},
local_name!("button") |
local_name!("meta") |
local_name!("iframe") => {
// dbg!(attrs);
}
}
_ => {/**/}
};
};
},
_ => {},

1
test
View File

@ -1 +0,0 @@
[{"result":[{"accessed_at":"2024-08-25T20:07:25.969525156Z","crawled":false,"domain":"google.com","id":"website:fd46b0cr5f5y3d57eje8","path":"/","url":"https://google.com"}],"status":"OK","time":"205.7