internet_mapper/src/main.rs

extern crate markup5ever_rcdom as rcdom;
extern crate html5ever;

use std::{env, rc::Rc};
use db::connect;
use html5ever::{parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, ParseOpts};
use rcdom::{Node, RcDom};
use surrealdb::{engine::remote::ws::Client, Surreal};
use tracing::{debug, error, info, trace, warn};

mod db;

#[tokio::main]
async fn main() {
    tracing_subscriber::fmt::init();
    debug!("Starting...");

    let url = "https://oliveratkinson.net";
    let budget = "10";
    
    let dom = get(url).await;
    let db = connect().await.expect("Failed to connect to db, aborting.");

    warn!("Walking...");
    walk(&dom, &db, url).await;
}

async fn get(url: &str) -> Rc<Node> {
    let response = reqwest::get(url).await.unwrap();
    let data = response.text().await.unwrap();

    let opts = ParseOpts {
        tree_builder: TreeBuilderOpts {
            drop_doctype: true,
            ..Default::default()
        },
        ..Default::default()
    };

    let dom = parse_document(RcDom::default(), opts)
        .from_utf8()
        .read_from(&mut data.as_bytes())
        .unwrap();
    dom.document
}

async fn walk(node: &rcdom::Handle, db: &Surreal<Client> , site_name: &str) {


    // Should protentailly just check for the record first.
    let created: Option<db::Record> = match db.create("website").content(db::Website { url: site_name.to_string(), crawled: true } ).await {
        Ok(e) => e,
        Err(e) => {
            match e {
                surrealdb::Error::Db(_) => todo!(),
                surrealdb::Error::Api(api) => {
                    match api {
                        surrealdb::error::Api::Query(query) => {
                            error!(query);  
                            None
                        },
                        _ => todo!(),
                    }
                },
            }
        }
    };

    match &node.data {
        rcdom::NodeData::Document => (),
        rcdom::NodeData::Doctype { name, public_id, system_id } => debug!("doctype"),
        rcdom::NodeData::Text { contents } => {},
        rcdom::NodeData::Comment { contents } => debug!("comment"),
        rcdom::NodeData::Element { name, attrs, template_contents, mathml_annotation_xml_integration_point } => {
            let attrs = attrs.borrow().clone();
            for attr in attrs {
                let name = name.local.to_string();
                let link = &*attr.value;
                trace!("element: {name}, attr: {link}"); 

                if name == "a" {
                    if link.starts_with("mailto") {
                        // mailto link, lol
                        warn!("{link}");
                        
                        let created: Option<db::Record> = db.create("email").content(db::Email { email: link.to_owned() }).await.unwrap();
                        info!("{:?}", created)

                    } else if link.starts_with("http") {
                        // normal link
                        debug!("{link}")
                    } else if link.contains("/") {
                        // possibly a relative link?
                        //
                        // TODO This needs more logic handling. Needs to handle the following cases:
                        //
                        // Absolute links:
                        // /img.png
                        // /file-no-extension
                        //
                        // Realtive Links:
                        //
                        // img.png
                        // file-no-extnesion
                        //
                        let link_name = format!("{site_name}/{link}");
                        debug!("{link_name}");
                        let created: Option<db::Record> = db.create("website").content(db::Website { url: link_name, crawled: false } ).await.unwrap();
                    } else {
                        error!("Unhandled link type: {link}")
                    }
                }
            };
        },
        rcdom::NodeData::ProcessingInstruction { target, contents } => debug!("ProcessingInstruction"),
    };

    for child in &*node.children.borrow() {
        Box::pin(walk(&child, db, site_name)).await; 
    }
}