internet_mapper/src/main.rs

extern crate markup5ever_rcdom as rcdom;
extern crate html5ever;

use std::env;
use html5ever::{parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, ParseOpts};
use rcdom::RcDom;
use tracing::{debug, info, trace, warn};


#[tokio::main]
async fn main() {
    tracing_subscriber::fmt::init();
    debug!("Starting...");

    let url = "https://oliveratkinson.net";
    let budget = "10";
    
    let response = reqwest::get(url).await.unwrap();
    let data = response.text().await.unwrap();


    let opts = ParseOpts {
        tree_builder: TreeBuilderOpts {
            drop_doctype: true,
            ..Default::default()
        },
        ..Default::default()
    };

    let dom = parse_document(RcDom::default(), opts)
        .from_utf8()
        .read_from(&mut data.as_bytes())
        .unwrap();

    let a = &dom.document;
    warn!("Walking...");

    walk(a);
}

fn walk(node: &rcdom::Handle) {
    match &node.data {
        rcdom::NodeData::Document => (),
        rcdom::NodeData::Doctype { name, public_id, system_id } => debug!("doctype"),
        rcdom::NodeData::Text { contents } => {},
        rcdom::NodeData::Comment { contents } => debug!("comment"),
        rcdom::NodeData::Element { name, attrs, template_contents, mathml_annotation_xml_integration_point } => {
            attrs.borrow().iter().for_each(|attr| {
                let name = name.local.to_string();
                let internal = &*attr.value;
                debug!("element: {name}, attr: {internal}"); 
            });
        },
        rcdom::NodeData::ProcessingInstruction { target, contents } => debug!("ProcessingInstruction"),
    };

    node.children.borrow().iter().for_each(|n| walk(n));
}