Oliver/internet_mapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

minimize vec resizing

Browse Source
This commit is contained in:
Rushmore75 2025-03-18 15:07:50 -06:00
parent bd0b946245
commit 3b4e6a40ce
1 changed files with 8 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/parser.rs
View File

@ -71,7 +71,7 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
site.store(db).await; site.store(db).await;
// prep work // prep work
let mut other_sites = Vec::new(); let mut other_sites: Vec<Website> = Vec::new();
{ // using blocks to prevent compiler's async worries { // using blocks to prevent compiler's async worries
let _t = Timer::start("Parsed page"); let _t = Timer::start("Parsed page");
@ -84,8 +84,9 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default()); let tokenizer = Tokenizer::new(site.clone(), TokenizerOpts::default());
// go thru buffer // go thru buffer
while let TokenizerResult::Script(sites) = tokenizer.feed(&mut token_buffer) { while let TokenizerResult::Script(mut sites) = tokenizer.feed(&mut token_buffer) {
other_sites.push(sites); other_sites.append(&mut sites);
// other_sites.push(sites);
} }
assert!(token_buffer.is_empty()); assert!(token_buffer.is_empty());
@ -93,18 +94,14 @@ pub async fn parse(db: &Surreal<Client>, site: &mut Website, data: &str) {
} }
{ {
let mut links_to = Vec::new(); let mut links_to = Vec::with_capacity(other_sites.len());
// this is a 2d vec accidentally
for a in other_sites { for a in other_sites {
for b in a { let other = a.store(db).await;
// TODO this can become a JoinSet later
let other = b.store(db).await;
if let Some(o) = other { if let Some(o) = other {
links_to.push(o); links_to.push(o);
} }
} }
}
site.links_to(links_to, db).await; site.links_to(links_to, db).await;
} }