use metrics::counter; use rusqlite::Connection; use std::fmt::Debug; use serde::{Deserialize, Serialize}; use tracing::{error, instrument, trace}; use url::Url; use crate::Config; const STORE: &str = "surql_store_calls"; #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] pub struct Website { /// The url that this data is found at pub site: Url, /// Wether or not this link has been crawled yet pub crawled: bool, } // manual impl to make tracing look nicer impl Debug for Website { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Website").field("site", &self.site).finish() } } impl Website { /// Creates a blank site (assumes that url param is site's root) pub fn new(url: &str, crawled: bool) -> Self { let site = match Url::parse(url) { Ok(a) => a, Err(_) => todo!(), }; Self { crawled, site } } pub fn set_crawled(&mut self) { trace!("Set crawled to true"); self.crawled = true } // Insert ever item in the vec into surreal, crawled state will be preserved as TRUE // if already in the database as such or incoming data is TRUE. pub async fn store_all(all: Vec, db: &Connection) { counter!(STORE).increment(1); let mut things = Vec::with_capacity(all.len()); rusqlite::ParamsFromIter; db.execute("", params![] ); match db .query( "INSERT INTO website $array ON DUPLICATE KEY UPDATE accessed_at = time::now(), crawled = crawled OR $input.crawled RETURN VALUE id; ", ) .bind(("array", all)) .await { Ok(mut id) => match id.take::>(0) { Ok(mut x) => things.append(&mut x), Err(err) => error!("{:?}", err), }, Err(err) => { error!("{:?}", err); } } things } } #[derive(Debug, Serialize)] pub struct Email { pub email: String, pub on: String, } #[derive(Debug, Deserialize)] pub struct Record { #[allow(dead_code)] pub id: Thing, } #[instrument(skip_all, name = "sqlite_connect")] pub async fn connect(config: &Config) -> Result { trace!("Establishing connection to sqlite..."); // Connect to the server Connection::open("./squeelite.db") }