100 lines
2.5 KiB
Rust
100 lines
2.5 KiB
Rust
use metrics::counter;
|
|
use rusqlite::Connection;
|
|
use std::fmt::Debug;
|
|
use serde::{Deserialize, Serialize};
|
|
use tracing::{error, instrument, trace};
|
|
use url::Url;
|
|
|
|
use crate::Config;
|
|
|
|
const STORE: &str = "surql_store_calls";
|
|
|
|
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
|
|
pub struct Website {
|
|
/// The url that this data is found at
|
|
pub site: Url,
|
|
/// Wether or not this link has been crawled yet
|
|
pub crawled: bool,
|
|
}
|
|
|
|
// manual impl to make tracing look nicer
|
|
impl Debug for Website {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("Website").field("site", &self.site).finish()
|
|
}
|
|
}
|
|
|
|
impl Website {
|
|
/// Creates a blank site (assumes that url param is site's root)
|
|
pub fn new(url: &str, crawled: bool) -> Self {
|
|
let site = match Url::parse(url) {
|
|
Ok(a) => a,
|
|
Err(_) => todo!(),
|
|
};
|
|
Self {
|
|
crawled,
|
|
site
|
|
}
|
|
}
|
|
|
|
pub fn set_crawled(&mut self) {
|
|
trace!("Set crawled to true");
|
|
self.crawled = true
|
|
}
|
|
|
|
// Insert ever item in the vec into surreal, crawled state will be preserved as TRUE
|
|
// if already in the database as such or incoming data is TRUE.
|
|
pub async fn store_all(all: Vec<Self>, db: &Connection) {
|
|
counter!(STORE).increment(1);
|
|
let mut things = Vec::with_capacity(all.len());
|
|
|
|
rusqlite::ParamsFromIter;
|
|
|
|
db.execute("",
|
|
params![]
|
|
);
|
|
|
|
match db
|
|
.query(
|
|
"INSERT INTO website $array
|
|
ON DUPLICATE KEY UPDATE
|
|
accessed_at = time::now(),
|
|
crawled = crawled OR $input.crawled
|
|
RETURN VALUE id;
|
|
",
|
|
)
|
|
.bind(("array", all))
|
|
.await
|
|
{
|
|
Ok(mut id) => match id.take::<Vec<Thing>>(0) {
|
|
Ok(mut x) => things.append(&mut x),
|
|
Err(err) => error!("{:?}", err),
|
|
},
|
|
Err(err) => {
|
|
error!("{:?}", err);
|
|
}
|
|
}
|
|
things
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub struct Email {
|
|
pub email: String,
|
|
pub on: String,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
pub struct Record {
|
|
#[allow(dead_code)]
|
|
pub id: Thing,
|
|
}
|
|
|
|
#[instrument(skip_all, name = "sqlite_connect")]
|
|
pub async fn connect(config: &Config) -> Result<Connection, rusqlite::Error> {
|
|
trace!("Establishing connection to sqlite...");
|
|
// Connect to the server
|
|
Connection::open("./squeelite.db")
|
|
}
|
|
|