surreal v1

Cargo.toml

@@ -18,3 +18,4 @@ tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 serde_json = "1.0.120"
 serde = { version = "1.0.204", features = ["derive"] }
+surrealdb = "1.5.4"

src/command.rs

@@ -1,9 +1,10 @@
-use std::{collections::HashMap, fmt::Display, fs, sync::Arc};
+use std::{collections::HashMap, fmt::Display, fs, hint::black_box, sync::Arc};
 
 use crate::Context;
 use anyhow::Error;
 use poise::{serenity_prelude::{Cache, CacheHttp, ChannelId, ChannelType, GetMessages, GuildChannel, Http, Message}, CreateReply};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+use surrealdb::{engine::remote::ws::Ws, opt::auth::Root, sql::Thing, Surreal};
 use tokio::time::Instant;
 use tracing::{debug, error, info, trace};
 
@@ -87,8 +88,9 @@ impl Display for Server {
 
 impl Server {
 
-    fn new() -> Self {
+    fn new(name: impl Into<String>) -> Self {
         Self {
+            name: name.into(),
             channels: Vec::new(),
             orphanage: Vec::new(), 
             needs_clean: false
@@ -213,6 +215,87 @@ impl Server {
         walk(&self.channels)
     }
 
+    async fn to_surreal(&self) -> surrealdb::Result<()> {
+        trace!("Connecting to surrealdb...");
+        // Connect to the server
+        let db = Surreal::new::<Ws>("127.0.0.1:8000").await?;
+        db.signin(Root {
+            username: "root",
+            password: "root",
+        }).await?;
+
+        db.use_ns("egress").use_db(self.name.clone()).await?;
+
+        // =========================================================
+        // Ingress data
+        trace!("Starting ingress...");
+        for cat in self.channels.iter() {
+
+            // TODO learn why this is a vec
+            // Do the first iteration of channels a bit different, so as to name it "category".
+            let new_category: Vec<Thing> = db
+                .create("category")
+                .content(&cat.this)
+                .await?;
+            import_messages(&cat.messages, &new_category[0], &db).await?;
+
+            // Ok, now automatically recurse the rest of the structure and auto import as channels
+            // and messages.
+            import_channel(&cat.children, &new_category[0], &db).await?;
+        }
+
+        async fn import_channel(channels: &Vec<Channel>, parent: &Thing, db: &Surreal<surrealdb::engine::remote::ws::Client>) -> surrealdb::Result<()> {
+            for channel in channels.iter() {
+                trace!("Importing channel \"{}\"", channel.this.name);
+                #[derive(Serialize)]
+                struct ChannelWrapper<'a, 'b> {
+                    channel: &'a GuildChannel,
+                    surreal_parent: &'b Thing,
+                }
+                let new_channel: Vec<Thing> = db
+                    .create("channel")
+                    .content(ChannelWrapper {
+                        channel: &channel.this,
+                        surreal_parent: &parent,
+                    })
+                    .await?;
+
+                import_messages(&channel.messages, &new_channel[0], &db).await?;
+                // async recursion - thus box
+                Box::pin(import_channel(&channel.children, &new_channel[0], &db)).await?;
+            }
+            Ok(())
+        }
+
+        async fn import_messages(msgs: &Vec<Message>, parent: &Thing, db: &Surreal<surrealdb::engine::remote::ws::Client>) -> surrealdb::Result<()> {
+            trace!("Importing {} messages...", msgs.len());
+            for msg in msgs.iter() {
+
+                #[derive(Serialize)]
+                struct MessageWrapper<'a, 'b> {
+                    message: &'a Message,
+                    surreal_parent: &'b Thing
+                }
+
+                let created: Vec<Thing> = db
+                    .create("message")
+                    .content(MessageWrapper {
+                        message: &msg,
+                        surreal_parent: &parent,
+                    })
+                .await?;
+
+                trace!("Imported message {:?}", created);
+            }
+            Ok(())
+        }
+
+        // Data is all in
+        // =========================================================
+
+        Ok(())
+    }
+
 }
 
 #[poise::command(slash_command, rename = "scrape_all", guild_only, owners_only)]
@@ -227,7 +310,7 @@ pub async fn scrape_all(ctx: Context<'_>, pretty_print: bool) -> Result<(), Erro
     }
 
     if let Ok(map) = guild.channels(ctx.http()).await {
-        let mut server = index(map).await;
+        let mut server = index(map, guild.name).await;
         match ctx.reply("Starting scrape...").await {
             Ok(ok) => {
                 let start = Instant::now();
@@ -235,22 +318,26 @@ pub async fn scrape_all(ctx: Context<'_>, pretty_print: bool) -> Result<(), Erro
                 let end = start.elapsed().as_millis();
                 let msg_count = server.message_count();
 
-                let print = if pretty_print {
+                if let Err(e) = server.to_surreal().await {
-                    serde_json::to_string_pretty(&server)
+                    error!("{e}");
-                } else {
-                    serde_json::to_string(&server)
                 };
 
-                match print {
+                // let print = if pretty_print {
-                    Ok(ok) => {
+                //     serde_json::to_string_pretty(&server)
-                        if let Err(e) = fs::write("server.json", ok) {
+                // } else {
-                            error!("Problem writing server to disk: {e}");
+                //     serde_json::to_string(&server)
-                        }
+                // };
-                    },
+
-                    Err(err) => {
+                // match print {
-                        error!("Trying to serialize server: {err}");
+                //     Ok(ok) => {
-                    },
+                //         if let Err(e) = fs::write("server.json", ok) {
-                }
+                //             error!("Problem writing server to disk: {e}");
+                //         }
+                //     },
+                //     Err(err) => {
+                //         error!("Trying to serialize server: {err}");
+                //     },
+                // }
 
                 // Done. Print stats.
                 let _ = ok.edit(ctx, CreateReply::default().content(
@@ -272,8 +359,8 @@ pub fn from_json() {
 }
 
 /// Get server's topology (and runs clean)
-async fn index(map: HashMap<ChannelId, GuildChannel>) -> Server {
+async fn index(map: HashMap<ChannelId, GuildChannel>, name: impl Into<String>) -> Server {
-    let mut server = Server::new();
+    let mut server = Server::new(name);
     // iterate thru all channels
     map.into_iter().for_each(|(_id, current)| {
         // println!("{} {} {:?}", current.name, current.id, current.parent_id);
@@ -298,10 +385,11 @@ pub async fn index_cmd(ctx: Context<'_>) -> Result<(), Error> {
 
     match guild.channels(ctx.http()).await {
         Ok(ok) => {
-            let server = index(ok).await;
+            let server = index(ok, guild.name).await;
             let _ = ctx.reply(server.to_string()).await;
         },
         Err(_) => todo!(),
     }
     Ok(())
 }
+

src/main.rs

@@ -15,6 +15,7 @@ static ENV: Lazy<BotEnv> = Lazy::new(|| {
 
 #[tokio::main]
 async fn main() {
+    command::from_json();
 
     // Start the tracing subscriber
     let filter = EnvFilter::builder()