don't serialize empty vecs

I don't want normies tyring to scrape the whole server

.gitea/workflows/rust.yaml

@@ -0,0 +1,20 @@
+name: Test Rust project
+on: [push]
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        rust: [stable]
+
+    steps:
+    - uses: https://git.oliveratkinson.net/Oliver/setup-rust-action@master
+      with:
+        rust-version: ${{ matrix.rust }}
+    - uses: actions/checkout@master
+    - name: Run tests
+      run: cargo test --verbose
+    - name: Clippy
+      run: cargo clippy
+

.gitignore

@@ -1,3 +1,4 @@
 /target
 /.vscode
 .env
+server.json

Cargo.lock

@@ -354,6 +354,8 @@ dependencies = [
  "dotenv",
  "once_cell",
  "poise",
+ "serde",
+ "serde_json",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -933,25 +935,25 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 [[package]]
 name = "poise"
 version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/Rushmore75/poise.git?rev=6afaf44a791cd3ff590c89fda26c94b4411b3266#6afaf44a791cd3ff590c89fda26c94b4411b3266"
-checksum = "1819d5a45e3590ef33754abce46432570c54a120798bdbf893112b4211fa09a6"
 dependencies = [
  "async-trait",
  "derivative",
  "futures-util",
+ "indexmap",
  "parking_lot",
  "poise_macros",
  "regex",
  "serenity",
  "tokio",
  "tracing",
+ "trim-in-place",
 ]
 
 [[package]]
 name = "poise_macros"
 version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/Rushmore75/poise.git?rev=6afaf44a791cd3ff590c89fda26c94b4411b3266#6afaf44a791cd3ff590c89fda26c94b4411b3266"
-checksum = "8fa2c123c961e78315cd3deac7663177f12be4460f5440dbf62a7ed37b1effea"
 dependencies = [
  "darling",
  "proc-macro2",
@@ -1328,8 +1330,7 @@ dependencies = [
 [[package]]
 name = "serenity"
 version = "0.12.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/serenity-rs/serenity.git?rev=e34f4491ee3a0b20ea8dc30cbc77b257f402f692#e34f4491ee3a0b20ea8dc30cbc77b257f402f692"
-checksum = "880a04106592d0a8f5bdacb1d935889bfbccb4a14f7074984d9cd857235d34ac"
 dependencies = [
  "arrayvec",
  "async-trait",
@@ -1725,6 +1726,12 @@ dependencies = [
  "tracing-log",
 ]
 
+[[package]]
+name = "trim-in-place"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc"
+
 [[package]]
 name = "triomphe"
 version = "0.1.13"

Cargo.toml

@@ -10,10 +10,11 @@ edition = "2021"
 
 [dependencies]
 tokio = { version = "1.21.2", features = ["macros", "rt-multi-thread"] }
-# songbird = { version = "0.3.2", features = ["yt-dlp"] }
+poise = { features = ["cache"], git="https://github.com/Rushmore75/poise.git", rev="6afaf44a791cd3ff590c89fda26c94b4411b3266" }
-poise = { version = "0.6", features = ["cache"] }
 dotenv = "0.15.0"
 anyhow = "1.0.75"
 once_cell = "1.19.0"
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+serde_json = "1.0.120"
+serde = { version = "1.0.204", features = ["derive"] }

Dockerfile

@@ -1,15 +0,0 @@
-# Build the execuitible 
-FROM rustlang/rust:nightly as builder
-ENV RUSTFLAGS=""
-
-WORKDIR /bot
-COPY . .
-RUN cargo build --release
-
-# Now make the runtime container
-FROM debian:bookworm-slim
-
-COPY --from=builder /bot/target/release/discord_egress /usr/local/bin/discord_egress
-COPY Cargo.lock /
-
-CMD ["/usr/local/bin/discord_egress"]

README.md

@@ -2,6 +2,26 @@
 
 egress a discord server
 
+### NOTE:
+Using custom version of poise (the discord library) because I need to use a specific version of serenity (the discord library for poise). Here's how to set it up:
+
+1) Clone Poise next to this repo:
+
+```shell
+git clone https://github.com/serenity-rs/poise.git
+```
+
+2) Change the version of serenity that poise uses:
+
+```diff
+[dependencies.serenity]
+- version = "x.xx.x"
++ git = "https://github.com/serenity-rs/serenity.git"
++ rev = "e34f4491ee3a0b20ea8dc30cbc77b257f402f692"
+```
+
+> Just make sure that the rev is a sha1 commit hash sometime after 7/25/24
+
 ## Getting started
 * goto: [Discord applicatoins](https://discord.com/developers/applications) and
 create an application.

docker-compose.yaml

@@ -1,30 +0,0 @@
-version: '3.1'
-name: discord-egress
-services:
-  bot:
-    container_name: discord-egress_bot
-    build: https://git.oliveratkinson.net/Oliver/discord-egress.git
-    restart: always
-    environment:
-      # These will read from the .env file
-      DISCORD_INTENTS: ${DISCORD_INTENTS}
-      DISCORD_TOKEN: ${DISCORD_TOKEN}
-      DISCORD_ID: ${DISCORD_ID}
-    networks:
-      - external
-      - internal
-  redis:
-    container_name: discord-egress_redis
-    image: redis
-    restart: always
-    ports:
-      - 6379:6379
-    networks:
-      - internal
-
-networks:
-  internal:
-    driver: bridge
-    internal: true
-  external:
-    driver: bridge

src/command.rs

@@ -1,19 +1,27 @@
-use std::{collections::HashMap, fmt::Display, sync::Arc};
+use std::{collections::HashMap, fmt::Display, fs, sync::Arc};
 
 use crate::Context;
 use anyhow::Error;
 use poise::{serenity_prelude::{Cache, CacheHttp, ChannelId, ChannelType, GetMessages, GuildChannel, Http, Message}, CreateReply};
-use tracing::{debug, error, trace};
+use serde::Serialize;
+use tokio::time::Instant;
+use tracing::{debug, error, info, trace};
 
+#[derive(Serialize)]
 struct Server {
     channels: Vec<Channel>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     orphanage: Vec<GuildChannel>,
+    #[serde(skip_serializing)]
     needs_clean: bool,
 }
 
+#[derive(Serialize)]
 struct Channel {
     this: GuildChannel,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     children: Vec<Channel>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     messages: Vec<Message>,
 }
 
@@ -86,9 +94,8 @@ impl Server {
             if child.this.id == *find {
                 return Some(child);
             }
-            match Self::search_by_id(&mut child.children, find) {
+            if let Some(x) = Self::search_by_id(&mut child.children, find) {
-                Some(x) => return Some(x),
+                return Some(x);
-                None => {},
             }
        }
         None
@@ -99,7 +106,7 @@ impl Server {
         if let Some(parent_id) = &insert.parent_id {
             // find the parent (needs to go thru all nodes)
 
-            match Self::search_by_id(&mut self.channels, &parent_id) {
+            match Self::search_by_id(&mut self.channels, parent_id) {
                 Some(parent_node) => {
                     parent_node.children.push(Channel::new(insert));
                 },
@@ -136,12 +143,16 @@ impl Server {
 
     /// Scrapes messages for all the channels in `self`.
     async fn scrape_all(&mut self) {
+        // TODO add status from this command, just because it's cool
         let cache: (&Arc<Cache>, &Http) = (&Arc::new(Cache::new()), &Http::new(&crate::ENV.token));
         walk_channels(&mut self.channels, cache).await;
 
         /// Recursive walk thru the channels
         async fn walk_channels(all: &mut Vec<Channel>, cache: impl CacheHttp + Clone) {
-            let settings = GetMessages::default().limit(5);
+            // Qty of messages to take at a time, max=100
+            let batch_size = 100;
+
+            let settings = GetMessages::default().limit(batch_size);
             for channel in all {
                 // Clone *should* be cheap - it's Arc under the hood
                 // get the messages
@@ -169,7 +180,11 @@ impl Server {
                                 }
                             }                        
                         },
-                        Err(e) => error!("Error while trying to get messages - {e}"),
+                        Err(e) => {
+                            error!("While reading messages in \"{}\" before `{}` - {e}", channel.this.name, last);
+                            // Stop reading this channel on an error.
+                            last_id = None;
+                        },
                     }
                 }
                 // Then recurse into children channels
@@ -191,17 +206,51 @@ impl Server {
         }
         walk(&self.channels)
     }
+
 }
 
-#[poise::command(slash_command, rename = "scrape_all", guild_only)]
+#[poise::command(slash_command, rename = "scrape_all", guild_only, owners_only)]
-pub async fn scrape_all(ctx: Context<'_>) -> Result<(), Error> {
+pub async fn scrape_all(ctx: Context<'_>, pretty_print: bool) -> Result<(), Error> {
     let guild = ctx.guild_id().unwrap().to_partial_guild(ctx.serenity_context()).await.unwrap();
+
+    let invoker = ctx.author().name.clone();
+    if let Some(nickname) = ctx.author().nick_in(ctx.http(), guild.id).await {
+        info!("{invoker} ({nickname}) started a scrape of {}", guild.name);
+    } else {
+        info!("{invoker} started a scrape of {}", guild.name);
+    }
+
     if let Ok(map) = guild.channels(ctx.http()).await {
         let mut server = index(map).await;
         match ctx.reply("Starting scrape...").await {
             Ok(ok) => {
+                let start = Instant::now();
                 server.scrape_all().await;
-                let _ = ok.edit(ctx, CreateReply::default().content(&format!("Scraped {} messages", server.message_count()))).await;
+                let end = start.elapsed().as_millis();
+                let msg_count = server.message_count();
+
+                let print = if pretty_print {
+                    serde_json::to_string_pretty(&server)
+                } else {
+                    serde_json::to_string(&server)
+                };
+
+                match print {
+                    Ok(ok) => {
+                        if let Err(e) = fs::write("server.json", ok) {
+                            error!("Problem writing server to disk: {e}");
+                        }
+                    },
+                    Err(err) => {
+                        error!("Trying to serialize server: {err}");
+                    },
+                }
+
+                // Done. Print stats.
+                let _ = ok.edit(ctx, CreateReply::default().content(
+                    &format!("Done. Stats: \n```toml\nMessages saved: {msg_count}\nElapsed time: {end}ms\n```")
+                )).await;
+                debug!("Scraped server in {}ms", end);
             },
             Err(e) => error!("{e} - While trying to reply to scrape command"),
         }
@@ -226,8 +275,14 @@ async fn index(map: HashMap<ChannelId, GuildChannel>) -> Server {
 // NOTE!!! Make sure these names in quotes are lowercase!
 #[poise::command(slash_command, rename = "index", guild_only)]
 pub async fn index_cmd(ctx: Context<'_>) -> Result<(), Error> {
-
     let guild = ctx.guild_id().unwrap().to_partial_guild(ctx.serenity_context()).await.unwrap();
+    let invoker = ctx.author().name.clone();
+    if let Some(nickname) = ctx.author().nick_in(ctx.http(), guild.id).await {
+        info!("{invoker} ({nickname}) is indexing {}", guild.name);
+    } else {
+        info!("{invoker} is indexing {}", guild.name);
+    }
+
     match guild.channels(ctx.http()).await {
         Ok(ok) => {
             let server = index(ok).await;

src/main.rs

@@ -1,5 +1,7 @@
+use std::collections::HashSet;
+
 use once_cell::sync::Lazy;
-use poise::serenity_prelude::{self as serenity, GatewayIntents};
+use poise::serenity_prelude::{self as serenity, GatewayIntents, UserId};
 use tracing::{debug, error, info, warn, Level};
 use tracing_subscriber::EnvFilter;
 mod command;
@@ -26,19 +28,17 @@ async fn main() {
         .with_env_filter(filter)
         .with_thread_ids(false)
         .with_file(false)
-        .without_time()
         .init();
 
     // Generate sick text like this: 
     // http://www.patorjk.com/software/taag/#p=testall&f=Graffiti&t=hello%20world
-    info!(r#"
+    info!("Invite with: https://discord.com/api/oauth2/authorize?client_id={}&permissions={}&scope=bot",
-    Invite this bot with:
-    "#);
-    info!("https://discord.com/api/oauth2/authorize?client_id={}&permissions={}&scope=bot",
         ENV.id,
         ENV.intents.bits(),
     );
-    info!("\n");
+
+    let mut owners = HashSet::new();
+    owners.insert(UserId::new(423970006334832650));
 
     // Setup framework
     let framework = poise::Framework::builder()
@@ -47,6 +47,7 @@ async fn main() {
                 command::index_cmd(),
                 command::scrape_all(),
             ],
+            owners,
             ..Default::default()
         })
         .setup(|ctx, _ready, framework| {