From a0fd81d956da28a5bfca2aa4aa02634425602508 Mon Sep 17 00:00:00 2001
From: Rushmore75 <oliveratkinsoneng@gmail.com>
Date: Thu, 17 Jul 2025 08:58:30 -0600
Subject: [PATCH] better config file

---
 Crawler.toml      |  8 ++++++++
 src/filesystem.rs |  2 +-
 src/main.rs       | 22 +++++++++++++---------
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/Crawler.toml b/Crawler.toml
index a7cdb42..b194a5a 100644
--- a/Crawler.toml
+++ b/Crawler.toml
@@ -1,3 +1,11 @@
+# Visability config
+# Alloy (for Tempo)
+tracing_endpoint = "http://localhost:4317"
+# Prometheus
+metrics_endpoint = "http://localhost:9090/api/v1/otlp/v1/metrics"
+# Alloy (for Loki)
+log_file = "./docker/logs/tracing.log"
+
 # Surreal config
 surreal_url = "localhost:8000"
 surreal_username = "root"
diff --git a/src/filesystem.rs b/src/filesystem.rs
index 84a3a01..64f5242 100644
--- a/src/filesystem.rs
+++ b/src/filesystem.rs
@@ -2,7 +2,7 @@ use std::{io::ErrorKind, path::PathBuf};
 
 use reqwest::header::HeaderValue;
 use tokio::fs;
-use tracing::{error, event, trace, warn, Level};
+use tracing::{error, trace, warn};
 use url::Url;
 
 pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {
diff --git a/src/main.rs b/src/main.rs
index 53fad8a..b7875d9 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -72,6 +72,10 @@ static CONFIG: LazyLock<Config> = LazyLock::new(|| {
 
 #[derive(Deserialize)]
 struct Config {
+    tracing_endpoint: String,
+    metrics_endpoint: String,
+    log_file: String,
+
     surreal_ns: String,
     surreal_db: String,
     surreal_url: String,
@@ -89,9 +93,9 @@ async fn main() {
     println!("Logs and metrics are provided to the Grafana dashboard");
 
     // Start TRACE / LOGGING / METRICS
-    load_tracing();
-    load_logging();
-    load_metrics();
+    load_tracing(&CONFIG);
+    load_logging(&CONFIG);
+    load_metrics(&CONFIG);
 
     // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
     // let crawl_filter = "en.wikipedia.org/";
@@ -279,11 +283,11 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
     BEING_PROCESSED.add(-1, &[]);
 }
 
-fn load_tracing() {
+fn load_tracing(config: &Config) {
     // Send spans to Alloy (which will send them to Tempo)
     let otlp_span = opentelemetry_otlp::SpanExporter::builder()
         .with_tonic()
-        .with_endpoint("http://localhost:4317")
+        .with_endpoint(config.tracing_endpoint.clone())
         .build()
         .unwrap();
     let tracer_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
@@ -293,7 +297,7 @@ fn load_tracing() {
     global::set_tracer_provider(tracer_provider);
 }
 
-fn load_logging() {
+fn load_logging(config: &Config) {
 //    let otlp_log = opentelemetry_otlp::LogExporter::builder()
 //        .with_tonic()
 //        .with_endpoint(endpoint)
@@ -305,7 +309,7 @@ fn load_logging() {
     let writer = std::fs::OpenOptions::new()
         .append(true)
         .create(true)
-        .open("./docker/logs/tracing.log")
+        .open(config.log_file.clone())
         .expect("Couldn't make log file!");
 
     let filter = EnvFilter::builder()
@@ -325,12 +329,12 @@ fn load_logging() {
     tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber");
 }
 
-fn load_metrics() {
+fn load_metrics(config: &Config) {
     //  Send metrics to Prometheus
     let otlp_metrics = opentelemetry_otlp::MetricExporter::builder()
         .with_http()
         .with_protocol(Protocol::HttpBinary)
-        .with_endpoint("http://localhost:9090/api/v1/otlp/v1/metrics")
+        .with_endpoint(config.metrics_endpoint.clone())
         .build()
         .unwrap();
     let metrics_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()