better config file

Crawler.toml

@@ -1,3 +1,11 @@
+# Visability config
+# Alloy (for Tempo)
+tracing_endpoint = "http://localhost:4317"
+# Prometheus
+metrics_endpoint = "http://localhost:9090/api/v1/otlp/v1/metrics"
+# Alloy (for Loki)
+log_file = "./docker/logs/tracing.log"
+
 # Surreal config
 surreal_url = "localhost:8000"
 surreal_username = "root"

src/filesystem.rs

@@ -2,7 +2,7 @@ use std::{io::ErrorKind, path::PathBuf};
 
 use reqwest::header::HeaderValue;
 use tokio::fs;
-use tracing::{error, event, trace, warn, Level};
+use tracing::{error, trace, warn};
 use url::Url;
 
 pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {

src/main.rs

@@ -72,6 +72,10 @@ static CONFIG: LazyLock<Config> = LazyLock::new(|| {
 
 #[derive(Deserialize)]
 struct Config {
+    tracing_endpoint: String,
+    metrics_endpoint: String,
+    log_file: String,
+
     surreal_ns: String,
     surreal_db: String,
     surreal_url: String,
@@ -89,9 +93,9 @@ async fn main() {
     println!("Logs and metrics are provided to the Grafana dashboard");
 
     // Start TRACE / LOGGING / METRICS
-    load_tracing();
-    load_logging();
-    load_metrics();
+    load_tracing(&CONFIG);
+    load_logging(&CONFIG);
+    load_metrics(&CONFIG);
 
     // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
     // let crawl_filter = "en.wikipedia.org/";
@@ -279,11 +283,11 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
     BEING_PROCESSED.add(-1, &[]);
 }
 
-fn load_tracing() {
+fn load_tracing(config: &Config) {
     // Send spans to Alloy (which will send them to Tempo)
     let otlp_span = opentelemetry_otlp::SpanExporter::builder()
         .with_tonic()
-        .with_endpoint("http://localhost:4317")
+        .with_endpoint(config.tracing_endpoint.clone())
         .build()
         .unwrap();
     let tracer_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
@@ -293,7 +297,7 @@ fn load_tracing() {
     global::set_tracer_provider(tracer_provider);
 }
 
-fn load_logging() {
+fn load_logging(config: &Config) {
 //    let otlp_log = opentelemetry_otlp::LogExporter::builder()
 //        .with_tonic()
 //        .with_endpoint(endpoint)
@@ -305,7 +309,7 @@ fn load_logging() {
     let writer = std::fs::OpenOptions::new()
         .append(true)
         .create(true)
-        .open("./docker/logs/tracing.log")
+        .open(config.log_file.clone())
         .expect("Couldn't make log file!");
 
     let filter = EnvFilter::builder()
@@ -325,12 +329,12 @@ fn load_logging() {
     tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber");
 }
 
-fn load_metrics() {
+fn load_metrics(config: &Config) {
     //  Send metrics to Prometheus
     let otlp_metrics = opentelemetry_otlp::MetricExporter::builder()
         .with_http()
         .with_protocol(Protocol::HttpBinary)
-        .with_endpoint("http://localhost:9090/api/v1/otlp/v1/metrics")
+        .with_endpoint(config.metrics_endpoint.clone())
         .build()
         .unwrap();
     let metrics_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()