Oliver/internet_mapper
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

better config file

Browse Source
This commit is contained in:
Rushmore75 2025-07-17 08:58:30 -06:00
parent 5cbba33a09
commit a0fd81d956
3 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Crawler.toml
View File

@ -1,3 +1,11 @@
# Visability config
# Alloy (for Tempo)
tracing_endpoint = "http://localhost:4317"
# Prometheus
metrics_endpoint = "http://localhost:9090/api/v1/otlp/v1/metrics"
# Alloy (for Loki)
log_file = "./docker/logs/tracing.log"
# Surreal config # Surreal config
surreal_url = "localhost:8000" surreal_url = "localhost:8000"
surreal_username = "root" surreal_username = "root"

2
src/filesystem.rs
View File

@ -2,7 +2,7 @@ use std::{io::ErrorKind, path::PathBuf};
use reqwest::header::HeaderValue; use reqwest::header::HeaderValue;
use tokio::fs; use tokio::fs;
use tracing::{error, event, trace, warn, Level}; use tracing::{error, trace, warn};
use url::Url; use url::Url;
pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf { pub fn as_path(url: &Url, content_type: &HeaderValue) -> PathBuf {

22
src/main.rs
View File

@ -72,6 +72,10 @@ static CONFIG: LazyLock<Config> = LazyLock::new(|| {
#[derive(Deserialize)] #[derive(Deserialize)]
struct Config { struct Config {
tracing_endpoint: String,
metrics_endpoint: String,
log_file: String,
surreal_ns: String, surreal_ns: String,
surreal_db: String, surreal_db: String,
surreal_url: String, surreal_url: String,
@ -89,9 +93,9 @@ async fn main() {
println!("Logs and metrics are provided to the Grafana dashboard"); println!("Logs and metrics are provided to the Grafana dashboard");
// Start TRACE / LOGGING / METRICS // Start TRACE / LOGGING / METRICS
load_tracing(); load_tracing(&CONFIG);
load_logging(); load_logging(&CONFIG);
load_metrics(); load_metrics(&CONFIG);
// When getting uncrawled pages, name must contain this variable. "" will effectively get ignored. // When getting uncrawled pages, name must contain this variable. "" will effectively get ignored.
// let crawl_filter = "en.wikipedia.org/"; // let crawl_filter = "en.wikipedia.org/";
@ -279,11 +283,11 @@ async fn process(mut site: Website, db: Surreal<Client>, reqwest: reqwest::Clien
BEING_PROCESSED.add(-1, &[]); BEING_PROCESSED.add(-1, &[]);
} }
fn load_tracing() { fn load_tracing(config: &Config) {
// Send spans to Alloy (which will send them to Tempo) // Send spans to Alloy (which will send them to Tempo)
let otlp_span = opentelemetry_otlp::SpanExporter::builder() let otlp_span = opentelemetry_otlp::SpanExporter::builder()
.with_tonic() .with_tonic()
.with_endpoint("http://localhost:4317") .with_endpoint(config.tracing_endpoint.clone())
.build() .build()
.unwrap(); .unwrap();
let tracer_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() let tracer_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
@ -293,7 +297,7 @@ fn load_tracing() {
global::set_tracer_provider(tracer_provider); global::set_tracer_provider(tracer_provider);
} }
fn load_logging() { fn load_logging(config: &Config) {
// let otlp_log = opentelemetry_otlp::LogExporter::builder() // let otlp_log = opentelemetry_otlp::LogExporter::builder()
// .with_tonic() // .with_tonic()
// .with_endpoint(endpoint) // .with_endpoint(endpoint)
@ -305,7 +309,7 @@ fn load_logging() {
let writer = std::fs::OpenOptions::new() let writer = std::fs::OpenOptions::new()
.append(true) .append(true)
.create(true) .create(true)
.open("./docker/logs/tracing.log") .open(config.log_file.clone())
.expect("Couldn't make log file!"); .expect("Couldn't make log file!");
let filter = EnvFilter::builder() let filter = EnvFilter::builder()
@ -325,12 +329,12 @@ fn load_logging() {
tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber"); tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber");
} }
fn load_metrics() { fn load_metrics(config: &Config) {
// Send metrics to Prometheus // Send metrics to Prometheus
let otlp_metrics = opentelemetry_otlp::MetricExporter::builder() let otlp_metrics = opentelemetry_otlp::MetricExporter::builder()
.with_http() .with_http()
.with_protocol(Protocol::HttpBinary) .with_protocol(Protocol::HttpBinary)
.with_endpoint("http://localhost:9090/api/v1/otlp/v1/metrics") .with_endpoint(config.metrics_endpoint.clone())
.build() .build()
.unwrap(); .unwrap();
let metrics_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder() let metrics_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()