diff --git a/.gitignore b/.gitignore index 301630e..dbaa546 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /.minio perf.data flamegraph.svg -perf.data.old \ No newline at end of file +perf.data.old +/docker/logs/* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index cbcede6..8c6a73f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4364,6 +4364,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.19" @@ -4374,6 +4384,8 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", @@ -4381,6 +4393,7 @@ dependencies = [ "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b04036c..8499bea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,5 +13,5 @@ serde = { version = "1.0", features = ["derive"] } surrealdb = "2.2" tokio = { version="1.41.0", features = ["full"] } tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "local-time", "json"] } url = { version = "2.5", features = ["serde"] } diff --git a/compose.yml b/compose.yml deleted file mode 100644 index 797688f..0000000 --- a/compose.yml +++ /dev/null @@ -1,31 +0,0 @@ -services: - surreal: - image: surrealdb/surrealdb:latest-dev - ports: - - 8000:8000 - volumes: - - ./.surrealdb/:/mydata - command: - - start - - --log - - debug - - --user - - root - - --pass - - root - - rocksdb:/mydata/database.db - minio: - image: quay.io/minio/minio - ports: - - 9000:9000 - - 9001:9001 - environment: - - MINIO_ROOT_USER=root - - MINIO_ROOT_PASSWORD=an8charpassword - volumes: - - ./.minio/:/data - command: - - server - - /data - - --console-address - - ":9001" diff --git a/docker/alloy.conf b/docker/alloy.conf new file mode 100644 index 0000000..35babb3 --- /dev/null +++ b/docker/alloy.conf @@ -0,0 +1,14 @@ +local.file_match "tmplogs" { + path_targets = [{"__path__" = "/tmp/alloy-logs/*.log"}] +} + +loki.source.file "local_files" { + targets = local.file_match.tmplogs.targets + forward_to = [loki.write.local_loki.receiver] +} + +loki.write "local_loki" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} \ No newline at end of file diff --git a/docker/compose.yml b/docker/compose.yml new file mode 100644 index 0000000..48c1a90 --- /dev/null +++ b/docker/compose.yml @@ -0,0 +1,71 @@ +services: + surreal: + image: surrealdb/surrealdb:latest-dev + ports: + - 8000:8000 + volumes: + - surrealdb_storage:/mydata + command: + - start + - --log + - debug + - --user + - root + - --pass + - root + - rocksdb:/mydata/database.db + minio: + image: quay.io/minio/minio + ports: + - 9000:9000 + - 9001:9001 + environment: + - MINIO_ROOT_USER=root + - MINIO_ROOT_PASSWORD=an8charpassword + volumes: + - minio_storage:/data + command: + - server + - /data + - --console-address + - ":9001" + + alloy: + image: grafana/alloy:latest + ports: + - 12345:12345 + volumes: + # if you change this, you also need to change it in the alloy config file + - ./logs/:/tmp/alloy-logs + - ./alloy.conf:/etc/alloy/config.alloy + - alloy_storage:/var/lib/alloy + command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy + + #logs + loki: + image: grafana/loki:latest + ports: + - 3100:3100 + command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./loki.yaml:/etc/loki/local-config.yaml + + # Everything viewer + grafana: + image: grafana/grafana:latest + volumes: + - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - grafana_storage:/var/lib/grafana + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor + ports: + - 3000:3000 + +volumes: + grafana_storage: + alloy_storage: + surrealdb_storage: + minio_storage: \ No newline at end of file diff --git a/docker/grafana.yaml b/docker/grafana.yaml new file mode 100644 index 0000000..be79737 --- /dev/null +++ b/docker/grafana.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: true + version: 1 + editable: false diff --git a/docker/load-me-into-grafana.json b/docker/load-me-into-grafana.json new file mode 100644 index 0000000..198e006 --- /dev/null +++ b/docker/load-me-into-grafana.json @@ -0,0 +1,223 @@ +{ + "__inputs": [ + { + "name": "DS_LOKI", + "label": "Loki", + "description": "", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.3.1" + }, + { + "type": "panel", + "id": "logs", + "name": "Logs", + "version": "" + }, + { + "type": "datasource", + "id": "loki", + "name": "Loki", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `ERROR` | line_format \"{{.fields_message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Errors", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `WARN` | line_format \"{{.fields_message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Warnings", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `DEBUG` | line_format \"{{.fields_message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Debug", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{filename=\"/tmp/alloy-logs/tracing.log\"} | json | level = `TRACE` | line_format \"{{.fields_message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Trace", + "type": "logs" + } + ], + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "New dashboard", + "uid": "ceg90x34pqgowd", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/loki.yaml b/docker/loki.yaml new file mode 100644 index 0000000..819d266 --- /dev/null +++ b/docker/loki.yaml @@ -0,0 +1,62 @@ +# this is mostly the default config from grafana's website + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +frontend: + encoding: protobuf + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false diff --git a/src/main.rs b/src/main.rs index 2475cb5..bd97892 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,9 +8,9 @@ use std::time::Instant; use db::{connect, Website}; use s3::S3; use surrealdb::{engine::remote::ws::Client, Surreal}; -use tokio::task::JoinSet; +use tokio::{task::JoinSet}; use tracing::{debug, info, instrument, trace, trace_span, warn}; -use tracing_subscriber::{fmt::time::LocalTime, EnvFilter}; +use tracing_subscriber::{fmt::{self, time::LocalTime}, layer::{Filter, SubscriberExt}, EnvFilter, Layer, Registry}; mod db; mod parser; @@ -33,13 +33,33 @@ struct Config<'a> { async fn main() { let total_runtime = Timer::start("Completed"); - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .with_line_number(true) - .with_thread_ids(true) - .with_file(true) - .with_timer(LocalTime::rfc_3339()) - .init(); + let writer = std::fs::OpenOptions::new() + .append(true) + .create(true) + .open("./docker/logs/tracing.log") + .expect("Couldn't make log file!"); + + let registry = Registry::default() + .with( + fmt::layer() + .with_line_number(true) + .with_thread_ids(true) + .with_file(true) + // .with_timer(LocalTime::rfc_3339()) // Loki or alloy does this automatically + .json() + .with_writer(writer) + ); + + tracing::subscriber::set_global_default(registry).expect("Failed to set default subscriber"); + + // tracing_subscriber::fmt() + // .with_env_filter(EnvFilter::from_default_env()) + // .with_line_number(true) + // .with_thread_ids(true) + // .with_file(true) + // .with_timer(LocalTime::rfc_3339()) + // .init(); + debug!("Starting..."); // Would probably take these in as parameters from a cli let starting_url = "https://en.wikipedia.org/"; @@ -56,8 +76,8 @@ async fn main() { surreal_db: "v1.12", s3_bucket: "v1.12", s3_url: "http://localhost:9000", - s3_access_key: "8UO76z8wCs9DnpxSbQUY", - s3_secret_key: "xwKVMpf2jzgprsdo85Dvo74UmO84y0aRrAUorYY5", + s3_access_key: "p8gXIZEO2FnWqWBiJYwo", + s3_secret_key: "1mRO0EYA2YAQ0xsKrlbkIIz4AT8KNXy6QIQPtxUu", }; @@ -66,7 +86,7 @@ async fn main() { .expect("Failed to connect to surreal, aborting."); let s3 = S3::connect(&config) .await - .expect("Failed to connect to minio, aborting."); + .expect("Failed to connect to minio, aborting.\n\nThis probably means you need to login to the minio console and get a new access key!\n\n(Probably here) http://localhost:9001/access-keys/new-account\n\n"); let reqwest = reqwest::Client::builder() // .use_rustls_tls()