traces and new metrics work

docker/alloy.conf

@@ -3,12 +3,34 @@ local.file_match "tmplogs" {
 }
 
 loki.source.file "local_files" {
-    targets    = local.file_match.tmplogs.targets
-    forward_to = [loki.write.local_loki.receiver]
+    targets    	= local.file_match.tmplogs.targets
+    forward_to 	= [loki.write.local_loki.receiver]
 }
 
 loki.write "local_loki" {
     endpoint {
         url = "http://loki:3100/loki/api/v1/push"
     }
-}
+}
+
+otelcol.receiver.otlp "otlp_receiver" {
+  grpc {
+    endpoint = "0.0.0.0:4317"
+  }
+  http {
+    endpoint = "0.0.0.0:4318"
+  }
+
+  output {
+    traces = [otelcol.exporter.otlp.tempo.input,]
+  }
+}
+
+otelcol.exporter.otlp "tempo" {
+  client {
+    endpoint = "tempo:4317"
+    tls {
+			insecure = true
+		}
+  }
+}

docker/compose.yml

@@ -1,4 +1,6 @@
 services:
+
+  # Database
   surreal:
     image: surrealdb/surrealdb:latest-dev
     ports:
@@ -15,6 +17,18 @@ services:
       - root
       - rocksdb:/mydata/database.db
 
+  # Tracing
+  tempo:
+    image: grafana/tempo:latest
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml
+      - tempo_storage:/var/tempo
+    ports:
+      - 3200:3200 # self metrics for prometheus
+      - 4317:4317 # otlp grpc - (alloy) 
+
+  # Log scraper
   alloy:
     image: grafana/alloy:latest
     ports:
@@ -24,9 +38,13 @@ services:
       - ./logs/:/tmp/alloy-logs
       - ./alloy.conf:/etc/alloy/config.alloy
       - alloy_storage:/var/lib/alloy
-    command:  run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
 
-  #logs
+  # Log storage / analysis
   loki:
     image: grafana/loki:latest
     ports:
@@ -35,16 +53,20 @@ services:
     volumes:
       - ./loki.yaml:/etc/loki/local-config.yaml
 
-  # Metrics collector
+  # Metrics
   prometheus:
     image: prom/prometheus:latest
-    expose:
-      - 9090
+    ports:
+      - 9090:9090
     volumes:
       - ./prometheus.yaml:/etc/prometheus/prometheus.yml
       # persist data
-      - prometheus_storage:/prometheus
-    command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
+      # - prometheus_storage:/prometheus
+    command:
+      - --enable-feature=native-histograms
+      - --web.enable-remote-write-receiver
+      - --web.enable-lifecycle
+      - --config.file=/etc/prometheus/prometheus.yml
 
   # Everything viewer
   grafana:
@@ -66,3 +88,4 @@ volumes:
   grafana_storage:
   alloy_storage:
   surrealdb_storage:
+  tempo_storage:

docker/grafana.yaml

@@ -22,3 +22,20 @@ datasources:
   editable: false
   jsonData:
     httpMethod: GET
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: true
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+    httpMethod: GET
+    serviceMap:
+      datasourceUid: prometheus
+    streamingEnabled:
+      search: true

docker/prometheus.yaml

@@ -1,17 +1,15 @@
 global:
-  scrape_interval: 5s 
+  scrape_interval: 60s 
   query_log_file: /etc/prometheus/query.log
 
 scrape_configs:
-  - job_name: crawler
+  # Crawler configs get pushed with OTLP
+#  - job_name: 'loki'
+#    static_configs:
+#      - targets: ['loki:3100']
+#  - job_name: 'prometheus'
+#    static_configs:
+#      - targets: ['localhost:9090']
+  - job_name: 'tempo'
     static_configs:
-    # change this your machine's ip, localhost won't work
-    # because localhost refers to the docker container.
-      - targets: ['172.20.239.48:2500']
-        #- targets: ['192.168.8.209:2500']
-  - job_name: loki
-    static_configs:
-      - targets: ['loki:3100']
-  - job_name: prometheus
-    static_configs:
-      - targets: ['localhost:9090']
+      - targets: ['tempo:3200']

docker/tempo.yaml

@@ -0,0 +1,48 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+      duration_slo: 5s
+      throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "tempo:4317"
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local # backend configuration to use
+    wal:
+      path: /var/tempo/wal # where to store the wal locally
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
+      generate_native_histograms: both