S078 · Prometheus, Grafana, OpenTelemetry — Hands-on

# docker-compose.yml — Prometheus + Grafana + OTel Collector
# Run: docker compose up -d
# Then: http://localhost:3000 (grafana) / http://localhost:9090 (prometheus)
version: "3.9"
services:
  prometheus:
    image: prom/prometheus:v2.53.0
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
    ports: ["9090:9090"]
 
  grafana:
    image: grafana/grafana:11.1.0
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    ports: ["3000:3000"]
 
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.104.0
    volumes:
      - ./otel-config.yaml:/etc/otelcol/config.yaml:ro
    command: ["--config=/etc/otelcol/config.yaml"]
    ports:
      - "4317:4317"  # OTLP gRPC
      - "4318:4318"  # OTLP HTTP
      - "8889:8889"  # /metrics for Prometheus to scrape

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s
 
scrape_configs:
  - job_name: "app"
    static_configs:
      - targets: ["host.docker.internal:5000"]  # your app's /metrics
  - job_name: "otel-collector"
    static_configs:
      - targets: ["otel-collector:8889"]  # OTel's metrics passthrough
 
rule_files:
  - "alert_rules.yml"

# otel-config.yaml
receivers:
  otlp:
    protocols:
      grpc: { endpoint: "0.0.0.0:4317" }
      http: { endpoint: "0.0.0.0:4318" }
 
processors:
  batch:
    timeout: 5s
  attributes:
    actions:
      - key: env
        value: dev
        action: insert
 
exporters:
  prometheus:
    endpoint: "0.0.0.0:8889"
  debug:
    verbosity: basic
 
service:
  pipelines:
    metrics:
      receivers: [otlp]
      processors: [batch, attributes]
      exporters: [prometheus, debug]
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [debug]

# alert_rules.yml — the ONE alert every service should have
groups:
  - name: SLO-alerts
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          /
          sum(rate(http_requests_total[5m]))
          > 0.01
        for: 5m
        labels:
          severity: page
        annotations:
          summary: "Error rate above 1% for 5m"
          description: "The 5xx ratio has exceeded 1% for the last 5 minutes."

#!/usr/bin/env python3
"""app.py — a Flask app that exposes Prometheus metrics AND sends
traces to the OTel Collector over OTLP.
 
Run:
  pip install flask prometheus-client opentelemetry-api \\
              opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc
  python app.py
  # then hammer it:
  while true; do curl -s localhost:5000/order/$RANDOM > /dev/null; sleep 0.1; done
"""
from __future__ import annotations
import random, time
from flask import Flask
from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
 
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 
# --- Tracing setup: emit OTLP to the local Collector ---
trace.set_tracer_provider(TracerProvider())
trace.get_tracer_provider().add_span_processor(
    BatchSpanProcessor(
        OTLPSpanExporter(endpoint="localhost:4317", insecure=True)
    )
)
tracer = trace.get_tracer(__name__)
 
# --- Metrics: scraped by Prometheus at /metrics ---
REQUESTS = Counter("http_requests_total",
                   "HTTP requests", ["method", "endpoint", "status"])
LATENCY = Histogram("http_request_duration_seconds",
                    "HTTP latency", ["endpoint"],
                    buckets=(.005, .01, .025, .05, .1, .25, .5, 1, 2.5))
 
app = Flask(__name__)
 
 
@app.route("/order/<int:order_id>")
def get_order(order_id: int):
    start = time.time()
    with tracer.start_as_current_span("get_order") as span:
        span.set_attribute("order.id", order_id)
        # Simulate variable work
        with tracer.start_as_current_span("db_lookup"):
            time.sleep(random.uniform(0.005, 0.05))
        with tracer.start_as_current_span("compute"):
            time.sleep(random.uniform(0.001, 0.02))
        # 5% error rate
        if random.random() < 0.05:
            status = "500"
        else:
            status = "200"
    REQUESTS.labels("GET", "/order/<id>", status).inc()
    LATENCY.labels("/order/<id>").observe(time.time() - start)
    return ({"order_id": order_id}, int(status))
 
 
@app.route("/metrics")
def metrics():
    return generate_latest(), 200, {"Content-Type": CONTENT_TYPE_LATEST}
 
 
if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5000)