From b0abd815be8b147eacc979bb89b7300a716c4f31 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 28 Dec 2025 16:29:46 +0200 Subject: Add Grafana Tempo distributed tracing with demo application MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Deploy Grafana Tempo in monolithic mode for distributed tracing - Configure Tempo with OTLP receivers (gRPC:4317, HTTP:4318) - Set up 10Gi filesystem storage with 7-day retention - Integrate Tempo datasource in Grafana with traces-to-logs and traces-to-metrics correlation - Update Grafana Alloy to collect and forward traces - Add OTLP receiver configuration to alloy-values.yaml - Configure batch processor for efficient trace forwarding to Tempo - Patch Alloy service to expose OTLP ports 4317/4318 - Create demo tracing application (frontend, middleware, backend) - Implement three-tier Python Flask application with OpenTelemetry instrumentation - Auto-instrument with OpenTelemetry for Flask and requests libraries - Push Docker images to private registry (registry.lan.buetow.org:30001) - Deploy via Helm chart with Traefik ingress at tracing-demo.f3s.buetow.org - Update Grafana configuration in prometheus/persistence-values.yaml - Add Tempo to additionalDataSources for automatic provisioning Files added: - tempo/values.yaml: Tempo Helm chart configuration - tempo/persistent-volumes.yaml: Storage configuration (10Gi PV/PVC) - tempo/datasource-configmap.yaml: Grafana datasource with correlations - tempo/Justfile: Installation automation - tempo/README.md: Documentation - tracing-demo/docker/frontend/: Python Flask frontend with OTel - tracing-demo/docker/middleware/: Python Flask middleware with OTel - tracing-demo/docker/backend/: Python Flask backend with OTel - tracing-demo/helm-chart/: Kubernetes deployments, services, ingress - tracing-demo/docker-image-Justfile: Docker build/push automation - tracing-demo/Justfile: Helm deployment automation - tracing-demo/README.md: Documentation 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- f3s/tracing-demo/docker/backend/Dockerfile | 16 ++++ f3s/tracing-demo/docker/backend/app.py | 115 +++++++++++++++++++++++ f3s/tracing-demo/docker/backend/requirements.txt | 4 + 3 files changed, 135 insertions(+) create mode 100644 f3s/tracing-demo/docker/backend/Dockerfile create mode 100644 f3s/tracing-demo/docker/backend/app.py create mode 100644 f3s/tracing-demo/docker/backend/requirements.txt (limited to 'f3s/tracing-demo/docker/backend') diff --git a/f3s/tracing-demo/docker/backend/Dockerfile b/f3s/tracing-demo/docker/backend/Dockerfile new file mode 100644 index 0000000..5018e8f --- /dev/null +++ b/f3s/tracing-demo/docker/backend/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Copy and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app.py . + +# Expose port for Flask application +EXPOSE 5002 + +# Run the application +CMD ["python", "app.py"] diff --git a/f3s/tracing-demo/docker/backend/app.py b/f3s/tracing-demo/docker/backend/app.py new file mode 100644 index 0000000..2c9e88a --- /dev/null +++ b/f3s/tracing-demo/docker/backend/app.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +""" +Tracing Demo - Backend Service +Final service in the chain that returns data. +Simulates database queries and demonstrates end-to-end tracing. +""" +from flask import Flask, jsonify +import os +import logging +import time +from datetime import datetime + +# OpenTelemetry imports for distributed tracing +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.flask import FlaskInstrumentor +from opentelemetry.sdk.resources import Resource + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize OpenTelemetry tracing with resource attributes +# These attributes identify this service in traces +resource = Resource(attributes={ + "service.name": "backend", + "service.namespace": "tracing-demo", + "service.version": "1.0.0", + "deployment.environment": "production" +}) + +provider = TracerProvider(resource=resource) + +# Configure OTLP exporter to send traces to Alloy +otlp_exporter = OTLPSpanExporter( + endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', + 'http://alloy.monitoring.svc.cluster.local:4317'), + insecure=True +) + +# Batch spans for efficient export +processor = BatchSpanProcessor(otlp_exporter) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) + +# Get tracer for manual instrumentation +tracer = trace.get_tracer(__name__) + +# Create Flask application +app = Flask(__name__) + +# Auto-instrument Flask +FlaskInstrumentor().instrument_app(app) + +@app.route('/') +def index(): + """ + Health check and service information endpoint. + Returns service metadata. + """ + return jsonify({ + "service": "backend", + "version": "1.0.0", + "message": "Tracing demo backend service" + }) + +@app.route('/health') +def health(): + """ + Kubernetes health check endpoint. + Used by readiness and liveness probes. + """ + return jsonify({"status": "healthy"}), 200 + +@app.route('/api/data', methods=['GET']) +def get_data(): + """ + Return data endpoint that simulates a database query. + Creates custom spans to track query execution. + This is the final service in the trace chain. + """ + # Create a custom span for the database query simulation + with tracer.start_as_current_span("backend-get-data") as span: + # Add custom attributes to the span + span.set_attribute("backend.handler", "get_data") + + # Simulate database query delay + query_time = 0.1 + time.sleep(query_time) + + # Record query duration in span + span.set_attribute("backend.query.duration_ms", query_time * 1000) + span.set_attribute("backend.query.type", "simulated_database_query") + + # Prepare response data + data = { + "service": "backend", + "data": { + "id": 12345, + "value": "Sample data from backend service", + "timestamp": datetime.utcnow().isoformat(), + "query_time_ms": query_time * 1000 + } + } + + logger.info(f"Returning data: {data['data']['id']}") + + return jsonify(data), 200 + +if __name__ == '__main__': + logger.info("Starting backend service on port 5002") + logger.info(f"OTLP endpoint: {os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}") + app.run(host='0.0.0.0', port=5002, debug=False) diff --git a/f3s/tracing-demo/docker/backend/requirements.txt b/f3s/tracing-demo/docker/backend/requirements.txt new file mode 100644 index 0000000..6022d6c --- /dev/null +++ b/f3s/tracing-demo/docker/backend/requirements.txt @@ -0,0 +1,4 @@ +flask==3.0.0 +opentelemetry-distro==0.49b0 +opentelemetry-exporter-otlp==1.28.0 +opentelemetry-instrumentation-flask==0.49b0 -- cgit v1.2.3