From b0abd815be8b147eacc979bb89b7300a716c4f31 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 28 Dec 2025 16:29:46 +0200 Subject: Add Grafana Tempo distributed tracing with demo application MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Deploy Grafana Tempo in monolithic mode for distributed tracing - Configure Tempo with OTLP receivers (gRPC:4317, HTTP:4318) - Set up 10Gi filesystem storage with 7-day retention - Integrate Tempo datasource in Grafana with traces-to-logs and traces-to-metrics correlation - Update Grafana Alloy to collect and forward traces - Add OTLP receiver configuration to alloy-values.yaml - Configure batch processor for efficient trace forwarding to Tempo - Patch Alloy service to expose OTLP ports 4317/4318 - Create demo tracing application (frontend, middleware, backend) - Implement three-tier Python Flask application with OpenTelemetry instrumentation - Auto-instrument with OpenTelemetry for Flask and requests libraries - Push Docker images to private registry (registry.lan.buetow.org:30001) - Deploy via Helm chart with Traefik ingress at tracing-demo.f3s.buetow.org - Update Grafana configuration in prometheus/persistence-values.yaml - Add Tempo to additionalDataSources for automatic provisioning Files added: - tempo/values.yaml: Tempo Helm chart configuration - tempo/persistent-volumes.yaml: Storage configuration (10Gi PV/PVC) - tempo/datasource-configmap.yaml: Grafana datasource with correlations - tempo/Justfile: Installation automation - tempo/README.md: Documentation - tracing-demo/docker/frontend/: Python Flask frontend with OTel - tracing-demo/docker/middleware/: Python Flask middleware with OTel - tracing-demo/docker/backend/: Python Flask backend with OTel - tracing-demo/helm-chart/: Kubernetes deployments, services, ingress - tracing-demo/docker-image-Justfile: Docker build/push automation - tracing-demo/Justfile: Helm deployment automation - tracing-demo/README.md: Documentation 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- f3s/tracing-demo/docker/frontend/Dockerfile | 16 +++ f3s/tracing-demo/docker/frontend/app.py | 149 ++++++++++++++++++++++ f3s/tracing-demo/docker/frontend/requirements.txt | 6 + 3 files changed, 171 insertions(+) create mode 100644 f3s/tracing-demo/docker/frontend/Dockerfile create mode 100644 f3s/tracing-demo/docker/frontend/app.py create mode 100644 f3s/tracing-demo/docker/frontend/requirements.txt (limited to 'f3s/tracing-demo/docker/frontend') diff --git a/f3s/tracing-demo/docker/frontend/Dockerfile b/f3s/tracing-demo/docker/frontend/Dockerfile new file mode 100644 index 0000000..dd28e97 --- /dev/null +++ b/f3s/tracing-demo/docker/frontend/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Copy and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app.py . + +# Expose port for Flask application +EXPOSE 5000 + +# Run the application +CMD ["python", "app.py"] diff --git a/f3s/tracing-demo/docker/frontend/app.py b/f3s/tracing-demo/docker/frontend/app.py new file mode 100644 index 0000000..65ab3f3 --- /dev/null +++ b/f3s/tracing-demo/docker/frontend/app.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Tracing Demo - Frontend Service +Receives user requests and forwards to middleware service. +Demonstrates OpenTelemetry auto-instrumentation with Flask. +""" +from flask import Flask, jsonify, request +import requests +import os +import logging + +# OpenTelemetry imports for distributed tracing +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.flask import FlaskInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor +from opentelemetry.sdk.resources import Resource + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize OpenTelemetry tracing with resource attributes +# These attributes identify this service in traces +resource = Resource(attributes={ + "service.name": "frontend", + "service.namespace": "tracing-demo", + "service.version": "1.0.0", + "deployment.environment": "production" +}) + +provider = TracerProvider(resource=resource) + +# Configure OTLP exporter to send traces to Alloy +otlp_exporter = OTLPSpanExporter( + endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', + 'http://alloy.monitoring.svc.cluster.local:4317'), + insecure=True +) + +# Batch spans for efficient export +processor = BatchSpanProcessor(otlp_exporter) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) + +# Get tracer for manual instrumentation if needed +tracer = trace.get_tracer(__name__) + +# Create Flask application +app = Flask(__name__) + +# Auto-instrument Flask to create spans for HTTP requests +FlaskInstrumentor().instrument_app(app) + +# Auto-instrument requests library to propagate trace context +RequestsInstrumentor().instrument() + +# Configuration for downstream services +MIDDLEWARE_URL = os.getenv('MIDDLEWARE_URL', + 'http://middleware-service.services.svc.cluster.local:5001') + +@app.route('/') +def index(): + """ + Health check and service information endpoint. + Returns service metadata. + """ + return jsonify({ + "service": "frontend", + "version": "1.0.0", + "message": "Tracing demo frontend service", + "trace_enabled": True, + "middleware_url": MIDDLEWARE_URL + }) + +@app.route('/health') +def health(): + """ + Kubernetes health check endpoint. + Used by readiness and liveness probes. + """ + return jsonify({"status": "healthy"}), 200 + +@app.route('/api/process', methods=['GET', 'POST']) +def process(): + """ + Main processing endpoint that demonstrates distributed tracing. + Forwards request to middleware service and returns combined response. + Creates a custom span to track the processing logic. + """ + # Create a custom span for the processing logic + with tracer.start_as_current_span("frontend-process") as span: + # Add custom attributes to the span for better observability + span.set_attribute("frontend.handler", "process") + + # Get request data (supports both GET and POST) + if request.method == 'POST': + data = request.get_json() or {} + else: + data = {"source": "GET request"} + + span.set_attribute("frontend.request.method", request.method) + + try: + # Call middleware service + # The requests library auto-instrumentation will create a span + # and propagate the trace context via W3C Trace Context headers + logger.info(f"Calling middleware at {MIDDLEWARE_URL}/api/transform") + + response = requests.post( + f'{MIDDLEWARE_URL}/api/transform', + json=data, + timeout=10 + ) + + response.raise_for_status() + middleware_data = response.json() + + # Record successful call in span + span.set_attribute("frontend.middleware.status", response.status_code) + + return jsonify({ + "service": "frontend", + "status": "success", + "request_data": data, + "middleware_response": middleware_data + }), 200 + + except requests.exceptions.RequestException as e: + # Log error and record in span + logger.error(f"Error calling middleware: {e}") + span.set_attribute("frontend.error", str(e)) + + # Set span status to error + span.set_status(trace.Status(trace.StatusCode.ERROR, str(e))) + + return jsonify({ + "service": "frontend", + "status": "error", + "error": str(e) + }), 500 + +if __name__ == '__main__': + logger.info("Starting frontend service on port 5000") + logger.info(f"Middleware URL: {MIDDLEWARE_URL}") + logger.info(f"OTLP endpoint: {os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}") + app.run(host='0.0.0.0', port=5000, debug=False) diff --git a/f3s/tracing-demo/docker/frontend/requirements.txt b/f3s/tracing-demo/docker/frontend/requirements.txt new file mode 100644 index 0000000..cb10687 --- /dev/null +++ b/f3s/tracing-demo/docker/frontend/requirements.txt @@ -0,0 +1,6 @@ +flask==3.0.0 +requests==2.31.0 +opentelemetry-distro==0.49b0 +opentelemetry-exporter-otlp==1.28.0 +opentelemetry-instrumentation-flask==0.49b0 +opentelemetry-instrumentation-requests==0.49b0 -- cgit v1.2.3