Add Grafana Tempo distributed tracing with demo application

- Deploy Grafana Tempo in monolithic mode for distributed tracing
- Configure Tempo with OTLP receivers (gRPC:4317, HTTP:4318)
- Set up 10Gi filesystem storage with 7-day retention
- Integrate Tempo datasource in Grafana with traces-to-logs and traces-to-metrics correlation

- Update Grafana Alloy to collect and forward traces
- Add OTLP receiver configuration to alloy-values.yaml
- Configure batch processor for efficient trace forwarding to Tempo
- Patch Alloy service to expose OTLP ports 4317/4318

- Create demo tracing application (frontend, middleware, backend)
- Implement three-tier Python Flask application with OpenTelemetry instrumentation
- Auto-instrument with OpenTelemetry for Flask and requests libraries
- Push Docker images to private registry (registry.lan.buetow.org:30001)
- Deploy via Helm chart with Traefik ingress at tracing-demo.f3s.buetow.org

- Update Grafana configuration in prometheus/persistence-values.yaml
- Add Tempo to additionalDataSources for automatic provisioning

Files added:
- tempo/values.yaml: Tempo Helm chart configuration
- tempo/persistent-volumes.yaml: Storage configuration (10Gi PV/PVC)
- tempo/datasource-configmap.yaml: Grafana datasource with correlations
- tempo/Justfile: Installation automation
- tempo/README.md: Documentation

- tracing-demo/docker/frontend/: Python Flask frontend with OTel
- tracing-demo/docker/middleware/: Python Flask middleware with OTel
- tracing-demo/docker/backend/: Python Flask backend with OTel
- tracing-demo/helm-chart/: Kubernetes deployments, services, ingress
- tracing-demo/docker-image-Justfile: Docker build/push automation
- tracing-demo/Justfile: Helm deployment automation
- tracing-demo/README.md: Documentation

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

9 files changed, 475 insertions, 0 deletions

diff --git a/f3s/tracing-demo/docker/backend/Dockerfile b/f3s/tracing-demo/docker/backend/Dockerfile
new file mode 100644
index 0000000..5018e8f
--- /dev/null
+++ b/f3s/tracing-demo/docker/backend/Dockerfile
@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Copy and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY app.py .
+
+# Expose port for Flask application
+EXPOSE 5002
+
+# Run the application
+CMD ["python", "app.py"]
diff --git a/f3s/tracing-demo/docker/backend/app.py b/f3s/tracing-demo/docker/backend/app.py
new file mode 100644
index 0000000..2c9e88a
--- /dev/null
+++ b/f3s/tracing-demo/docker/backend/app.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""
+Tracing Demo - Backend Service
+Final service in the chain that returns data.
+Simulates database queries and demonstrates end-to-end tracing.
+"""
+from flask import Flask, jsonify
+import os
+import logging
+import time
+from datetime import datetime
+
+# OpenTelemetry imports for distributed tracing
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.sdk.resources import Resource
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Initialize OpenTelemetry tracing with resource attributes
+# These attributes identify this service in traces
+resource = Resource(attributes={
+    "service.name": "backend",
+    "service.namespace": "tracing-demo",
+    "service.version": "1.0.0",
+    "deployment.environment": "production"
+})
+
+provider = TracerProvider(resource=resource)
+
+# Configure OTLP exporter to send traces to Alloy
+otlp_exporter = OTLPSpanExporter(
+    endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT',
+                      'http://alloy.monitoring.svc.cluster.local:4317'),
+    insecure=True
+)
+
+# Batch spans for efficient export
+processor = BatchSpanProcessor(otlp_exporter)
+provider.add_span_processor(processor)
+trace.set_tracer_provider(provider)
+
+# Get tracer for manual instrumentation
+tracer = trace.get_tracer(__name__)
+
+# Create Flask application
+app = Flask(__name__)
+
+# Auto-instrument Flask
+FlaskInstrumentor().instrument_app(app)
+
+@app.route('/')
+def index():
+    """
+    Health check and service information endpoint.
+    Returns service metadata.
+    """
+    return jsonify({
+        "service": "backend",
+        "version": "1.0.0",
+        "message": "Tracing demo backend service"
+    })
+
+@app.route('/health')
+def health():
+    """
+    Kubernetes health check endpoint.
+    Used by readiness and liveness probes.
+    """
+    return jsonify({"status": "healthy"}), 200
+
+@app.route('/api/data', methods=['GET'])
+def get_data():
+    """
+    Return data endpoint that simulates a database query.
+    Creates custom spans to track query execution.
+    This is the final service in the trace chain.
+    """
+    # Create a custom span for the database query simulation
+    with tracer.start_as_current_span("backend-get-data") as span:
+        # Add custom attributes to the span
+        span.set_attribute("backend.handler", "get_data")
+
+        # Simulate database query delay
+        query_time = 0.1
+        time.sleep(query_time)
+
+        # Record query duration in span
+        span.set_attribute("backend.query.duration_ms", query_time * 1000)
+        span.set_attribute("backend.query.type", "simulated_database_query")
+
+        # Prepare response data
+        data = {
+            "service": "backend",
+            "data": {
+                "id": 12345,
+                "value": "Sample data from backend service",
+                "timestamp": datetime.utcnow().isoformat(),
+                "query_time_ms": query_time * 1000
+            }
+        }
+
+        logger.info(f"Returning data: {data['data']['id']}")
+
+        return jsonify(data), 200
+
+if __name__ == '__main__':
+    logger.info("Starting backend service on port 5002")
+    logger.info(f"OTLP endpoint: {os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}")
+    app.run(host='0.0.0.0', port=5002, debug=False)
diff --git a/f3s/tracing-demo/docker/backend/requirements.txt b/f3s/tracing-demo/docker/backend/requirements.txt
new file mode 100644
index 0000000..6022d6c
--- /dev/null
+++ b/f3s/tracing-demo/docker/backend/requirements.txt
@@ -0,0 +1,4 @@
+flask==3.0.0
+opentelemetry-distro==0.49b0
+opentelemetry-exporter-otlp==1.28.0
+opentelemetry-instrumentation-flask==0.49b0
diff --git a/f3s/tracing-demo/docker/frontend/Dockerfile b/f3s/tracing-demo/docker/frontend/Dockerfile
new file mode 100644
index 0000000..dd28e97
--- /dev/null
+++ b/f3s/tracing-demo/docker/frontend/Dockerfile
@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Copy and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY app.py .
+
+# Expose port for Flask application
+EXPOSE 5000
+
+# Run the application
+CMD ["python", "app.py"]
diff --git a/f3s/tracing-demo/docker/frontend/app.py b/f3s/tracing-demo/docker/frontend/app.py
new file mode 100644
index 0000000..65ab3f3
--- /dev/null
+++ b/f3s/tracing-demo/docker/frontend/app.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""
+Tracing Demo - Frontend Service
+Receives user requests and forwards to middleware service.
+Demonstrates OpenTelemetry auto-instrumentation with Flask.
+"""
+from flask import Flask, jsonify, request
+import requests
+import os
+import logging
+
+# OpenTelemetry imports for distributed tracing
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk.resources import Resource
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Initialize OpenTelemetry tracing with resource attributes
+# These attributes identify this service in traces
+resource = Resource(attributes={
+    "service.name": "frontend",
+    "service.namespace": "tracing-demo",
+    "service.version": "1.0.0",
+    "deployment.environment": "production"
+})
+
+provider = TracerProvider(resource=resource)
+
+# Configure OTLP exporter to send traces to Alloy
+otlp_exporter = OTLPSpanExporter(
+    endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT',
+                      'http://alloy.monitoring.svc.cluster.local:4317'),
+    insecure=True
+)
+
+# Batch spans for efficient export
+processor = BatchSpanProcessor(otlp_exporter)
+provider.add_span_processor(processor)
+trace.set_tracer_provider(provider)
+
+# Get tracer for manual instrumentation if needed
+tracer = trace.get_tracer(__name__)
+
+# Create Flask application
+app = Flask(__name__)
+
+# Auto-instrument Flask to create spans for HTTP requests
+FlaskInstrumentor().instrument_app(app)
+
+# Auto-instrument requests library to propagate trace context
+RequestsInstrumentor().instrument()
+
+# Configuration for downstream services
+MIDDLEWARE_URL = os.getenv('MIDDLEWARE_URL',
+                          'http://middleware-service.services.svc.cluster.local:5001')
+
+@app.route('/')
+def index():
+    """
+    Health check and service information endpoint.
+    Returns service metadata.
+    """
+    return jsonify({
+        "service": "frontend",
+        "version": "1.0.0",
+        "message": "Tracing demo frontend service",
+        "trace_enabled": True,
+        "middleware_url": MIDDLEWARE_URL
+    })
+
+@app.route('/health')
+def health():
+    """
+    Kubernetes health check endpoint.
+    Used by readiness and liveness probes.
+    """
+    return jsonify({"status": "healthy"}), 200
+
+@app.route('/api/process', methods=['GET', 'POST'])
+def process():
+    """
+    Main processing endpoint that demonstrates distributed tracing.
+    Forwards request to middleware service and returns combined response.
+    Creates a custom span to track the processing logic.
+    """
+    # Create a custom span for the processing logic
+    with tracer.start_as_current_span("frontend-process") as span:
+        # Add custom attributes to the span for better observability
+        span.set_attribute("frontend.handler", "process")
+
+        # Get request data (supports both GET and POST)
+        if request.method == 'POST':
+            data = request.get_json() or {}
+        else:
+            data = {"source": "GET request"}
+
+        span.set_attribute("frontend.request.method", request.method)
+
+        try:
+            # Call middleware service
+            # The requests library auto-instrumentation will create a span
+            # and propagate the trace context via W3C Trace Context headers
+            logger.info(f"Calling middleware at {MIDDLEWARE_URL}/api/transform")
+
+            response = requests.post(
+                f'{MIDDLEWARE_URL}/api/transform',
+                json=data,
+                timeout=10
+            )
+
+            response.raise_for_status()
+            middleware_data = response.json()
+
+            # Record successful call in span
+            span.set_attribute("frontend.middleware.status", response.status_code)
+
+            return jsonify({
+                "service": "frontend",
+                "status": "success",
+                "request_data": data,
+                "middleware_response": middleware_data
+            }), 200
+
+        except requests.exceptions.RequestException as e:
+            # Log error and record in span
+            logger.error(f"Error calling middleware: {e}")
+            span.set_attribute("frontend.error", str(e))
+
+            # Set span status to error
+            span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
+
+            return jsonify({
+                "service": "frontend",
+                "status": "error",
+                "error": str(e)
+            }), 500
+
+if __name__ == '__main__':
+    logger.info("Starting frontend service on port 5000")
+    logger.info(f"Middleware URL: {MIDDLEWARE_URL}")
+    logger.info(f"OTLP endpoint: {os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}")
+    app.run(host='0.0.0.0', port=5000, debug=False)
diff --git a/f3s/tracing-demo/docker/frontend/requirements.txt b/f3s/tracing-demo/docker/frontend/requirements.txt
new file mode 100644
index 0000000..cb10687
--- /dev/null
+++ b/f3s/tracing-demo/docker/frontend/requirements.txt
@@ -0,0 +1,6 @@
+flask==3.0.0
+requests==2.31.0
+opentelemetry-distro==0.49b0
+opentelemetry-exporter-otlp==1.28.0
+opentelemetry-instrumentation-flask==0.49b0
+opentelemetry-instrumentation-requests==0.49b0
diff --git a/f3s/tracing-demo/docker/middleware/Dockerfile b/f3s/tracing-demo/docker/middleware/Dockerfile
new file mode 100644
index 0000000..60272f7
--- /dev/null
+++ b/f3s/tracing-demo/docker/middleware/Dockerfile
@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Copy and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY app.py .
+
+# Expose port for Flask application
+EXPOSE 5001
+
+# Run the application
+CMD ["python", "app.py"]
diff --git a/f3s/tracing-demo/docker/middleware/app.py b/f3s/tracing-demo/docker/middleware/app.py
new file mode 100644
index 0000000..9c0ad30
--- /dev/null
+++ b/f3s/tracing-demo/docker/middleware/app.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""
+Tracing Demo - Middleware Service
+Transforms data and calls backend service.
+Demonstrates trace context propagation in a multi-tier architecture.
+"""
+from flask import Flask, jsonify, request
+import requests
+import os
+import logging
+import time
+
+# OpenTelemetry imports for distributed tracing
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk.resources import Resource
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Initialize OpenTelemetry tracing with resource attributes
+# These attributes identify this service in traces
+resource = Resource(attributes={
+    "service.name": "middleware",
+    "service.namespace": "tracing-demo",
+    "service.version": "1.0.0",
+    "deployment.environment": "production"
+})
+
+provider = TracerProvider(resource=resource)
+
+# Configure OTLP exporter to send traces to Alloy
+otlp_exporter = OTLPSpanExporter(
+    endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT',
+                      'http://alloy.monitoring.svc.cluster.local:4317'),
+    insecure=True
+)
+
+# Batch spans for efficient export
+processor = BatchSpanProcessor(otlp_exporter)
+provider.add_span_processor(processor)
+trace.set_tracer_provider(provider)
+
+# Get tracer for manual instrumentation
+tracer = trace.get_tracer(__name__)
+
+# Create Flask application
+app = Flask(__name__)
+
+# Auto-instrument Flask and requests library
+FlaskInstrumentor().instrument_app(app)
+RequestsInstrumentor().instrument()
+
+# Configuration for downstream services
+BACKEND_URL = os.getenv('BACKEND_URL',
+                       'http://backend-service.services.svc.cluster.local:5002')
+
+@app.route('/')
+def index():
+    """
+    Health check and service information endpoint.
+    Returns service metadata.
+    """
+    return jsonify({
+        "service": "middleware",
+        "version": "1.0.0",
+        "message": "Tracing demo middleware service",
+        "backend_url": BACKEND_URL
+    })
+
+@app.route('/health')
+def health():
+    """
+    Kubernetes health check endpoint.
+    Used by readiness and liveness probes.
+    """
+    return jsonify({"status": "healthy"}), 200
+
+@app.route('/api/transform', methods=['POST'])
+def transform():
+    """
+    Transform data and fetch additional data from backend.
+    Demonstrates trace context propagation through multiple services.
+    Creates custom spans to track transformation logic.
+    """
+    # Create a custom span for the transformation logic
+    with tracer.start_as_current_span("middleware-transform") as span:
+        # Add custom attributes to the span
+        span.set_attribute("middleware.handler", "transform")
+
+        # Get request data from frontend
+        data = request.get_json() or {}
+        span.set_attribute("middleware.input.keys", str(list(data.keys())))
+
+        # Simulate some data transformation processing
+        time.sleep(0.05)
+
+        try:
+            # Call backend service to fetch additional data
+            # The trace context is automatically propagated via HTTP headers
+            logger.info(f"Calling backend at {BACKEND_URL}/api/data")
+
+            response = requests.get(
+                f'{BACKEND_URL}/api/data',
+                timeout=10
+            )
+
+            response.raise_for_status()
+            backend_data = response.json()
+
+            # Record successful call in span
+            span.set_attribute("middleware.backend.status", response.status_code)
+
+            # Transform and combine the data
+            transformed = {
+                "middleware_processed": True,
+                "original_data": data,
+                "backend_data": backend_data,
+                "transformation_time_ms": 50
+            }
+
+            return jsonify(transformed), 200
+
+        except requests.exceptions.RequestException as e:
+            # Log error and record in span
+            logger.error(f"Error calling backend: {e}")
+            span.set_attribute("middleware.error", str(e))
+
+            # Set span status to error
+            span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
+
+            return jsonify({
+                "service": "middleware",
+                "status": "error",
+                "error": str(e)
+            }), 500
+
+if __name__ == '__main__':
+    logger.info("Starting middleware service on port 5001")
+    logger.info(f"Backend URL: {BACKEND_URL}")
+    logger.info(f"OTLP endpoint: {os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}")
+    app.run(host='0.0.0.0', port=5001, debug=False)
diff --git a/f3s/tracing-demo/docker/middleware/requirements.txt b/f3s/tracing-demo/docker/middleware/requirements.txt
new file mode 100644
index 0000000..cb10687
--- /dev/null
+++ b/f3s/tracing-demo/docker/middleware/requirements.txt
@@ -0,0 +1,6 @@
+flask==3.0.0
+requests==2.31.0
+opentelemetry-distro==0.49b0
+opentelemetry-exporter-otlp==1.28.0
+opentelemetry-instrumentation-flask==0.49b0
+opentelemetry-instrumentation-requests==0.49b0