diff options
Diffstat (limited to 'f3s/tracing-demo')
20 files changed, 1091 insertions, 0 deletions
diff --git a/f3s/tracing-demo/Justfile b/f3s/tracing-demo/Justfile new file mode 100644 index 0000000..d1bc474 --- /dev/null +++ b/f3s/tracing-demo/Justfile @@ -0,0 +1,93 @@ +# Tracing Demo Application deployment automation +# Three-tier Python application demonstrating distributed tracing + +NAMESPACE := "services" +RELEASE_NAME := "tracing-demo" +CHART_PATH := "./helm-chart" + +# Build all Docker images (use docker-image-Justfile for build/push to registry) +build: + just -f docker-image-Justfile build + +# Push images to private registry +push: + just -f docker-image-Justfile push + +# Build and push images +build-push: build push + +# Install Helm chart +install: + helm install {{RELEASE_NAME}} {{CHART_PATH}} --namespace {{NAMESPACE}} --create-namespace + +# Upgrade Helm chart +upgrade: + helm upgrade {{RELEASE_NAME}} {{CHART_PATH}} --namespace {{NAMESPACE}} + +# Delete Helm release +delete: + helm uninstall {{RELEASE_NAME}} --namespace {{NAMESPACE}} + +# Rebuild images, import, and upgrade deployment +rebuild: build import upgrade + +# Check deployment status +status: + kubectl get pods -n {{NAMESPACE}} | grep tracing-demo + kubectl get svc -n {{NAMESPACE}} | grep -E '(frontend|middleware|backend)-service' + kubectl get ingress -n {{NAMESPACE}} tracing-demo-ingress + +# View logs from all services +logs: + @echo "=== Frontend logs ===" + kubectl logs -n {{NAMESPACE}} -l app=tracing-demo-frontend --tail=20 + @echo "" + @echo "=== Middleware logs ===" + kubectl logs -n {{NAMESPACE}} -l app=tracing-demo-middleware --tail=20 + @echo "" + @echo "=== Backend logs ===" + kubectl logs -n {{NAMESPACE}} -l app=tracing-demo-backend --tail=20 + +# Follow logs from frontend +logs-frontend: + kubectl logs -n {{NAMESPACE}} -l app=tracing-demo-frontend -f + +# Follow logs from middleware +logs-middleware: + kubectl logs -n {{NAMESPACE}} -l app=tracing-demo-middleware -f + +# Follow logs from backend +logs-backend: + kubectl logs -n {{NAMESPACE}} -l app=tracing-demo-backend -f + +# Test the application +test: + @echo "Testing frontend health endpoint..." + curl http://tracing-demo.f3s.buetow.org/ + @echo "" + @echo "Testing API process endpoint..." + curl http://tracing-demo.f3s.buetow.org/api/process + +# Load test - generate multiple traces +load-test: + @echo "Generating 50 requests with 0.5s delay..." + @for i in {1..50}; do \ + curl -s http://tracing-demo.f3s.buetow.org/api/process >/dev/null && echo "Request $$i complete"; \ + sleep 0.5; \ + done + @echo "Load test complete!" + +# Port forward to services for local testing +port-forward-frontend: + kubectl port-forward -n {{NAMESPACE}} svc/frontend-service 5000:5000 + +port-forward-middleware: + kubectl port-forward -n {{NAMESPACE}} svc/middleware-service 5001:5001 + +port-forward-backend: + kubectl port-forward -n {{NAMESPACE}} svc/backend-service 5002:5002 + +# Check if traces are being generated +check-traces: + @echo "Check Grafana Tempo for traces with:" + @echo " { resource.service.namespace = \"tracing-demo\" }" diff --git a/f3s/tracing-demo/README.md b/f3s/tracing-demo/README.md new file mode 100644 index 0000000..5934c00 --- /dev/null +++ b/f3s/tracing-demo/README.md @@ -0,0 +1,250 @@ +# Tracing Demo Application + +Three-tier Python Flask application demonstrating distributed tracing with OpenTelemetry and Grafana Tempo. + +## Overview + +This demo application shows how distributed tracing works across multiple microservices: + +- **Frontend**: Receives HTTP requests, forwards to middleware +- **Middleware**: Transforms data, calls backend +- **Backend**: Returns data (simulates database queries) + +Each service is instrumented with OpenTelemetry and sends traces to Grafana Tempo via Alloy. + +## Architecture + +``` +User → Frontend (Flask:5000) → Middleware (Flask:5001) → Backend (Flask:5002) + ↓ ↓ ↓ + Alloy (OTLP:4317) → Tempo → Grafana +``` + +## Components + +### Frontend Service +- Port: 5000 +- Endpoints: + - `GET /` - Service info and health + - `GET /health` - Kubernetes health probe + - `GET|POST /api/process` - Main processing endpoint +- Calls: Middleware service + +### Middleware Service +- Port: 5001 +- Endpoints: + - `GET /` - Service info and health + - `GET /health` - Kubernetes health probe + - `POST /api/transform` - Data transformation endpoint +- Calls: Backend service + +### Backend Service +- Port: 5002 +- Endpoints: + - `GET /` - Service info and health + - `GET /health` - Kubernetes health probe + - `GET /api/data` - Data retrieval endpoint (simulates DB query) +- Calls: None (leaf service) + +## OpenTelemetry Instrumentation + +All services use: +- **Auto-instrumentation**: Flask and Requests libraries automatically create spans +- **Manual spans**: Custom spans for business logic with attributes +- **OTLP export**: Traces sent to Alloy via gRPC on port 4317 +- **Resource attributes**: Service name, namespace, version identify each service + +## Build and Deploy + +### Prerequisites + +1. Tempo must be deployed and running in `monitoring` namespace +2. Alloy must be configured with OTLP receivers +3. Docker installed for building images +4. Access to k3s cluster (SSH to r0) + +### Quick Start + +```bash +# Build Docker images +just build + +# Import images to k3s +just import + +# Deploy with Helm +just install + +# Check status +just status +``` + +### Rebuild and Update + +```bash +# Rebuild images, import, and upgrade deployment +just rebuild +``` + +## Testing + +### Basic Test + +```bash +# Test health endpoint +curl http://tracing-demo.f3s.buetow.org/ + +# Test API endpoint (generates a trace) +curl http://tracing-demo.f3s.buetow.org/api/process +``` + +### Load Test + +Generate 50 requests to create multiple traces: + +```bash +just load-test +``` + +### View Logs + +```bash +# View logs from all services +just logs + +# Follow frontend logs +just logs-frontend + +# Follow middleware logs +just logs-middleware + +# Follow backend logs +just logs-backend +``` + +## Viewing Traces in Grafana + +1. Navigate to Grafana: https://grafana.f3s.buetow.org +2. Go to Explore → Select "Tempo" datasource +3. Use TraceQL queries: + +``` +# All traces from demo app +{ resource.service.namespace = "tracing-demo" } + +# Slow requests (>200ms) +{ duration > 200ms } + +# Traces from specific service +{ resource.service.name = "frontend" } + +# Errors +{ status = error } +``` + +4. View Service Graph to see connections between services + +## Trace Features Demonstrated + +### Distributed Context Propagation +Traces automatically span all three services, showing: +- Frontend span (root) +- Middleware span (child of frontend) +- Backend span (child of middleware) + +### Custom Attributes +Each service adds custom attributes: +- `service.name` - Service identifier +- `service.namespace` - Application namespace +- Custom business logic attributes + +### Trace Correlation +- **Traces-to-Logs**: Click on a span to see related logs in Loki +- **Traces-to-Metrics**: View Prometheus metrics for services in the trace +- **Service Graph**: Visualize service dependencies + +## Development + +### Local Testing with Port Forwarding + +```bash +# Forward frontend +just port-forward-frontend +curl http://localhost:5000/ + +# Forward middleware +just port-forward-middleware +curl http://localhost:5001/ + +# Forward backend +just port-forward-backend +curl http://localhost:5002/ +``` + +### Modifying the Application + +1. Edit Python code in `docker/*/app.py` +2. Rebuild: `just build` +3. Import: `just import` +4. Upgrade: `just upgrade` + +Or use the combined command: `just rebuild` + +## Troubleshooting + +### No traces appearing in Grafana + +1. Check pods are running: +```bash +kubectl get pods -n services | grep tracing-demo +``` + +2. Check Alloy is receiving traces: +```bash +kubectl logs -n monitoring -l app.kubernetes.io/name=alloy | grep -i otlp +``` + +3. Check Tempo is storing traces: +```bash +kubectl logs -n monitoring -l app.kubernetes.io/name=tempo | grep -i trace +``` + +4. Verify OTLP endpoint is accessible: +```bash +kubectl exec -n services $(kubectl get pod -n services -l app=tracing-demo-frontend -o jsonpath='{.items[0].metadata.name}') -- wget -qO- http://alloy.monitoring.svc.cluster.local:4317 +``` + +### Pods not starting + +Check events and logs: +```bash +kubectl describe pod -n services -l app=tracing-demo-frontend +kubectl logs -n services -l app=tracing-demo-frontend +``` + +### Images not found + +Verify images are imported to k3s: +```bash +ssh r0 'k3s crictl images | grep tracing-demo' +``` + +If missing, run: +```bash +just import +``` + +## Cleanup + +Remove the demo application: + +```bash +just delete +``` + +## References + +- [OpenTelemetry Python Documentation](https://opentelemetry.io/docs/languages/python/) +- [Flask Instrumentation](https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/flask/flask.html) +- [Grafana Tempo Documentation](https://grafana.com/docs/tempo/latest/) +- [TraceQL Query Language](https://grafana.com/docs/tempo/latest/traceql/) diff --git a/f3s/tracing-demo/docker-image-Justfile b/f3s/tracing-demo/docker-image-Justfile new file mode 100644 index 0000000..7b263b1 --- /dev/null +++ b/f3s/tracing-demo/docker-image-Justfile @@ -0,0 +1,38 @@ +# Docker image build and push automation for tracing demo +# Similar to anki-sync-server pattern + +REGISTRY := "r0.lan.buetow.org:30001" + +# Build all images +build: + docker build -t tracing-demo-frontend:latest ./docker/frontend + docker build -t tracing-demo-middleware:latest ./docker/middleware + docker build -t tracing-demo-backend:latest ./docker/backend + +# Tag and push all images to private registry +push: + docker tag tracing-demo-frontend:latest {{REGISTRY}}/tracing-demo-frontend:latest + docker tag tracing-demo-middleware:latest {{REGISTRY}}/tracing-demo-middleware:latest + docker tag tracing-demo-backend:latest {{REGISTRY}}/tracing-demo-backend:latest + docker push {{REGISTRY}}/tracing-demo-frontend:latest + docker push {{REGISTRY}}/tracing-demo-middleware:latest + docker push {{REGISTRY}}/tracing-demo-backend:latest + +# Build and push in one command +all: build push + +# Build and push specific service +frontend: + docker build -t tracing-demo-frontend:latest ./docker/frontend + docker tag tracing-demo-frontend:latest {{REGISTRY}}/tracing-demo-frontend:latest + docker push {{REGISTRY}}/tracing-demo-frontend:latest + +middleware: + docker build -t tracing-demo-middleware:latest ./docker/middleware + docker tag tracing-demo-middleware:latest {{REGISTRY}}/tracing-demo-middleware:latest + docker push {{REGISTRY}}/tracing-demo-middleware:latest + +backend: + docker build -t tracing-demo-backend:latest ./docker/backend + docker tag tracing-demo-backend:latest {{REGISTRY}}/tracing-demo-backend:latest + docker push {{REGISTRY}}/tracing-demo-backend:latest diff --git a/f3s/tracing-demo/docker/backend/Dockerfile b/f3s/tracing-demo/docker/backend/Dockerfile new file mode 100644 index 0000000..5018e8f --- /dev/null +++ b/f3s/tracing-demo/docker/backend/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Copy and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app.py . + +# Expose port for Flask application +EXPOSE 5002 + +# Run the application +CMD ["python", "app.py"] diff --git a/f3s/tracing-demo/docker/backend/app.py b/f3s/tracing-demo/docker/backend/app.py new file mode 100644 index 0000000..2c9e88a --- /dev/null +++ b/f3s/tracing-demo/docker/backend/app.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +""" +Tracing Demo - Backend Service +Final service in the chain that returns data. +Simulates database queries and demonstrates end-to-end tracing. +""" +from flask import Flask, jsonify +import os +import logging +import time +from datetime import datetime + +# OpenTelemetry imports for distributed tracing +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.flask import FlaskInstrumentor +from opentelemetry.sdk.resources import Resource + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize OpenTelemetry tracing with resource attributes +# These attributes identify this service in traces +resource = Resource(attributes={ + "service.name": "backend", + "service.namespace": "tracing-demo", + "service.version": "1.0.0", + "deployment.environment": "production" +}) + +provider = TracerProvider(resource=resource) + +# Configure OTLP exporter to send traces to Alloy +otlp_exporter = OTLPSpanExporter( + endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', + 'http://alloy.monitoring.svc.cluster.local:4317'), + insecure=True +) + +# Batch spans for efficient export +processor = BatchSpanProcessor(otlp_exporter) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) + +# Get tracer for manual instrumentation +tracer = trace.get_tracer(__name__) + +# Create Flask application +app = Flask(__name__) + +# Auto-instrument Flask +FlaskInstrumentor().instrument_app(app) + +@app.route('/') +def index(): + """ + Health check and service information endpoint. + Returns service metadata. + """ + return jsonify({ + "service": "backend", + "version": "1.0.0", + "message": "Tracing demo backend service" + }) + +@app.route('/health') +def health(): + """ + Kubernetes health check endpoint. + Used by readiness and liveness probes. + """ + return jsonify({"status": "healthy"}), 200 + +@app.route('/api/data', methods=['GET']) +def get_data(): + """ + Return data endpoint that simulates a database query. + Creates custom spans to track query execution. + This is the final service in the trace chain. + """ + # Create a custom span for the database query simulation + with tracer.start_as_current_span("backend-get-data") as span: + # Add custom attributes to the span + span.set_attribute("backend.handler", "get_data") + + # Simulate database query delay + query_time = 0.1 + time.sleep(query_time) + + # Record query duration in span + span.set_attribute("backend.query.duration_ms", query_time * 1000) + span.set_attribute("backend.query.type", "simulated_database_query") + + # Prepare response data + data = { + "service": "backend", + "data": { + "id": 12345, + "value": "Sample data from backend service", + "timestamp": datetime.utcnow().isoformat(), + "query_time_ms": query_time * 1000 + } + } + + logger.info(f"Returning data: {data['data']['id']}") + + return jsonify(data), 200 + +if __name__ == '__main__': + logger.info("Starting backend service on port 5002") + logger.info(f"OTLP endpoint: {os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}") + app.run(host='0.0.0.0', port=5002, debug=False) diff --git a/f3s/tracing-demo/docker/backend/requirements.txt b/f3s/tracing-demo/docker/backend/requirements.txt new file mode 100644 index 0000000..6022d6c --- /dev/null +++ b/f3s/tracing-demo/docker/backend/requirements.txt @@ -0,0 +1,4 @@ +flask==3.0.0 +opentelemetry-distro==0.49b0 +opentelemetry-exporter-otlp==1.28.0 +opentelemetry-instrumentation-flask==0.49b0 diff --git a/f3s/tracing-demo/docker/frontend/Dockerfile b/f3s/tracing-demo/docker/frontend/Dockerfile new file mode 100644 index 0000000..dd28e97 --- /dev/null +++ b/f3s/tracing-demo/docker/frontend/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Copy and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app.py . + +# Expose port for Flask application +EXPOSE 5000 + +# Run the application +CMD ["python", "app.py"] diff --git a/f3s/tracing-demo/docker/frontend/app.py b/f3s/tracing-demo/docker/frontend/app.py new file mode 100644 index 0000000..65ab3f3 --- /dev/null +++ b/f3s/tracing-demo/docker/frontend/app.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Tracing Demo - Frontend Service +Receives user requests and forwards to middleware service. +Demonstrates OpenTelemetry auto-instrumentation with Flask. +""" +from flask import Flask, jsonify, request +import requests +import os +import logging + +# OpenTelemetry imports for distributed tracing +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.flask import FlaskInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor +from opentelemetry.sdk.resources import Resource + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize OpenTelemetry tracing with resource attributes +# These attributes identify this service in traces +resource = Resource(attributes={ + "service.name": "frontend", + "service.namespace": "tracing-demo", + "service.version": "1.0.0", + "deployment.environment": "production" +}) + +provider = TracerProvider(resource=resource) + +# Configure OTLP exporter to send traces to Alloy +otlp_exporter = OTLPSpanExporter( + endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', + 'http://alloy.monitoring.svc.cluster.local:4317'), + insecure=True +) + +# Batch spans for efficient export +processor = BatchSpanProcessor(otlp_exporter) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) + +# Get tracer for manual instrumentation if needed +tracer = trace.get_tracer(__name__) + +# Create Flask application +app = Flask(__name__) + +# Auto-instrument Flask to create spans for HTTP requests +FlaskInstrumentor().instrument_app(app) + +# Auto-instrument requests library to propagate trace context +RequestsInstrumentor().instrument() + +# Configuration for downstream services +MIDDLEWARE_URL = os.getenv('MIDDLEWARE_URL', + 'http://middleware-service.services.svc.cluster.local:5001') + +@app.route('/') +def index(): + """ + Health check and service information endpoint. + Returns service metadata. + """ + return jsonify({ + "service": "frontend", + "version": "1.0.0", + "message": "Tracing demo frontend service", + "trace_enabled": True, + "middleware_url": MIDDLEWARE_URL + }) + +@app.route('/health') +def health(): + """ + Kubernetes health check endpoint. + Used by readiness and liveness probes. + """ + return jsonify({"status": "healthy"}), 200 + +@app.route('/api/process', methods=['GET', 'POST']) +def process(): + """ + Main processing endpoint that demonstrates distributed tracing. + Forwards request to middleware service and returns combined response. + Creates a custom span to track the processing logic. + """ + # Create a custom span for the processing logic + with tracer.start_as_current_span("frontend-process") as span: + # Add custom attributes to the span for better observability + span.set_attribute("frontend.handler", "process") + + # Get request data (supports both GET and POST) + if request.method == 'POST': + data = request.get_json() or {} + else: + data = {"source": "GET request"} + + span.set_attribute("frontend.request.method", request.method) + + try: + # Call middleware service + # The requests library auto-instrumentation will create a span + # and propagate the trace context via W3C Trace Context headers + logger.info(f"Calling middleware at {MIDDLEWARE_URL}/api/transform") + + response = requests.post( + f'{MIDDLEWARE_URL}/api/transform', + json=data, + timeout=10 + ) + + response.raise_for_status() + middleware_data = response.json() + + # Record successful call in span + span.set_attribute("frontend.middleware.status", response.status_code) + + return jsonify({ + "service": "frontend", + "status": "success", + "request_data": data, + "middleware_response": middleware_data + }), 200 + + except requests.exceptions.RequestException as e: + # Log error and record in span + logger.error(f"Error calling middleware: {e}") + span.set_attribute("frontend.error", str(e)) + + # Set span status to error + span.set_status(trace.Status(trace.StatusCode.ERROR, str(e))) + + return jsonify({ + "service": "frontend", + "status": "error", + "error": str(e) + }), 500 + +if __name__ == '__main__': + logger.info("Starting frontend service on port 5000") + logger.info(f"Middleware URL: {MIDDLEWARE_URL}") + logger.info(f"OTLP endpoint: {os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}") + app.run(host='0.0.0.0', port=5000, debug=False) diff --git a/f3s/tracing-demo/docker/frontend/requirements.txt b/f3s/tracing-demo/docker/frontend/requirements.txt new file mode 100644 index 0000000..cb10687 --- /dev/null +++ b/f3s/tracing-demo/docker/frontend/requirements.txt @@ -0,0 +1,6 @@ +flask==3.0.0 +requests==2.31.0 +opentelemetry-distro==0.49b0 +opentelemetry-exporter-otlp==1.28.0 +opentelemetry-instrumentation-flask==0.49b0 +opentelemetry-instrumentation-requests==0.49b0 diff --git a/f3s/tracing-demo/docker/middleware/Dockerfile b/f3s/tracing-demo/docker/middleware/Dockerfile new file mode 100644 index 0000000..60272f7 --- /dev/null +++ b/f3s/tracing-demo/docker/middleware/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Copy and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app.py . + +# Expose port for Flask application +EXPOSE 5001 + +# Run the application +CMD ["python", "app.py"] diff --git a/f3s/tracing-demo/docker/middleware/app.py b/f3s/tracing-demo/docker/middleware/app.py new file mode 100644 index 0000000..9c0ad30 --- /dev/null +++ b/f3s/tracing-demo/docker/middleware/app.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +Tracing Demo - Middleware Service +Transforms data and calls backend service. +Demonstrates trace context propagation in a multi-tier architecture. +""" +from flask import Flask, jsonify, request +import requests +import os +import logging +import time + +# OpenTelemetry imports for distributed tracing +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.flask import FlaskInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor +from opentelemetry.sdk.resources import Resource + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize OpenTelemetry tracing with resource attributes +# These attributes identify this service in traces +resource = Resource(attributes={ + "service.name": "middleware", + "service.namespace": "tracing-demo", + "service.version": "1.0.0", + "deployment.environment": "production" +}) + +provider = TracerProvider(resource=resource) + +# Configure OTLP exporter to send traces to Alloy +otlp_exporter = OTLPSpanExporter( + endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', + 'http://alloy.monitoring.svc.cluster.local:4317'), + insecure=True +) + +# Batch spans for efficient export +processor = BatchSpanProcessor(otlp_exporter) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) + +# Get tracer for manual instrumentation +tracer = trace.get_tracer(__name__) + +# Create Flask application +app = Flask(__name__) + +# Auto-instrument Flask and requests library +FlaskInstrumentor().instrument_app(app) +RequestsInstrumentor().instrument() + +# Configuration for downstream services +BACKEND_URL = os.getenv('BACKEND_URL', + 'http://backend-service.services.svc.cluster.local:5002') + +@app.route('/') +def index(): + """ + Health check and service information endpoint. + Returns service metadata. + """ + return jsonify({ + "service": "middleware", + "version": "1.0.0", + "message": "Tracing demo middleware service", + "backend_url": BACKEND_URL + }) + +@app.route('/health') +def health(): + """ + Kubernetes health check endpoint. + Used by readiness and liveness probes. + """ + return jsonify({"status": "healthy"}), 200 + +@app.route('/api/transform', methods=['POST']) +def transform(): + """ + Transform data and fetch additional data from backend. + Demonstrates trace context propagation through multiple services. + Creates custom spans to track transformation logic. + """ + # Create a custom span for the transformation logic + with tracer.start_as_current_span("middleware-transform") as span: + # Add custom attributes to the span + span.set_attribute("middleware.handler", "transform") + + # Get request data from frontend + data = request.get_json() or {} + span.set_attribute("middleware.input.keys", str(list(data.keys()))) + + # Simulate some data transformation processing + time.sleep(0.05) + + try: + # Call backend service to fetch additional data + # The trace context is automatically propagated via HTTP headers + logger.info(f"Calling backend at {BACKEND_URL}/api/data") + + response = requests.get( + f'{BACKEND_URL}/api/data', + timeout=10 + ) + + response.raise_for_status() + backend_data = response.json() + + # Record successful call in span + span.set_attribute("middleware.backend.status", response.status_code) + + # Transform and combine the data + transformed = { + "middleware_processed": True, + "original_data": data, + "backend_data": backend_data, + "transformation_time_ms": 50 + } + + return jsonify(transformed), 200 + + except requests.exceptions.RequestException as e: + # Log error and record in span + logger.error(f"Error calling backend: {e}") + span.set_attribute("middleware.error", str(e)) + + # Set span status to error + span.set_status(trace.Status(trace.StatusCode.ERROR, str(e))) + + return jsonify({ + "service": "middleware", + "status": "error", + "error": str(e) + }), 500 + +if __name__ == '__main__': + logger.info("Starting middleware service on port 5001") + logger.info(f"Backend URL: {BACKEND_URL}") + logger.info(f"OTLP endpoint: {os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}") + app.run(host='0.0.0.0', port=5001, debug=False) diff --git a/f3s/tracing-demo/docker/middleware/requirements.txt b/f3s/tracing-demo/docker/middleware/requirements.txt new file mode 100644 index 0000000..cb10687 --- /dev/null +++ b/f3s/tracing-demo/docker/middleware/requirements.txt @@ -0,0 +1,6 @@ +flask==3.0.0 +requests==2.31.0 +opentelemetry-distro==0.49b0 +opentelemetry-exporter-otlp==1.28.0 +opentelemetry-instrumentation-flask==0.49b0 +opentelemetry-instrumentation-requests==0.49b0 diff --git a/f3s/tracing-demo/helm-chart/Chart.yaml b/f3s/tracing-demo/helm-chart/Chart.yaml new file mode 100644 index 0000000..c884ea0 --- /dev/null +++ b/f3s/tracing-demo/helm-chart/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: tracing-demo +description: A Helm chart for deploying distributed tracing demo application (Frontend, Middleware, Backend) +version: 0.1.0 +appVersion: "1.0.0" diff --git a/f3s/tracing-demo/helm-chart/templates/backend-deployment.yaml b/f3s/tracing-demo/helm-chart/templates/backend-deployment.yaml new file mode 100644 index 0000000..0a1f831 --- /dev/null +++ b/f3s/tracing-demo/helm-chart/templates/backend-deployment.yaml @@ -0,0 +1,51 @@ +# Backend Service Deployment +# Returns data (simulates database queries) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tracing-demo-backend + namespace: services + labels: + app: tracing-demo-backend + component: backend +spec: + replicas: 1 + selector: + matchLabels: + app: tracing-demo-backend + template: + metadata: + labels: + app: tracing-demo-backend + component: backend + spec: + containers: + - name: backend + image: registry.lan.buetow.org:30001/tracing-demo-backend:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5002 + name: http + protocol: TCP + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://alloy.monitoring.svc.cluster.local:4317" + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + livenessProbe: + httpGet: + path: /health + port: 5002 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: 5002 + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/f3s/tracing-demo/helm-chart/templates/backend-service.yaml b/f3s/tracing-demo/helm-chart/templates/backend-service.yaml new file mode 100644 index 0000000..a7f6e61 --- /dev/null +++ b/f3s/tracing-demo/helm-chart/templates/backend-service.yaml @@ -0,0 +1,17 @@ +# Backend Service +# Exposes the backend deployment within the cluster +apiVersion: v1 +kind: Service +metadata: + name: backend-service + namespace: services + labels: + app: tracing-demo-backend +spec: + ports: + - name: http + port: 5002 + protocol: TCP + targetPort: 5002 + selector: + app: tracing-demo-backend diff --git a/f3s/tracing-demo/helm-chart/templates/frontend-deployment.yaml b/f3s/tracing-demo/helm-chart/templates/frontend-deployment.yaml new file mode 100644 index 0000000..f607b01 --- /dev/null +++ b/f3s/tracing-demo/helm-chart/templates/frontend-deployment.yaml @@ -0,0 +1,53 @@ +# Frontend Service Deployment +# Receives HTTP requests and forwards to middleware +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tracing-demo-frontend + namespace: services + labels: + app: tracing-demo-frontend + component: frontend +spec: + replicas: 1 + selector: + matchLabels: + app: tracing-demo-frontend + template: + metadata: + labels: + app: tracing-demo-frontend + component: frontend + spec: + containers: + - name: frontend + image: registry.lan.buetow.org:30001/tracing-demo-frontend:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5000 + name: http + protocol: TCP + env: + - name: MIDDLEWARE_URL + value: "http://middleware-service.services.svc.cluster.local:5001" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://alloy.monitoring.svc.cluster.local:4317" + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + livenessProbe: + httpGet: + path: /health + port: 5000 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: 5000 + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/f3s/tracing-demo/helm-chart/templates/frontend-service.yaml b/f3s/tracing-demo/helm-chart/templates/frontend-service.yaml new file mode 100644 index 0000000..d45dd2a --- /dev/null +++ b/f3s/tracing-demo/helm-chart/templates/frontend-service.yaml @@ -0,0 +1,17 @@ +# Frontend Service +# Exposes the frontend deployment within the cluster +apiVersion: v1 +kind: Service +metadata: + name: frontend-service + namespace: services + labels: + app: tracing-demo-frontend +spec: + ports: + - name: http + port: 5000 + protocol: TCP + targetPort: 5000 + selector: + app: tracing-demo-frontend diff --git a/f3s/tracing-demo/helm-chart/templates/ingress.yaml b/f3s/tracing-demo/helm-chart/templates/ingress.yaml new file mode 100644 index 0000000..f080761 --- /dev/null +++ b/f3s/tracing-demo/helm-chart/templates/ingress.yaml @@ -0,0 +1,22 @@ +# Ingress for Frontend Service +# Exposes the tracing demo application at tracing-demo.f3s.buetow.org +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: tracing-demo-ingress + namespace: services + annotations: + spec.ingressClassName: traefik + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + rules: + - host: tracing-demo.f3s.buetow.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-service + port: + number: 5000 diff --git a/f3s/tracing-demo/helm-chart/templates/middleware-deployment.yaml b/f3s/tracing-demo/helm-chart/templates/middleware-deployment.yaml new file mode 100644 index 0000000..cae0c59 --- /dev/null +++ b/f3s/tracing-demo/helm-chart/templates/middleware-deployment.yaml @@ -0,0 +1,53 @@ +# Middleware Service Deployment +# Transforms data and calls backend +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tracing-demo-middleware + namespace: services + labels: + app: tracing-demo-middleware + component: middleware +spec: + replicas: 1 + selector: + matchLabels: + app: tracing-demo-middleware + template: + metadata: + labels: + app: tracing-demo-middleware + component: middleware + spec: + containers: + - name: middleware + image: registry.lan.buetow.org:30001/tracing-demo-middleware:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5001 + name: http + protocol: TCP + env: + - name: BACKEND_URL + value: "http://backend-service.services.svc.cluster.local:5002" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://alloy.monitoring.svc.cluster.local:4317" + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + livenessProbe: + httpGet: + path: /health + port: 5001 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: 5001 + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/f3s/tracing-demo/helm-chart/templates/middleware-service.yaml b/f3s/tracing-demo/helm-chart/templates/middleware-service.yaml new file mode 100644 index 0000000..08c325b --- /dev/null +++ b/f3s/tracing-demo/helm-chart/templates/middleware-service.yaml @@ -0,0 +1,17 @@ +# Middleware Service +# Exposes the middleware deployment within the cluster +apiVersion: v1 +kind: Service +metadata: + name: middleware-service + namespace: services + labels: + app: tracing-demo-middleware +spec: + ports: + - name: http + port: 5001 + protocol: TCP + targetPort: 5001 + selector: + app: tracing-demo-middleware |