summaryrefslogtreecommitdiff
path: root/f3s/prometheus
diff options
context:
space:
mode:
Diffstat (limited to 'f3s/prometheus')
-rw-r--r--f3s/prometheus/README.md58
-rw-r--r--f3s/prometheus/epimetheus-dashboard.yaml479
-rw-r--r--f3s/prometheus/persistence-values.yaml9
3 files changed, 546 insertions, 0 deletions
diff --git a/f3s/prometheus/README.md b/f3s/prometheus/README.md
index 9a5fb8b..9cd6a19 100644
--- a/f3s/prometheus/README.md
+++ b/f3s/prometheus/README.md
@@ -23,3 +23,61 @@ This ConfigMap is directly mounted to `/etc/grafana/provisioning/datasources/` i
- ✅ **Tempo** (uid=tempo) - Distributed tracing with traces-to-logs and traces-to-metrics correlation
**Note:** The sidecar-based provisioning is disabled in favor of direct ConfigMap mounting (following the pattern from /home/paul/git/x-rag/infra/k8s/monitoring/). See `problem.md` for the complete debugging journey and resolution.
+
+## Historic Data Ingestion
+
+Prometheus is configured to accept historic data with custom timestamps via the Remote Write API. This enables backfilling test data for ad-hoc troubleshooting and development purposes.
+
+### Configuration
+
+The following features are enabled in `persistence-values.yaml`:
+
+```yaml
+prometheus:
+ prometheusSpec:
+ # Enable Remote Write receiver endpoint
+ additionalArgs:
+ - name: web.enable-remote-write-receiver
+ value: ""
+
+ # Enable out-of-order ingestion for backfilling
+ enableFeatures:
+ - exemplar-storage
+ - otlp-write-receiver
+
+ # Allow backfilling up to 30 days in the past
+ tsdb:
+ outOfOrderTimeWindow: 720h # 30 days
+```
+
+### What This Enables
+
+- **Remote Write API**: HTTP endpoint at `/api/v1/write` for ingesting metrics with custom timestamps
+- **Out-of-Order Ingestion**: Allows writing data points older than existing data for the same time series
+- **30-Day Window**: Can backfill data up to 30 days in the past (configured via `outOfOrderTimeWindow`)
+
+### Use Cases
+
+This configuration is designed for:
+- **Testing**: Populating Grafana dashboards with synthetic historic data
+- **Development**: Simulating various time-series scenarios
+- **Troubleshooting**: Backfilling gaps in metric collection
+
+Example: The [Epimetheus](https://github.com/pbuetow/epimetheus) tool uses this to push test metrics with historic timestamps.
+
+### Performance Considerations
+
+**Important**: This is NOT a production-ready configuration. Enabling these features has trade-offs:
+
+- **Increased Memory Usage**: Out-of-order ingestion requires additional memory for buffering and sorting time series
+- **Higher TSDB Overhead**: Prometheus TSDB needs to handle non-sequential writes, increasing disk I/O
+- **Query Performance**: Queries may be slower due to fragmented data blocks
+- **Storage Amplification**: Out-of-order samples can trigger additional compactions, increasing storage usage
+
+**Recommendation**: For production environments:
+- Keep `outOfOrderTimeWindow` as small as possible (or disabled)
+- Monitor Prometheus memory and disk usage closely
+- Use Remote Write only when necessary
+- Consider using dedicated testing/development Prometheus instances
+
+**Note**: This setup is optimized for ad-hoc troubleshooting and development workflows, not for production monitoring at scale.
diff --git a/f3s/prometheus/epimetheus-dashboard.yaml b/f3s/prometheus/epimetheus-dashboard.yaml
new file mode 100644
index 0000000..abd7718
--- /dev/null
+++ b/f3s/prometheus/epimetheus-dashboard.yaml
@@ -0,0 +1,479 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: epimetheus-dashboard
+ namespace: monitoring
+ labels:
+ grafana_dashboard: '1'
+ app.kubernetes.io/instance: prometheus
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/part-of: kube-prometheus-stack
+ release: prometheus
+data:
+ epimetheus.json: |-
+ {
+ "id": null,
+ "uid": "epimetheus-test",
+ "title": "Epimetheus Test Metrics",
+ "tags": [
+ "epimetheus",
+ "test",
+ "demo"
+ ],
+ "timezone": "browser",
+ "schemaVersion": 38,
+ "version": 1,
+ "refresh": "10s",
+ "panels": [
+ {
+ "id": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "type": "timeseries",
+ "title": "Data Timeline - All Metrics Over Time",
+ "description": "Shows when metrics data is available - helps identify realtime vs historic data",
+ "targets": [
+ {
+ "expr": "epimetheus_test_requests_total",
+ "legendFormat": "Requests Total",
+ "refId": "A"
+ },
+ {
+ "expr": "epimetheus_test_active_connections",
+ "legendFormat": "Active Connections",
+ "refId": "B"
+ },
+ {
+ "expr": "epimetheus_test_temperature_celsius",
+ "legendFormat": "Temperature",
+ "refId": "C"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "barAlignment": 0,
+ "lineWidth": 1,
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "auto",
+ "pointSize": 5,
+ "stacking": {
+ "mode": "none",
+ "group": "A"
+ },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "hideFrom": {
+ "tooltip": false,
+ "viz": false,
+ "legend": false
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "color": {
+ "mode": "palette-classic"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ },
+ "legend": {
+ "showLegend": true,
+ "displayMode": "list",
+ "placement": "bottom",
+ "calcs": []
+ }
+ }
+ },
+ {
+ "id": 1,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 6
+ },
+ "type": "graph",
+ "title": "Request Rate",
+ "targets": [
+ {
+ "expr": "rate(epimetheus_test_requests_total[5m])",
+ "legendFormat": "Requests/sec",
+ "refId": "A"
+ }
+ ],
+ "yaxes": [
+ {
+ "format": "reqps",
+ "label": "Requests per Second"
+ },
+ {
+ "format": "short"
+ }
+ ],
+ "xaxis": {
+ "mode": "time"
+ },
+ "lines": true,
+ "fill": 1,
+ "linewidth": 2,
+ "pointradius": 5,
+ "tooltip": {
+ "shared": true
+ }
+ },
+ {
+ "id": 2,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 6
+ },
+ "type": "stat",
+ "title": "Total Requests",
+ "targets": [
+ {
+ "expr": "epimetheus_test_requests_total",
+ "legendFormat": "Total",
+ "refId": "A"
+ }
+ ],
+ "options": {
+ "graphMode": "area",
+ "colorMode": "value",
+ "justifyMode": "auto",
+ "textMode": "auto",
+ "reduceOptions": {
+ "values": false,
+ "calcs": [
+ "lastNotNull"
+ ]
+ }
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "unit": "short"
+ }
+ }
+ },
+ {
+ "id": 3,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 14
+ },
+ "type": "gauge",
+ "title": "Active Connections",
+ "targets": [
+ {
+ "expr": "epimetheus_test_active_connections",
+ "legendFormat": "Connections",
+ "refId": "A"
+ }
+ ],
+ "options": {
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "fieldConfig": {
+ "defaults": {
+ "min": 0,
+ "max": 100,
+ "color": {
+ "mode": "thresholds"
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": 0,
+ "color": "green"
+ },
+ {
+ "value": 50,
+ "color": "yellow"
+ },
+ {
+ "value": 80,
+ "color": "red"
+ }
+ ]
+ },
+ "unit": "short"
+ }
+ }
+ },
+ {
+ "id": 4,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 14
+ },
+ "type": "gauge",
+ "title": "Temperature (°C)",
+ "targets": [
+ {
+ "expr": "epimetheus_test_temperature_celsius",
+ "legendFormat": "°C",
+ "refId": "A"
+ }
+ ],
+ "options": {
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "fieldConfig": {
+ "defaults": {
+ "min": 0,
+ "max": 50,
+ "color": {
+ "mode": "thresholds"
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": 0,
+ "color": "blue"
+ },
+ {
+ "value": 20,
+ "color": "green"
+ },
+ {
+ "value": 30,
+ "color": "yellow"
+ },
+ {
+ "value": 35,
+ "color": "red"
+ }
+ ]
+ },
+ "unit": "celsius"
+ }
+ }
+ },
+ {
+ "id": 5,
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 22
+ },
+ "type": "graph",
+ "title": "Request Duration Histogram",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.50, rate(epimetheus_test_request_duration_seconds_bucket[5m]))",
+ "legendFormat": "p50",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.90, rate(epimetheus_test_request_duration_seconds_bucket[5m]))",
+ "legendFormat": "p90",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.99, rate(epimetheus_test_request_duration_seconds_bucket[5m]))",
+ "legendFormat": "p99",
+ "refId": "C"
+ }
+ ],
+ "yaxes": [
+ {
+ "format": "s",
+ "label": "Duration"
+ },
+ {
+ "format": "short"
+ }
+ ],
+ "xaxis": {
+ "mode": "time"
+ },
+ "lines": true,
+ "fill": 1,
+ "linewidth": 2,
+ "tooltip": {
+ "shared": true
+ }
+ },
+ {
+ "id": 6,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 30
+ },
+ "type": "stat",
+ "title": "Average Request Duration",
+ "targets": [
+ {
+ "expr": "rate(epimetheus_test_request_duration_seconds_sum[5m]) / rate(epimetheus_test_request_duration_seconds_count[5m])",
+ "legendFormat": "Avg Duration",
+ "refId": "A"
+ }
+ ],
+ "options": {
+ "graphMode": "area",
+ "colorMode": "value",
+ "textMode": "auto",
+ "reduceOptions": {
+ "values": false,
+ "calcs": [
+ "lastNotNull"
+ ]
+ }
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "unit": "s",
+ "decimals": 3
+ }
+ }
+ },
+ {
+ "id": 7,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "type": "bargauge",
+ "title": "Jobs Processed by Type",
+ "targets": [
+ {
+ "expr": "sum by (job_type) (epimetheus_test_jobs_processed_total)",
+ "legendFormat": "{{job_type}}",
+ "refId": "A"
+ }
+ ],
+ "options": {
+ "orientation": "horizontal",
+ "displayMode": "gradient",
+ "showUnfilled": true
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "unit": "short"
+ }
+ }
+ },
+ {
+ "id": 8,
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 38
+ },
+ "type": "table",
+ "title": "Jobs Status Breakdown",
+ "targets": [
+ {
+ "expr": "epimetheus_test_jobs_processed_total",
+ "legendFormat": "",
+ "refId": "A",
+ "format": "table",
+ "instant": true
+ }
+ ],
+ "options": {
+ "showHeader": true
+ },
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": true,
+ "__name__": true,
+ "instance": true,
+ "job": true
+ },
+ "indexByName": {},
+ "renameByName": {
+ "job_type": "Job Type",
+ "status": "Status",
+ "Value": "Count"
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m"
+ ]
+ },
+ "templating": {
+ "list": []
+ },
+ "annotations": {
+ "list": []
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
+ "links": [],
+ "liveNow": false,
+ "style": "dark"
+ }
diff --git a/f3s/prometheus/persistence-values.yaml b/f3s/prometheus/persistence-values.yaml
index 35119c9..588e28a 100644
--- a/f3s/prometheus/persistence-values.yaml
+++ b/f3s/prometheus/persistence-values.yaml
@@ -31,6 +31,15 @@ prometheus:
additionalArgs:
- name: web.enable-remote-write-receiver
value: ""
+ # Enable out-of-order ingestion to allow backfilling historic data
+ # even when newer data already exists for the same time series
+ enableFeatures:
+ - exemplar-storage
+ - otlp-write-receiver
+ # Set out-of-order time window to 30 days
+ additionalScrapeConfigs: []
+ tsdb:
+ outOfOrderTimeWindow: 720h # 30 days
additionalScrapeConfigsSecret:
enabled: true
name: additional-scrape-configs
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-11 08:26:14 +0000