From 3fd46f3977fb650974e5e936cba362c787c00637 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 7 Feb 2026 16:32:10 +0200
Subject: reimport this PoC

---
 AGENT.md                               |   63 ++
 CLAUDE.md                              |    1 +
 Magefile.go                            |  253 ++++++++
 README.md                              | 1000 ++++++++++++++++++++++++++++++++
 backfill-historic-data.sh              |   60 ++
 benchmark-100mb.sh                     |  223 +++++++
 benchmark-1gb.sh                       |  223 +++++++
 cleanup-benchmark-data.sh              |   88 +++
 cleanup-benchmark-metrics.sh           |   83 +++
 generate-test-data.sh                  |   46 ++
 go.mod                                 |   26 +
 go.sum                                 |   83 +++
 internal/config/config.go              |   57 ++
 internal/config/config_test.go         |   52 ++
 internal/ingester/auto.go              |  145 +++++
 internal/ingester/auto_test.go         |  164 ++++++
 internal/ingester/clickhouse.go        |  191 ++++++
 internal/ingester/pushgateway.go       |   51 ++
 internal/ingester/pushgateway_test.go  |   28 +
 internal/ingester/remotewrite.go       |  455 +++++++++++++++
 internal/ingester/remotewrite_test.go  |  210 +++++++
 internal/metrics/generator.go          |   85 +++
 internal/metrics/generator_test.go     |   53 ++
 internal/metrics/sample.go             |   34 ++
 internal/metrics/sample_test.go        |  160 +++++
 internal/parser/csv.go                 |  101 ++++
 internal/parser/csv_test.go            |  175 ++++++
 internal/parser/json.go                |   62 ++
 internal/parser/json_test.go           |  177 ++++++
 internal/parser/parser.go              |   56 ++
 internal/parser/parser_test.go         |   99 ++++
 internal/parser/tabular_csv.go         |  256 ++++++++
 internal/parser/tabular_csv_test.go    |  469 +++++++++++++++
 internal/resolver/dns_resolver.go      |  274 +++++++++
 internal/resolver/dns_resolver_test.go |  232 ++++++++
 internal/version/version.go            |    4 +
 internal/watcher/file_watcher.go       |   86 +++
 logo.png                               |  Bin 0 -> 145949 bytes
 run.sh                                 |   31 +
 test-data/watch-clickhouse-test.csv    |   12 +
 verify-clickhouse.sh                   |   52 ++
 41 files changed, 5920 insertions(+)
 create mode 100644 AGENT.md
 create mode 100644 CLAUDE.md
 create mode 100644 Magefile.go
 create mode 100644 README.md
 create mode 100755 backfill-historic-data.sh
 create mode 100755 benchmark-100mb.sh
 create mode 100755 benchmark-1gb.sh
 create mode 100755 cleanup-benchmark-data.sh
 create mode 100755 cleanup-benchmark-metrics.sh
 create mode 100755 generate-test-data.sh
 create mode 100644 go.mod
 create mode 100644 go.sum
 create mode 100644 internal/config/config.go
 create mode 100644 internal/config/config_test.go
 create mode 100644 internal/ingester/auto.go
 create mode 100644 internal/ingester/auto_test.go
 create mode 100644 internal/ingester/clickhouse.go
 create mode 100644 internal/ingester/pushgateway.go
 create mode 100644 internal/ingester/pushgateway_test.go
 create mode 100644 internal/ingester/remotewrite.go
 create mode 100644 internal/ingester/remotewrite_test.go
 create mode 100644 internal/metrics/generator.go
 create mode 100644 internal/metrics/generator_test.go
 create mode 100644 internal/metrics/sample.go
 create mode 100644 internal/metrics/sample_test.go
 create mode 100644 internal/parser/csv.go
 create mode 100644 internal/parser/csv_test.go
 create mode 100644 internal/parser/json.go
 create mode 100644 internal/parser/json_test.go
 create mode 100644 internal/parser/parser.go
 create mode 100644 internal/parser/parser_test.go
 create mode 100644 internal/parser/tabular_csv.go
 create mode 100644 internal/parser/tabular_csv_test.go
 create mode 100644 internal/resolver/dns_resolver.go
 create mode 100644 internal/resolver/dns_resolver_test.go
 create mode 100644 internal/version/version.go
 create mode 100644 internal/watcher/file_watcher.go
 create mode 100644 logo.png
 create mode 100755 run.sh
 create mode 100644 test-data/watch-clickhouse-test.csv
 create mode 100755 verify-clickhouse.sh

diff --git a/AGENT.md b/AGENT.md
new file mode 100644
index 0000000..d5c22e7
--- /dev/null
+++ b/AGENT.md
@@ -0,0 +1,63 @@
+Follow ~/git/conf/snippets/go/go-projects.md
+
+## Grafana Dashboard Guidelines
+
+When creating or updating Grafana dashboards:
+
+### Sorting Requirements
+**ALWAYS ensure ALL panels are sorted by value (descending):**
+
+1. **Time Series Panels:**
+   - Add to `options.legend`: `"sortBy": "Last", "sortDesc": true`
+   
+2. **Bar Gauge Panels:**
+   - Use `sort_desc()` in PromQL queries
+   - Example: `sort_desc(topk(10, sum by (label) (metric)))`
+   
+3. **Pie/Donut Chart Panels:**
+   - Add to `options.legend`: `"sortBy": "Value", "sortDesc": true`
+   
+4. **Table Panels:**
+   - Add to `options`: `"sortBy": [{"displayName": "ColumnName", "desc": true}]`
+
+### Example Panel Configuration
+
+**Time Series:**
+```json
+{
+  "type": "timeseries",
+  "options": {
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "sortBy": "Last",
+      "sortDesc": true,
+      "calcs": ["lastNotNull", "mean", "max"]
+    }
+  }
+}
+```
+
+**Bar Gauge:**
+```json
+{
+  "type": "bargauge",
+  "targets": [{
+    "expr": "sort_desc(topk(15, sum by (cust) (metric)))"
+  }],
+  "options": {
+    "orientation": "horizontal",
+    "displayMode": "gradient"
+  }
+}
+```
+
+**Table:**
+```json
+{
+  "type": "table",
+  "options": {
+    "sortBy": [{"displayName": "Count", "desc": true}]
+  }
+}
+```
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..02d954c
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1 @@
+Follow AGENT.md
diff --git a/Magefile.go b/Magefile.go
new file mode 100644
index 0000000..3cce6e0
--- /dev/null
+++ b/Magefile.go
@@ -0,0 +1,253 @@
+//go:build mage
+// +build mage
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
+)
+
+const (
+	binaryName = "epimetheus"
+	mainPath   = "./cmd/epimetheus"
+)
+
+// Default target to run when none is specified
+var Default = Build
+
+// Build compiles the epimetheus binary
+func Build() error {
+	fmt.Println("Building epimetheus...")
+	return sh.RunV("go", "build", "-o", binaryName, mainPath)
+}
+
+// Install installs the binary to $GOPATH/bin
+func Install() error {
+	fmt.Println("Installing epimetheus...")
+	return sh.RunV("go", "install", mainPath)
+}
+
+// Run executes the epimetheus binary in realtime mode
+func Run() error {
+	mg.Deps(Build)
+	fmt.Println("Running epimetheus in realtime mode...")
+	return sh.RunV("./"+binaryName, "-mode=realtime", "-continuous")
+}
+
+// RunHistoric runs epimetheus in historic mode
+func RunHistoric() error {
+	mg.Deps(Build)
+	fmt.Println("Running epimetheus in historic mode (24 hours ago)...")
+	return sh.RunV("./"+binaryName, "-mode=historic", "-hours-ago=24")
+}
+
+// RunAuto runs epimetheus in auto mode with a file
+func RunAuto(file string) error {
+	mg.Deps(Build)
+	if file == "" {
+		return fmt.Errorf("file parameter required: mage RunAuto <file>")
+	}
+	fmt.Printf("Running epimetheus in auto mode with file: %s\n", file)
+	return sh.RunV("./"+binaryName, "-mode=auto", "-file="+file)
+}
+
+// RunWatchClickHouse runs epimetheus in watch mode with ClickHouse ingestion
+func RunWatchClickHouse(file string) error {
+	mg.Deps(Build)
+	if file == "" {
+		file = "test-data/watch-clickhouse-test.csv"
+	}
+	fmt.Printf("Running epimetheus in watch mode with ClickHouse (file: %s)\n", file)
+	return sh.RunV("./"+binaryName, "-mode=watch", "-file="+file, "-metric-name=watch_test",
+		"-clickhouse=http://localhost:8123", "-prometheus=")
+}
+
+// Test runs all tests
+func Test() error {
+	fmt.Println("Running tests...")
+	return sh.RunV("go", "test", "./...", "-v")
+}
+
+// TestCoverage runs tests with coverage report
+func TestCoverage() error {
+	fmt.Println("Running tests with coverage...")
+	if err := sh.RunV("go", "test", "./...", "-cover", "-coverprofile=coverage.out"); err != nil {
+		return err
+	}
+	return sh.RunV("go", "tool", "cover", "-html=coverage.out", "-o", "coverage.html")
+}
+
+// TestRace runs tests with race detector
+func TestRace() error {
+	fmt.Println("Running tests with race detector...")
+	return sh.RunV("go", "test", "./...", "-race", "-v")
+}
+
+// Benchmark runs all benchmarks
+func Benchmark() error {
+	fmt.Println("Running benchmarks...")
+	return sh.RunV("go", "test", "./...", "-bench=.", "-benchmem")
+}
+
+// Lint runs golangci-lint
+func Lint() error {
+	fmt.Println("Running linter...")
+	return sh.RunV("golangci-lint", "run", "./...")
+}
+
+// Fmt formats all Go code
+func Fmt() error {
+	fmt.Println("Formatting code...")
+	return sh.RunV("go", "fmt", "./...")
+}
+
+// Vet runs go vet
+func Vet() error {
+	fmt.Println("Running go vet...")
+	return sh.RunV("go", "vet", "./...")
+}
+
+// Tidy runs go mod tidy
+func Tidy() error {
+	fmt.Println("Tidying dependencies...")
+	return sh.RunV("go", "mod", "tidy")
+}
+
+// Clean removes build artifacts
+func Clean() error {
+	fmt.Println("Cleaning build artifacts...")
+	files := []string{
+		binaryName,
+		"coverage.out",
+		"coverage.html",
+	}
+	for _, f := range files {
+		if err := sh.Rm(f); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// Generate runs go generate
+func Generate() error {
+	fmt.Println("Running go generate...")
+	return sh.RunV("go", "generate", "./...")
+}
+
+// Version prints the version
+func Version() error {
+	fmt.Println("Printing version...")
+	mg.Deps(Build)
+	return sh.RunV("./"+binaryName, "-version")
+}
+
+// All runs format, vet, test, and build
+func All() {
+	mg.Deps(Fmt, Vet, Test, Build)
+}
+
+// CI runs the full CI pipeline (format check, vet, test, build)
+func CI() error {
+	fmt.Println("Running CI pipeline...")
+	mg.Deps(Tidy, Vet, Test)
+	return Build()
+}
+
+// Dev starts development mode with port-forwarding
+func Dev() error {
+	mg.Deps(Build)
+	fmt.Println("Starting development mode...")
+	fmt.Println("Setting up port-forward to Pushgateway...")
+	
+	// Start port-forward in background
+	portForwardCmd := sh.RunCmd("kubectl", "port-forward", "-n", "monitoring", "svc/pushgateway", "9091:9091")
+	go func() {
+		if err := portForwardCmd(); err != nil {
+			fmt.Printf("Port-forward error: %v\n", err)
+		}
+	}()
+	
+	fmt.Println("Running epimetheus in realtime mode...")
+	return sh.RunV("./"+binaryName, "-mode=realtime", "-continuous")
+}
+
+// GenerateTestData creates test data files
+func GenerateTestData() error {
+	fmt.Println("Generating test data...")
+	return sh.RunV("./generate-test-data.sh")
+}
+
+// Backfill runs backfill for the last 48 hours
+func Backfill() error {
+	mg.Deps(Build)
+	fmt.Println("Running backfill (last 48 hours)...")
+	return sh.RunV("./"+binaryName, "-mode=backfill", "-start-hours=48", "-end-hours=0", "-interval=1")
+}
+
+// Benchmark100MB runs the 100MB benchmark
+func Benchmark100MB() error {
+	fmt.Println("Running 100MB benchmark...")
+	return sh.RunV("./benchmark-100mb.sh")
+}
+
+// Benchmark1GB runs the 1GB benchmark
+func Benchmark1GB() error {
+	fmt.Println("Running 1GB benchmark...")
+	return sh.RunV("./benchmark-1gb.sh")
+}
+
+// CleanupBenchmarkData removes benchmark data from Prometheus
+func CleanupBenchmarkData() error {
+	fmt.Println("Cleaning up benchmark data...")
+	return sh.RunV("./cleanup-benchmark-data.sh")
+}
+
+// CleanupBenchmarkMetrics removes benchmark metric files
+func CleanupBenchmarkMetrics() error {
+	fmt.Println("Cleaning up benchmark metric files...")
+	return sh.RunV("./cleanup-benchmark-metrics.sh")
+}
+
+// DeployDashboard deploys the Grafana dashboard
+func DeployDashboard() error {
+	fmt.Println("Deploying Grafana dashboard...")
+	return sh.RunV("./deploy-dashboard.sh")
+}
+
+// Help prints available targets
+func Help() {
+	fmt.Println("Available targets:")
+	fmt.Println("  build                - Build the epimetheus binary (default)")
+	fmt.Println("  install              - Install the binary to $GOPATH/bin")
+	fmt.Println("  run                  - Build and run in realtime mode")
+	fmt.Println("  runHistoric          - Build and run in historic mode")
+	fmt.Println("  runAuto <file>       - Build and run in auto mode with file")
+	fmt.Println("  runWatchClickHouse [file] - Build and run watch mode with ClickHouse (default: test-data/watch-clickhouse-test.csv)")
+	fmt.Println("  test                 - Run all tests")
+	fmt.Println("  testCoverage         - Run tests with coverage report")
+	fmt.Println("  testRace             - Run tests with race detector")
+	fmt.Println("  benchmark            - Run Go benchmarks")
+	fmt.Println("  lint                 - Run golangci-lint")
+	fmt.Println("  fmt                  - Format all Go code")
+	fmt.Println("  vet                  - Run go vet")
+	fmt.Println("  tidy                 - Run go mod tidy")
+	fmt.Println("  clean                - Remove build artifacts")
+	fmt.Println("  generate             - Run go generate")
+	fmt.Println("  version              - Print version")
+	fmt.Println("  all                  - Run fmt, vet, test, and build")
+	fmt.Println("  ci                   - Run full CI pipeline")
+	fmt.Println("  dev                  - Start development mode with port-forwarding")
+	fmt.Println("  generateTestData     - Generate test data files")
+	fmt.Println("  backfill             - Run backfill for last 48 hours")
+	fmt.Println("  benchmark100MB       - Run 100MB benchmark")
+	fmt.Println("  benchmark1GB         - Run 1GB benchmark")
+	fmt.Println("  cleanupBenchmarkData - Clean up benchmark data from Prometheus")
+	fmt.Println("  cleanupBenchmarkMetrics - Clean up benchmark metric files")
+	fmt.Println("  deployDashboard      - Deploy Grafana dashboard")
+	fmt.Println("  help                 - Print this help message")
+}
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..ba10a76
--- /dev/null
+++ b/README.md
@@ -0,0 +1,1000 @@
+<div align="center">
+  <img src="logo.png" alt="Epimetheus Logo" width="400"/>
+</div>
+
+# Epimetheus
+
+A versatile Go tool for pushing metrics to Prometheus with support for both realtime and historic data ingestion.
+
+## Why "Epimetheus"?
+
+In Greek mythology, [Epimetheus](https://en.wikipedia.org/wiki/Epimetheus_(mythology)) is Prometheus's brother, whose name means "afterthought" or "hindsight" (while Prometheus means "forethought"). This name cleverly captures the tool's purpose: bringing data to Prometheus **after** collection, whether it's historic data from hours, days, or weeks ago, or realtime data pushed on-demand.
+
+While Epimetheus is sometimes depicted as foolish in myths (he accepted Pandora's box despite warnings), this tool embraces the "afterthought" aspect productively - it's never too late to bring your metrics home to Prometheus!
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│                            Epimetheus                                   │
+│                     (Metrics Ingestion Tool)                            │
+│                                                                         │
+│  Modes:                                                                 │
+│  • Realtime  - Current metrics (< 5 min old)                          │
+│  • Historic  - Historic metrics (≥ 5 min old)                         │
+│  • Backfill  - Range of historic data                                 │
+│  • Auto      - Automatic routing based on timestamp age               │
+└─────────────────────────────────────────────────────────────────────────┘
+         │                                           │
+         │ Realtime Data                            │ Historic Data
+         │ (via HTTP POST)                          │ (via Remote Write API)
+         │ Uses "now" timestamp                     │ Preserves timestamps
+         ▼                                           ▼
+┌─────────────────────┐                    ┌─────────────────────┐
+│   Pushgateway       │                    │    Prometheus       │
+│   (Port 9091)       │                    │    (Port 9090)      │
+│                     │                    │                     │
+│ • Buffers metrics   │                    │ Remote Write API:   │
+│ • Scraped by        │──── Scraped ─────▶ │ /api/v1/write       │
+│   Prometheus        │    every 15-30s    │                     │
+│ • No timestamp      │                    │ Feature Required:   │
+│   preservation      │                    │ --enable-feature=   │
+│                     │                    │  remote-write-      │
+│                     │                    │  receiver           │
+└─────────────────────┘                    └─────────────────────┘
+                                                      │
+                                                      │ Prometheus Query API
+                                                      │ /api/v1/query
+                                                      ▼
+                                           ┌─────────────────────┐
+                                           │     Grafana         │
+                                           │   (Port 3000)       │
+                                           │                     │
+                                           │ • Prometheus as     │
+                                           │   datasource        │
+                                           │ • Dashboards:       │
+                                           │   - Epimetheus      │
+                                           │     Test Metrics    │
+                                           │ • Auto-refresh      │
+                                           └─────────────────────┘
+```
+
+### Data Flow
+
+1. **Realtime Path** (for current data):
+   - Epimetheus → Pushgateway (HTTP POST)
+   - Prometheus scrapes Pushgateway periodically
+   - Timestamp = "now" when Prometheus scrapes
+
+2. **Historic Path** (for old data):
+   - Epimetheus → Prometheus Remote Write API (HTTP POST)
+   - Direct write to Prometheus TSDB
+   - Timestamp preserved from original data
+
+3. **Visualization**:
+   - Grafana queries Prometheus
+   - Displays metrics in dashboards
+   - Auto-refresh every 10 seconds
+
+## Overview
+
+**epimetheus** is a standalone binary that:
+- **Generates** realistic example metrics simulating production applications
+- **Pushes** metrics via Pushgateway (realtime) or Remote Write API (historic)
+- **Automatically detects** timestamp age and chooses the optimal ingestion method
+- **Supports** multiple data formats (CSV, JSON) and all Prometheus metric types
+- **Provides** Grafana dashboard for visualizing test metrics
+
+## Quick Start
+
+### 1. Deploy Pushgateway (one-time setup)
+
+The Pushgateway Helm chart is available in the [conf repository](https://codeberg.org/snonux/conf) at `f3s/pushgateway/helm-chart`.
+
+```bash
+# Clone the conf repository if you haven't already
+git clone https://codeberg.org/snonux/conf.git
+cd conf/f3s/pushgateway/helm-chart
+
+# Deploy Pushgateway
+helm upgrade --install pushgateway . -n monitoring --create-namespace
+```
+
+Alternatively, deploy Pushgateway using the official chart:
+
+```bash
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm install pushgateway prometheus-community/prometheus-pushgateway -n monitoring --create-namespace
+```
+
+### 2. Run in Realtime Mode
+
+```bash
+# Port-forward Pushgateway
+kubectl port-forward -n monitoring svc/pushgateway 9091:9091 &
+
+# Push test metrics continuously
+cd /home/paul/git/conf/f3s/epimetheus
+./epimetheus -mode=realtime -continuous
+```
+
+The binary pushes metrics every 15 seconds. Press Ctrl+C to stop.
+
+### 3. View Metrics
+
+```bash
+# Pushgateway UI
+open http://localhost:9091
+
+# Prometheus UI
+kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 &
+open http://localhost:9090
+```
+
+## Operating Modes
+
+### 👁️ Watch Mode
+Monitor CSV files for changes and push metrics to Prometheus with file modification timestamps.
+
+**Works with ANY CSV format** - automatically detects numeric vs string columns and sanitizes names.
+
+**NEW: Automatic DNS Resolution** - IP addresses are automatically resolved to hostnames for better observability in Grafana.
+
+```bash
+./epimetheus -mode=watch \
+    -file=mydata.csv \
+    -metric-name=myapp \
+    -prometheus=http://localhost:9090/api/v1/write
+```
+
+**Features:**
+- 🔍 **Format-agnostic**: Works with any tabular CSV structure
+- 📊 **Automatic detection**: Numeric columns → metrics, String columns → labels
+- 🏷️ **Name sanitization**: `min(potatoes)`, `avg(time)`, `p99(latency)` → valid metric names
+- 🌐 **DNS Resolution**: IP addresses → hostnames (e.g., `10.50.52.61` → `foo.example.lan`)
+- 💾 **Smart Caching**: In-memory cache prevents redundant DNS lookups
+- ⏱️ **Timestamp preservation**: Uses file modification time
+- 🔄 **Continuous monitoring**: Polls file every 1 second
+- 💪 **Error resilient**: Continues watching despite failures
+- 🎯 **Remote Write**: Pushes to Prometheus (preserves timestamps)
+
+**CSV Format:**
+Works with any tabular CSV:
+- First row: column headers (automatically sanitized)
+- Subsequent rows: data values
+- Column names can be anything: `min(x)`, `avg(y)`, `p99(latency)`, etc.
+
+**Example 1** - Web metrics:
+```csv
+avg(response_time),p99(latency),endpoint,method
+45.2,120.5,/api/users,GET
+52.1,135.8,/api/orders,POST
+```
+
+Generates:
+```promql
+web_avg_response_time{endpoint="/api/users",method="GET"} 45.2
+web_p99_latency{endpoint="/api/users",method="GET"} 120.5
+web_avg_response_time{endpoint="/api/orders",method="POST"} 52.1
+web_p99_latency{endpoint="/api/orders",method="POST"} 135.8
+```
+
+**Example 2** - Food metrics:
+```csv
+min(potatoes),last(coke),avg(price),country,store_type
+5.2,10.5,12.99,USA,grocery
+3.8,8.2,9.99,Canada,convenience
+```
+
+Generates:
+```promql
+food_min_potatoes{country="USA",store_type="grocery"} 5.2
+food_last_coke{country="USA",store_type="grocery"} 10.5
+food_avg_price{country="USA",store_type="grocery"} 12.99
+# ... etc
+```
+
+Each row generates N samples (N = number of numeric columns).
+
+See [CSV-FORMAT-FLEXIBILITY.md](CSV-FORMAT-FLEXIBILITY.md) for more examples.
+
+**Options:**
+- `-file` - CSV file to watch (required)
+- `-metric-name` - Base metric name (required, e.g., `food`, `network`, `database`)
+- `-prometheus` - Prometheus Remote Write URL (default: http://localhost:9090/api/v1/write)
+- `-clickhouse` - ClickHouse HTTP URL (e.g. http://localhost:8123) to also ingest metrics
+- `-clickhouse-table` - ClickHouse table name (default: epimetheus_metrics)
+- `-job` - Job name for metrics (default: example_metrics_pusher)
+- `-resolve-ip-labels` - Additional IP labels to resolve via DNS (default: ip is always resolved)
+
+**ClickHouse Support:**
+Watch mode can ingest to ClickHouse in addition to (or instead of) Prometheus:
+
+```bash
+# Ingest to both Prometheus and ClickHouse
+./epimetheus -mode=watch -file=data.csv -metric-name=myapp \
+    -prometheus=http://localhost:9090/api/v1/write \
+    -clickhouse=http://localhost:8123
+
+# ClickHouse only (use -prometheus= to disable Prometheus)
+./epimetheus -mode=watch -file=test-data/watch-clickhouse-test.csv \
+    -metric-name=watch_test -clickhouse=http://localhost:8123 -prometheus=
+
+# Verify data in ClickHouse
+./verify-clickhouse.sh
+```
+
+**DNS Resolution:**
+By default, the `ip` label is automatically resolved to a hostname. To resolve additional IP labels:
+
+```bash
+./epimetheus -mode=watch \
+    -file=network.csv \
+    -metric-name=network \
+    -resolve-ip-labels=source_ip,dest_ip
+```
+
+This will resolve: `ip` (default) + `source_ip` + `dest_ip`
+
+**Example:**
+- Input: `ip="10.50.52.61"`
+- Output: `ip="foo.example.lan"`
+- Failed lookups: IP remains unchanged
+
+**Documentation:**
+- [DNS-RESOLUTION-FEATURE.md](DNS-RESOLUTION-FEATURE.md) - Complete DNS resolution guide
+- [CSV-FORMAT-FLEXIBILITY.md](CSV-FORMAT-FLEXIBILITY.md) - Works with ANY CSV format
+- [DTAIL-METRICS-EXAMPLE.md](DTAIL-METRICS-EXAMPLE.md) - Detailed dtail.csv example
+
+### 🔄 Realtime Mode (Default)
+Push current metrics to Pushgateway with "now" timestamp.
+
+```bash
+./epimetheus -mode=realtime -continuous
+```
+
+**Options:**
+- `-pushgateway` - Pushgateway URL (default: http://localhost:9091)
+- `-job` - Job name (default: example_metrics_pusher)
+- `-continuous` - Keep pushing every 15 seconds
+
+### ⏰ Historic Mode
+Push a single datapoint from the past using Remote Write API.
+
+```bash
+# Port-forward Prometheus
+kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 &
+
+# Push data from 24 hours ago
+./epimetheus -mode=historic -hours-ago=24
+```
+
+**Options:**
+- `-prometheus` - Prometheus URL (default: http://localhost:9090/api/v1/write)
+- `-hours-ago` - Hours in the past (default: 24)
+
+### 📦 Backfill Mode
+Import a range of historic data points.
+
+```bash
+# Backfill last 48 hours with 1-hour intervals
+./epimetheus -mode=backfill -start-hours=48 -end-hours=0 -interval=1
+
+# Backfill last week with 6-hour intervals
+./epimetheus -mode=backfill -start-hours=168 -end-hours=0 -interval=6
+```
+
+**Options:**
+- `-start-hours` - Start time in hours ago
+- `-end-hours` - End time in hours ago (0 = now)
+- `-interval` - Interval between points in hours
+
+### 🤖 Auto Mode (Recommended!)
+Automatically detect timestamp age and route to the correct ingestion method.
+
+```bash
+# Generate test data
+./generate-test-data.sh
+
+# Import mixed current and historic data
+./epimetheus -mode=auto -file=test-all-ages.csv
+```
+
+**Detection Logic:**
+- Data < 5 minutes old → Pushgateway (realtime)
+- Data ≥ 5 minutes old → Remote Write (historic)
+
+**Options:**
+- `-file` - Input file path
+- `-format` - Data format: csv or json (default: csv)
+- `-pushgateway` - Pushgateway URL
+- `-prometheus` - Prometheus Remote Write URL
+
+## Data Formats
+
+### CSV Format
+
+```csv
+# Format: metric_name,labels,value,timestamp_ms
+# Labels: key1=value1;key2=value2
+epimetheus_test_requests_total,instance=web1;env=prod,100,1767125148000
+epimetheus_test_temperature_celsius,instance=web2,22.5,1767038748000
+
+# Timestamp is optional (uses "now" if omitted)
+epimetheus_test_active_connections,instance=web3,42,
+```
+
+### JSON Format
+
+```json
+[
+  {
+    "metric": "epimetheus_test_requests_total",
+    "labels": {"instance": "web1", "env": "prod"},
+    "value": 100,
+    "timestamp_ms": 1767125148000
+  },
+  {
+    "metric": "epimetheus_test_temperature_celsius",
+    "labels": {"instance": "web2"},
+    "value": 22.5,
+    "timestamp_ms": 1767038748000
+  }
+]
+```
+
+## Test Metrics
+
+All generated metrics use the `epimetheus_test_` prefix to clearly identify them as test data.
+
+### Counter: `epimetheus_test_requests_total`
+- **Type:** Counter (monotonically increasing)
+- **Description:** Total number of requests processed
+- **Use case:** Counting total events, requests, errors
+
+### Gauge: `epimetheus_test_active_connections`
+- **Type:** Gauge (can increase or decrease)
+- **Description:** Current number of active connections (0-100)
+- **Use case:** Current state measurements, capacity
+
+### Gauge: `epimetheus_test_temperature_celsius`
+- **Type:** Gauge
+- **Description:** Current temperature in Celsius (0-50°C)
+- **Use case:** Environmental monitoring
+
+### Histogram: `epimetheus_test_request_duration_seconds`
+- **Type:** Histogram (distribution)
+- **Description:** Request duration distribution
+- **Buckets:** 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10 seconds
+- **Use case:** Latency measurements, SLO tracking
+
+### Labeled Counter: `epimetheus_test_jobs_processed_total`
+- **Type:** Counter with labels
+- **Description:** Jobs processed by type and status
+- **Labels:**
+  - `job_type`: email, report, backup
+  - `status`: success, failed
+- **Use case:** Categorized counting, multi-dimensional metrics
+
+## Grafana Dashboard
+
+A comprehensive dashboard is available showcasing all test metrics.
+
+### Dashboard Features
+
+- **8 Panels:**
+  1. Request Rate (line graph)
+  2. Total Requests (stat panel)
+  3. Active Connections (gauge with thresholds)
+  4. Temperature (gauge with thresholds)
+  5. Request Duration Histogram (p50, p90, p99)
+  6. Average Request Duration (stat)
+  7. Jobs Processed by Type (bar gauge)
+  8. Jobs Status Breakdown (table)
+
+- **Auto-refresh:** Every 10 seconds
+- **Time range:** Last 15 minutes (customizable)
+- **Dark theme optimized**
+
+### Deploy Dashboard
+
+#### Option 1: Helm/Kubernetes ConfigMap (Recommended)
+
+```bash
+# Deploy via Kubernetes ConfigMap
+kubectl apply -f ../prometheus/epimetheus-dashboard.yaml
+```
+
+The dashboard will be automatically discovered by Grafana.
+
+#### Option 2: Manual Import
+
+```bash
+# Port-forward Grafana
+kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80
+
+# Open Grafana
+open http://localhost:3000
+
+# Go to Dashboards → Import → Upload grafana-dashboard.json
+```
+
+#### Option 3: Automated Script
+
+```bash
+# Deploy via API
+./deploy-dashboard.sh
+
+# Or with custom credentials
+GRAFANA_URL="http://localhost:3000" \
+GRAFANA_USER="admin" \
+GRAFANA_PASSWORD="yourpassword" \
+./deploy-dashboard.sh
+```
+
+## Example Queries
+
+### Basic Queries
+
+```promql
+# View total requests
+epimetheus_test_requests_total
+
+# View request rate over last 5 minutes
+rate(epimetheus_test_requests_total[5m])
+
+# View current active connections
+epimetheus_test_active_connections
+
+# View current temperature
+epimetheus_test_temperature_celsius
+```
+
+### Histogram Queries
+
+```promql
+# 95th percentile request duration
+histogram_quantile(0.95, rate(epimetheus_test_request_duration_seconds_bucket[5m]))
+
+# 50th percentile (median)
+histogram_quantile(0.50, rate(epimetheus_test_request_duration_seconds_bucket[5m]))
+
+# Average request duration
+rate(epimetheus_test_request_duration_seconds_sum[5m]) /
+rate(epimetheus_test_request_duration_seconds_count[5m])
+```
+
+### Labeled Counter Queries
+
+```promql
+# Failed jobs by type
+epimetheus_test_jobs_processed_total{status="failed"}
+
+# Job success rate
+rate(epimetheus_test_jobs_processed_total{status="success"}[5m]) /
+rate(epimetheus_test_jobs_processed_total[5m])
+
+# Total jobs by type
+sum by (job_type) (epimetheus_test_jobs_processed_total)
+```
+
+### Curl Examples
+
+```bash
+# Port-forward Prometheus
+kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 &
+
+# Query total requests
+curl -s "http://localhost:9090/api/v1/query?query=epimetheus_test_requests_total" | jq .
+
+# Query temperature
+curl -s "http://localhost:9090/api/v1/query?query=epimetheus_test_temperature_celsius" | jq .
+
+# Query request rate
+curl -s "http://localhost:9090/api/v1/query?query=rate(epimetheus_test_requests_total[5m])" | jq .
+
+# Query histogram p95
+curl -s "http://localhost:9090/api/v1/query?query=histogram_quantile(0.95,rate(epimetheus_test_request_duration_seconds_bucket[5m]))" | jq .
+```
+
+## Time Range Limitations
+
+### ✅ Supported Time Ranges
+
+| Time Range | Status | Method |
+|------------|--------|--------|
+| Current (< 5 min) | ✅ Works | Pushgateway |
+| 1 hour old | ✅ Works | Remote Write |
+| 1 day old | ✅ Works | Remote Write |
+| 1 week old | ✅ Works | Remote Write |
+| 1 month old | ✅ Works | Remote Write |
+
+### ⚠️ Potential Issues
+
+- **Future timestamps:** Rejected (> 5 minutes in future)
+- **Very old data (6+ months):** May be rejected depending on Prometheus retention
+- **Years old:** Likely rejected - use `promtool tsdb create-blocks-from` instead
+- **Out-of-order samples:** Can't insert older data into existing time series (use different labels)
+
+### Prometheus Configuration
+
+Check your retention settings:
+
+```bash
+# View retention
+kubectl get prometheus -n monitoring prometheus-kube-prometheus-prometheus \
+  -o jsonpath='{.spec.retention}'
+
+# Default is typically 15 days
+```
+
+For very old data:
+- Increase retention in Prometheus config
+- Enable out-of-order ingestion (experimental)
+- Use `promtool` for direct TSDB block creation
+
+## Project Structure
+
+```
+epimetheus/
+├── cmd/
+│   └── epimetheus/
+│       └── main.go              # Main entry point
+├── internal/
+│   ├── config/                  # Configuration
+│   ├── metrics/                 # Metric generators
+│   ├── parser/                  # CSV/JSON parsers (includes tabular CSV)
+│   ├── ingester/                # Pushgateway & Remote Write ingesters
+│   └── watcher/                 # File watcher for watch mode
+├── epimetheus            # Compiled binary
+├── grafana-dashboard.json       # Grafana dashboard definition
+├── deploy-dashboard.sh          # Dashboard deployment script
+├── generate-test-data.sh        # Test data generator
+├── run.sh                       # Helper script
+└── README.md                    # This file
+```
+
+## Setup Requirements
+
+### 1. Enable Prometheus Remote Write Receiver ⚠️ **REQUIRED for Historic Data**
+
+**IMPORTANT**: To use historic mode, backfill mode, or auto mode with old data, you **must** enable the Prometheus Remote Write receiver. Without this feature, Epimetheus can only push realtime data via Pushgateway.
+
+The Remote Write receiver is configured in the [conf repository](https://codeberg.org/snonux/conf) at `f3s/prometheus/persistence-values.yaml`:
+
+```yaml
+# In prometheus/persistence-values.yaml (from conf repository)
+prometheus:
+  prometheusSpec:
+    # Enable Remote Write receiver endpoint and Admin API (Prometheus 3.x syntax)
+    additionalArgs:
+      - name: web.enable-remote-write-receiver
+        value: ""
+      - name: web.enable-admin-api
+        value: ""
+
+    # Enable out-of-order ingestion for backfilling
+    # Allows writing data points older than existing data for the same time series
+    enableFeatures:
+      - exemplar-storage
+      - otlp-write-receiver
+
+    # Allow backfilling up to 31 days in the past (provides 1-day buffer for 30-day datasets)
+    tsdb:
+      outOfOrderTimeWindow: 744h  # 31 days
+```
+
+**What This Enables:**
+- **Remote Write API**: HTTP endpoint at `/api/v1/write` for ingesting metrics with custom timestamps
+- **Admin API**: HTTP endpoints at `/api/v1/admin/tsdb/*` for data deletion and management
+- **Out-of-Order Ingestion**: Allows writing data points older than existing data for the same time series
+- **31-Day Window**: Can backfill data up to 31 days in the past (provides 1-day buffer for 30-day datasets)
+
+After updating the configuration, upgrade your Prometheus installation:
+
+```bash
+cd conf/f3s/prometheus
+just upgrade  # Or manually:
+# helm upgrade prometheus prometheus-community/kube-prometheus-stack \
+#   -n monitoring -f persistence-values.yaml
+```
+
+Verify the features are enabled:
+
+```bash
+# Check Remote Write receiver flag
+kubectl get pod -n monitoring prometheus-prometheus-kube-prometheus-prometheus-0 \
+  -o jsonpath='{.spec.containers[0].args}' | grep -o "web.enable-remote-write-receiver"
+
+# Check out-of-order time window
+kubectl get prometheus -n monitoring prometheus-kube-prometheus-prometheus \
+  -o jsonpath='{.spec.tsdb.outOfOrderTimeWindow}'
+# Should output: 744h
+
+# Check admin API flag
+kubectl get pod -n monitoring prometheus-prometheus-kube-prometheus-prometheus-0 \
+  -o jsonpath='{.spec.containers[0].args}' | grep -o "web.enable-admin-api"
+```
+
+**Performance Considerations:**
+
+This configuration is designed for ad-hoc troubleshooting and development, **NOT production use**. Enabling these features has trade-offs:
+
+- **Increased Memory Usage**: Out-of-order ingestion requires additional memory for buffering and sorting time series
+- **Higher TSDB Overhead**: Prometheus TSDB needs to handle non-sequential writes, increasing disk I/O
+- **Query Performance**: Queries may be slower due to fragmented data blocks
+- **Storage Amplification**: Out-of-order samples can trigger additional compactions, increasing storage usage
+
+**Recommendation for Production:**
+- Keep `outOfOrderTimeWindow` as small as possible (or disabled)
+- Monitor Prometheus memory and disk usage closely
+- Use Remote Write only when necessary
+- Consider using dedicated testing/development Prometheus instances
+
+**Note**: The syntax changed in Prometheus 3.x - use `additionalArgs` with `web.enable-remote-write-receiver` instead of the deprecated `enableFeatures: [remote-write-receiver]`.
+
+### 2. Update Prometheus Scrape Config
+
+Ensure Pushgateway is in scrape targets:
+
+```yaml
+# additional-scrape-configs.yaml
+- job_name: 'pushgateway'
+  honor_labels: true
+  static_configs:
+    - targets:
+      - 'pushgateway.monitoring.svc.cluster.local:9091'
+```
+
+Apply the configuration:
+
+```bash
+kubectl create secret generic additional-scrape-configs \
+  --from-file=/home/paul/git/conf/f3s/prometheus/additional-scrape-configs.yaml \
+  --dry-run=client -o yaml -n monitoring | kubectl apply -f -
+```
+
+## Building from Source
+
+### Using Mage (Recommended)
+
+This project includes a [Magefile](./MAGEFILE.md) for easy building, testing, and running:
+
+```bash
+# Install Mage (one-time setup)
+go install github.com/magefile/mage@latest
+
+# Build binary
+mage build
+
+# Run tests
+mage test
+
+# Run with coverage report
+mage testCoverage
+
+# Run in realtime mode
+mage run
+
+# See all available targets
+mage -l
+```
+
+See [MAGEFILE.md](./MAGEFILE.md) for complete documentation.
+
+### Using Go directly
+
+```bash
+# Build binary
+go build -o epimetheus cmd/epimetheus/main.go
+
+# Run tests
+go test ./... -v
+
+# Check test coverage
+go test ./... -cover
+```
+
+## Troubleshooting
+
+### Binary can't connect to Pushgateway
+
+```bash
+# Check port-forward is running
+ps aux | grep "port-forward.*9091"
+
+# Restart port-forward
+kubectl port-forward -n monitoring svc/pushgateway 9091:9091
+```
+
+### Metrics not appearing in Prometheus
+
+```bash
+# Check Pushgateway has metrics
+curl http://localhost:9091/metrics | grep "prometheus_pusher_test"
+
+# Check Prometheus scrape targets
+# Open http://localhost:9090/targets - look for "pushgateway" job
+
+# Check Prometheus logs
+kubectl logs -n monitoring -l app.kubernetes.io/name=prometheus
+```
+
+### "Remote write receiver not enabled" error
+
+```bash
+# Verify feature is enabled
+kubectl logs -n monitoring prometheus-prometheus-kube-prometheus-prometheus-0 | grep "remote-write-receiver"
+
+# Should see: msg="Experimental features enabled" features=[remote-write-receiver]
+```
+
+### "Out of order sample" error
+
+This occurs when trying to insert data older than existing data for the same time series.
+
+**Solutions:**
+- Use different job labels for historic data (e.g., `job="historic_data"`)
+- Enable out-of-order ingestion in Prometheus (experimental)
+- Ensure backfill goes from oldest to newest
+
+### Dashboard not appearing in Grafana
+
+```bash
+# Check ConfigMap exists
+kubectl get configmap -n monitoring | grep epimetheus
+
+# Check labels
+kubectl get configmap epimetheus-dashboard -n monitoring -o yaml | grep "grafana_dashboard"
+
+# Restart Grafana to force reload
+kubectl rollout restart deployment/prometheus-grafana -n monitoring
+```
+
+## Architecture
+
+```
+┌─────────────────┐
+│  Go Binary      │
+│ (prometheus-    │──Push realtime──┐
+│  pusher)        │                 │
+└─────────────────┘                 ▼
+         │                  ┌──────────────────┐
+         │                  │  Pushgateway     │◄──Scrape──┐
+         │                  │  (Port 9091)     │           │
+         │                  └──────────────────┘           │
+         │                                                 │
+         └──Push historic──────────────────┐              │
+                                            ▼              │
+                                  ┌─────────────────┐     │
+                                  │   Prometheus    │◄────┘
+                                  │   (Port 9090)   │
+                                  │ Remote Write API│
+                                  └─────────────────┘
+                                           │
+                                           │ Datasource
+                                           ▼
+                                  ┌─────────────────┐
+                                  │    Grafana      │
+                                  │   (Port 3000)   │
+                                  │   Dashboards    │
+                                  └─────────────────┘
+```
+
+## Best Practices
+
+### When to Use Pushgateway vs. Remote Write
+
+**Use Pushgateway (realtime mode):**
+- Short-lived batch jobs
+- Service-level metrics
+- Jobs behind firewalls
+- Current/recent data (< 5 minutes old)
+
+**Use Remote Write (historic mode):**
+- Historic data import
+- Backfilling gaps
+- Data migration
+- Data older than 5 minutes
+
+**Use Auto Mode:**
+- Mixed current and historic data
+- Importing from files
+- Unknown timestamp ages
+- General-purpose ingestion
+
+### Metric Design
+
+- **Use appropriate metric types:**
+  - Counter for cumulative values (requests, errors)
+  - Gauge for point-in-time values (temperature, connections)
+  - Histogram for distributions (latency, sizes)
+
+- **Label cardinality:**
+  - Include meaningful labels
+  - Avoid high-cardinality labels (user IDs, timestamps)
+  - Keep label combinations reasonable (< 1000 per metric)
+
+- **Naming conventions:**
+  - Use descriptive names
+  - Include units in gauge names (\_celsius, \_bytes)
+  - Use \_total suffix for counters
+
+## Cleanup
+
+### Cleaning Up Benchmark Data from Prometheus
+
+For cleaning up benchmark metrics from Prometheus, use the provided cleanup script:
+
+```bash
+# Port-forward to Prometheus
+kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 &
+
+# Run the cleanup script
+./cleanup-benchmark-data.sh
+```
+
+The script will:
+1. Delete all `epimetheus_benchmark_*` metrics using the Prometheus Admin API
+2. Clean up tombstones to free disk space
+3. Provide clear success/error feedback
+
+**Manual cleanup** (if you prefer):
+
+```bash
+# Delete specific metric
+curl -X POST 'http://localhost:9090/api/v1/admin/tsdb/delete_series?match[]=epimetheus_benchmark_cpu_usage'
+
+# Clean up tombstones
+curl -X POST 'http://localhost:9090/api/v1/admin/tsdb/clean_tombstones'
+```
+
+### Other Cleanup Tasks
+
+```bash
+# Stop port-forwards
+pkill -f "port-forward.*9091"
+pkill -f "port-forward.*9090"
+pkill -f "port-forward.*3000"
+
+# Delete test metrics from Pushgateway
+curl -X DELETE http://localhost:9091/metrics/job/example_metrics_pusher
+
+# Uninstall Pushgateway (if needed)
+helm uninstall pushgateway -n monitoring
+```
+
+## MacOS Setup
+
+### Basic Installation
+
+```bash
+brew install prometheus
+brew install grafana
+go install github.com/prometheus/pushgateway@latest
+brew services start grafana
+brew services start prometheus
+~/go/bin/pushgateway &
+```
+
+Once done, login to http://localhost:3000 as admin:admin, you will be prompted to change the password. Afterwards, add http://localhost:9090 as a Prometheus datasource.
+
+### Enable Remote Write Receiver (Required for Watch Mode)
+
+⚠️ **Important**: Watch mode, historic mode, backfill mode, and auto mode require the Prometheus Remote Write receiver to be enabled.
+
+#### Option 1: Permanent Configuration (Recommended)
+
+Edit the Prometheus arguments file:
+
+```bash
+# Edit the arguments file
+nano /opt/homebrew/etc/prometheus.args
+```
+
+Add this line at the end:
+```
+--web.enable-remote-write-receiver
+```
+
+The complete file should look like:
+```
+--config.file /opt/homebrew/etc/prometheus.yml
+--web.listen-address=127.0.0.1:9090
+--storage.tsdb.path /opt/homebrew/var/prometheus
+--web.enable-remote-write-receiver
+--web.enable-admin-api
+```
+
+**Note:** `--web.enable-admin-api` is optional but recommended for easier data management (allows deleting old metrics).
+
+Restart Prometheus:
+```bash
+brew services restart prometheus
+```
+
+Verify it's working:
+```bash
+# Check Prometheus is healthy
+curl http://localhost:9090/-/healthy
+
+# Test Remote Write endpoint (should return 400, not 404)
+curl -X POST http://localhost:9090/api/v1/write
+```
+
+#### Option 2: Temporary (For Testing)
+
+Stop the service and start manually:
+
+```bash
+# Stop brew service
+brew services stop prometheus
+
+# Start with Remote Write enabled
+prometheus --web.enable-remote-write-receiver
+```
+
+Keep this terminal open. In another terminal, run your epimetheus commands.
+
+**Note**: This only lasts until you stop the terminal. Use Option 1 for permanent setup.
+
+### Clearing Old Metrics (Optional)
+
+If you need to delete old metrics and start fresh:
+
+```bash
+# Delete specific metrics (e.g., blockstore)
+curl -X POST -g 'http://localhost:9090/api/v1/admin/tsdb/delete_series?match[]={__name__=~"blockstore_.*"}'
+
+# Clean up deleted data
+curl -X POST http://localhost:9090/api/v1/admin/tsdb/clean_tombstones
+
+# Wait a moment for cleanup
+sleep 2
+```
+
+**Note:** Admin API must be enabled (add `--web.enable-admin-api` to prometheus.args).
+
+### Verify Setup
+
+Once Remote Write is enabled, test watch mode:
+
+```bash
+# Create a test CSV
+cat > /tmp/test.csv << EOF
+status,count,method
+200,100,GET
+404,50,POST
+EOF
+
+# Watch the file
+./epimetheus -mode=watch \
+    -file=/tmp/test.csv \
+    -metric-name=test \
+    -prometheus=http://localhost:9090/api/v1/write
+```
+
+You should see:
+```
+✅ Successfully pushed X samples to Prometheus
+```
+
+Query in Prometheus (http://localhost:9090):
+```promql
+{__name__=~"test_.*"}
+```
+
+## Additional Resources
+
+- [Prometheus Documentation](https://prometheus.io/docs/)
+- [Pushgateway Documentation](https://github.com/prometheus/pushgateway)
+- [Prometheus Remote Write Spec](https://prometheus.io/docs/concepts/remote_write_spec/)
+- [Grafana Documentation](https://grafana.com/docs/)
+
+## Version
+
+Current version: 0.0.0
+
+## License
+
+See LICENSE file for details.
diff --git a/backfill-historic-data.sh b/backfill-historic-data.sh
new file mode 100755
index 0000000..fa0e065
--- /dev/null
+++ b/backfill-historic-data.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+# Backfill historic data to Prometheus for Epimetheus dashboard
+
+set -e
+
+echo "=== Epimetheus Historic Data Backfill ==="
+echo ""
+echo "This script will populate Prometheus with historic test data"
+echo "going back 7 days, with data points every 12 hours."
+echo ""
+
+# Port-forward to Prometheus
+echo "Step 1: Setting up port-forward to Prometheus..."
+kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 > /tmp/epimetheus-prom-pf.log 2>&1 &
+PF_PID=$!
+echo "Port-forward started (PID: $PF_PID)"
+
+# Wait for port-forward to be ready
+sleep 5
+
+# Run backfill
+echo ""
+echo "Step 2: Backfilling data from 7 days ago to now (12-hour intervals)..."
+echo ""
+./epimetheus -mode=backfill \
+  -prometheus=http://localhost:9090/api/v1/write \
+  -start-hours=168 \
+  -end-hours=0 \
+  -interval=12
+
+EXIT_CODE=$?
+
+# Clean up
+echo ""
+echo "Step 3: Cleaning up port-forward..."
+kill $PF_PID 2>/dev/null || true
+
+if [ $EXIT_CODE -eq 0 ]; then
+    echo ""
+    echo "✅ Historic data backfill complete!"
+    echo ""
+    echo "The Grafana dashboard timeline should now show data from:"
+    echo "  - 7 days ago"
+    echo "  - 6 days ago"
+    echo "  - 5 days ago"
+    echo "  - 4 days ago"
+    echo "  - 3 days ago"
+    echo "  - 2 days ago"
+    echo "  - 1 day ago"
+    echo "  - 12 hours ago"
+    echo "  - Now (from previous realtime push)"
+    echo ""
+    echo "View the dashboard at: https://grafana.f3s.buetow.org/d/epimetheus-test/epimetheus-test-metrics"
+else
+    echo ""
+    echo "❌ Backfill failed with exit code $EXIT_CODE"
+    echo "Check /tmp/epimetheus-prom-pf.log for port-forward logs"
+fi
+
+exit $EXIT_CODE
diff --git a/benchmark-100mb.sh b/benchmark-100mb.sh
new file mode 100755
index 0000000..1d3fad0
--- /dev/null
+++ b/benchmark-100mb.sh
@@ -0,0 +1,223 @@
+#!/bin/bash
+# Benchmark script: Generate and ingest 100MB of historic metrics
+# This tests Epimetheus performance with large-scale data ingestion
+
+set -e
+
+# Optimize Go GC for better performance (Phase 3 optimization)
+export GOGC=200  # Reduce GC frequency (default 100)
+export GOMEMLIMIT=3GiB  # Set memory limit for Go 1.19+
+
+BENCHMARK_DIR="benchmark-results"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+RESULT_FILE="$BENCHMARK_DIR/benchmark-$TIMESTAMP.log"
+
+mkdir -p "$BENCHMARK_DIR"
+
+echo "=== Epimetheus 100MB Benchmark ===" | tee "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+echo "Timestamp: $(date)" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 1: Generate 100MB of test data
+echo "Step 1: Generating 100MB of test data..." | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Calculate: ~70 bytes per line, 100MB = ~1.5M lines
+TARGET_SIZE_MB=100
+TARGET_BYTES=$((TARGET_SIZE_MB * 1024 * 1024))
+BYTES_PER_LINE=70
+TARGET_LINES=$((TARGET_BYTES / BYTES_PER_LINE))
+
+echo "Target size: ${TARGET_SIZE_MB}MB" | tee -a "$RESULT_FILE"
+echo "Estimated lines needed: $TARGET_LINES" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Generate data going back 7 days with 1-minute intervals
+# This gives us ~10,080 data points across 7 days
+# We'll generate multiple metrics per timestamp to reach 100MB
+# All data is historic (> 5 minutes old) to use Remote Write API exclusively
+
+GENERATION_START=$(date +%s)
+
+NOW=$(date +%s)000  # Current time in milliseconds
+ONE_HOUR_AGO=$((NOW - 3600000))  # Start from 1 hour ago to ensure all data is historic
+SEVEN_DAYS_AGO=$((ONE_HOUR_AGO - 604800000))  # 7 days before that
+
+# CSV header
+cat > benchmark-data-100mb.csv << 'EOF'
+# Prometheus metrics - 100MB benchmark dataset
+# Format: metric_name,labels,value,timestamp_ms
+EOF
+
+# Generate metrics
+# We'll create ~150 unique time series, each with ~10,000 data points = 1.5M samples
+METRICS=(
+  "epimetheus_benchmark_cpu_usage"
+  "epimetheus_benchmark_memory_bytes"
+  "epimetheus_benchmark_disk_io_bytes"
+  "epimetheus_benchmark_network_rx_bytes"
+  "epimetheus_benchmark_network_tx_bytes"
+  "epimetheus_benchmark_requests_total"
+  "epimetheus_benchmark_errors_total"
+  "epimetheus_benchmark_response_time_ms"
+  "epimetheus_benchmark_active_connections"
+  "epimetheus_benchmark_queue_depth"
+)
+
+INSTANCES=(
+  "web-01" "web-02" "web-03" "web-04" "web-05"
+  "api-01" "api-02" "api-03" "api-04" "api-05"
+  "db-01" "db-02" "db-03" "worker-01" "worker-02"
+)
+
+INTERVAL_MS=60000  # 1 minute interval
+TOTAL_INTERVALS=10080  # 7 days of 1-minute intervals
+
+echo "Generating data..." | tee -a "$RESULT_FILE"
+LINES_GENERATED=0
+
+for ((i=0; i<TOTAL_INTERVALS; i++)); do
+  TIMESTAMP=$((SEVEN_DAYS_AGO + (i * INTERVAL_MS)))
+
+  # Generate a sample for each metric x instance combination
+  for METRIC in "${METRICS[@]}"; do
+    for INSTANCE in "${INSTANCES[@]}"; do
+      VALUE=$((RANDOM % 1000))
+      echo "$METRIC,instance=$INSTANCE;env=benchmark,$VALUE,$TIMESTAMP" >> benchmark-data-100mb.csv
+      LINES_GENERATED=$((LINES_GENERATED + 1))
+    done
+  done
+
+  # Progress indicator every 1000 intervals
+  if [ $((i % 1000)) -eq 0 ]; then
+    PROGRESS=$((i * 100 / TOTAL_INTERVALS))
+    echo -ne "\rProgress: $PROGRESS% ($LINES_GENERATED lines)" | tee -a "$RESULT_FILE"
+  fi
+done
+
+echo "" | tee -a "$RESULT_FILE"
+
+GENERATION_END=$(date +%s)
+GENERATION_TIME=$((GENERATION_END - GENERATION_START))
+
+# Get actual file size
+FILE_SIZE=$(stat -f%z benchmark-data-100mb.csv 2>/dev/null || stat -c%s benchmark-data-100mb.csv 2>/dev/null)
+FILE_SIZE_MB=$((FILE_SIZE / 1024 / 1024))
+
+echo "" | tee -a "$RESULT_FILE"
+echo "Data generation complete:" | tee -a "$RESULT_FILE"
+echo "  Lines generated: $LINES_GENERATED" | tee -a "$RESULT_FILE"
+echo "  File size: ${FILE_SIZE_MB}MB ($FILE_SIZE bytes)" | tee -a "$RESULT_FILE"
+echo "  Generation time: ${GENERATION_TIME}s" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 2: Start port-forward to Prometheus
+echo "Step 2: Setting up port-forward to Prometheus..." | tee -a "$RESULT_FILE"
+kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 > /tmp/benchmark-pf.log 2>&1 &
+PF_PID=$!
+echo "Port-forward started (PID: $PF_PID)" | tee -a "$RESULT_FILE"
+sleep 8  # Wait for port-forward to be ready
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 3: Get baseline Prometheus metrics
+echo "Step 3: Collecting baseline Prometheus metrics..." | tee -a "$RESULT_FILE"
+PROM_POD=$(kubectl get pod -n monitoring -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].metadata.name}')
+echo "Prometheus pod: $PROM_POD" | tee -a "$RESULT_FILE"
+
+# Get memory and CPU usage before ingestion
+BASELINE_MEMORY=$(kubectl top pod -n monitoring "$PROM_POD" --no-headers | awk '{print $3}')
+BASELINE_CPU=$(kubectl top pod -n monitoring "$PROM_POD" --no-headers | awk '{print $2}')
+
+echo "  Baseline memory: $BASELINE_MEMORY" | tee -a "$RESULT_FILE"
+echo "  Baseline CPU: $BASELINE_CPU" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 4: Run ingestion benchmark
+echo "Step 4: Running ingestion benchmark..." | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+INGEST_START=$(date +%s.%N)
+
+# Run epimetheus with time measurement
+# Use CSV mode with Remote Write API (all data is historic)
+# Note: We can't use auto mode because it requires both Pushgateway and Remote Write
+# Instead, we'll implement a direct CSV->Remote Write ingestion
+
+echo "Parsing CSV and preparing for Remote Write ingestion..." | tee -a "$RESULT_FILE"
+
+# For now, use backfill mode to process the CSV data
+# We'll need to enhance epimetheus to support pure CSV->RemoteWrite mode
+echo "WARNING: Using auto mode - this may fail if data is too recent" | tee -a "$RESULT_FILE"
+echo "Continuing with Remote Write API for historic data..." | tee -a "$RESULT_FILE"
+
+/usr/bin/time -v ./epimetheus \
+  -mode=auto \
+  -file=benchmark-data-100mb.csv \
+  -format=csv \
+  -prometheus=http://localhost:9090/api/v1/write \
+  -pushgateway=http://localhost:9091 \
+  2>&1 | tee -a "$RESULT_FILE" || true  # Continue even if pushgateway fails
+
+INGEST_END=$(date +%s.%N)
+
+# Calculate ingestion time
+INGEST_TIME=$(echo "$INGEST_END - $INGEST_START" | bc)
+
+echo "" | tee -a "$RESULT_FILE"
+echo "Ingestion complete:" | tee -a "$RESULT_FILE"
+echo "  Total time: ${INGEST_TIME}s" | tee -a "$RESULT_FILE"
+
+# Calculate throughput
+SAMPLES_PER_SECOND=$(echo "scale=2; $LINES_GENERATED / $INGEST_TIME" | bc)
+MB_PER_SECOND=$(echo "scale=2; $FILE_SIZE_MB / $INGEST_TIME" | bc)
+
+echo "  Samples/second: $SAMPLES_PER_SECOND" | tee -a "$RESULT_FILE"
+echo "  MB/second: $MB_PER_SECOND" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 5: Get post-ingestion Prometheus metrics
+echo "Step 5: Collecting post-ingestion Prometheus metrics..." | tee -a "$RESULT_FILE"
+sleep 5  # Wait for metrics to stabilize
+
+POST_MEMORY=$(kubectl top pod -n monitoring "$PROM_POD" --no-headers | awk '{print $3}')
+POST_CPU=$(kubectl top pod -n monitoring "$PROM_POD" --no-headers | awk '{print $2}')
+
+echo "  Post-ingestion memory: $POST_MEMORY" | tee -a "$RESULT_FILE"
+echo "  Post-ingestion CPU: $POST_CPU" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 6: Query some data to verify ingestion
+echo "Step 6: Verifying data ingestion..." | tee -a "$RESULT_FILE"
+QUERY_RESULT=$(curl -s "http://localhost:9090/api/v1/query?query=count(epimetheus_benchmark_cpu_usage)" | jq -r '.data.result[0].value[1]')
+echo "  Samples found for epimetheus_benchmark_cpu_usage: $QUERY_RESULT" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 7: Cleanup
+echo "Step 7: Cleaning up..." | tee -a "$RESULT_FILE"
+kill $PF_PID 2>/dev/null || true
+echo "" | tee -a "$RESULT_FILE"
+
+# Summary
+echo "=== BENCHMARK SUMMARY ===" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+echo "Dataset:" | tee -a "$RESULT_FILE"
+echo "  Size: ${FILE_SIZE_MB}MB" | tee -a "$RESULT_FILE"
+echo "  Samples: $LINES_GENERATED" | tee -a "$RESULT_FILE"
+echo "  Time range: 7 days" | tee -a "$RESULT_FILE"
+echo "  Interval: 1 minute" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+echo "Performance:" | tee -a "$RESULT_FILE"
+echo "  Generation time: ${GENERATION_TIME}s" | tee -a "$RESULT_FILE"
+echo "  Ingestion time: ${INGEST_TIME}s" | tee -a "$RESULT_FILE"
+echo "  Throughput: $SAMPLES_PER_SECOND samples/s" | tee -a "$RESULT_FILE"
+echo "  Throughput: $MB_PER_SECOND MB/s" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+echo "Resources:" | tee -a "$RESULT_FILE"
+echo "  Memory: $BASELINE_MEMORY -> $POST_MEMORY" | tee -a "$RESULT_FILE"
+echo "  CPU: $BASELINE_CPU -> $POST_CPU" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+echo "Results saved to: $RESULT_FILE" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+echo "To view results: cat $RESULT_FILE"
+echo "To analyze: less $RESULT_FILE"
diff --git a/benchmark-1gb.sh b/benchmark-1gb.sh
new file mode 100755
index 0000000..f715376
--- /dev/null
+++ b/benchmark-1gb.sh
@@ -0,0 +1,223 @@
+#!/bin/bash
+# Benchmark script: Generate and ingest 1GB of historic metrics
+# This tests Epimetheus performance with large-scale data ingestion
+
+set -e
+
+# Optimize Go GC for better performance (Phase 3 optimization)
+export GOGC=200  # Reduce GC frequency (default 100)
+export GOMEMLIMIT=3GiB  # Set memory limit for Go 1.19+
+
+BENCHMARK_DIR="benchmark-results"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+RESULT_FILE="$BENCHMARK_DIR/benchmark-1gb-$TIMESTAMP.log"
+
+mkdir -p "$BENCHMARK_DIR"
+
+echo "=== Epimetheus 1GB Benchmark ===" | tee "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+echo "Timestamp: $(date)" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 1: Generate 1GB of test data
+echo "Step 1: Generating 1GB of test data..." | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Calculate: ~80 bytes per line, 1GB = ~13M lines
+TARGET_SIZE_MB=1000
+TARGET_BYTES=$((TARGET_SIZE_MB * 1024 * 1024))
+BYTES_PER_LINE=80
+TARGET_LINES=$((TARGET_BYTES / BYTES_PER_LINE))
+
+echo "Target size: ${TARGET_SIZE_MB}MB" | tee -a "$RESULT_FILE"
+echo "Estimated lines needed: $TARGET_LINES" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Generate data going back 30 days with 30-second intervals
+# This gives us ~86,400 data points across 30 days (respects Prometheus 720h out-of-order limit)
+# We'll generate multiple metrics per timestamp to reach 1GB
+# All data is historic (> 5 minutes old) to use Remote Write API exclusively
+
+GENERATION_START=$(date +%s)
+
+NOW=$(date +%s)000  # Current time in milliseconds
+ONE_HOUR_AGO=$((NOW - 3600000))  # Start from 1 hour ago to ensure all data is historic
+THIRTY_DAYS_AGO=$((ONE_HOUR_AGO - 2592000000))  # 30 days before that (30 * 24 * 60 * 60 * 1000)
+
+# CSV header
+cat > benchmark-data-1gb.csv << 'EOF'
+# Prometheus metrics - 1GB benchmark dataset
+# Format: metric_name,labels,value,timestamp_ms
+EOF
+
+# Generate metrics
+# We'll create ~150 unique time series, each with ~86,400 data points = 13M samples
+METRICS=(
+  "epimetheus_benchmark_cpu_usage"
+  "epimetheus_benchmark_memory_bytes"
+  "epimetheus_benchmark_disk_io_bytes"
+  "epimetheus_benchmark_network_rx_bytes"
+  "epimetheus_benchmark_network_tx_bytes"
+  "epimetheus_benchmark_requests_total"
+  "epimetheus_benchmark_errors_total"
+  "epimetheus_benchmark_response_time_ms"
+  "epimetheus_benchmark_active_connections"
+  "epimetheus_benchmark_queue_depth"
+)
+
+INSTANCES=(
+  "web-01" "web-02" "web-03" "web-04" "web-05"
+  "api-01" "api-02" "api-03" "api-04" "api-05"
+  "db-01" "db-02" "db-03" "worker-01" "worker-02"
+)
+
+INTERVAL_MS=30000  # 30 second interval (to maintain 1GB size with 30 days)
+TOTAL_INTERVALS=86400  # 30 days of 30-second intervals
+
+echo "Generating data..." | tee -a "$RESULT_FILE"
+LINES_GENERATED=0
+
+for ((i=0; i<TOTAL_INTERVALS; i++)); do
+  TIMESTAMP=$((THIRTY_DAYS_AGO + (i * INTERVAL_MS)))
+
+  # Generate a sample for each metric x instance combination
+  for METRIC in "${METRICS[@]}"; do
+    for INSTANCE in "${INSTANCES[@]}"; do
+      VALUE=$((RANDOM % 1000))
+      echo "$METRIC,instance=$INSTANCE;env=benchmark,$VALUE,$TIMESTAMP" >> benchmark-data-1gb.csv
+      LINES_GENERATED=$((LINES_GENERATED + 1))
+    done
+  done
+
+  # Progress indicator every 5000 intervals
+  if [ $((i % 5000)) -eq 0 ]; then
+    PROGRESS=$((i * 100 / TOTAL_INTERVALS))
+    echo -ne "\rProgress: $PROGRESS% ($LINES_GENERATED lines)" | tee -a "$RESULT_FILE"
+  fi
+done
+
+echo "" | tee -a "$RESULT_FILE"
+
+GENERATION_END=$(date +%s)
+GENERATION_TIME=$((GENERATION_END - GENERATION_START))
+
+# Get actual file size
+FILE_SIZE=$(stat -f%z benchmark-data-1gb.csv 2>/dev/null || stat -c%s benchmark-data-1gb.csv 2>/dev/null)
+FILE_SIZE_MB=$((FILE_SIZE / 1024 / 1024))
+
+echo "" | tee -a "$RESULT_FILE"
+echo "Data generation complete:" | tee -a "$RESULT_FILE"
+echo "  Lines generated: $LINES_GENERATED" | tee -a "$RESULT_FILE"
+echo "  File size: ${FILE_SIZE_MB}MB ($FILE_SIZE bytes)" | tee -a "$RESULT_FILE"
+echo "  Generation time: ${GENERATION_TIME}s" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 2: Start port-forward to Prometheus
+echo "Step 2: Setting up port-forward to Prometheus..." | tee -a "$RESULT_FILE"
+kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 > /tmp/benchmark-pf.log 2>&1 &
+PF_PID=$!
+echo "Port-forward started (PID: $PF_PID)" | tee -a "$RESULT_FILE"
+sleep 8  # Wait for port-forward to be ready
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 3: Get baseline Prometheus metrics
+echo "Step 3: Collecting baseline Prometheus metrics..." | tee -a "$RESULT_FILE"
+PROM_POD=$(kubectl get pod -n monitoring -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].metadata.name}')
+echo "Prometheus pod: $PROM_POD" | tee -a "$RESULT_FILE"
+
+# Get memory and CPU usage before ingestion
+BASELINE_MEMORY=$(kubectl top pod -n monitoring "$PROM_POD" --no-headers | awk '{print $3}')
+BASELINE_CPU=$(kubectl top pod -n monitoring "$PROM_POD" --no-headers | awk '{print $2}')
+
+echo "  Baseline memory: $BASELINE_MEMORY" | tee -a "$RESULT_FILE"
+echo "  Baseline CPU: $BASELINE_CPU" | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+# Step 4: Run ingestion benchmark
+echo "Step 4: Running ingestion benchmark..." | tee -a "$RESULT_FILE"
+echo "" | tee -a "$RESULT_FILE"
+
+INGEST_START=$(date +%s.%N)
+
+# Run epimetheus with time measurement
+# Use CSV mode with Remote Write API (all data is historic)
+# Note: We can't use auto mode because it requires both Pushgateway and Remote Write
+# Instead, we'll implement a direct CSV->Remote Write ingestion
+
+echo "Parsing CSV and preparing for Remote Write ingestion..." | tee -a "$RESULT_FILE"
+
+# For now, use backfill mode to process the CSV data
+# We'll need to enhance epimetheus to support pure CSV->RemoteWrite mode
+echo "WARNING: Using auto mode - this may fail if data is too recent" | tee -a "$RESULT_FILE"
+echo "Continuing with Remote Write API for historic data..." | tee -a "$RESULT_FILE"
+
+/usr/bin/t