Enable Prometheus historic data ingestion with out-of-order support

This commit configures Prometheus to accept historic data via the Remote
Write API, enabling backfilling of test metrics for development and
troubleshooting purposes.

Changes:
- Enable Remote Write receiver (--web.enable-remote-write-receiver)
- Enable out-of-order ingestion with 30-day window (720h)
- Enable exemplar-storage and otlp-write-receiver features
- Add Epimetheus dashboard ConfigMap for Grafana provisioning
- Remove old prometheus-pusher directory (moved to separate repo)
- Document configuration, use cases, and performance considerations

Configuration allows backfilling data up to 30 days in the past, supporting
tools like Epimetheus for generating synthetic historic metrics.

Performance note: This is optimized for ad-hoc troubleshooting, not
production use. Out-of-order ingestion increases memory usage, TSDB overhead,
and may impact query performance.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

6 files changed, 0 insertions, 670 deletions

diff --git a/f3s/prometheus-pusher/internal/parser/csv.go b/f3s/prometheus-pusher/internal/parser/csv.go
deleted file mode 100644
index a59b6e6..0000000
--- a/f3s/prometheus-pusher/internal/parser/csv.go
+++ /dev/null
@@ -1,101 +0,0 @@
-package parser
-
-import (
-	"context"
-	"encoding/csv"
-	"fmt"
-	"io"
-	"strconv"
-	"strings"
-	"time"
-
-	"prometheus-pusher/internal/metrics"
-)
-
-// CSVParser parses metrics from CSV format
-type CSVParser struct{}
-
-// NewCSVParser creates a new CSV parser
-func NewCSVParser() *CSVParser {
-	return &CSVParser{}
-}
-
-// Parse reads metrics from CSV format
-// Format: metric_name,label1=value1;label2=value2,value,timestamp_ms
-func (p *CSVParser) Parse(ctx context.Context, reader io.Reader) ([]metrics.Sample, error) {
-	var samples []metrics.Sample
-
-	csvReader := csv.NewReader(reader)
-	csvReader.Comment = '#'
-
-	lineNum := 0
-	for {
-		select {
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		default:
-		}
-
-		record, err := csvReader.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, fmt.Errorf("line %d: %w", lineNum, err)
-		}
-		lineNum++
-
-		if len(record) < 3 {
-			continue // Skip invalid records
-		}
-
-		sample, err := p.parseRecord(record, lineNum)
-		if err != nil {
-			continue // Skip records with errors
-		}
-
-		samples = append(samples, sample)
-	}
-
-	return samples, nil
-}
-
-func (p *CSVParser) parseRecord(record []string, lineNum int) (metrics.Sample, error) {
-	metricName := strings.TrimSpace(record[0])
-	if metricName == "" {
-		return metrics.Sample{}, fmt.Errorf("empty metric name")
-	}
-
-	labels := parseLabels(record[1])
-
-	value, err := strconv.ParseFloat(strings.TrimSpace(record[2]), 64)
-	if err != nil {
-		return metrics.Sample{}, fmt.Errorf("invalid value: %w", err)
-	}
-
-	timestamp := time.Now()
-	if len(record) > 3 && record[3] != "" {
-		timestampMs, err := strconv.ParseInt(strings.TrimSpace(record[3]), 10, 64)
-		if err == nil {
-			timestamp = time.UnixMilli(timestampMs)
-		}
-	}
-
-	return metrics.NewSample(metricName, labels, value, timestamp), nil
-}
-
-func parseLabels(labelStr string) map[string]string {
-	labels := make(map[string]string)
-	if labelStr == "" {
-		return labels
-	}
-
-	labelPairs := strings.Split(labelStr, ";")
-	for _, pair := range labelPairs {
-		parts := strings.SplitN(pair, "=", 2)
-		if len(parts) == 2 {
-			labels[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
-		}
-	}
-	return labels
-}
diff --git a/f3s/prometheus-pusher/internal/parser/csv_test.go b/f3s/prometheus-pusher/internal/parser/csv_test.go
deleted file mode 100644
index a06e7d9..0000000
--- a/f3s/prometheus-pusher/internal/parser/csv_test.go
+++ /dev/null
@@ -1,175 +0,0 @@
-package parser
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestCSVParser_Parse(t *testing.T) {
-	tests := []struct {
-		name      string
-		input     string
-		wantCount int
-		wantErr   bool
-	}{
-		{
-			name: "valid single line",
-			input: `test_metric,env=prod;host=server1,42.5,1234567890000`,
-			wantCount: 1,
-			wantErr:   false,
-		},
-		{
-			name: "multiple lines",
-			input: `metric1,label1=value1,100,1234567890000
-metric2,label2=value2,200,1234567891000
-metric3,label3=value3,300,1234567892000`,
-			wantCount: 3,
-			wantErr:   false,
-		},
-		{
-			name: "with comments",
-			input: `# This is a comment
-metric1,env=test,50,1234567890000
-# Another comment
-metric2,env=prod,75,1234567891000`,
-			wantCount: 2,
-			wantErr:   false,
-		},
-		{
-			name: "no timestamp defaults to now",
-			input: `metric1,env=test,100`,
-			wantCount: 1,
-			wantErr:   false,
-		},
-		{
-			name: "no labels",
-			input: `metric1,,100,1234567890000`,
-			wantCount: 1,
-			wantErr:   false,
-		},
-		{
-			name:      "empty input",
-			input:     "",
-			wantCount: 0,
-			wantErr:   false,
-		},
-		{
-			name: "invalid line causes error",
-			input: `metric1,env=test,100,1234567890000
-invalid
-metric2,env=prod,200,1234567891000`,
-			wantCount: 0,
-			wantErr:   true,
-		},
-		{
-			name: "invalid value skipped",
-			input: `metric1,env=test,not_a_number,1234567890000
-metric2,env=prod,200,1234567891000`,
-			wantCount: 1,
-			wantErr:   false,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			parser := NewCSVParser()
-			reader := strings.NewReader(tt.input)
-			ctx := context.Background()
-
-			samples, err := parser.Parse(ctx, reader)
-
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if len(samples) != tt.wantCount {
-				t.Errorf("Parse() returned %d samples, want %d", len(samples), tt.wantCount)
-			}
-		})
-	}
-}
-
-func TestCSVParser_ParseLabels(t *testing.T) {
-	tests := []struct {
-		name  string
-		input string
-		want  map[string]string
-	}{
-		{
-			name:  "single label",
-			input: "env=prod",
-			want:  map[string]string{"env": "prod"},
-		},
-		{
-			name:  "multiple labels",
-			input: "env=prod;host=server1;region=us-west",
-			want:  map[string]string{"env": "prod", "host": "server1", "region": "us-west"},
-		},
-		{
-			name:  "empty string",
-			input: "",
-			want:  map[string]string{},
-		},
-		{
-			name:  "invalid label format skipped",
-			input: "env=prod;invalid;host=server1",
-			want:  map[string]string{"env": "prod", "host": "server1"},
-		},
-		{
-			name:  "with spaces",
-			input: " env = prod ; host = server1 ",
-			want:  map[string]string{"env": "prod", "host": "server1"},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := parseLabels(tt.input)
-			if len(got) != len(tt.want) {
-				t.Errorf("parseLabels() returned %d labels, want %d", len(got), len(tt.want))
-			}
-			for k, v := range tt.want {
-				if got[k] != v {
-					t.Errorf("parseLabels()[%s] = %v, want %v", k, got[k], v)
-				}
-			}
-		})
-	}
-}
-
-func TestCSVParser_ParseWithContext(t *testing.T) {
-	t.Run("context cancellation", func(t *testing.T) {
-		parser := NewCSVParser()
-		ctx, cancel := context.WithCancel(context.Background())
-		cancel() // Cancel immediately
-
-		input := strings.NewReader(`metric1,env=test,100,1234567890000`)
-		_, err := parser.Parse(ctx, input)
-
-		if err != context.Canceled {
-			t.Errorf("Expected context.Canceled error, got %v", err)
-		}
-	})
-}
-
-func TestCSVParser_ParseTimestamp(t *testing.T) {
-	parser := NewCSVParser()
-	input := `metric1,env=test,100,1234567890000`
-	reader := strings.NewReader(input)
-	ctx := context.Background()
-
-	samples, err := parser.Parse(ctx, reader)
-	if err != nil {
-		t.Fatalf("Parse() error = %v", err)
-	}
-	if len(samples) != 1 {
-		t.Fatalf("Expected 1 sample, got %d", len(samples))
-	}
-
-	expectedTime := time.UnixMilli(1234567890000)
-	if !samples[0].Timestamp.Equal(expectedTime) {
-		t.Errorf("Timestamp = %v, want %v", samples[0].Timestamp, expectedTime)
-	}
-}
diff --git a/f3s/prometheus-pusher/internal/parser/json.go b/f3s/prometheus-pusher/internal/parser/json.go
deleted file mode 100644
index b6b230c..0000000
--- a/f3s/prometheus-pusher/internal/parser/json.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package parser
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"time"
-
-	"prometheus-pusher/internal/metrics"
-)
-
-// JSONParser parses metrics from JSON format
-type JSONParser struct{}
-
-// NewJSONParser creates a new JSON parser
-func NewJSONParser() *JSONParser {
-	return &JSONParser{}
-}
-
-type jsonSample struct {
-	Metric      string            `json:"metric"`
-	Labels      map[string]string `json:"labels"`
-	Value       float64           `json:"value"`
-	TimestampMs int64             `json:"timestamp_ms,omitempty"`
-}
-
-// Parse reads metrics from JSON format
-func (p *JSONParser) Parse(ctx context.Context, reader io.Reader) ([]metrics.Sample, error) {
-	var rawSamples []jsonSample
-
-	decoder := json.NewDecoder(reader)
-	if err := decoder.Decode(&rawSamples); err != nil {
-		return nil, fmt.Errorf("failed to parse JSON: %w", err)
-	}
-
-	samples := make([]metrics.Sample, 0, len(rawSamples))
-	for _, raw := range rawSamples {
-		select {
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		default:
-		}
-
-		if raw.Metric == "" {
-			continue
-		}
-
-		timestamp := time.Now()
-		if raw.TimestampMs > 0 {
-			timestamp = time.UnixMilli(raw.TimestampMs)
-		}
-
-		if raw.Labels == nil {
-			raw.Labels = make(map[string]string)
-		}
-
-		samples = append(samples, metrics.NewSample(raw.Metric, raw.Labels, raw.Value, timestamp))
-	}
-
-	return samples, nil
-}
diff --git a/f3s/prometheus-pusher/internal/parser/json_test.go b/f3s/prometheus-pusher/internal/parser/json_test.go
deleted file mode 100644
index d521942..0000000
--- a/f3s/prometheus-pusher/internal/parser/json_test.go
+++ /dev/null
@@ -1,177 +0,0 @@
-package parser
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestJSONParser_Parse(t *testing.T) {
-	tests := []struct {
-		name      string
-		input     string
-		wantCount int
-		wantErr   bool
-	}{
-		{
-			name: "valid single sample",
-			input: `[
-				{
-					"metric": "test_metric",
-					"labels": {"env": "prod", "host": "server1"},
-					"value": 42.5,
-					"timestamp_ms": 1234567890000
-				}
-			]`,
-			wantCount: 1,
-			wantErr:   false,
-		},
-		{
-			name: "multiple samples",
-			input: `[
-				{"metric": "metric1", "labels": {"env": "prod"}, "value": 100, "timestamp_ms": 1234567890000},
-				{"metric": "metric2", "labels": {"env": "test"}, "value": 200, "timestamp_ms": 1234567891000},
-				{"metric": "metric3", "labels": {"env": "dev"}, "value": 300, "timestamp_ms": 1234567892000}
-			]`,
-			wantCount: 3,
-			wantErr:   false,
-		},
-		{
-			name: "no timestamp defaults to now",
-			input: `[
-				{"metric": "test_metric", "labels": {"env": "prod"}, "value": 100}
-			]`,
-			wantCount: 1,
-			wantErr:   false,
-		},
-		{
-			name: "no labels",
-			input: `[
-				{"metric": "test_metric", "value": 100, "timestamp_ms": 1234567890000}
-			]`,
-			wantCount: 1,
-			wantErr:   false,
-		},
-		{
-			name: "empty metric skipped",
-			input: `[
-				{"metric": "", "labels": {"env": "prod"}, "value": 100},
-				{"metric": "valid_metric", "labels": {"env": "test"}, "value": 200}
-			]`,
-			wantCount: 1,
-			wantErr:   false,
-		},
-		{
-			name:      "empty array",
-			input:     `[]`,
-			wantCount: 0,
-			wantErr:   false,
-		},
-		{
-			name:      "invalid json",
-			input:     `{not valid json}`,
-			wantCount: 0,
-			wantErr:   true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			parser := NewJSONParser()
-			reader := strings.NewReader(tt.input)
-			ctx := context.Background()
-
-			samples, err := parser.Parse(ctx, reader)
-
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if len(samples) != tt.wantCount {
-				t.Errorf("Parse() returned %d samples, want %d", len(samples), tt.wantCount)
-			}
-		})
-	}
-}
-
-func TestJSONParser_ParseWithContext(t *testing.T) {
-	t.Run("context check during parse", func(t *testing.T) {
-		parser := NewJSONParser()
-		ctx, cancel := context.WithCancel(context.Background())
-
-		// Create valid input with empty metrics that will be filtered
-		input := `[
-			{"metric": "", "value": 1},
-			{"metric": "", "value": 2},
-			{"metric": "", "value": 3}
-		]`
-
-		cancel() // Cancel before parsing
-
-		reader := strings.NewReader(input)
-		_, err := parser.Parse(ctx, reader)
-
-		// Context cancellation should be detected during sample processing
-		if err != context.Canceled {
-			// Note: JSON decoder may finish before context is checked
-			// This test verifies context support exists, but timing is not guaranteed
-			t.Logf("Got error: %v (context may not be checked until after JSON decode)", err)
-		}
-	})
-}
-
-func TestJSONParser_ParseTimestamp(t *testing.T) {
-	parser := NewJSONParser()
-	input := `[{"metric": "test_metric", "value": 100, "timestamp_ms": 1234567890000}]`
-	reader := strings.NewReader(input)
-	ctx := context.Background()
-
-	samples, err := parser.Parse(ctx, reader)
-	if err != nil {
-		t.Fatalf("Parse() error = %v", err)
-	}
-	if len(samples) != 1 {
-		t.Fatalf("Expected 1 sample, got %d", len(samples))
-	}
-
-	expectedTime := time.UnixMilli(1234567890000)
-	if !samples[0].Timestamp.Equal(expectedTime) {
-		t.Errorf("Timestamp = %v, want %v", samples[0].Timestamp, expectedTime)
-	}
-}
-
-func TestJSONParser_ParseLabels(t *testing.T) {
-	parser := NewJSONParser()
-	input := `[{
-		"metric": "test_metric",
-		"labels": {"env": "prod", "host": "server1", "region": "us-west"},
-		"value": 100
-	}]`
-	reader := strings.NewReader(input)
-	ctx := context.Background()
-
-	samples, err := parser.Parse(ctx, reader)
-	if err != nil {
-		t.Fatalf("Parse() error = %v", err)
-	}
-	if len(samples) != 1 {
-		t.Fatalf("Expected 1 sample, got %d", len(samples))
-	}
-
-	expectedLabels := map[string]string{
-		"env":    "prod",
-		"host":   "server1",
-		"region": "us-west",
-	}
-
-	if len(samples[0].Labels) != len(expectedLabels) {
-		t.Errorf("Got %d labels, want %d", len(samples[0].Labels), len(expectedLabels))
-	}
-
-	for k, v := range expectedLabels {
-		if samples[0].Labels[k] != v {
-			t.Errorf("Label[%s] = %v, want %v", k, samples[0].Labels[k], v)
-		}
-	}
-}
diff --git a/f3s/prometheus-pusher/internal/parser/parser.go b/f3s/prometheus-pusher/internal/parser/parser.go
deleted file mode 100644
index 6a7732d..0000000
--- a/f3s/prometheus-pusher/internal/parser/parser.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package parser
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"os"
-
-	"prometheus-pusher/internal/metrics"
-)
-
-// Parser defines the interface for metric parsers.
-type Parser interface {
-	Parse(ctx context.Context, reader io.Reader) ([]metrics.Sample, error)
-}
-
-// ParseFile parses metrics from a file.
-func ParseFile(ctx context.Context, filename, format string) ([]metrics.Sample, error) {
-	file, err := os.Open(filename)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open file: %w", err)
-	}
-	defer file.Close()
-
-	return parseWithFormat(ctx, file, format)
-}
-
-// ParseStdin parses metrics from standard input.
-func ParseStdin(ctx context.Context, format string) ([]metrics.Sample, error) {
-	return parseWithFormat(ctx, os.Stdin, format)
-}
-
-// parseWithFormat parses metrics using the specified format.
-func parseWithFormat(ctx context.Context, reader io.Reader, format string) ([]metrics.Sample, error) {
-	var parser Parser
-
-	switch format {
-	case "csv":
-		parser = NewCSVParser()
-	case "json":
-		parser = NewJSONParser()
-	default:
-		return nil, fmt.Errorf("unsupported format: %s (use csv or json)", format)
-	}
-
-	samples, err := parser.Parse(ctx, reader)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse metrics: %w", err)
-	}
-
-	if len(samples) == 0 {
-		return nil, fmt.Errorf("no valid samples found")
-	}
-
-	return samples, nil
-}
diff --git a/f3s/prometheus-pusher/internal/parser/parser_test.go b/f3s/prometheus-pusher/internal/parser/parser_test.go
deleted file mode 100644
index 05255a5..0000000
--- a/f3s/prometheus-pusher/internal/parser/parser_test.go
+++ /dev/null
@@ -1,99 +0,0 @@
-package parser
-
-import (
-	"context"
-	"strings"
-	"testing"
-)
-
-func TestParseFile_CSV(t *testing.T) {
-	// We can't easily test file operations without creating temp files
-	// So we'll test the error case
-	ctx := context.Background()
-	_, err := ParseFile(ctx, "/nonexistent/file.csv", "csv")
-
-	if err == nil {
-		t.Error("Expected error for nonexistent file")
-	}
-}
-
-func TestParseWithFormat_CSV(t *testing.T) {
-	ctx := context.Background()
-	input := `test_metric,env=prod,100,1234567890000`
-	reader := strings.NewReader(input)
-
-	samples, err := parseWithFormat(ctx, reader, "csv")
-	if err != nil {
-		t.Fatalf("parseWithFormat(csv) error = %v", err)
-	}
-	if len(samples) != 1 {
-		t.Errorf("Expected 1 sample, got %d", len(samples))
-	}
-}
-
-func TestParseWithFormat_JSON(t *testing.T) {
-	ctx := context.Background()
-	input := `[{"metric": "test_metric", "value": 100, "timestamp_ms": 1234567890000}]`
-	reader := strings.NewReader(input)
-
-	samples, err := parseWithFormat(ctx, reader, "json")
-	if err != nil {
-		t.Fatalf("parseWithFormat(json) error = %v", err)
-	}
-	if len(samples) != 1 {
-		t.Errorf("Expected 1 sample, got %d", len(samples))
-	}
-}
-
-func TestParseWithFormat_UnsupportedFormat(t *testing.T) {
-	ctx := context.Background()
-	reader := strings.NewReader("")
-
-	_, err := parseWithFormat(ctx, reader, "xml")
-	if err == nil {
-		t.Error("Expected error for unsupported format")
-	}
-	if err.Error() != "unsupported format: xml (use csv or json)" {
-		t.Errorf("Unexpected error message: %v", err)
-	}
-}
-
-func TestParseWithFormat_EmptyResult(t *testing.T) {
-	ctx := context.Background()
-	input := `[]` // Empty JSON array
-	reader := strings.NewReader(input)
-
-	_, err := parseWithFormat(ctx, reader, "json")
-	if err == nil {
-		t.Error("Expected error for empty samples")
-	}
-	if err.Error() != "no valid samples found" {
-		t.Errorf("Expected 'no valid samples found' error, got: %v", err)
-	}
-}
-
-func TestParseStdin_Format(t *testing.T) {
-	// We can't easily test stdin without mocking,
-	// but we can verify the error path
-	ctx := context.Background()
-
-	// Test with invalid format
-	_, err := parseWithFormat(ctx, strings.NewReader(""), "invalid_format")
-	if err == nil {
-		t.Error("Expected error for invalid format")
-	}
-}
-
-func TestNewCSVParser(t *testing.T) {
-	parser := NewCSVParser()
-	if parser == nil {
-		t.Error("NewCSVParser() returned nil")
-	}
-}
-
-func TestNewJSONParser(t *testing.T) {
-	parser := NewJSONParser()
-	if parser == nil {
-		t.Error("NewJSONParser() returned nil")
-	}
-}