diff options
Diffstat (limited to 'f3s')
| -rw-r--r-- | f3s/prometheus-pusher/README.md | 151 | ||||
| -rw-r--r-- | f3s/prometheus-pusher/SUMMARY.md | 215 | ||||
| -rw-r--r-- | f3s/prometheus-pusher/USAGE.md | 231 | ||||
| -rw-r--r-- | f3s/prometheus-pusher/example-metrics.txt | 57 | ||||
| -rw-r--r-- | f3s/prometheus-pusher/go.mod | 16 | ||||
| -rw-r--r-- | f3s/prometheus-pusher/go.sum | 22 | ||||
| -rw-r--r-- | f3s/prometheus-pusher/main.go | 138 | ||||
| -rwxr-xr-x | f3s/prometheus-pusher/prometheus-pusher | bin | 0 -> 12149010 bytes | |||
| -rwxr-xr-x | f3s/prometheus-pusher/run.sh | 31 | ||||
| -rw-r--r-- | f3s/prometheus/additional-scrape-configs.yaml | 6 | ||||
| -rw-r--r-- | f3s/pushgateway/README.md | 166 | ||||
| -rw-r--r-- | f3s/pushgateway/helm-chart/Chart.yaml | 6 | ||||
| -rw-r--r-- | f3s/pushgateway/helm-chart/templates/deployment.yaml | 24 | ||||
| -rw-r--r-- | f3s/pushgateway/helm-chart/templates/service.yaml | 16 | ||||
| -rw-r--r-- | f3s/pushgateway/helm-chart/values.yaml | 20 |
15 files changed, 1099 insertions, 0 deletions
diff --git a/f3s/prometheus-pusher/README.md b/f3s/prometheus-pusher/README.md new file mode 100644 index 0000000..d107299 --- /dev/null +++ b/f3s/prometheus-pusher/README.md @@ -0,0 +1,151 @@ +# Prometheus Pusher + +A standalone Go binary that pushes metrics to Prometheus via Pushgateway. + +## Quick Start + +```bash +# 1. Deploy Pushgateway (one-time - see /home/paul/git/conf/f3s/pushgateway/) +cd /home/paul/git/conf/f3s/pushgateway/helm-chart +helm upgrade --install pushgateway . -n monitoring + +# 2. Run the binary +cd /home/paul/git/conf/f3s/prometheus-pusher +./run.sh +``` + +That's it! The binary will push metrics every 15 seconds. Press Ctrl+C to stop. + +## Overview + +This project consists of: +1. **Pushgateway** - A Kubernetes service that receives pushed metrics +2. **prometheus-pusher** - A standalone Go binary that generates and pushes example metrics + +## Metric Types Demonstrated + +The application pushes the following types of metrics: + +### Counter (`app_requests_total`) +- Monotonically increasing value +- Example: Total number of requests processed +- Use case: Counting events, total requests, errors, etc. + +### Gauge (`app_active_connections`, `app_temperature_celsius`) +- Value that can increase or decrease +- Examples: Active connections, temperature, memory usage +- Use case: Current state measurements + +### Histogram (`app_request_duration_seconds`) +- Samples observations and counts them in configurable buckets +- Example: Request duration distribution +- Use case: Latency measurements, response times + +### Counter with Labels (`app_jobs_processed_total`) +- Counter with dimensional labels +- Labels: `job_type` (email, report, backup), `status` (success, failed) +- Use case: Categorized counting + +## Project Structure + +``` +prometheus-pusher/ +├── main.go # Go source code +├── go.mod / go.sum # Go dependencies +├── prometheus-pusher # Compiled binary (standalone executable) +├── run.sh # Helper script to run the binary +├── example-metrics.txt # Example of metrics format +├── USAGE.md # Detailed usage guide +└── README.md # This file + +Note: Pushgateway Helm chart is located at /home/paul/git/conf/f3s/pushgateway/ +``` + +## What It Does + +The `prometheus-pusher` binary: +- **Generates** realistic example metrics simulating a production application +- **Pushes** metrics to Pushgateway every 15 seconds using HTTP POST +- **Demonstrates** all major Prometheus metric types with practical examples + +The metrics flow: `Go Binary → Pushgateway → Prometheus → Grafana` + +## Example Metrics Format + +The pusher sends metrics in Prometheus format to the Pushgateway. Here's what the data looks like: + +``` +# HELP app_requests_total Total number of requests processed +# TYPE app_requests_total counter +app_requests_total{instance="example-app",job="example_metrics_pusher"} 42 + +# HELP app_active_connections Number of currently active connections +# TYPE app_active_connections gauge +app_active_connections{instance="example-app",job="example_metrics_pusher"} 67 + +# HELP app_temperature_celsius Current temperature in Celsius +# TYPE app_temperature_celsius gauge +app_temperature_celsius{instance="example-app",job="example_metrics_pusher"} 23.5 + +# HELP app_request_duration_seconds Histogram of request duration in seconds +# TYPE app_request_duration_seconds histogram +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.005"} 2 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.01"} 3 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="+Inf"} 10 +app_request_duration_seconds_sum{instance="example-app",job="example_metrics_pusher"} 8.5 +app_request_duration_seconds_count{instance="example-app",job="example_metrics_pusher"} 10 + +# HELP app_jobs_processed_total Total number of jobs processed by type +# TYPE app_jobs_processed_total counter +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="email",status="success"} 15 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="email",status="failed"} 2 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="report",status="success"} 8 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="backup",status="success"} 12 +``` + +## Querying Metrics in Prometheus + +Once configured, you can query these metrics in Prometheus: + +```promql +# View request rate +rate(app_requests_total[5m]) + +# View current active connections +app_active_connections + +# View 95th percentile request duration +histogram_quantile(0.95, rate(app_request_duration_seconds_bucket[5m])) + +# View failed jobs by type +app_jobs_processed_total{status="failed"} + +# View job success rate +rate(app_jobs_processed_total{status="success"}[5m]) / rate(app_jobs_processed_total[5m]) +``` + +## Configuration + +The pusher is configured to: +- Push metrics every 15 seconds +- Use job name: `example_metrics_pusher` +- Use instance label: `example-app` +- Connect to Pushgateway at: `http://pushgateway.monitoring.svc.cluster.local:9091` + +## How It Works + +1. The Go application generates random example metrics simulating a real application +2. Metrics are pushed to the Pushgateway via HTTP POST +3. Prometheus scrapes the Pushgateway periodically +4. Metrics become available in Prometheus for querying and alerting +5. Grafana can visualize these metrics + +## Best Practices + +- Use Pushgateway for batch jobs, short-lived processes, or service-level metrics +- For long-running applications, prefer exposing a `/metrics` endpoint for Prometheus to scrape +- Include meaningful labels but avoid high-cardinality labels (e.g., user IDs, timestamps) +- Use appropriate metric types: + - Counter for cumulative values + - Gauge for point-in-time values + - Histogram/Summary for distributions diff --git a/f3s/prometheus-pusher/SUMMARY.md b/f3s/prometheus-pusher/SUMMARY.md new file mode 100644 index 0000000..d71138c --- /dev/null +++ b/f3s/prometheus-pusher/SUMMARY.md @@ -0,0 +1,215 @@ +# Prometheus Data Ingestion - Summary + +## What Was Created + +A complete Prometheus data ingestion solution consisting of: + +### 1. **Standalone Go Binary** (`prometheus-pusher`) +- **Size**: ~12MB standalone executable +- **Language**: Go 1.21 +- **Dependencies**: Prometheus client library +- **Function**: Generates and pushes metrics to Pushgateway every 15 seconds + +### 2. **Pushgateway Deployment** +- **Type**: Kubernetes deployment in `monitoring` namespace +- **Image**: `prom/pushgateway:v1.10.0` +- **Port**: 9091 +- **Function**: Receives metrics from the Go binary and exposes them for Prometheus to scrape + +### 3. **Prometheus Configuration** +- Updated `/home/paul/git/conf/f3s/prometheus/additional-scrape-configs.yaml` +- Added Pushgateway as a scrape target +- Prometheus automatically scrapes Pushgateway every 15-30 seconds + +## Data Format + +The binary pushes metrics in **Prometheus text format** via HTTP POST to the Pushgateway. This is the standard format for all Prometheus metrics. + +Example: +``` +# HELP app_requests_total Total number of requests processed +# TYPE app_requests_total counter +app_requests_total{instance="example-app",job="example_metrics_pusher"} 42 +``` + +## Metric Types Demonstrated + +### 1. **Counter**: `app_requests_total` +- Monotonically increasing value +- Best for: Total requests, errors, events + +### 2. **Gauge**: `app_active_connections`, `app_temperature_celsius` +- Value that can increase or decrease +- Best for: Current state (connections, temperature, memory) + +### 3. **Histogram**: `app_request_duration_seconds` +- Distribution of values in buckets +- Best for: Latency, response times, sizes +- Automatically provides percentile calculations + +### 4. **Counter with Labels**: `app_jobs_processed_total` +- Counter with multiple dimensions +- Labels: `job_type` (email, report, backup), `status` (success, failed) +- Best for: Categorized counting + +## Why This Format? + +The Prometheus text format was chosen because: + +1. **Standard**: Universal format understood by all Prometheus components +2. **Human-readable**: Easy to debug and understand +3. **Efficient**: Compact representation +4. **Type-safe**: Explicit metric types prevent errors +5. **Labeled**: Supports multi-dimensional data + +## How to Use + +### Quick Start +```bash +cd /home/paul/git/conf/f3s/prometheus-pusher +./run.sh +``` + +### Manual Operation +```bash +# 1. Port-forward Pushgateway +kubectl port-forward -n monitoring svc/pushgateway 9091:9091 + +# 2. Run binary (in another terminal) +./prometheus-pusher +``` + +### Query Metrics +```bash +# Port-forward Prometheus +kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 + +# Open http://localhost:9090 and query: +app_requests_total +app_active_connections +rate(app_requests_total[5m]) +histogram_quantile(0.95, rate(app_request_duration_seconds_bucket[5m])) +``` + +## Example Data + +See `example-metrics.txt` for a complete example of all metric types with sample values. + +## Testing Results + +✅ **Binary compilation**: Success (12MB executable) +✅ **Pushgateway deployment**: Running in monitoring namespace +✅ **Metrics push**: Successfully pushing every 15 seconds +✅ **Prometheus scraping**: Confirmed metrics visible in Prometheus +✅ **Query testing**: All metric types queryable + +### Sample Query Results + +```json +{ + "status": "success", + "data": { + "result": [{ + "metric": { + "__name__": "app_requests_total", + "instance": "example-app", + "job": "example_metrics_pusher" + }, + "value": [1767121623.654, "10"] + }] + } +} +``` + +## Architecture + +``` +┌──────────────────────┐ +│ prometheus-pusher │ Standalone Go binary +│ (your machine) │ - Generates metrics +└──────────┬───────────┘ - Pushes via HTTP POST + │ + │ HTTP POST + │ :9091/metrics/job/<jobname> + ▼ +┌──────────────────────┐ +│ Pushgateway │ Kubernetes pod +│ (monitoring ns) │ - Receives pushed metrics +└──────────┬───────────┘ - Exposes /metrics endpoint + │ + │ HTTP GET (scrape) + │ Every 15-30s + ▼ +┌──────────────────────┐ +│ Prometheus │ Kubernetes pod +│ (monitoring ns) │ - Scrapes Pushgateway +└──────────┬───────────┘ - Stores time-series data + │ + │ HTTP API + │ PromQL queries + ▼ +┌──────────────────────┐ +│ Grafana / Users │ Visualization & Alerts +│ │ - Query metrics +└──────────────────────┘ - Create dashboards +``` + +## Files Created + +``` +/home/paul/git/conf/f3s/prometheus-pusher/ +├── main.go # Go source code +├── go.mod, go.sum # Go dependencies +├── prometheus-pusher # Compiled binary (12MB) +├── run.sh # Helper script +├── README.md # Project overview +├── USAGE.md # Detailed usage guide +├── SUMMARY.md # This file +├── example-metrics.txt # Example metrics format +└── Dockerfile # Docker build (optional) + +/home/paul/git/conf/f3s/pushgateway/ +└── helm-chart/ # Kubernetes deployment + ├── Chart.yaml # Helm chart metadata + ├── values.yaml # Configuration values + ├── README.md # Chart documentation + └── templates/ + ├── deployment.yaml # Pushgateway pod + └── service.yaml # Pushgateway service + +/home/paul/git/conf/f3s/prometheus/ +└── additional-scrape-configs.yaml # Prometheus config (updated) +``` + +## Next Steps + +### For Production Use + +1. **Modify metrics** in `main.go` to track your actual application data +2. **Adjust push interval** (currently 15 seconds) +3. **Add authentication** if Pushgateway is exposed externally +4. **Set up Grafana dashboards** to visualize the metrics +5. **Configure alerts** in Prometheus for critical thresholds + +### For Learning + +1. Experiment with different metric types +2. Try querying with different PromQL expressions +3. Create Grafana dashboards +4. Set up alerting rules +5. Compare Pushgateway approach vs. direct scraping + +## Key Concepts + +- **Push vs. Pull**: Pushgateway allows pushing metrics (vs. Prometheus scraping) +- **Labels**: Enable multi-dimensional metrics +- **Metric Types**: Different types for different use cases +- **Aggregation**: Histograms automatically calculate percentiles +- **Time Series**: Prometheus stores timestamped values + +## References + +- Prometheus text format: https://prometheus.io/docs/instrumenting/exposition_formats/ +- Prometheus client library: https://github.com/prometheus/client_golang +- Pushgateway: https://github.com/prometheus/pushgateway +- Metric types: https://prometheus.io/docs/concepts/metric_types/ diff --git a/f3s/prometheus-pusher/USAGE.md b/f3s/prometheus-pusher/USAGE.md new file mode 100644 index 0000000..be5837f --- /dev/null +++ b/f3s/prometheus-pusher/USAGE.md @@ -0,0 +1,231 @@ +# Prometheus Pusher - Usage Guide + +## Quick Start + +### 1. Deploy Pushgateway (One-time setup) + +```bash +cd /home/paul/git/conf/f3s/pushgateway/helm-chart +helm upgrade --install pushgateway . -n monitoring --create-namespace +``` + +### 2. Update Prometheus Configuration (One-time setup) + +The Prometheus scrape configuration has already been updated in `/home/paul/git/conf/f3s/prometheus/additional-scrape-configs.yaml` to include: + +```yaml +- job_name: 'pushgateway' + honor_labels: true + static_configs: + - targets: + - 'pushgateway.monitoring.svc.cluster.local:9091' +``` + +Apply it: +```bash +kubectl create secret generic additional-scrape-configs \ + --from-file=/home/paul/git/conf/f3s/prometheus/additional-scrape-configs.yaml \ + --dry-run=client -o yaml -n monitoring | kubectl apply -f - +``` + +### 3. Run the Standalone Binary + +First, port-forward the Pushgateway: +```bash +kubectl port-forward -n monitoring svc/pushgateway 9091:9091 +``` + +In another terminal, run the binary: +```bash +cd /home/paul/git/conf/f3s/prometheus-pusher +./prometheus-pusher +``` + +The binary will: +- Push metrics immediately on startup +- Continue pushing metrics every 15 seconds +- Generate random example data to simulate a real application + +## Viewing Metrics + +### View Pushgateway UI +```bash +kubectl port-forward -n monitoring svc/pushgateway 9091:9091 +# Open http://localhost:9091 +``` + +### Query Prometheus +```bash +kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090 +# Open http://localhost:9090 +``` + +Example queries: +```promql +# View total requests +app_requests_total + +# View request rate over last 5 minutes +rate(app_requests_total[5m]) + +# View current active connections +app_active_connections + +# View current temperature +app_temperature_celsius + +# View 95th percentile request duration +histogram_quantile(0.95, rate(app_request_duration_seconds_bucket[5m])) + +# View failed jobs by type +app_jobs_processed_total{status="failed"} + +# View job success rate +rate(app_jobs_processed_total{status="success"}[5m]) / rate(app_jobs_processed_total[5m]) +``` + +## Metric Types Explained + +### Counter: `app_requests_total` +- **Type**: Counter +- **Description**: Total number of requests processed +- **Value behavior**: Only increases (monotonically increasing) +- **Use case**: Counting total events, requests, errors + +### Gauge: `app_active_connections`, `app_temperature_celsius` +- **Type**: Gauge +- **Description**: Current value that can go up or down +- **Value behavior**: Can increase or decrease +- **Use cases**: + - Active connections + - Current temperature + - Memory usage + - Queue length + +### Histogram: `app_request_duration_seconds` +- **Type**: Histogram +- **Description**: Distribution of request durations +- **Value behavior**: Samples observations into buckets +- **Use cases**: + - Request latency + - Response times + - Data sizes +- **Buckets**: .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 seconds + +### Counter with Labels: `app_jobs_processed_total` +- **Type**: Counter with labels +- **Description**: Jobs processed by type and status +- **Labels**: + - `job_type`: email, report, backup + - `status`: success, failed +- **Use cases**: Categorized counting, multi-dimensional metrics + +## Prometheus Format Example + +The metrics are sent in Prometheus text format: + +``` +# HELP app_requests_total Total number of requests processed +# TYPE app_requests_total counter +app_requests_total{instance="example-app",job="example_metrics_pusher"} 42 + +# HELP app_active_connections Number of currently active connections +# TYPE app_active_connections gauge +app_active_connections{instance="example-app",job="example_metrics_pusher"} 67 + +# HELP app_temperature_celsius Current temperature in Celsius +# TYPE app_temperature_celsius gauge +app_temperature_celsius{instance="example-app",job="example_metrics_pusher"} 23.5 + +# HELP app_jobs_processed_total Total number of jobs processed by type +# TYPE app_jobs_processed_total counter +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="email",status="success"} 15 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="email",status="failed"} 2 +``` + +## Customizing the Binary + +Edit `main.go` to: +1. Change the Pushgateway URL +2. Modify the push interval (currently 15 seconds) +3. Add your own metrics +4. Change label values + +Then rebuild: +```bash +go build -o prometheus-pusher main.go +``` + +## Architecture + +``` +┌─────────────────┐ +│ Go Binary │ +│ (prometheus- │──Push metrics──┐ +│ pusher) │ │ +└─────────────────┘ │ + ▼ + ┌──────────────────┐ + │ Pushgateway │◄──Scrape──┐ + │ (Port 9091) │ │ + └──────────────────┘ │ + │ + ┌─────────────────┐ + │ Prometheus │ + │ (Port 9090) │ + └─────────────────┘ +``` + +## When to Use Pushgateway vs. Scraping + +**Use Pushgateway (what we're doing) for:** +- Batch jobs +- Short-lived processes +- Service-level metrics +- Jobs behind firewalls + +**Use Prometheus scraping (alternative approach) for:** +- Long-running applications +- Services with consistent endpoints +- Applications that can expose `/metrics` endpoint + +## Troubleshooting + +### Binary can't connect to Pushgateway +```bash +# Check port-forward is running +ps aux | grep "port-forward.*9091" + +# Restart port-forward +kubectl port-forward -n monitoring svc/pushgateway 9091:9091 +``` + +### Metrics not appearing in Prometheus +```bash +# Check Pushgateway has metrics +curl http://localhost:9091/metrics | grep "app_" + +# Check Prometheus scrape targets +# Open http://localhost:9090/targets +# Look for "pushgateway" job + +# Check Prometheus logs +kubectl logs -n monitoring -l app.kubernetes.io/name=prometheus +``` + +### Reload Prometheus config manually +```bash +# The Prometheus Operator should auto-reload, but if needed: +kubectl delete pod -n monitoring -l app.kubernetes.io/name=prometheus +``` + +## Clean Up + +```bash +# Stop port-forwards +pkill -f "port-forward.*9091" +pkill -f "port-forward.*9090" + +# Remove deployment (if you want to uninstall) +helm uninstall pushgateway-only -n monitoring +``` diff --git a/f3s/prometheus-pusher/example-metrics.txt b/f3s/prometheus-pusher/example-metrics.txt new file mode 100644 index 0000000..a30b617 --- /dev/null +++ b/f3s/prometheus-pusher/example-metrics.txt @@ -0,0 +1,57 @@ +# Example Metrics Output +# This shows what the Prometheus pusher sends to the Pushgateway + +# COUNTER: Monotonically increasing value +# Use for: Total requests, total errors, total events +# ------------------------------------------------------- +# HELP app_requests_total Total number of requests processed +# TYPE app_requests_total counter +app_requests_total{instance="example-app",job="example_metrics_pusher"} 10 + +# GAUGE: Value that can increase or decrease +# Use for: Current connections, temperature, memory usage +# ------------------------------------------------------- +# HELP app_active_connections Number of currently active connections +# TYPE app_active_connections gauge +app_active_connections{instance="example-app",job="example_metrics_pusher"} 58 + +# HELP app_temperature_celsius Current temperature in Celsius +# TYPE app_temperature_celsius gauge +app_temperature_celsius{instance="example-app",job="example_metrics_pusher"} 23.456789 + +# HISTOGRAM: Distribution of values +# Use for: Request duration, response sizes +# ------------------------------------------------------- +# HELP app_request_duration_seconds Histogram of request duration in seconds +# TYPE app_request_duration_seconds histogram +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.005"} 0 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.01"} 0 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.025"} 0 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.05"} 0 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.1"} 0 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.25"} 0 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="0.5"} 0 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="1"} 2 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="2.5"} 6 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="5"} 6 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="10"} 6 +app_request_duration_seconds_bucket{instance="example-app",job="example_metrics_pusher",le="+Inf"} 6 +app_request_duration_seconds_sum{instance="example-app",job="example_metrics_pusher"} 4.123456 +app_request_duration_seconds_count{instance="example-app",job="example_metrics_pusher"} 6 + +# COUNTER WITH LABELS: Categorized counting +# Use for: Categorizing events by type, status, etc. +# ------------------------------------------------------- +# HELP app_jobs_processed_total Total number of jobs processed by type +# TYPE app_jobs_processed_total counter +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="email",status="success"} 2 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="email",status="failed"} 0 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="report",status="success"} 4 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="report",status="failed"} 0 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="backup",status="success"} 0 +app_jobs_processed_total{instance="example-app",job="example_metrics_pusher",job_type="backup",status="failed"} 4 + +# Pushgateway metadata +# ------------------------------------------------------- +push_time_seconds{instance="example-app",job="example_metrics_pusher"} 1767121612.254 +push_failure_time_seconds{instance="example-app",job="example_metrics_pusher"} 0 diff --git a/f3s/prometheus-pusher/go.mod b/f3s/prometheus-pusher/go.mod new file mode 100644 index 0000000..4576c38 --- /dev/null +++ b/f3s/prometheus-pusher/go.mod @@ -0,0 +1,16 @@ +module prometheus-pusher + +go 1.21 + +require github.com/prometheus/client_golang v1.20.5 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.22.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) diff --git a/f3s/prometheus-pusher/go.sum b/f3s/prometheus-pusher/go.sum new file mode 100644 index 0000000..d3e9db5 --- /dev/null +++ b/f3s/prometheus-pusher/go.sum @@ -0,0 +1,22 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/f3s/prometheus-pusher/main.go b/f3s/prometheus-pusher/main.go new file mode 100644 index 0000000..46a896a --- /dev/null +++ b/f3s/prometheus-pusher/main.go @@ -0,0 +1,138 @@ +package main + +import ( + "fmt" + "log" + "math/rand" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/push" +) + +// Define metrics +var ( + // Counter: Monotonically increasing value (e.g., total requests processed) + requestsTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "app_requests_total", + Help: "Total number of requests processed", + }, + ) + + // Gauge: Value that can go up or down (e.g., current temperature, active connections) + activeConnections = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "app_active_connections", + Help: "Number of currently active connections", + }, + ) + + // Gauge for temperature simulation + temperatureCelsius = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "app_temperature_celsius", + Help: "Current temperature in Celsius", + }, + ) + + // Histogram: Distribution of values (e.g., request duration) + requestDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "app_request_duration_seconds", + Help: "Histogram of request duration in seconds", + Buckets: prometheus.DefBuckets, // Default buckets: .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 + }, + ) + + // Counter with labels + jobsProcessed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "app_jobs_processed_total", + Help: "Total number of jobs processed by type", + }, + []string{"job_type", "status"}, + ) +) + +// simulateMetrics generates example metric data +func simulateMetrics() { + // Increment request counter + requestsTotal.Add(float64(rand.Intn(10) + 1)) + + // Update active connections (random number between 0-100) + activeConnections.Set(float64(rand.Intn(100))) + + // Simulate temperature (random between 15-35 Celsius) + temperatureCelsius.Set(15 + rand.Float64()*20) + + // Record some request durations + for i := 0; i < rand.Intn(5)+1; i++ { + duration := rand.Float64() * 2 // 0-2 seconds + requestDuration.Observe(duration) + } + + // Record job completions with labels + jobTypes := []string{"email", "report", "backup"} + statuses := []string{"success", "failed"} + + for _, jobType := range jobTypes { + status := statuses[rand.Intn(len(statuses))] + jobsProcessed.WithLabelValues(jobType, status).Add(float64(rand.Intn(5))) + } +} + +// pushMetrics pushes all metrics to the Pushgateway +func pushMetrics(pushgatewayURL, jobName string) error { + // Create a new pusher + pusher := push.New(pushgatewayURL, jobName). + Collector(requestsTotal). + Collector(activeConnections). + Collector(temperatureCelsius). + Collector(requestDuration). + Collector(jobsProcessed). + Grouping("instance", "example-app") + + // Push metrics to the Pushgateway + if err := pusher.Push(); err != nil { + return fmt.Errorf("failed to push metrics: %w", err) + } + + return nil +} + +func main() { + // Configuration - use localhost:9091 when port-forwarding + // kubectl port-forward -n monitoring svc/pushgateway 9091:9091 + pushgatewayURL := "http://localhost:9091" + jobName := "example_metrics_pusher" + + // Seed random number generator + rand.Seed(time.Now().UnixNano()) + + log.Printf("Starting Prometheus metrics pusher") + log.Printf("Pushgateway URL: %s", pushgatewayURL) + log.Printf("Job name: %s", jobName) + + // Push metrics every 15 seconds + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + + // Push immediately on start + simulateMetrics() + if err := pushMetrics(pushgatewayURL, jobName); err != nil { + log.Printf("Error pushing metrics: %v", err) + } else { + log.Printf("Successfully pushed metrics to Pushgateway") + } + + // Continue pushing periodically + for range ticker.C { + simulateMetrics() + if err := pushMetrics(pushgatewayURL, jobName); err != nil { + log.Printf("Error pushing metrics: %v", err) + } else { + log.Printf("Successfully pushed metrics to Pushgateway") + } + } +} diff --git a/f3s/prometheus-pusher/prometheus-pusher b/f3s/prometheus-pusher/prometheus-pusher Binary files differnew file mode 100755 index 0000000..27a4196 --- /dev/null +++ b/f3s/prometheus-pusher/prometheus-pusher diff --git a/f3s/prometheus-pusher/run.sh b/f3s/prometheus-pusher/run.sh new file mode 100755 index 0000000..18811c8 --- /dev/null +++ b/f3s/prometheus-pusher/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Simple script to run the prometheus pusher +# Automatically sets up port-forwarding and runs the binary + +set -e + +echo "Starting Prometheus Pusher..." +echo "" +echo "Step 1: Setting up port-forward to Pushgateway..." +kubectl port-forward -n monitoring svc/pushgateway 9091:9091 > /tmp/pushgateway-port-forward.log 2>&1 & +PF_PID=$! + +# Wait for port-forward to be ready +sleep 2 + +echo "Step 2: Running prometheus-pusher binary..." +echo "Press Ctrl+C to stop" +echo "" + +# Run the binary and capture its exit status +./prometheus-pusher +EXIT_CODE=$? + +# Clean up port-forward +echo "" +echo "Cleaning up port-forward..." +kill $PF_PID 2>/dev/null || true + +echo "Done!" +exit $EXIT_CODE diff --git a/f3s/prometheus/additional-scrape-configs.yaml b/f3s/prometheus/additional-scrape-configs.yaml index 93035d8..0ea8d69 100644 --- a/f3s/prometheus/additional-scrape-configs.yaml +++ b/f3s/prometheus/additional-scrape-configs.yaml @@ -11,3 +11,9 @@ - '192.168.2.111:9100' # fishfinger via WireGuard labels: os: openbsd + +- job_name: 'pushgateway' + honor_labels: true + static_configs: + - targets: + - 'pushgateway.monitoring.svc.cluster.local:9091' diff --git a/f3s/pushgateway/README.md b/f3s/pushgateway/README.md new file mode 100644 index 0000000..a2cce54 --- /dev/null +++ b/f3s/pushgateway/README.md @@ -0,0 +1,166 @@ +# Pushgateway |