From d9671ba9c6ba158cd4516626c4627d38d6478110 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Thu, 8 Jan 2026 22:22:10 +0200
Subject: Add special handling for Prometheus Watchdog alert

- Treat firing Watchdog as OK status to confirm Alertmanager is working
- Treat absent/non-firing Watchdog as CRITICAL to alert on Alertmanager issues
- Add comprehensive tests for both scenarios
---
 internal/prometheus.go | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

(limited to 'internal/prometheus.go')

diff --git a/internal/prometheus.go b/internal/prometheus.go
index 1c06aaf..c73f1cf 100644
--- a/internal/prometheus.go
+++ b/internal/prometheus.go
@@ -44,12 +44,33 @@ func mergePrometheusAlerts(ctx context.Context, state state, conf config) state
 
 	log.Printf("Fetched %d firing alerts from Prometheus host %s", len(alerts), host)
 
+	// Check if Watchdog alert is firing
+	watchdogFiring := false
+
 	for _, alert := range alerts {
+		alertname := alert.Labels["alertname"]
+
+		// Special handling for Prometheus Watchdog alert
+		if alertname == "Watchdog" {
+			if alert.State == "firing" {
+				watchdogFiring = true
+				// Watchdog is firing as expected, treat as OK
+				cs := checkResult{
+					name:   "Prometheus: Watchdog",
+					output: "OK [none]: Alertmanager is working properly",
+					epoch:  time.Now().Unix(),
+					status: nagiosOk,
+				}
+				state.update(cs)
+			}
+			continue
+		}
+
 		if alert.State != "firing" {
 			continue
 		}
 
-		name := fmt.Sprintf("Prometheus: %s", alert.Labels["alertname"])
+		name := fmt.Sprintf("Prometheus: %s", alertname)
 		severity := alert.Labels["severity"]
 		description := alert.Annotations["summary"]
 		if description == "" {
@@ -66,13 +87,24 @@ func mergePrometheusAlerts(ctx context.Context, state state, conf config) state
 
 		cs := checkResult{
 			name:   name,
-			output: fmt.Sprintf("%s [%s]: %s", alert.Labels["alertname"], severity, description),
+			output: fmt.Sprintf("%s [%s]: %s", alertname, severity, description),
 			epoch:  time.Now().Unix(),
 			status: status,
 		}
 		state.update(cs)
 	}
 
+	// If Watchdog is not firing, alert as critical
+	if !watchdogFiring {
+		cs := checkResult{
+			name:   "Prometheus: Watchdog",
+			output: "CRITICAL [none]: Watchdog alert is not firing, Alertmanager may not be working",
+			epoch:  time.Now().Unix(),
+			status: nagiosCritical,
+		}
+		state.update(cs)
+	}
+
 	return state
 }
 
-- 
cgit v1.2.3