can report stale alerts

4 files changed, 50 insertions, 18 deletions

diff --git a/README.md b/README.md
index fc0eb92..ae596e3 100644
--- a/README.md
+++ b/README.md
@@ -8,10 +8,10 @@ You can also read about it in this blog post: https://foo.zone/gemfeed/2023-06-0
 
 ## Example alert
 
-This is an example alert report received via E-Mail. Whereas, `[C:2 W:0 U:0 OK:51]` means that we've got two alerts in status critical, 0 warnings, 0 unknowns and 51 OKs.
+This is an example alert report received via E-Mail. Whereas, `[C:2 W:0 U:0 S:0 OK:51]` means that we've got two alerts in status critical, 0 warnings, 0 unknowns, 0 stale alerts (last check too far in the past) and 51 OKs.
 
 ```
-Subject: GOGIOS Report [C:2 W:0 U:0 OK:51]
+Subject: GOGIOS Report [C:2 W:0 U:0 S:0 OK:51]
 
 This is the recent Gogios report!
 
@@ -25,6 +25,10 @@ OK->CRITICAL: Check ICMP6 vulcan.buetow.org: Check command timed out
 CRITICAL: Check ICMP4 vulcan.buetow.org: Check command timed out
 CRITICAL: Check ICMP6 vulcan.buetow.org: Check command timed out
 
+# Stale alerts:
+
+There are no stale alerts...
+
 Have a nice day!
 ```
 
@@ -106,6 +110,7 @@ To configure Gogios, create a JSON configuration file (e.g., `/etc/gogios.json`)
   "EmailFrom": "gogios@buetow.org",
   "CheckTimeoutS": 10,
   "CheckConcurrency": 2,
+  "StaleThreshold": 3600,
   "StateDir": "/var/run/gogios",
   "Checks": {
     "Check ICMP4 www.foo.zone": {
@@ -142,6 +147,7 @@ To configure Gogios, create a JSON configuration file (e.g., `/etc/gogios.json`)
 * `EmailFrom`: Indicates the sender's email address for email notifications.
 * `CheckTimeoutS`: Sets the timeout for checks in seconds.
 * `CheckConcurrency`: Determines the number of concurrent checks that can run simultaneously.
+* `StaleThreshold`: Defines the threshold in seconds for considering a check stale if it hasn't been updated within this time frame.
 * `StateDir`: Specifies the directory where Gogios stores its persistent state in a `state.json` file. 
 * `Checks`: Defines a list of checks to be performed, each with a unique name, plugin path, and arguments. 
 
diff --git a/internal/config.go b/internal/config.go
index 9d43e5c..ccc3f36 100644
--- a/internal/config.go
+++ b/internal/config.go
@@ -15,6 +15,7 @@ type config struct {
 	StateDir         string `json:"StateDir,omitempty"`
 	CheckTimeoutS    int
 	CheckConcurrency int
+	StaleThreshold   int `json:"StaleThreshold,omitempty"`
 	Checks           map[string]check
 }
 
@@ -51,6 +52,10 @@ func newConfig(configFile string) (config, error) {
 		log.Println("Set StateDir to " + conf.StateDir)
 	}
 
+	if conf.StaleThreshold == 0 {
+		conf.StaleThreshold = 3600 // Default to 1 hour
+	}
+
 	return conf, nil
 }
 
diff --git a/internal/run.go b/internal/run.go
index b48c19c..a4fbf68 100644
--- a/internal/run.go
+++ b/internal/run.go
@@ -35,12 +35,12 @@ func Run(ctx context.Context, configFile string, renotify, force bool) {
 			return
 		}
 	}
-	if err := persistReport(body, conf); err != nil {
+	if err := persistReport(subject, body, conf); err != nil {
 		notifyError(conf, err)
 	}
 }
 
-func persistReport(body string, conf config) error {
+func persistReport(subject, body string, conf config) error {
 	reportFile := fmt.Sprintf("%s/report.txt", conf.StateDir)
 	tmpFile := fmt.Sprintf("%s.tmp", reportFile)
 
@@ -50,6 +50,9 @@ func persistReport(body string, conf config) error {
 	}
 	defer f.Close()
 
+	if _, err = f.WriteString(fmt.Sprintf("%s\n\n", subject)); err != nil {
+		return err
+	}
 	if _, err = f.WriteString(body); err != nil {
 		return err
 	}
diff --git a/internal/state.go b/internal/state.go
index d37ecf4..062a698 100644
--- a/internal/state.go
+++ b/internal/state.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 type checkState struct {
@@ -22,14 +23,16 @@ func (cs checkState) changed() bool {
 }
 
 type state struct {
-	stateFile string
-	checks    map[string]checkState
+	stateFile  string
+	checks     map[string]checkState
+	staleEpoch int64
 }
 
 func newState(conf config) (state, error) {
 	s := state{
-		stateFile: fmt.Sprintf("%s/state.json", conf.StateDir),
-		checks:    make(map[string]checkState),
+		stateFile:  fmt.Sprintf("%s/state.json", conf.StateDir),
+		checks:     make(map[string]checkState),
+		staleEpoch: time.Now().Unix() - int64(conf.StaleThreshold),
 	}
 
 	if _, err := os.Stat(s.stateFile); err != nil {
@@ -124,35 +127,41 @@ func (s state) report(renotify, force bool) (string, string, bool) {
 		sb.WriteString("There are no unhandled alerts...\n\n")
 	}
 
+	sb.WriteString("# Stale alerts:\n\n")
+	numStale := s.reportStaleAlerts(&sb)
+	if numStale == 0 {
+		sb.WriteString("There are no stale alerts...\n\n")
+	}
+
 	sb.WriteString("Have a nice day!\n")
 
-	subject := fmt.Sprintf("GOGIOS Report [C:%d W:%d U:%d OK:%d]",
-		numCriticals, numWarnings, numUnknown, numOK)
+	subject := fmt.Sprintf("GOGIOS Report [C:%d W:%d U:%d S:%d OK:%d]",
+		numCriticals, numWarnings, numUnknown, numStale, numOK)
 
 	doNotify := force || (changed || (renotify && hasUnhandled))
 	return subject, sb.String(), doNotify
 }
 
 func (s state) reportChanged(sb *strings.Builder) (changed bool) {
-	if 0 < s.reportBy(sb, true, func(cs checkState) bool {
+	if 0 < s.reportBy(sb, true, false, func(cs checkState) bool {
 		return cs.Status == nagiosCritical && cs.changed()
 	}) {
 		changed = true
 	}
 
-	if 0 < s.reportBy(sb, true, func(cs checkState) bool {
+	if 0 < s.reportBy(sb, true, false, func(cs checkState) bool {
 		return cs.Status == nagiosWarning && cs.changed()
 	}) {
 		changed = true
 	}
 
-	if 0 < s.reportBy(sb, true, func(cs checkState) bool {
+	if 0 < s.reportBy(sb, true, false, func(cs checkState) bool {
 		return cs.Status == nagiosUnknown && cs.changed()
 	}) {
 		changed = true
 	}
 
-	if 0 < s.reportBy(sb, true, func(cs checkState) bool {
+	if 0 < s.reportBy(sb, true, false, func(cs checkState) bool {
 		return cs.Status == nagiosOk && cs.changed()
 	}) {
 		changed = true
@@ -164,15 +173,15 @@ func (s state) reportChanged(sb *strings.Builder) (changed bool) {
 func (s state) reportUnhandled(sb *strings.Builder) (numCriticals, numWarnings,
 	numUnknown, numOK int) {
 
-	numCriticals = s.reportBy(sb, false, func(cs checkState) bool {
+	numCriticals = s.reportBy(sb, false, false, func(cs checkState) bool {
 		return cs.Status == nagiosCritical
 	})
 
-	numWarnings = s.reportBy(sb, false, func(cs checkState) bool {
+	numWarnings = s.reportBy(sb, false, false, func(cs checkState) bool {
 		return cs.Status == nagiosWarning
 	})
 
-	numUnknown = s.reportBy(sb, false, func(cs checkState) bool {
+	numUnknown = s.reportBy(sb, false, false, func(cs checkState) bool {
 		return cs.Status == nagiosUnknown
 	})
 
@@ -183,13 +192,22 @@ func (s state) reportUnhandled(sb *strings.Builder) (numCriticals, numWarnings,
 	return
 }
 
-func (s state) reportBy(sb *strings.Builder, showStatusChange bool,
+func (s state) reportStaleAlerts(sb *strings.Builder) int {
+	return s.reportBy(sb, false, true, func(cs checkState) bool {
+		return cs.Epoch < s.staleEpoch
+	})
+}
+
+func (s state) reportBy(sb *strings.Builder, showStatusChange, isStaleReport bool,
 	filter func(cs checkState) bool) (count int) {
 
 	for name, cs := range s.checks {
 		if !filter(cs) {
 			continue
 		}
+		if !isStaleReport && cs.Epoch < s.staleEpoch {
+			continue // skip stale checks in non-stale report
+		}
 		count++
 
 		if showStatusChange && cs.changed() {