From ce6844995eaa3b1e2d50825d5490f3593cae2b8e Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 29 May 2025 09:23:11 +0300 Subject: can report stale alerts --- README.md | 10 ++++++++-- internal/config.go | 5 +++++ internal/run.go | 7 +++++-- internal/state.go | 46 ++++++++++++++++++++++++++++++++-------------- 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index fc0eb92..ae596e3 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ You can also read about it in this blog post: https://foo.zone/gemfeed/2023-06-0 ## Example alert -This is an example alert report received via E-Mail. Whereas, `[C:2 W:0 U:0 OK:51]` means that we've got two alerts in status critical, 0 warnings, 0 unknowns and 51 OKs. +This is an example alert report received via E-Mail. Whereas, `[C:2 W:0 U:0 S:0 OK:51]` means that we've got two alerts in status critical, 0 warnings, 0 unknowns, 0 stale alerts (last check too far in the past) and 51 OKs. ``` -Subject: GOGIOS Report [C:2 W:0 U:0 OK:51] +Subject: GOGIOS Report [C:2 W:0 U:0 S:0 OK:51] This is the recent Gogios report! @@ -25,6 +25,10 @@ OK->CRITICAL: Check ICMP6 vulcan.buetow.org: Check command timed out CRITICAL: Check ICMP4 vulcan.buetow.org: Check command timed out CRITICAL: Check ICMP6 vulcan.buetow.org: Check command timed out +# Stale alerts: + +There are no stale alerts... + Have a nice day! ``` @@ -106,6 +110,7 @@ To configure Gogios, create a JSON configuration file (e.g., `/etc/gogios.json`) "EmailFrom": "gogios@buetow.org", "CheckTimeoutS": 10, "CheckConcurrency": 2, + "StaleThreshold": 3600, "StateDir": "/var/run/gogios", "Checks": { "Check ICMP4 www.foo.zone": { @@ -142,6 +147,7 @@ To configure Gogios, create a JSON configuration file (e.g., `/etc/gogios.json`) * `EmailFrom`: Indicates the sender's email address for email notifications. * `CheckTimeoutS`: Sets the timeout for checks in seconds. * `CheckConcurrency`: Determines the number of concurrent checks that can run simultaneously. +* `StaleThreshold`: Defines the threshold in seconds for considering a check stale if it hasn't been updated within this time frame. * `StateDir`: Specifies the directory where Gogios stores its persistent state in a `state.json` file. * `Checks`: Defines a list of checks to be performed, each with a unique name, plugin path, and arguments. diff --git a/internal/config.go b/internal/config.go index 9d43e5c..ccc3f36 100644 --- a/internal/config.go +++ b/internal/config.go @@ -15,6 +15,7 @@ type config struct { StateDir string `json:"StateDir,omitempty"` CheckTimeoutS int CheckConcurrency int + StaleThreshold int `json:"StaleThreshold,omitempty"` Checks map[string]check } @@ -51,6 +52,10 @@ func newConfig(configFile string) (config, error) { log.Println("Set StateDir to " + conf.StateDir) } + if conf.StaleThreshold == 0 { + conf.StaleThreshold = 3600 // Default to 1 hour + } + return conf, nil } diff --git a/internal/run.go b/internal/run.go index b48c19c..a4fbf68 100644 --- a/internal/run.go +++ b/internal/run.go @@ -35,12 +35,12 @@ func Run(ctx context.Context, configFile string, renotify, force bool) { return } } - if err := persistReport(body, conf); err != nil { + if err := persistReport(subject, body, conf); err != nil { notifyError(conf, err) } } -func persistReport(body string, conf config) error { +func persistReport(subject, body string, conf config) error { reportFile := fmt.Sprintf("%s/report.txt", conf.StateDir) tmpFile := fmt.Sprintf("%s.tmp", reportFile) @@ -50,6 +50,9 @@ func persistReport(body string, conf config) error { } defer f.Close() + if _, err = f.WriteString(fmt.Sprintf("%s\n\n", subject)); err != nil { + return err + } if _, err = f.WriteString(body); err != nil { return err } diff --git a/internal/state.go b/internal/state.go index d37ecf4..062a698 100644 --- a/internal/state.go +++ b/internal/state.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "time" ) type checkState struct { @@ -22,14 +23,16 @@ func (cs checkState) changed() bool { } type state struct { - stateFile string - checks map[string]checkState + stateFile string + checks map[string]checkState + staleEpoch int64 } func newState(conf config) (state, error) { s := state{ - stateFile: fmt.Sprintf("%s/state.json", conf.StateDir), - checks: make(map[string]checkState), + stateFile: fmt.Sprintf("%s/state.json", conf.StateDir), + checks: make(map[string]checkState), + staleEpoch: time.Now().Unix() - int64(conf.StaleThreshold), } if _, err := os.Stat(s.stateFile); err != nil { @@ -124,35 +127,41 @@ func (s state) report(renotify, force bool) (string, string, bool) { sb.WriteString("There are no unhandled alerts...\n\n") } + sb.WriteString("# Stale alerts:\n\n") + numStale := s.reportStaleAlerts(&sb) + if numStale == 0 { + sb.WriteString("There are no stale alerts...\n\n") + } + sb.WriteString("Have a nice day!\n") - subject := fmt.Sprintf("GOGIOS Report [C:%d W:%d U:%d OK:%d]", - numCriticals, numWarnings, numUnknown, numOK) + subject := fmt.Sprintf("GOGIOS Report [C:%d W:%d U:%d S:%d OK:%d]", + numCriticals, numWarnings, numUnknown, numStale, numOK) doNotify := force || (changed || (renotify && hasUnhandled)) return subject, sb.String(), doNotify } func (s state) reportChanged(sb *strings.Builder) (changed bool) { - if 0 < s.reportBy(sb, true, func(cs checkState) bool { + if 0 < s.reportBy(sb, true, false, func(cs checkState) bool { return cs.Status == nagiosCritical && cs.changed() }) { changed = true } - if 0 < s.reportBy(sb, true, func(cs checkState) bool { + if 0 < s.reportBy(sb, true, false, func(cs checkState) bool { return cs.Status == nagiosWarning && cs.changed() }) { changed = true } - if 0 < s.reportBy(sb, true, func(cs checkState) bool { + if 0 < s.reportBy(sb, true, false, func(cs checkState) bool { return cs.Status == nagiosUnknown && cs.changed() }) { changed = true } - if 0 < s.reportBy(sb, true, func(cs checkState) bool { + if 0 < s.reportBy(sb, true, false, func(cs checkState) bool { return cs.Status == nagiosOk && cs.changed() }) { changed = true @@ -164,15 +173,15 @@ func (s state) reportChanged(sb *strings.Builder) (changed bool) { func (s state) reportUnhandled(sb *strings.Builder) (numCriticals, numWarnings, numUnknown, numOK int) { - numCriticals = s.reportBy(sb, false, func(cs checkState) bool { + numCriticals = s.reportBy(sb, false, false, func(cs checkState) bool { return cs.Status == nagiosCritical }) - numWarnings = s.reportBy(sb, false, func(cs checkState) bool { + numWarnings = s.reportBy(sb, false, false, func(cs checkState) bool { return cs.Status == nagiosWarning }) - numUnknown = s.reportBy(sb, false, func(cs checkState) bool { + numUnknown = s.reportBy(sb, false, false, func(cs checkState) bool { return cs.Status == nagiosUnknown }) @@ -183,13 +192,22 @@ func (s state) reportUnhandled(sb *strings.Builder) (numCriticals, numWarnings, return } -func (s state) reportBy(sb *strings.Builder, showStatusChange bool, +func (s state) reportStaleAlerts(sb *strings.Builder) int { + return s.reportBy(sb, false, true, func(cs checkState) bool { + return cs.Epoch < s.staleEpoch + }) +} + +func (s state) reportBy(sb *strings.Builder, showStatusChange, isStaleReport bool, filter func(cs checkState) bool) (count int) { for name, cs := range s.checks { if !filter(cs) { continue } + if !isStaleReport && cs.Epoch < s.staleEpoch { + continue // skip stale checks in non-stale report + } count++ if showStatusChange && cs.changed() { -- cgit v1.2.3