diff options
| -rw-r--r-- | README.md | 37 | ||||
| -rw-r--r-- | internal/cli/cmd_daemon.go | 5 | ||||
| -rw-r--r-- | internal/daemon/daemon.go | 13 | ||||
| -rw-r--r-- | internal/daemon/daemon_test.go | 8 | ||||
| -rw-r--r-- | internal/daemon/metrics.go | 83 | ||||
| -rw-r--r-- | internal/daemon/metrics_test.go | 51 | ||||
| -rw-r--r-- | internal/daemon/upload_test.go | 2 | ||||
| -rw-r--r-- | internal/version/version.go | 2 |
8 files changed, 161 insertions, 40 deletions
@@ -252,6 +252,43 @@ sudo loginctl enable-linger "$USER" If there are **no** keys in the auth database, uploads are accepted without **`Authorization`** (useful for local testing only). +### Excluding hosts from /metrics alerts + +Hosts that no longer send updates can be marked as excluded so Prometheus +alert rules can filter them out. Excluded hosts still appear in `/metrics` +with `excluded="true"`. + +**Exclude a host** + +```bash +# daemon on Kubernetes — auth DB lives on the stats PVC +kubectl exec -n services deployment/goprecords -- \ + goprecords exclude -db=/data/stats/goprecords-auth.db -reason="decommissioned" myhost + +# local daemon +goprecords exclude -db=/var/lib/goprecords/goprecords-auth.db -reason="decommissioned" myhost +``` + +**List excluded hosts** + +```bash +kubectl exec -n services deployment/goprecords -- \ + goprecords list-excluded -db=/data/stats/goprecords-auth.db +``` + +**Remove an exclusion manually** + +```bash +kubectl exec -n services deployment/goprecords -- \ + goprecords unexclude -db=/data/stats/goprecords-auth.db myhost +``` + +**Auto-unexclude** + +When a host resumes sending uploads its records file mtime becomes newer than +`excluded_at`. On the next `/metrics` scrape the daemon detects this and +automatically removes the exclusion — no manual intervention needed. + ### Manual upload client (all operating systems) The unified script **`scripts/goprecords-upload-client.sh`** works on diff --git a/internal/cli/cmd_daemon.go b/internal/cli/cmd_daemon.go index 111968f..00071cf 100644 --- a/internal/cli/cmd_daemon.go +++ b/internal/cli/cmd_daemon.go @@ -24,8 +24,7 @@ func runDaemon(args []string) error { fs.SetOutput(os.Stdout) statsDir := fs.String("stats-dir", os.Getenv("GOPRECORDS_STATS_DIR"), "Uptimed stats directory (required; env GOPRECORDS_STATS_DIR)") listen := fs.String("listen", defaultListenFromEnv(), "TCP listen address (env GOPRECORDS_LISTEN, default :8080)") - authDB := fs.String("auth-db", "", "SQLite file for upload API keys (default: <stats-dir>/goprecords-auth.db)") - db := fs.String("db", os.Getenv("GOPRECORDS_DB"), "SQLite database path for excluded hosts used by /metrics (env GOPRECORDS_DB)") + authDB := fs.String("auth-db", "", "SQLite file for upload API keys and excluded hosts (default: <stats-dir>/goprecords-auth.db)") if err := fs.Parse(args); err != nil { return err } @@ -36,7 +35,7 @@ func runDaemon(args []string) error { } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - err := daemon.Run(ctx, daemon.Config{StatsDir: *statsDir, Addr: *listen, AuthDB: *authDB, DB: *db}) + err := daemon.Run(ctx, daemon.Config{StatsDir: *statsDir, Addr: *listen, AuthDB: *authDB}) if err != nil && !errors.Is(err, context.Canceled) { return err } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 326224e..6346df6 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -31,7 +31,6 @@ type Config struct { StatsDir string Addr string AuthDB string - DB string LogOutput io.Writer } @@ -42,17 +41,17 @@ func NewHandler(statsDir string) (http.Handler, error) { if err != nil { return nil, fmt.Errorf("auth db: %w", err) } - return routes(statsDir, "", "", store), nil + return routes(statsDir, authkeys.DefaultPath(statsDir), store), nil } -func routes(statsDir, authDB, db string, store *authkeys.Store) http.Handler { +func routes(statsDir, authDB string, store *authkeys.Store) http.Handler { mux := http.NewServeMux() mux.HandleFunc("/", root(statsDir)) mux.HandleFunc("/health", health) mux.HandleFunc("/livez", health) mux.HandleFunc("/readyz", readiness(statsDir, authDB)) mux.HandleFunc("/report", report(statsDir)) - mux.HandleFunc("/metrics", metricsHandler(statsDir, db)) + mux.HandleFunc("/metrics", metricsHandler(statsDir, authDB)) mux.Handle("/upload/", uploadHandler(statsDir, store)) return mux } @@ -148,12 +147,16 @@ func Run(ctx context.Context, cfg Config) error { } w := logWriter(cfg) slogLog, textHandler := newDaemonLogger(w) + resolvedAuthDB := cfg.AuthDB + if resolvedAuthDB == "" { + resolvedAuthDB = authkeys.DefaultPath(cfg.StatsDir) + } store, err := openAuthStore(ctx, cfg.StatsDir, cfg.AuthDB) if err != nil { return fmt.Errorf("auth db: %w", err) } defer store.Close() - srv := newDaemonHTTPServer(cfg.Addr, withAccessLog(slogLog, routes(cfg.StatsDir, cfg.AuthDB, cfg.DB, store)), + srv := newDaemonHTTPServer(cfg.Addr, withAccessLog(slogLog, routes(cfg.StatsDir, resolvedAuthDB, store)), slog.NewLogLogger(textHandler, slog.LevelError)) slogLog.Info("daemon_listen", "addr", cfg.Addr) errCh := make(chan error, 1) diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 462e322..44f9888 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -542,7 +542,7 @@ func TestAccessLogLineToWriter(t *testing.T) { t.Fatal(err) } defer store.Close() - srv := httptest.NewServer(withAccessLog(log, routes(statsDir, "", "", store))) + srv := httptest.NewServer(withAccessLog(log, routes(statsDir, "", store))) defer srv.Close() res, err := http.Get(srv.URL + "/health") if err != nil { @@ -591,7 +591,7 @@ func TestUploadRequiresBearerWhenKeysExist(t *testing.T) { if _, err := store.CreateKey(ctx, "myhost"); err != nil { t.Fatal(err) } - srv := httptest.NewServer(routes(statsDir, "", "", store)) + srv := httptest.NewServer(routes(statsDir, "", store)) defer srv.Close() req, _ := http.NewRequest(http.MethodPut, srv.URL+"/upload/myhost/txt", strings.NewReader("x")) res, err := http.DefaultClient.Do(req) @@ -616,7 +616,7 @@ func TestUploadWithValidBearer(t *testing.T) { if err != nil { t.Fatal(err) } - srv := httptest.NewServer(routes(statsDir, "", "", store)) + srv := httptest.NewServer(routes(statsDir, "", store)) defer srv.Close() req, _ := http.NewRequest(http.MethodPut, srv.URL+"/upload/myhost/os.txt", strings.NewReader("os")) req.Header.Set("Authorization", "Bearer "+tok) @@ -642,7 +642,7 @@ func TestUploadWrongHostForbidden(t *testing.T) { if err != nil { t.Fatal(err) } - srv := httptest.NewServer(routes(statsDir, "", "", store)) + srv := httptest.NewServer(routes(statsDir, "", store)) defer srv.Close() req, _ := http.NewRequest(http.MethodPut, srv.URL+"/upload/other/txt", strings.NewReader("x")) req.Header.Set("Authorization", "Bearer "+tok) diff --git a/internal/daemon/metrics.go b/internal/daemon/metrics.go index 192db77..21701db 100644 --- a/internal/daemon/metrics.go +++ b/internal/daemon/metrics.go @@ -35,7 +35,11 @@ func buildMetrics(ctx context.Context, statsDir, dbPath string) ([]byte, error) if err != nil { return nil, fmt.Errorf("list records: %w", err) } - excluded, err := loadExcludedSet(ctx, dbPath) + db := openMetricsDB(ctx, dbPath) + if db != nil { + defer db.Close() + } + excluded, err := loadExcludedMap(ctx, db) if err != nil { return nil, fmt.Errorf("load excluded hosts: %w", err) } @@ -46,12 +50,63 @@ func buildMetrics(ctx context.Context, statsDir, dbPath string) ([]byte, error) if err != nil { continue } - isExcluded := excluded[e.Host] + isExcluded := resolveExcluded(ctx, db, e.Host, mtime, excluded) writeHostMetric(&sb, e.Host, mtime, isExcluded) } return []byte(sb.String()), nil } +// openMetricsDB opens (and schema-initialises) the SQLite DB at dbPath. +// Returns nil silently on empty path or any open/schema error so the metrics +// endpoint degrades gracefully when no DB is configured. +func openMetricsDB(ctx context.Context, dbPath string) *sql.DB { + if dbPath == "" { + return nil + } + db, err := storage.Open(ctx, dbPath) + if err != nil { + return nil + } + if err := storage.CreateSchema(ctx, db); err != nil { + db.Close() + return nil + } + return db +} + +// loadExcludedMap returns a map of host→excluded_at for all excluded hosts. +func loadExcludedMap(ctx context.Context, db *sql.DB) (map[string]int64, error) { + if db == nil { + return map[string]int64{}, nil + } + hosts, err := storage.LoadExcludedHosts(ctx, db) + if err != nil { + return nil, err + } + m := make(map[string]int64, len(hosts)) + for _, h := range hosts { + m[h.Host] = h.ExcludedAt + } + return m, nil +} + +// resolveExcluded returns true if the host is currently excluded. +// If the records file mtime is newer than excluded_at the exclusion is +// automatically lifted in the DB and the host is treated as active again. +func resolveExcluded(ctx context.Context, db *sql.DB, host string, mtime int64, excluded map[string]int64) bool { + excludedAt, isExcluded := excluded[host] + if !isExcluded { + return false + } + if mtime > excludedAt { + if db != nil { + _ = storage.RemoveExcludedHost(ctx, db, host) + } + return false + } + return true +} + func writeMetricHeader(sb *strings.Builder) { sb.WriteString("# HELP goprecords_host_records_last_update_timestamp_seconds Unix timestamp of the last records file update for each host.\n") sb.WriteString("# TYPE goprecords_host_records_last_update_timestamp_seconds gauge\n") @@ -74,27 +129,3 @@ func recordsMtime(statsDir, host string) (int64, error) { } return fi.ModTime().Unix(), nil } - -func loadExcludedSet(ctx context.Context, dbPath string) (map[string]bool, error) { - if dbPath == "" { - return map[string]bool{}, nil - } - db, err := storage.Open(ctx, dbPath) - if err != nil { - return map[string]bool{}, nil - } - defer db.Close() - return loadExcludedHosts(ctx, db) -} - -func loadExcludedHosts(ctx context.Context, db *sql.DB) (map[string]bool, error) { - hosts, err := storage.LoadExcludedHosts(ctx, db) - if err != nil { - return nil, err - } - set := make(map[string]bool, len(hosts)) - for _, h := range hosts { - set[h.Host] = true - } - return set, nil -} diff --git a/internal/daemon/metrics_test.go b/internal/daemon/metrics_test.go index 536f72c..3c390f8 100644 --- a/internal/daemon/metrics_test.go +++ b/internal/daemon/metrics_test.go @@ -109,6 +109,57 @@ func TestMetricsMethodNotAllowed(t *testing.T) { } } +func TestMetricsAutoUnexclude(t *testing.T) { + statsDir := t.TempDir() + dbPath := filepath.Join(t.TempDir(), "test.db") + recordsFile := filepath.Join(statsDir, "comeback.records") + if err := os.WriteFile(recordsFile, []byte("1:1:Linux 5.0\n"), 0o644); err != nil { + t.Fatal(err) + } + ctx := context.Background() + db, err := storage.Open(ctx, dbPath) + if err != nil { + t.Fatal(err) + } + if err := storage.CreateSchema(ctx, db); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(recordsFile) + if err != nil { + t.Fatal(err) + } + pastTime := fi.ModTime().Unix() - 3600 + _, err = db.ExecContext(ctx, + "INSERT OR REPLACE INTO excluded_host (host, reason, excluded_at) VALUES (?, ?, ?)", + "comeback", "old exclusion", pastTime) + if err != nil { + t.Fatal(err) + } + db.Close() + body, err := buildMetrics(ctx, statsDir, dbPath) + if err != nil { + t.Fatal(err) + } + s := string(body) + if !strings.Contains(s, `excluded="false"`) { + t.Fatalf("expected auto-unexclude (excluded=false) but got: %q", s) + } + db2, err := storage.Open(ctx, dbPath) + if err != nil { + t.Fatal(err) + } + defer db2.Close() + hosts, err := storage.LoadExcludedHosts(ctx, db2) + if err != nil { + t.Fatal(err) + } + for _, h := range hosts { + if h.Host == "comeback" { + t.Fatalf("expected comeback to be removed from exclusion list after auto-unexclude") + } + } +} + func TestBuildMetricsNoDBPath(t *testing.T) { statsDir := t.TempDir() if err := os.WriteFile(filepath.Join(statsDir, "h1.records"), []byte("1:1:Linux 5.0\n"), 0o644); err != nil { diff --git a/internal/daemon/upload_test.go b/internal/daemon/upload_test.go index 761fd62..2cc52f3 100644 --- a/internal/daemon/upload_test.go +++ b/internal/daemon/upload_test.go @@ -207,7 +207,7 @@ func TestUploadAuthBearerNegativeTable(t *testing.T) { if _, err := store.CreateKey(ctx, "myhost"); err != nil { t.Fatal(err) } - srv := httptest.NewServer(routes(statsDir, "", "", store)) + srv := httptest.NewServer(routes(statsDir, "", store)) defer srv.Close() url := srv.URL + "/upload/myhost/txt" tests := []struct { diff --git a/internal/version/version.go b/internal/version/version.go index 3328b71..c847147 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -1,4 +1,4 @@ package version // Tag is the application release version. -const Tag = "0.5.0" +const Tag = "0.5.1" |