From 849951be1d1a7ee9f9302006ccb187bf5b4e36f3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 22 Jul 2026 23:51:18 +0300 Subject: =?UTF-8?q?feat:=20DTail=20fork=20=E2=80=94=20server/client=20feat?= =?UTF-8?q?ure=20development?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed development of the snonux/dtail fork's product code (internal/, cmd/) since diverging from mimecast/dtail. Major areas: - Read/output path: the former "turbo" channel-less path is now the single, default server-side read/output path for cat/grep/tail and MapReduce; the old channel-based path and its config/env toggles were removed. - MapReduce: single aggregate implementation (server + serverless) fed directly by a processor pipeline, with input-exhausted finalization via the shutdown coordinator; high-concurrency and data-race fixes. - Journal source reads (journal:unit.service) via journalctl, Linux-gated behind a journal-v1 capability. - Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys, registered over an authenticated session (AUTHKEY), checked before authorized_keys. - Interactive query reload (--interactive-query) with SESSION START/UPDATE generation boundaries and capability negotiation. - Client-side deadlines: --timeout / --shutdownAfter as context deadlines; follow shutdown handling. - Client logging: diagnostics-only daily log by default, opt-in payload tee via --log-payload. - Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests. Co-Authored-By: Claude Opus 4.8 --- cmd/dserver/main.go | 60 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 12 deletions(-) (limited to 'cmd/dserver') diff --git a/cmd/dserver/main.go b/cmd/dserver/main.go index 3377273..936edce 100644 --- a/cmd/dserver/main.go +++ b/cmd/dserver/main.go @@ -3,15 +3,13 @@ package main import ( "context" "flag" - "net/http" - _ "net/http" - _ "net/http/pprof" "os" "os/signal" "sync" "syscall" "time" + "github.com/mimecast/dtail/internal/cli" "github.com/mimecast/dtail/internal/config" "github.com/mimecast/dtail/internal/io/dlog" "github.com/mimecast/dtail/internal/server" @@ -46,13 +44,31 @@ func main() { config.Setup(source.Server, &args, flag.Args()) if displayVersion { - version.PrintAndExit() + runtimeCfg := config.CurrentRuntime() + version.PrintAndExit(runtimeCfg.Client != nil && runtimeCfg.Client.TermColorsEnable) } - version.Print() + version.Print(false) + + // rootCtx is always cancelled on exit to ensure the internal goroutine + // spawned by context.WithCancel is released. When -shutdownAfter is set, + // ctx is replaced by a child WithTimeout context whose own cancel is also + // deferred, preventing the lostcancel leak flagged by go vet. + rootCtx, rootCancel := context.WithCancel(context.Background()) + defer rootCancel() + + ctx := rootCtx + cancel := context.CancelFunc(rootCancel) - ctx, cancel := context.WithCancel(context.Background()) if shutdownAfter > 0 { - ctx, cancel = context.WithTimeout(ctx, time.Duration(shutdownAfter)*time.Second) + // Override ctx with a timeout-bounded child; defer its cancel so the + // timeout goroutine is always cleaned up regardless of code path. + var timeoutCancel context.CancelFunc + ctx, timeoutCancel = context.WithTimeout(rootCtx, time.Duration(shutdownAfter)*time.Second) + defer timeoutCancel() + // Callers that invoke cancel() (e.g. the signal handler and post-serve + // cleanup) should trigger the timeout cancel so the server shuts down + // promptly even before the deadline fires. + cancel = timeoutCancel } sigCh := make(chan os.Signal, 10) @@ -70,16 +86,36 @@ func main() { wg.Add(1) dlog.Start(ctx, &wg, source.Server) + var pprofServer *cli.PProfServer if pprof != "" { - dlog.Client.Info("Starting PProf", pprof) - go func() { - panic(http.ListenAndServe(pprof, nil)) - }() + // Enable mutex and block profiling so the /debug/pprof/mutex and + // /debug/pprof/block endpoints actually contain samples. These rates + // are gated on --pprof so they cost nothing when profiling is off. + cli.EnableProfilingRates() + + // Assign to the outer pprofServer with '=' (declaring pprofErr + // separately) so it is NOT shadowed: the graceful Shutdown below relies + // on the outer var being non-nil to actually stop the pprof server. + var pprofErr error + pprofServer, pprofErr = cli.NewPProfServer(pprof) + if pprofErr != nil { + dlog.Client.Error("Unable to start PProf", pprofErr) + } else { + dlog.Client.Info("Starting PProf", pprofServer.Address()) + pprofServer.Start(nil) + } } - serv := server.New() + serv := server.New(config.CurrentRuntime()) status := serv.Start(ctx) cancel() + if pprofServer != nil { + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := pprofServer.Shutdown(shutdownCtx); err != nil { + dlog.Client.Error("Unable to stop PProf", err) + } + shutdownCancel() + } wg.Wait() os.Exit(status) -- cgit v1.2.3