From 849951be1d1a7ee9f9302006ccb187bf5b4e36f3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 22 Jul 2026 23:51:18 +0300 Subject: =?UTF-8?q?feat:=20DTail=20fork=20=E2=80=94=20server/client=20feat?= =?UTF-8?q?ure=20development?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed development of the snonux/dtail fork's product code (internal/, cmd/) since diverging from mimecast/dtail. Major areas: - Read/output path: the former "turbo" channel-less path is now the single, default server-side read/output path for cat/grep/tail and MapReduce; the old channel-based path and its config/env toggles were removed. - MapReduce: single aggregate implementation (server + serverless) fed directly by a processor pipeline, with input-exhausted finalization via the shutdown coordinator; high-concurrency and data-race fixes. - Journal source reads (journal:unit.service) via journalctl, Linux-gated behind a journal-v1 capability. - Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys, registered over an authenticated session (AUTHKEY), checked before authorized_keys. - Interactive query reload (--interactive-query) with SESSION START/UPDATE generation boundaries and capability negotiation. - Client-side deadlines: --timeout / --shutdownAfter as context deadlines; follow shutdown handling. - Client logging: diagnostics-only daily log by default, opt-in payload tee via --log-payload. - Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests. Co-Authored-By: Claude Opus 4.8 --- internal/cli/pprof.go | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 internal/cli/pprof.go (limited to 'internal/cli/pprof.go') diff --git a/internal/cli/pprof.go b/internal/cli/pprof.go new file mode 100644 index 0000000..57cb38c --- /dev/null +++ b/internal/cli/pprof.go @@ -0,0 +1,119 @@ +package cli + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/pprof" + "runtime" + "sync" + + "github.com/mimecast/dtail/internal/io/dlog" +) + +// Mutex and block profiling rates. The /debug/pprof/mutex and +// /debug/pprof/block endpoints return empty profiles unless these runtime +// rates are enabled first; they are off by default in the Go runtime. +// +// The values below were validated during task ss0: a mutex profile fraction of +// 5 (report ~1/5 of contention events) and a block profile rate of 100000ns +// (record blocking events longer than 100µs) were sufficient to measure +// outputManager mutex contention — 12.4ms total delay across three concurrent +// 100MB server-mode dcats, i.e. NOT a bottleneck; an idle follow session +// costs 0 CPU ticks/10s because the 1ms read poll only runs while direct output is +// active and the EOF-ack drops it back to 1s polling. Documented here so the +// locking design is not re-litigated. +const ( + mutexProfileFraction = 5 + blockProfileRateNanos = 100000 +) + +// PProfServer owns a dedicated pprof HTTP server lifecycle. +type PProfServer struct { + listener net.Listener + server *http.Server + done chan struct{} +} + +// NewPProfServer creates a pprof HTTP server bound to address. +func NewPProfServer(address string) (*PProfServer, error) { + listener, err := net.Listen("tcp", address) + if err != nil { + return nil, err + } + + return &PProfServer{ + listener: listener, + server: &http.Server{ + Handler: newPProfServeMux(), + }, + done: make(chan struct{}), + }, nil +} + +// EnableProfilingRates turns on mutex and block profiling collection so that +// the /debug/pprof/mutex and /debug/pprof/block endpoints actually contain +// samples. Without this the Go runtime keeps both rates at zero and those +// endpoints report empty profiles. Only call this when pprof is enabled so it +// costs nothing in the common (no --pprof) case. +func EnableProfilingRates() { + runtime.SetMutexProfileFraction(mutexProfileFraction) + runtime.SetBlockProfileRate(blockProfileRateNanos) +} + +// Address returns the bound pprof listener address. +func (s *PProfServer) Address() string { + if s == nil || s.listener == nil { + return "" + } + return s.listener.Addr().String() +} + +// Start serves the pprof HTTP endpoints until shutdown. +func (s *PProfServer) Start(wg *sync.WaitGroup) { + if s == nil { + return + } + + if wg != nil { + wg.Add(1) + } + + go func() { + if wg != nil { + defer wg.Done() + } + defer close(s.done) + + if err := s.server.Serve(s.listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + dlog.Client.Error("PProf server exited", err) + } + }() +} + +// Shutdown stops the pprof HTTP server and waits for Serve to return. +func (s *PProfServer) Shutdown(ctx context.Context) error { + if s == nil { + return nil + } + + err := s.server.Shutdown(ctx) + <-s.done + return err +} + +func newPProfServeMux() *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + for _, name := range []string{"allocs", "block", "goroutine", "heap", "mutex", "threadcreate"} { + mux.Handle("/debug/pprof/"+name, pprof.Handler(name)) + } + + return mux +} -- cgit v1.2.3