feat: DTail fork — server/client feature development

Squashed development of the snonux/dtail fork's product code (internal/, cmd/)
since diverging from mimecast/dtail. Major areas:

- Read/output path: the former "turbo" channel-less path is now the single,
  default server-side read/output path for cat/grep/tail and MapReduce; the old
  channel-based path and its config/env toggles were removed.
- MapReduce: single aggregate implementation (server + serverless) fed directly
  by a processor pipeline, with input-exhausted finalization via the shutdown
  coordinator; high-concurrency and data-race fixes.
- Journal source reads (journal:unit.service) via journalctl, Linux-gated behind
  a journal-v1 capability.
- Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys,
  registered over an authenticated session (AUTHKEY), checked before
  authorized_keys.
- Interactive query reload (--interactive-query) with SESSION START/UPDATE
  generation boundaries and capability negotiation.
- Client-side deadlines: --timeout / --shutdownAfter as context deadlines;
  follow shutdown handling.
- Client logging: diagnostics-only daily log by default, opt-in payload tee via
  --log-payload.
- Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel
  leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

1 files changed, 64 insertions, 0 deletions

diff --git a/internal/server/handlers/mapcommand_race_test.go b/internal/server/handlers/mapcommand_race_test.go
new file mode 100644
index 0000000..02c3078
--- /dev/null
+++ b/internal/server/handlers/mapcommand_race_test.go
@@ -0,0 +1,64 @@
+package handlers
+
+// TestAggregatePointerRaceWithShutdown is a negative test that exercises
+// the race between handleMapCommand writing h.aggregate and baseHandler
+// Shutdown reading that same pointer concurrently. Running this test with -race
+// detects unsynchronized access before the atomic.Pointer fix is applied, and
+// passes cleanly after. The regular-aggregate counterpart was removed with the
+// regular server.Aggregate itself (task hv0); output is now the only aggregate.
+//
+// The test does not set up a real MapReduce query; instead it injects a real
+// output aggregate via the atomic accessors so the test is self-contained.
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/io/line"
+	maprserver "github.com/mimecast/dtail/internal/mapr/server"
+	userserver "github.com/mimecast/dtail/internal/user/server"
+)
+
+func TestAggregatePointerRaceWithShutdown(t *testing.T) {
+	resetServerLogger(t)
+
+	const iterations = 200
+
+	for i := 0; i < iterations; i++ {
+		h := &baseHandler{
+			done:           internal.NewDone(),
+			lines:          make(chan *line.Line, 4),
+			serverMessages: make(chan string, 8),
+			maprMessages:   make(chan string, 4),
+			user:           &userserver.User{Name: "race-test-output-user"},
+		}
+
+		ta, err := maprserver.NewAggregate("select count($0) from .", "")
+		if err != nil {
+			t.Skipf("could not create output aggregate: %v", err)
+		}
+
+		var wg sync.WaitGroup
+
+		// Writer goroutine – simulates handleMapCommand setting the output aggregate.
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			h.setAggregate(ta)
+		}()
+
+		// Reader goroutine – simulates Aggregate() / Shutdown reading.
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_ = h.getAggregate()
+		}()
+
+		wg.Wait()
+		// Cleanup: abort the output aggregate so its internal goroutines exit.
+		if got := h.getAggregate(); got != nil {
+			got.Abort()
+		}
+	}
+}