From 849951be1d1a7ee9f9302006ccb187bf5b4e36f3 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 22 Jul 2026 23:51:18 +0300
Subject: =?UTF-8?q?feat:=20DTail=20fork=20=E2=80=94=20server/client=20feat?=
 =?UTF-8?q?ure=20development?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Squashed development of the snonux/dtail fork's product code (internal/, cmd/)
since diverging from mimecast/dtail. Major areas:

- Read/output path: the former "turbo" channel-less path is now the single,
  default server-side read/output path for cat/grep/tail and MapReduce; the old
  channel-based path and its config/env toggles were removed.
- MapReduce: single aggregate implementation (server + serverless) fed directly
  by a processor pipeline, with input-exhausted finalization via the shutdown
  coordinator; high-concurrency and data-race fixes.
- Journal source reads (journal:unit.service) via journalctl, Linux-gated behind
  a journal-v1 capability.
- Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys,
  registered over an authenticated session (AUTHKEY), checked before
  authorized_keys.
- Interactive query reload (--interactive-query) with SESSION START/UPDATE
  generation boundaries and capability negotiation.
- Client-side deadlines: --timeout / --shutdownAfter as context deadlines;
  follow shutdown handling.
- Client logging: diagnostics-only daily log by default, opt-in payload tee via
  --log-payload.
- Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel
  leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 internal/server/handlers/lineprocessor_test.go | 82 ++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 internal/server/handlers/lineprocessor_test.go

(limited to 'internal/server/handlers/lineprocessor_test.go')

diff --git a/internal/server/handlers/lineprocessor_test.go b/internal/server/handlers/lineprocessor_test.go
new file mode 100644
index 0000000..0c8adf3
--- /dev/null
+++ b/internal/server/handlers/lineprocessor_test.go
@@ -0,0 +1,82 @@
+package handlers
+
+import (
+	"bytes"
+	"testing"
+)
+
+// TestGrepLineProcessor_StatsBytesWrittenMatchesOutput verifies that the
+// bytesWritten stat counts each line's formatted bytes exactly once even
+// though writeBuf accumulates many lines before flushing. Regression test:
+// the stat used to add the whole accumulated buffer length per line, so it
+// grew quadratically with the number of buffered lines.
+func TestGrepLineProcessor_StatsBytesWrittenMatchesOutput(t *testing.T) {
+	tests := []struct {
+		name       string
+		plain      bool
+		serverless bool
+	}{
+		{name: "plain", plain: true, serverless: false},
+		{name: "network formatted", plain: false, serverless: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var buf bytes.Buffer
+			p := NewGrepLineProcessor(&buf, "testhost", tt.plain, tt.serverless)
+
+			// All lines stay buffered (well below the 64KB flush threshold)
+			// until Flush, so bytesWritten must equal the flushed output size.
+			const numLines = 50
+			for i := uint64(1); i <= numLines; i++ {
+				line := bytes.NewBufferString("grep match line")
+				if err := p.ProcessLine(line, i, "source.log"); err != nil {
+					t.Fatalf("ProcessLine failed on line %d: %v", i, err)
+				}
+			}
+			if err := p.Flush(); err != nil {
+				t.Fatalf("Flush failed: %v", err)
+			}
+
+			lines, bytesWritten := p.Stats()
+			if lines != numLines {
+				t.Errorf("Expected %d lines processed, got %d", numLines, lines)
+			}
+			if want := uint64(buf.Len()); bytesWritten != want {
+				t.Errorf("Expected bytesWritten %d (flushed output size), got %d", want, bytesWritten)
+			}
+		})
+	}
+}
+
+// TestGrepLineProcessor_StatsBytesWrittenAcrossFlushThreshold verifies the
+// bytesWritten stat stays exact when the buffer crosses the flush threshold
+// several times during processing.
+func TestGrepLineProcessor_StatsBytesWrittenAcrossFlushThreshold(t *testing.T) {
+	var buf bytes.Buffer
+	p := NewGrepLineProcessor(&buf, "testhost", true, false)
+	p.bufSize = 32 // Force several intermediate flushes.
+
+	const numLines = 20
+	content := "0123456789" // 11 bytes with the message delimiter.
+	for i := uint64(1); i <= numLines; i++ {
+		if err := p.ProcessLine(bytes.NewBufferString(content), i, "source.log"); err != nil {
+			t.Fatalf("ProcessLine failed on line %d: %v", i, err)
+		}
+	}
+	if err := p.Flush(); err != nil {
+		t.Fatalf("Flush failed: %v", err)
+	}
+
+	lines, bytesWritten := p.Stats()
+	if lines != numLines {
+		t.Errorf("Expected %d lines processed, got %d", numLines, lines)
+	}
+	want := uint64(numLines * (len(content) + 1))
+	if bytesWritten != want {
+		t.Errorf("Expected bytesWritten %d, got %d", want, bytesWritten)
+	}
+	if got := uint64(buf.Len()); got != want {
+		t.Errorf("Expected flushed output size %d, got %d", want, got)
+	}
+}
-- 
cgit v1.2.3