From 849951be1d1a7ee9f9302006ccb187bf5b4e36f3 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 22 Jul 2026 23:51:18 +0300
Subject: =?UTF-8?q?feat:=20DTail=20fork=20=E2=80=94=20server/client=20feat?=
 =?UTF-8?q?ure=20development?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Squashed development of the snonux/dtail fork's product code (internal/, cmd/)
since diverging from mimecast/dtail. Major areas:

- Read/output path: the former "turbo" channel-less path is now the single,
  default server-side read/output path for cat/grep/tail and MapReduce; the old
  channel-based path and its config/env toggles were removed.
- MapReduce: single aggregate implementation (server + serverless) fed directly
  by a processor pipeline, with input-exhausted finalization via the shutdown
  coordinator; high-concurrency and data-race fixes.
- Journal source reads (journal:unit.service) via journalctl, Linux-gated behind
  a journal-v1 capability.
- Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys,
  registered over an authenticated session (AUTHKEY), checked before
  authorized_keys.
- Interactive query reload (--interactive-query) with SESSION START/UPDATE
  generation boundaries and capability negotiation.
- Client-side deadlines: --timeout / --shutdownAfter as context deadlines;
  follow shutdown handling.
- Client logging: diagnostics-only daily log by default, opt-in payload tee via
  --log-payload.
- Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel
  leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 internal/io/pool/bytesbuffer.go  |  4 +-
 internal/io/pool/scanner_pool.go | 85 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 internal/io/pool/scanner_pool.go

(limited to 'internal/io/pool')

diff --git a/internal/io/pool/bytesbuffer.go b/internal/io/pool/bytesbuffer.go
index 3d48f2c..21871f1 100644
--- a/internal/io/pool/bytesbuffer.go
+++ b/internal/io/pool/bytesbuffer.go
@@ -10,7 +10,9 @@ import (
 var BytesBuffer = sync.Pool{
 	New: func() interface{} {
 		b := bytes.Buffer{}
-		b.Grow(128)
+		// Increase initial capacity to 4KB to reduce reallocations
+		// Most log lines are between 100-500 bytes, but some can be larger
+		b.Grow(4096)
 		return &b
 	},
 }
diff --git a/internal/io/pool/scanner_pool.go b/internal/io/pool/scanner_pool.go
new file mode 100644
index 0000000..228d65c
--- /dev/null
+++ b/internal/io/pool/scanner_pool.go
@@ -0,0 +1,85 @@
+package pool
+
+import (
+	"sync"
+)
+
+// ScannerBufferPool provides a pool of 1MB buffers for scanner operations
+// to reduce allocation overhead in the direct-output read path
+var ScannerBufferPool = sync.Pool{
+	New: func() interface{} {
+		// 1MB buffer for scanner operations
+		buf := make([]byte, 1024*1024)
+		return &buf
+	},
+}
+
+// MediumBufferPool provides a pool of 64KB buffers for tail mode reads
+var MediumBufferPool = sync.Pool{
+	New: func() interface{} {
+		// 64KB buffer for medium-sized operations
+		buf := make([]byte, 64*1024)
+		return &buf
+	},
+}
+
+// SmallBufferPool provides a pool of 4KB buffers for small operations
+var SmallBufferPool = sync.Pool{
+	New: func() interface{} {
+		// 4KB buffer for small operations
+		buf := make([]byte, 4*1024)
+		return &buf
+	},
+}
+
+// GetScannerBuffer gets a 1MB buffer from the pool
+func GetScannerBuffer() *[]byte {
+	return ScannerBufferPool.Get().(*[]byte)
+}
+
+// PutScannerBuffer returns a scanner buffer to the pool
+func PutScannerBuffer(buf *[]byte) {
+	// Clear the buffer before returning to pool to avoid memory leaks
+	if buf != nil && len(*buf) > 0 {
+		// Reset to original capacity but clear contents
+		*buf = (*buf)[:cap(*buf)]
+		for i := range *buf {
+			(*buf)[i] = 0
+		}
+	}
+	ScannerBufferPool.Put(buf)
+}
+
+// GetMediumBuffer gets a 64KB buffer from the pool
+func GetMediumBuffer() *[]byte {
+	return MediumBufferPool.Get().(*[]byte)
+}
+
+// PutMediumBuffer returns a medium buffer to the pool
+func PutMediumBuffer(buf *[]byte) {
+	// Clear the buffer before returning to pool
+	if buf != nil && len(*buf) > 0 {
+		*buf = (*buf)[:cap(*buf)]
+		for i := range *buf {
+			(*buf)[i] = 0
+		}
+	}
+	MediumBufferPool.Put(buf)
+}
+
+// GetSmallBuffer gets a 4KB buffer from the pool
+func GetSmallBuffer() *[]byte {
+	return SmallBufferPool.Get().(*[]byte)
+}
+
+// PutSmallBuffer returns a small buffer to the pool
+func PutSmallBuffer(buf *[]byte) {
+	// Clear the buffer before returning to pool
+	if buf != nil && len(*buf) > 0 {
+		*buf = (*buf)[:cap(*buf)]
+		for i := range *buf {
+			(*buf)[i] = 0
+		}
+	}
+	SmallBufferPool.Put(buf)
+}
\ No newline at end of file
-- 
cgit v1.2.3