feat: DTail fork — server/client feature development

Squashed development of the snonux/dtail fork's product code (internal/, cmd/)
since diverging from mimecast/dtail. Major areas:

- Read/output path: the former "turbo" channel-less path is now the single,
  default server-side read/output path for cat/grep/tail and MapReduce; the old
  channel-based path and its config/env toggles were removed.
- MapReduce: single aggregate implementation (server + serverless) fed directly
  by a processor pipeline, with input-exhausted finalization via the shutdown
  coordinator; high-concurrency and data-race fixes.
- Journal source reads (journal:unit.service) via journalctl, Linux-gated behind
  a journal-v1 capability.
- Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys,
  registered over an authenticated session (AUTHKEY), checked before
  authorized_keys.
- Interactive query reload (--interactive-query) with SESSION START/UPDATE
  generation boundaries and capability negotiation.
- Client-side deadlines: --timeout / --shutdownAfter as context deadlines;
  follow shutdown handling.
- Client logging: diagnostics-only daily log by default, opt-in payload tee via
  --log-payload.
- Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel
  leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

1 files changed, 76 insertions, 11 deletions

diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 9c3bf64..ba62aa4 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -29,17 +29,25 @@ func (in *initializer) parseConfig(args *Args) error {
 		return in.parseSpecificConfig(args.ConfigFile)
 	}
 
-	if homeDir, err := os.UserHomeDir(); err != nil {
-		var paths []string
-		paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir))
-		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
+	homeDir, err := os.UserHomeDir()
+	if err == nil && homeDir != "" {
+		// Search candidate paths in priority order. The first existing file
+		// wins: ~/.config/dtail/dtail.conf takes precedence over ~/.dtail.conf.
+		// Loading both would silently merge scalar fields (later-file wins),
+		// which is surprising and hard to debug.
+		paths := []string{
+			fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir),
+			fmt.Sprintf("%s/.dtail.conf", homeDir),
+		}
 		for _, configPath := range paths {
-			if _, err := os.Stat(configPath); os.IsNotExist(err) {
-				continue
-			}
-			if err := in.parseSpecificConfig(configPath); err != nil {
+			if _, err := os.Stat(configPath); err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
 				return err
 			}
+			// Stop after loading the first file that exists.
+			return in.parseSpecificConfig(configPath)
 		}
 	}
 
@@ -88,10 +96,39 @@ func (in *initializer) processEnvVars(args *Args) {
 		os.Setenv("DTAIL_HOSTNAME_OVERRIDE", "integrationtest")
 		in.Server.MaxLineLength = 1024
 	}
-	sshPrivateKeyPathFile := os.Getenv("DTAIL_SSH_PRIVATE_KEYFILE_PATH")
-	if len(sshPrivateKeyPathFile) > 0 && args.SSHPrivateKeyFilePath == "" {
-		args.SSHPrivateKeyFilePath = sshPrivateKeyPathFile
+
+	// Resolve SSH private key path from environment variables.
+	// DTAIL_AUTH_KEY_PATH is the documented alias and takes precedence.
+	// DTAIL_SSH_PRIVATE_KEYFILE_PATH is the legacy name and is only used when
+	// DTAIL_AUTH_KEY_PATH is not set, so that the documented env var always wins.
+	// Neither env var overrides an explicitly supplied CLI flag value.
+	args.SSHPrivateKeyFilePath = resolveSSHKeyPath(
+		args.SSHPrivateKeyFilePath,
+		os.Getenv("DTAIL_AUTH_KEY_PATH"),
+		os.Getenv("DTAIL_SSH_PRIVATE_KEYFILE_PATH"),
+	)
+
+	// Note: the direct-output read/aggregate path is now the one and only runtime
+	// path. The historical disable toggle (a former env var and its matching
+	// server config field) no longer exists and is not read here. Old configs
+	// that still set that JSON key, or callers that still export the old env var,
+	// keep working: unknown JSON keys are silently ignored by the lenient decoder
+	// and an unread env var has no effect.
+}
+
+// resolveSSHKeyPath returns the effective SSH private key file path, applying
+// the following precedence (highest to lowest):
+//  1. cliValue  — an explicit flag value supplied by the user
+//  2. authKeyEnv — DTAIL_AUTH_KEY_PATH (the documented alias)
+//  3. legacyEnv  — DTAIL_SSH_PRIVATE_KEYFILE_PATH (the legacy name)
+func resolveSSHKeyPath(cliValue, authKeyEnv, legacyEnv string) string {
+	if cliValue != "" {
+		return cliValue
+	}
+	if authKeyEnv != "" {
+		return authKeyEnv
 	}
+	return legacyEnv
 }
 
 func (in *initializer) setupConfig(sourceCb transformCb, args *Args,
@@ -108,12 +145,40 @@ func (in *initializer) setupConfig(sourceCb transformCb, args *Args,
 	if args.NoColor {
 		in.Client.TermColorsEnable = false
 	}
+	if args.NoAuthKey {
+		in.Client.AuthKeyDisable = true
+	}
+	if in.Client.AuthKeyDisable {
+		args.NoAuthKey = true
+	}
+	if args.SSHPrivateKeyFilePath == "" {
+		args.SSHPrivateKeyFilePath = in.Client.AuthKeyPath
+	}
+	if args.SSHPrivateKeyFilePath != "" {
+		in.Client.AuthKeyPath = args.SSHPrivateKeyFilePath
+	}
+	// Warn early when the auth-key path cannot be determined and auth-key is
+	// still enabled. The SSH stack does not expand '~', so a literal path
+	// would silently fail later; warning here points the operator at the fix.
+	if !in.Client.AuthKeyDisable && in.Client.AuthKeyPath == "" {
+		fmt.Fprintf(os.Stderr,
+			"WARN: cannot determine home directory; auth-key fast reconnect disabled. "+
+				"Set DTAIL_AUTH_KEY_PATH explicitly to re-enable it.\n")
+		in.Client.AuthKeyDisable = true
+		args.NoAuthKey = true
+	}
 	if args.LogDir != "" {
 		in.Common.LogDir = args.LogDir
 	}
 	if args.Logger != "" {
 		in.Common.Logger = args.Logger
 	}
+	// Opt in to teeing retrieved payload into the client log file. The flag can
+	// only turn it on; a config-file value (Client.LogPayload) is preserved when
+	// the flag is not given, since the flag defaults to false.
+	if args.LogPayload {
+		in.Client.LogPayload = true
+	}
 	if args.ConnectionsPerCPU == 0 {
 		args.ConnectionsPerCPU = DefaultConnectionsPerCPU
 	}