From 849951be1d1a7ee9f9302006ccb187bf5b4e36f3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 22 Jul 2026 23:51:18 +0300 Subject: =?UTF-8?q?feat:=20DTail=20fork=20=E2=80=94=20server/client=20feat?= =?UTF-8?q?ure=20development?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed development of the snonux/dtail fork's product code (internal/, cmd/) since diverging from mimecast/dtail. Major areas: - Read/output path: the former "turbo" channel-less path is now the single, default server-side read/output path for cat/grep/tail and MapReduce; the old channel-based path and its config/env toggles were removed. - MapReduce: single aggregate implementation (server + serverless) fed directly by a processor pipeline, with input-exhausted finalization via the shutdown coordinator; high-concurrency and data-race fixes. - Journal source reads (journal:unit.service) via journalctl, Linux-gated behind a journal-v1 capability. - Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys, registered over an authenticated session (AUTHKEY), checked before authorized_keys. - Interactive query reload (--interactive-query) with SESSION START/UPDATE generation boundaries and capability negotiation. - Client-side deadlines: --timeout / --shutdownAfter as context deadlines; follow shutdown handling. - Client logging: diagnostics-only daily log by default, opt-in payload tee via --log-payload. - Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests. Co-Authored-By: Claude Opus 4.8 --- internal/config/initializer.go | 87 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 11 deletions(-) (limited to 'internal/config/initializer.go') diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 9c3bf64..ba62aa4 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -29,17 +29,25 @@ func (in *initializer) parseConfig(args *Args) error { return in.parseSpecificConfig(args.ConfigFile) } - if homeDir, err := os.UserHomeDir(); err != nil { - var paths []string - paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir)) - paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir)) + homeDir, err := os.UserHomeDir() + if err == nil && homeDir != "" { + // Search candidate paths in priority order. The first existing file + // wins: ~/.config/dtail/dtail.conf takes precedence over ~/.dtail.conf. + // Loading both would silently merge scalar fields (later-file wins), + // which is surprising and hard to debug. + paths := []string{ + fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir), + fmt.Sprintf("%s/.dtail.conf", homeDir), + } for _, configPath := range paths { - if _, err := os.Stat(configPath); os.IsNotExist(err) { - continue - } - if err := in.parseSpecificConfig(configPath); err != nil { + if _, err := os.Stat(configPath); err != nil { + if os.IsNotExist(err) { + continue + } return err } + // Stop after loading the first file that exists. + return in.parseSpecificConfig(configPath) } } @@ -88,10 +96,39 @@ func (in *initializer) processEnvVars(args *Args) { os.Setenv("DTAIL_HOSTNAME_OVERRIDE", "integrationtest") in.Server.MaxLineLength = 1024 } - sshPrivateKeyPathFile := os.Getenv("DTAIL_SSH_PRIVATE_KEYFILE_PATH") - if len(sshPrivateKeyPathFile) > 0 && args.SSHPrivateKeyFilePath == "" { - args.SSHPrivateKeyFilePath = sshPrivateKeyPathFile + + // Resolve SSH private key path from environment variables. + // DTAIL_AUTH_KEY_PATH is the documented alias and takes precedence. + // DTAIL_SSH_PRIVATE_KEYFILE_PATH is the legacy name and is only used when + // DTAIL_AUTH_KEY_PATH is not set, so that the documented env var always wins. + // Neither env var overrides an explicitly supplied CLI flag value. + args.SSHPrivateKeyFilePath = resolveSSHKeyPath( + args.SSHPrivateKeyFilePath, + os.Getenv("DTAIL_AUTH_KEY_PATH"), + os.Getenv("DTAIL_SSH_PRIVATE_KEYFILE_PATH"), + ) + + // Note: the direct-output read/aggregate path is now the one and only runtime + // path. The historical disable toggle (a former env var and its matching + // server config field) no longer exists and is not read here. Old configs + // that still set that JSON key, or callers that still export the old env var, + // keep working: unknown JSON keys are silently ignored by the lenient decoder + // and an unread env var has no effect. +} + +// resolveSSHKeyPath returns the effective SSH private key file path, applying +// the following precedence (highest to lowest): +// 1. cliValue — an explicit flag value supplied by the user +// 2. authKeyEnv — DTAIL_AUTH_KEY_PATH (the documented alias) +// 3. legacyEnv — DTAIL_SSH_PRIVATE_KEYFILE_PATH (the legacy name) +func resolveSSHKeyPath(cliValue, authKeyEnv, legacyEnv string) string { + if cliValue != "" { + return cliValue + } + if authKeyEnv != "" { + return authKeyEnv } + return legacyEnv } func (in *initializer) setupConfig(sourceCb transformCb, args *Args, @@ -108,12 +145,40 @@ func (in *initializer) setupConfig(sourceCb transformCb, args *Args, if args.NoColor { in.Client.TermColorsEnable = false } + if args.NoAuthKey { + in.Client.AuthKeyDisable = true + } + if in.Client.AuthKeyDisable { + args.NoAuthKey = true + } + if args.SSHPrivateKeyFilePath == "" { + args.SSHPrivateKeyFilePath = in.Client.AuthKeyPath + } + if args.SSHPrivateKeyFilePath != "" { + in.Client.AuthKeyPath = args.SSHPrivateKeyFilePath + } + // Warn early when the auth-key path cannot be determined and auth-key is + // still enabled. The SSH stack does not expand '~', so a literal path + // would silently fail later; warning here points the operator at the fix. + if !in.Client.AuthKeyDisable && in.Client.AuthKeyPath == "" { + fmt.Fprintf(os.Stderr, + "WARN: cannot determine home directory; auth-key fast reconnect disabled. "+ + "Set DTAIL_AUTH_KEY_PATH explicitly to re-enable it.\n") + in.Client.AuthKeyDisable = true + args.NoAuthKey = true + } if args.LogDir != "" { in.Common.LogDir = args.LogDir } if args.Logger != "" { in.Common.Logger = args.Logger } + // Opt in to teeing retrieved payload into the client log file. The flag can + // only turn it on; a config-file value (Client.LogPayload) is preserved when + // the flag is not given, since the flag defaults to false. + if args.LogPayload { + in.Client.LogPayload = true + } if args.ConnectionsPerCPU == 0 { args.ConnectionsPerCPU = DefaultConnectionsPerCPU } -- cgit v1.2.3