feat: DTail fork — server/client feature development

Squashed development of the snonux/dtail fork's product code (internal/, cmd/)
since diverging from mimecast/dtail. Major areas:

- Read/output path: the former "turbo" channel-less path is now the single,
  default server-side read/output path for cat/grep/tail and MapReduce; the old
  channel-based path and its config/env toggles were removed.
- MapReduce: single aggregate implementation (server + serverless) fed directly
  by a processor pipeline, with input-exhausted finalization via the shutdown
  coordinator; high-concurrency and data-race fixes.
- Journal source reads (journal:unit.service) via journalctl, Linux-gated behind
  a journal-v1 capability.
- Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys,
  registered over an authenticated session (AUTHKEY), checked before
  authorized_keys.
- Interactive query reload (--interactive-query) with SESSION START/UPDATE
  generation boundaries and capability negotiation.
- Client-side deadlines: --timeout / --shutdownAfter as context deadlines;
  follow shutdown handling.
- Client logging: diagnostics-only daily log by default, opt-in payload tee via
  --log-payload.
- Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel
  leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

1 files changed, 107 insertions, 0 deletions

diff --git a/internal/mapr/logformat/variables.go b/internal/mapr/logformat/variables.go
new file mode 100644
index 0000000..541386b
--- /dev/null
+++ b/internal/mapr/logformat/variables.go
@@ -0,0 +1,107 @@
+package logformat
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mimecast/dtail/internal/mapr"
+)
+
+// The $-variable sets below enumerate every $-prefixed variable a given parser
+// is able to populate. They are the single source of truth for the plan-time
+// "unknown $-variable" diagnostic (see PlanVariableWarnings).
+//
+// Keep them in sync with defaultParser in default.go:
+//   - addDefaultFields populates commonVariables for every parser that embeds
+//     defaultParser (generic, generickv, csv and default itself).
+//   - defaultParser.MakeFields additionally populates defaultOnlyVariables from
+//     the positional fields of DTail's own MAPREDUCE log line layout; the
+//     lighter parsers (generic/generickv/csv) override MakeFields and therefore
+//     do NOT populate these.
+
+// commonVariables are the $-variables set by defaultParser.addDefaultFields and
+// are therefore available in every built-in log format.
+var commonVariables = map[string]struct{}{
+	"$line":       {},
+	"$empty":      {},
+	"$hostname":   {},
+	"$server":     {},
+	"$timezone":   {},
+	"$timeoffset": {},
+}
+
+// defaultOnlyVariables are the extra $-variables that only the "default" parser
+// extracts from DTail's MAPREDUCE log lines (see defaultParser.MakeFields).
+var defaultOnlyVariables = map[string]struct{}{
+	"$severity":   {},
+	"$loglevel":   {},
+	"$time":       {},
+	"$date":       {},
+	"$hour":       {},
+	"$minute":     {},
+	"$second":     {},
+	"$pid":        {},
+	"$caller":     {},
+	"$cpus":       {},
+	"$goroutines": {},
+	"$cgocalls":   {},
+	"$loadavg":    {},
+	"$uptime":     {},
+}
+
+// knownVariables returns the set of $-variables the named parser can populate,
+// and whether that set is enumerable at all. For log formats whose variable set
+// cannot be determined statically (proprietary, stub or unknown formats) it
+// returns (nil, false) so callers skip the diagnostic entirely rather than emit
+// false-positive warnings for variables that might in fact be valid.
+func knownVariables(logFormatName string) (map[string]struct{}, bool) {
+	switch logFormatName {
+	case "default":
+		known := make(map[string]struct{}, len(commonVariables)+len(defaultOnlyVariables))
+		for name := range commonVariables {
+			known[name] = struct{}{}
+		}
+		for name := range defaultOnlyVariables {
+			known[name] = struct{}{}
+		}
+		return known, true
+	case "generic", "generickv", "csv":
+		return commonVariables, true
+	default:
+		return nil, false
+	}
+}
+
+// PlanVariableWarnings returns one warning per unknown $-variable referenced by
+// the query, given the parser selected by logFormatName. A $-variable is
+// "unknown" when the selected parser cannot populate it and the query does not
+// define it via a `set` clause; at runtime such a variable silently resolves to
+// the empty string, collapsing an aggregation into a single empty group with no
+// other diagnostic. This is a WARNING and not an error: sparse dynamic fields
+// and built-ins like $empty are legitimate, so resolution behaviour is
+// unchanged.
+//
+// No warnings are produced when the parser's variable set is not enumerable
+// (see knownVariables); guessing there would risk false positives, which are
+// worse than none because they train users to ignore warnings.
+func PlanVariableWarnings(query *mapr.Query, logFormatName string) []string {
+	if query == nil {
+		return nil
+	}
+	known, enumerable := knownVariables(logFormatName)
+	if !enumerable {
+		return nil
+	}
+
+	var warnings []string
+	for _, variable := range query.ReferencedVariables() {
+		if _, ok := known[variable]; ok {
+			continue
+		}
+		warnings = append(warnings, fmt.Sprintf(
+			"warning: %s is not a known variable for log format %q; "+
+				"did you mean bareword %s?",
+			variable, logFormatName, strings.TrimPrefix(variable, "$")))
+	}
+	return warnings
+}