From 849951be1d1a7ee9f9302006ccb187bf5b4e36f3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 22 Jul 2026 23:51:18 +0300 Subject: =?UTF-8?q?feat:=20DTail=20fork=20=E2=80=94=20server/client=20feat?= =?UTF-8?q?ure=20development?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed development of the snonux/dtail fork's product code (internal/, cmd/) since diverging from mimecast/dtail. Major areas: - Read/output path: the former "turbo" channel-less path is now the single, default server-side read/output path for cat/grep/tail and MapReduce; the old channel-based path and its config/env toggles were removed. - MapReduce: single aggregate implementation (server + serverless) fed directly by a processor pipeline, with input-exhausted finalization via the shutdown coordinator; high-concurrency and data-race fixes. - Journal source reads (journal:unit.service) via journalctl, Linux-gated behind a journal-v1 capability. - Auth-key fast reconnect: in-memory per-user public-key cache with TTL/max-keys, registered over an authenticated session (AUTHKEY), checked before authorized_keys. - Interactive query reload (--interactive-query) with SESSION START/UPDATE generation boundaries and capability negotiation. - Client-side deadlines: --timeout / --shutdownAfter as context deadlines; follow shutdown handling. - Client logging: diagnostics-only daily log by default, opt-in payload tee via --log-payload. - Numerous correctness fixes (buffer-pool double-recycle races, EOF-sentinel leaks, glob-expansion cap, TOCTOU in CSV parsing) with accompanying unit tests. Co-Authored-By: Claude Opus 4.8 --- internal/mapr/logformat/variables.go | 107 +++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 internal/mapr/logformat/variables.go (limited to 'internal/mapr/logformat/variables.go') diff --git a/internal/mapr/logformat/variables.go b/internal/mapr/logformat/variables.go new file mode 100644 index 0000000..541386b --- /dev/null +++ b/internal/mapr/logformat/variables.go @@ -0,0 +1,107 @@ +package logformat + +import ( + "fmt" + "strings" + + "github.com/mimecast/dtail/internal/mapr" +) + +// The $-variable sets below enumerate every $-prefixed variable a given parser +// is able to populate. They are the single source of truth for the plan-time +// "unknown $-variable" diagnostic (see PlanVariableWarnings). +// +// Keep them in sync with defaultParser in default.go: +// - addDefaultFields populates commonVariables for every parser that embeds +// defaultParser (generic, generickv, csv and default itself). +// - defaultParser.MakeFields additionally populates defaultOnlyVariables from +// the positional fields of DTail's own MAPREDUCE log line layout; the +// lighter parsers (generic/generickv/csv) override MakeFields and therefore +// do NOT populate these. + +// commonVariables are the $-variables set by defaultParser.addDefaultFields and +// are therefore available in every built-in log format. +var commonVariables = map[string]struct{}{ + "$line": {}, + "$empty": {}, + "$hostname": {}, + "$server": {}, + "$timezone": {}, + "$timeoffset": {}, +} + +// defaultOnlyVariables are the extra $-variables that only the "default" parser +// extracts from DTail's MAPREDUCE log lines (see defaultParser.MakeFields). +var defaultOnlyVariables = map[string]struct{}{ + "$severity": {}, + "$loglevel": {}, + "$time": {}, + "$date": {}, + "$hour": {}, + "$minute": {}, + "$second": {}, + "$pid": {}, + "$caller": {}, + "$cpus": {}, + "$goroutines": {}, + "$cgocalls": {}, + "$loadavg": {}, + "$uptime": {}, +} + +// knownVariables returns the set of $-variables the named parser can populate, +// and whether that set is enumerable at all. For log formats whose variable set +// cannot be determined statically (proprietary, stub or unknown formats) it +// returns (nil, false) so callers skip the diagnostic entirely rather than emit +// false-positive warnings for variables that might in fact be valid. +func knownVariables(logFormatName string) (map[string]struct{}, bool) { + switch logFormatName { + case "default": + known := make(map[string]struct{}, len(commonVariables)+len(defaultOnlyVariables)) + for name := range commonVariables { + known[name] = struct{}{} + } + for name := range defaultOnlyVariables { + known[name] = struct{}{} + } + return known, true + case "generic", "generickv", "csv": + return commonVariables, true + default: + return nil, false + } +} + +// PlanVariableWarnings returns one warning per unknown $-variable referenced by +// the query, given the parser selected by logFormatName. A $-variable is +// "unknown" when the selected parser cannot populate it and the query does not +// define it via a `set` clause; at runtime such a variable silently resolves to +// the empty string, collapsing an aggregation into a single empty group with no +// other diagnostic. This is a WARNING and not an error: sparse dynamic fields +// and built-ins like $empty are legitimate, so resolution behaviour is +// unchanged. +// +// No warnings are produced when the parser's variable set is not enumerable +// (see knownVariables); guessing there would risk false positives, which are +// worse than none because they train users to ignore warnings. +func PlanVariableWarnings(query *mapr.Query, logFormatName string) []string { + if query == nil { + return nil + } + known, enumerable := knownVariables(logFormatName) + if !enumerable { + return nil + } + + var warnings []string + for _, variable := range query.ReferencedVariables() { + if _, ok := known[variable]; ok { + continue + } + warnings = append(warnings, fmt.Sprintf( + "warning: %s is not a known variable for log format %q; "+ + "did you mean bareword %s?", + variable, logFormatName, strings.TrimPrefix(variable, "$"))) + } + return warnings +} -- cgit v1.2.3