1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
package logformat
import (
"fmt"
"strings"
"github.com/mimecast/dtail/internal/mapr"
)
// The $-variable sets below enumerate every $-prefixed variable a given parser
// is able to populate. They are the single source of truth for the plan-time
// "unknown $-variable" diagnostic (see PlanVariableWarnings).
//
// Keep them in sync with defaultParser in default.go:
// - addDefaultFields populates commonVariables for every parser that embeds
// defaultParser (generic, generickv, csv and default itself).
// - defaultParser.MakeFields additionally populates defaultOnlyVariables from
// the positional fields of DTail's own MAPREDUCE log line layout; the
// lighter parsers (generic/generickv/csv) override MakeFields and therefore
// do NOT populate these.
// commonVariables are the $-variables set by defaultParser.addDefaultFields and
// are therefore available in every built-in log format.
var commonVariables = map[string]struct{}{
"$line": {},
"$empty": {},
"$hostname": {},
"$server": {},
"$timezone": {},
"$timeoffset": {},
}
// defaultOnlyVariables are the extra $-variables that only the "default" parser
// extracts from DTail's MAPREDUCE log lines (see defaultParser.MakeFields).
var defaultOnlyVariables = map[string]struct{}{
"$severity": {},
"$loglevel": {},
"$time": {},
"$date": {},
"$hour": {},
"$minute": {},
"$second": {},
"$pid": {},
"$caller": {},
"$cpus": {},
"$goroutines": {},
"$cgocalls": {},
"$loadavg": {},
"$uptime": {},
}
// knownVariables returns the set of $-variables the named parser can populate,
// and whether that set is enumerable at all. For log formats whose variable set
// cannot be determined statically (proprietary, stub or unknown formats) it
// returns (nil, false) so callers skip the diagnostic entirely rather than emit
// false-positive warnings for variables that might in fact be valid.
func knownVariables(logFormatName string) (map[string]struct{}, bool) {
switch logFormatName {
case "default":
known := make(map[string]struct{}, len(commonVariables)+len(defaultOnlyVariables))
for name := range commonVariables {
known[name] = struct{}{}
}
for name := range defaultOnlyVariables {
known[name] = struct{}{}
}
return known, true
case "generic", "generickv", "csv":
return commonVariables, true
default:
return nil, false
}
}
// PlanVariableWarnings returns one warning per unknown $-variable referenced by
// the query, given the parser selected by logFormatName. A $-variable is
// "unknown" when the selected parser cannot populate it and the query does not
// define it via a `set` clause; at runtime such a variable silently resolves to
// the empty string, collapsing an aggregation into a single empty group with no
// other diagnostic. This is a WARNING and not an error: sparse dynamic fields
// and built-ins like $empty are legitimate, so resolution behaviour is
// unchanged.
//
// No warnings are produced when the parser's variable set is not enumerable
// (see knownVariables); guessing there would risk false positives, which are
// worse than none because they train users to ignore warnings.
func PlanVariableWarnings(query *mapr.Query, logFormatName string) []string {
if query == nil {
return nil
}
known, enumerable := knownVariables(logFormatName)
if !enumerable {
return nil
}
var warnings []string
for _, variable := range query.ReferencedVariables() {
if _, ok := known[variable]; ok {
continue
}
warnings = append(warnings, fmt.Sprintf(
"warning: %s is not a known variable for log format %q; "+
"did you mean bareword %s?",
variable, logFormatName, strings.TrimPrefix(variable, "$")))
}
return warnings
}
|
