From 23982f331c2154a66b86d596226c24454fd06be5 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 28 Aug 2021 20:26:32 +0100 Subject: 1. Major performance gain by not checking for file truncation aftter each bytes read. 2. Introduce field separator to the protocol package. --- internal/mapr/logformat/default.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'internal/mapr/logformat/default.go') diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index 44bf558..32a34bd 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -3,12 +3,14 @@ package logformat import ( "errors" "strings" + + "github.com/mimecast/dtail/internal/protocol" ) // MakeFieldsDEFAULT is the default log file mapreduce parser. func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { fields := make(map[string]string, 20) - splitted := strings.Split(maprLine, "|") + splitted := strings.Split(maprLine, protocol.FieldDelimiter) fields["*"] = "*" fields["$line"] = maprLine -- cgit v1.2.3 From f74a9e4b35feb8c07d8a70b5a581088a0a59889d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 7 Sep 2021 10:01:32 +0300 Subject: Produce MAPREDUCE lines, can aggregate these via default log format --- internal/mapr/logformat/default.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'internal/mapr/logformat/default.go') diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index 32a34bd..2881047 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -9,8 +9,8 @@ import ( // MakeFieldsDEFAULT is the default log file mapreduce parser. func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { - fields := make(map[string]string, 20) splitted := strings.Split(maprLine, protocol.FieldDelimiter) + fields := make(map[string]string, len(splitted)) fields["*"] = "*" fields["$line"] = maprLine @@ -19,10 +19,19 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { fields["$timezone"] = p.timeZoneName fields["$timeoffset"] = p.timeZoneOffset - for _, kv := range splitted { + kvStart := 0 + // DTail mapreduce format + if len(splitted) > 3 && strings.HasPrefix(splitted[3], "MAPREDUCE:") { + fields["$severity"] = splitted[0] + // TODO: Parse time like we do at Mimecast + fields["$time"] = splitted[1] + kvStart = 4 + } + + for _, kv := range splitted[kvStart:] { keyAndValue := strings.SplitN(kv, "=", 2) if len(keyAndValue) != 2 { - return fields, errors.New("Error parsing mapr token: " + kv) + return fields, errors.New("Error parsing mapreduce token: " + kv) } fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1] } -- cgit v1.2.3 From c83c9e61a08c7ea1cb528bc26dfab25b46faa866 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 8 Sep 2021 08:39:41 +0300 Subject: Don't fail parsing whole line when one kv could not be parsed --- internal/mapr/logformat/default.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'internal/mapr/logformat/default.go') diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index 2881047..da976bd 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -1,9 +1,9 @@ package logformat import ( - "errors" "strings" + "github.com/mimecast/dtail/internal/io/logger" "github.com/mimecast/dtail/internal/protocol" ) @@ -31,7 +31,8 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { for _, kv := range splitted[kvStart:] { keyAndValue := strings.SplitN(kv, "=", 2) if len(keyAndValue) != 2 { - return fields, errors.New("Error parsing mapreduce token: " + kv) + logger.Debug("Unable to parse key-value token, ignoring it", kv) + continue } fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1] } -- cgit v1.2.3 From 16dc57e1e1c28e9d762424e596223a980770e059 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 8 Sep 2021 19:10:50 +0300 Subject: mapreduce tables are in colors now too --- internal/mapr/logformat/default.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'internal/mapr/logformat/default.go') diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index da976bd..c67137e 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -1,16 +1,22 @@ package logformat import ( + "fmt" "strings" - "github.com/mimecast/dtail/internal/io/logger" "github.com/mimecast/dtail/internal/protocol" ) -// MakeFieldsDEFAULT is the default log file mapreduce parser. +// MakeFieldsDEFAULT is the default DTail log file key-value parser. func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { splitted := strings.Split(maprLine, protocol.FieldDelimiter) - fields := make(map[string]string, len(splitted)) + + if len(splitted) < 3 || !strings.HasPrefix(splitted[3], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") { + // Not a DTail mapreduce log line. + return nil, IgnoreFieldsErr + } + + fields := make(map[string]string, len(splitted)+8) fields["*"] = "*" fields["$line"] = maprLine @@ -19,20 +25,14 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { fields["$timezone"] = p.timeZoneName fields["$timeoffset"] = p.timeZoneOffset - kvStart := 0 - // DTail mapreduce format - if len(splitted) > 3 && strings.HasPrefix(splitted[3], "MAPREDUCE:") { - fields["$severity"] = splitted[0] - // TODO: Parse time like we do at Mimecast - fields["$time"] = splitted[1] - kvStart = 4 - } + fields["$severity"] = splitted[0] + // TODO: Parse time like we do at Mimecast + fields["$time"] = splitted[1] - for _, kv := range splitted[kvStart:] { + for _, kv := range splitted[4:] { keyAndValue := strings.SplitN(kv, "=", 2) if len(keyAndValue) != 2 { - logger.Debug("Unable to parse key-value token, ignoring it", kv) - continue + return fields, fmt.Errorf("Unable to parse key-value token '%s'", kv) } fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1] } -- cgit v1.2.3 From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 19 Sep 2021 13:22:59 +0300 Subject: move args to config package logger package rewrite as dlog --- internal/mapr/logformat/default.go | 1 + 1 file changed, 1 insertion(+) (limited to 'internal/mapr/logformat/default.go') diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index c67137e..e0bbc30 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -26,6 +26,7 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { fields["$timeoffset"] = p.timeZoneOffset fields["$severity"] = splitted[0] + fields["$loglevel"] = splitted[0] // TODO: Parse time like we do at Mimecast fields["$time"] = splitted[1] -- cgit v1.2.3 From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 26 Sep 2021 16:42:47 +0300 Subject: refactor config reader - also looks in additional search paths for config file unless NONE is specified --- internal/mapr/logformat/default.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'internal/mapr/logformat/default.go') diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index e0bbc30..7fb1700 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -27,7 +27,7 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { fields["$severity"] = splitted[0] fields["$loglevel"] = splitted[0] - // TODO: Parse time like we do at Mimecast + // NEXT: Parse time like we do at Mimecast fields["$time"] = splitted[1] for _, kv := range splitted[4:] { -- cgit v1.2.3 From 764ef99a3d779a0db1fb60679292af52425ba2f6 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 10:46:47 +0300 Subject: add more default fields to MAPREDUCE --- internal/mapr/logformat/default.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'internal/mapr/logformat/default.go') diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index 7fb1700..8016667 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -11,7 +11,7 @@ import ( func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { splitted := strings.Split(maprLine, protocol.FieldDelimiter) - if len(splitted) < 3 || !strings.HasPrefix(splitted[3], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") { + if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") { // Not a DTail mapreduce log line. return nil, IgnoreFieldsErr } @@ -27,10 +27,25 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { fields["$severity"] = splitted[0] fields["$loglevel"] = splitted[0] - // NEXT: Parse time like we do at Mimecast - fields["$time"] = splitted[1] - for _, kv := range splitted[4:] { + time := splitted[1] + fields["$time"] = time + if len(time) == 15 { + // Example: 20211002-071209 + fields["$date"] = time[0:8] + fields["$hour"] = time[9:11] + fields["$minute"] = time[11:13] + fields["$second"] = time[13:] + } + fields["$pid"] = splitted[2] + fields["$caller"] = splitted[3] + fields["$cpus"] = splitted[4] + fields["$goroutines"] = splitted[5] + fields["$cgocalls"] = splitted[6] + fields["$loadavg"] = splitted[7] + fields["$uptime"] = splitted[8] + + for _, kv := range splitted[10:] { keyAndValue := strings.SplitN(kv, "=", 2) if len(keyAndValue) != 2 { return fields, fmt.Errorf("Unable to parse key-value token '%s'", kv) -- cgit v1.2.3 From 97747ea0f3178f7f5890512d483fdccaa82846b0 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 21:10:29 +0300 Subject: vetting and linting and some code restyling --- internal/mapr/logformat/default.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'internal/mapr/logformat/default.go') diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index 8016667..9b6c855 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -11,9 +11,10 @@ import ( func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { splitted := strings.Split(maprLine, protocol.FieldDelimiter) - if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") { + if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") || + !strings.HasPrefix(splitted[0], "INFO") { // Not a DTail mapreduce log line. - return nil, IgnoreFieldsErr + return nil, ErrIgnoreFields } fields := make(map[string]string, len(splitted)+8) -- cgit v1.2.3