add support to run mapreduce queries on generic log lines, e.g. count all log lines containing a certain string, such as "ERROR"

6 files changed, 33 insertions, 8 deletions

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 32340b3..c4e445b 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -56,10 +56,10 @@ func NewMaprClient(args Args, queryStr string) (*MaprClient, error) {
 	}
 
 	switch c.query.Table {
+	case "", ".":
+		c.Regex = "."
 	case "*":
 		c.Regex = fmt.Sprintf("\\|MAPREDUCE:\\|")
-	case ".":
-		c.Regex = "."
 	default:
 		c.Regex = fmt.Sprintf("\\|MAPREDUCE:%s\\|", c.query.Table)
 	}
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index f0df5bc..3178dda 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -10,6 +10,9 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields := make(map[string]string, 20)
 	splitted := strings.Split(maprLine, "|")
 
+	fields["*"] = "*"
+	fields["$line"] = maprLine
+	fields["$empty"] = ""
 	fields["$hostname"] = p.hostname
 
 	for _, kv := range splitted {
diff --git a/internal/mapr/logformat/generic.go b/internal/mapr/logformat/generic.go
new file mode 100644
index 0000000..9102c0b
--- /dev/null
+++ b/internal/mapr/logformat/generic.go
@@ -0,0 +1,13 @@
+package logformat
+
+// MakeFieldsGENEROC is the generic log line parser.
+func (p *Parser) MakeFieldsGENERIC(maprLine string) (map[string]string, error) {
+	fields := make(map[string]string, 3)
+
+	fields["*"] = "*"
+	fields["$hostname"] = p.hostname
+	fields["$line"] = maprLine
+	fields["$empty"] = ""
+
+	return fields, nil
+}
diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go
index 09c706b..4671191 100644
--- a/internal/mapr/logformat/parser.go
+++ b/internal/mapr/logformat/parser.go
@@ -3,10 +3,11 @@ package logformat
 import (
 	"errors"
 	"fmt"
-	"github.com/mimecast/dtail/internal/io/logger"
 	"os"
 	"reflect"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/io/logger"
 )
 
 // Parser is used to parse the mapreduce information from the server log files.
diff --git a/internal/mapr/query.go b/internal/mapr/query.go
index ec93e14..be766d1 100644
--- a/internal/mapr/query.go
+++ b/internal/mapr/query.go
@@ -148,9 +148,12 @@ func (q *Query) parse(tokens []token) error {
 		}
 	}
 
-	if q.Table == "" {
-		return errors.New(invalidQuery + "Empty table specified in 'from' clause")
-	}
+	// Comment out for empty table support, which is "all" log lines.
+	/*
+		if q.Table == "" {
+			return errors.New(invalidQuery + "Empty table specified in 'from' clause")
+		}
+	*/
 	if len(q.Select) < 1 {
 		return errors.New(invalidQuery + "Expected at least one field in 'select' clause but got none")
 	}
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index fade689..9403aa9 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -44,8 +44,13 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 	}
 	s := strings.Split(fqdn, ".")
 
-	logger.Info("Creating mapr log format parser", config.Server.MapreduceLogFormat)
-	logParser, err := logformat.NewParser(config.Server.MapreduceLogFormat)
+	parserName := config.Server.MapreduceLogFormat
+	if query.Table == "" {
+		parserName = "generic"
+	}
+
+	logger.Info("Creating mapr log format parser", parserName)
+	logParser, err := logformat.NewParser(parserName)
 	if err != nil {
 		logger.FatalExit("Could not create mapr log format parser", err)
 	}