add dmap tests

8 files changed, 324 insertions, 12 deletions

diff --git a/integrationtests/commons.go b/integrationtests/commons.go
index 6422949..74eeac5 100644
--- a/integrationtests/commons.go
+++ b/integrationtests/commons.go
@@ -1,6 +1,7 @@
 package integrationtests
 
 import (
+	"bufio"
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
@@ -33,6 +34,60 @@ func runCommand(t *testing.T, cmd string, args []string, stdoutFile string) erro
 	return nil
 }
 
+// Checks whether both files have the same lines (order doesn't matter)
+func compareFilesContents(t *testing.T, fileA, fileB string) error {
+	mapFile := func(file string) (map[string]int, error) {
+		t.Log("Reading", file)
+		contents := make(map[string]int)
+		fd, err := os.Open(file)
+		if err != nil {
+			return contents, err
+		}
+		defer fd.Close()
+
+		scanner := bufio.NewScanner(fd)
+		scanner.Split(bufio.ScanLines)
+		for scanner.Scan() {
+			line := scanner.Text()
+			count, _ := contents[line]
+			contents[line] = count + 1
+		}
+
+		return contents, nil
+	}
+
+	compareMaps := func(a, b map[string]int) error {
+		for line, countA := range a {
+			countB, ok := b[line]
+			if !ok {
+				return fmt.Errorf("Files differ, line '%s' is missing in one of them", line)
+			}
+			if countA != countB {
+				return fmt.Errorf("Files differ, count of line '%s' is %d in one but %d in another", line, countA, countB)
+			}
+		}
+		return nil
+	}
+
+	a, err := mapFile(fileA)
+	if err != nil {
+		return err
+	}
+	b, err := mapFile(fileB)
+	if err != nil {
+		return err
+	}
+
+	if err := compareMaps(a, b); err != nil {
+		return err
+	}
+	if err := compareMaps(b, a); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func compareFiles(t *testing.T, fileA, fileB string) error {
 	t.Log("Comparing files", fileA, fileB)
 	shaFileA := shaOfFile(t, fileA)
diff --git a/integrationtests/dgrep_expected.txt b/integrationtests/dgrep.txt.expected
index d5df9c1..d5df9c1 100644
--- a/integrationtests/dgrep_expected.txt
+++ b/integrationtests/dgrep.txt.expected
diff --git a/integrationtests/dgrep_expected2.txt b/integrationtests/dgrep2.txt.expected
index fd18602..a14c045 100644
--- a/integrationtests/dgrep_expected2.txt
+++ b/integrationtests/dgrep2.txt.expected
@@ -591,4 +591,4 @@ INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|current
 INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
 INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
 INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
-INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
+INFO|20211002-071949|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
diff --git a/integrationtests/dgrep_test.go b/integrationtests/dgrep_test.go
index 691e2a1..32c0ace 100644
--- a/integrationtests/dgrep_test.go
+++ b/integrationtests/dgrep_test.go
@@ -6,16 +6,16 @@ import (
 )
 
 func TestDGrep(t *testing.T) {
-	testdataFile := "mapr_testdata.log"
-	stdoutFile := "dgrep.out"
-	expectedResultFile := "dgrep_expected.txt"
+	inFile := "mapr_testdata.log"
+	stdoutFile := "dgrep.stdout.tmp"
+	expectedStdoutFile := "dgrep.txt.expected"
 
-	if err := runCommand(t, "../dgrep", []string{"-spartan", "--grep", "20211002-071947", testdataFile}, stdoutFile); err != nil {
+	if err := runCommand(t, "../dgrep", []string{"-spartan", "--grep", "20211002-071947", inFile}, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
 
-	if err := compareFiles(t, stdoutFile, expectedResultFile); err != nil {
+	if err := compareFiles(t, stdoutFile, expectedStdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
@@ -23,16 +23,16 @@ func TestDGrep(t *testing.T) {
 }
 
 func TestDGrep2(t *testing.T) {
-	testdataFile := "mapr_testdata.log"
-	stdoutFile := "dgrep.out"
-	expectedResultFile := "dgrep_expected2.txt"
+	inFile := "mapr_testdata.log"
+	stdoutFile := "dgrep2.stdout.tmp"
+	expectedStdoutFile := "dgrep2.txt.expected"
 
-	if err := runCommand(t, "../dgrep", []string{"-spartan", "--grep", "20211002-071947", "--invert", testdataFile}, stdoutFile); err != nil {
+	if err := runCommand(t, "../dgrep", []string{"-spartan", "--grep", "20211002-071947", "--invert", inFile}, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
 
-	if err := compareFiles(t, stdoutFile, expectedResultFile); err != nil {
+	if err := compareFiles(t, stdoutFile, expectedStdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
diff --git a/integrationtests/dmap.csv.expected b/integrationtests/dmap.csv.expected
new file mode 100644
index 0000000..d4e6f0f
--- /dev/null
+++ b/integrationtests/dmap.csv.expected
@@ -0,0 +1,2 @@
+count($line),last($time),avg($goroutines),min(concurrentconnections),max(lifetimeconnections)
+597,20211002-071949,11.628141,0.000000,6.000000
diff --git a/integrationtests/dmap2.csv.expected b/integrationtests/dmap2.csv.expected
new file mode 100644
index 0000000..a637fdc
--- /dev/null
+++ b/integrationtests/dmap2.csv.expected
@@ -0,0 +1,204 @@
+count($time),$time,max($goroutines),avg($goroutines),min($goroutines)
+23,20211002-071147,16.000000,14.391304,12.000000
+20,20211002-071213,17.000000,14.100000,12.000000
+20,20211002-071143,17.000000,15.000000,13.000000
+11,20211002-071948,15.000000,14.272727,11.000000
+10,20211002-071913,13.000000,13.000000,13.000000
+10,20211002-071912,15.000000,15.000000,15.000000
+9,20211002-071921,15.000000,13.333333,12.000000
+7,20211002-071920,15.000000,15.000000,15.000000
+4,20211002-071922,13.000000,12.500000,12.000000
+3,20211002-071617,11.000000,11.000000,11.000000
+3,20211002-071927,11.000000,11.000000,11.000000
+3,20211002-071659,11.000000,11.000000,11.000000
+3,20211002-071737,11.000000,11.000000,11.000000
+3,20211002-071809,11.000000,11.000000,11.000000
+3,20211002-071456,11.000000,11.000000,11.000000
+3,20211002-071306,11.000000,11.000000,11.000000
+3,20211002-071909,11.000000,11.000000,11.000000
+3,20211002-071759,11.000000,11.000000,11.000000
+3,20211002-071248,11.000000,11.000000,11.000000
+3,20211002-071639,11.000000,11.000000,11.000000
+3,20211002-071356,11.000000,11.000000,11.000000
+3,20211002-071859,11.000000,11.000000,11.000000
+3,20211002-071336,11.000000,11.000000,11.000000
+3,20211002-071549,11.000000,11.000000,11.000000
+3,20211002-071448,11.000000,11.000000,11.000000
+3,20211002-071338,11.000000,11.000000,11.000000
+3,20211002-071657,11.000000,11.000000,11.000000
+3,20211002-071829,11.000000,11.000000,11.000000
+3,20211002-071146,11.000000,11.000000,11.000000
+3,20211002-071326,11.000000,11.000000,11.000000
+3,20211002-071739,11.000000,11.000000,11.000000
+3,20211002-071709,11.000000,11.000000,11.000000
+3,20211002-071246,11.000000,11.000000,11.000000
+3,20211002-071907,11.000000,11.000000,11.000000
+3,20211002-071536,11.000000,11.000000,11.000000
+3,20211002-071757,11.000000,11.000000,11.000000
+3,20211002-071516,11.000000,11.000000,11.000000
+3,20211002-071937,11.000000,11.000000,11.000000
+3,20211002-071318,11.000000,11.000000,11.000000
+3,20211002-071847,11.000000,11.000000,11.000000
+3,20211002-071438,11.000000,11.000000,11.000000
+3,20211002-071919,11.000000,11.000000,11.000000
+3,20211002-071216,11.000000,11.000000,11.000000
+3,20211002-071837,11.000000,11.000000,11.000000
+3,20211002-071827,11.000000,11.000000,11.000000
+3,20211002-071819,11.000000,11.000000,11.000000
+3,20211002-071519,11.000000,11.000000,11.000000
+3,20211002-071238,11.000000,11.000000,11.000000
+3,20211002-071348,11.000000,11.000000,11.000000
+3,20211002-071406,11.000000,11.000000,11.000000
+3,20211002-071717,11.000000,11.000000,11.000000
+3,20211002-071749,11.000000,11.000000,11.000000
+3,20211002-071506,11.000000,11.000000,11.000000
+3,20211002-071637,11.000000,11.000000,11.000000
+3,20211002-071839,11.000000,11.000000,11.000000
+3,20211002-071556,11.000000,11.000000,11.000000
+3,20211002-071747,11.000000,11.000000,11.000000
+3,20211002-071647,11.000000,11.000000,11.000000
+3,20211002-071629,11.000000,11.000000,11.000000
+3,20211002-071258,11.000000,11.000000,11.000000
+3,20211002-071156,11.000000,11.000000,11.000000
+3,20211002-071218,11.000000,11.000000,11.000000
+3,20211002-071358,11.000000,11.000000,11.000000
+3,20211002-071539,11.000000,11.000000,11.000000
+3,20211002-071436,11.000000,11.000000,11.000000
+3,20211002-071649,11.000000,11.000000,11.000000
+3,20211002-071308,11.000000,11.000000,11.000000
+3,20211002-071807,11.000000,11.000000,11.000000
+3,20211002-071408,11.000000,11.000000,11.000000
+3,20211002-071619,11.000000,11.000000,11.000000
+3,20211002-071526,11.000000,11.000000,11.000000
+3,20211002-071609,11.000000,11.000000,11.000000
+3,20211002-071328,11.000000,11.000000,11.000000
+3,20211002-071929,11.000000,11.000000,11.000000
+3,20211002-071346,11.000000,11.000000,11.000000
+3,20211002-071256,11.000000,11.000000,11.000000
+3,20211002-071559,11.000000,11.000000,11.000000
+3,20211002-071529,11.000000,11.000000,11.000000
+3,20211002-071817,11.000000,11.000000,11.000000
+3,20211002-071729,11.000000,11.000000,11.000000
+3,20211002-071508,11.000000,11.000000,11.000000
+3,20211002-071947,11.000000,11.000000,11.000000
+3,20211002-071426,11.000000,11.000000,11.000000
+3,20211002-071206,11.000000,11.000000,11.000000
+3,20211002-071416,11.000000,11.000000,11.000000
+3,20211002-071627,11.000000,11.000000,11.000000
+3,20211002-071458,11.000000,11.000000,11.000000
+3,20211002-071208,11.000000,11.000000,11.000000
+3,20211002-071226,11.000000,11.000000,11.000000
+3,20211002-071857,11.000000,11.000000,11.000000
+3,20211002-071727,11.000000,11.000000,11.000000
+3,20211002-071157,11.000000,11.000000,11.000000
+3,20211002-071428,11.000000,11.000000,11.000000
+3,20211002-071228,11.000000,11.000000,11.000000
+3,20211002-071446,11.000000,11.000000,11.000000
+3,20211002-071939,11.000000,11.000000,11.000000
+3,20211002-071719,11.000000,11.000000,11.000000
+3,20211002-071418,11.000000,11.000000,11.000000
+3,20211002-071707,11.000000,11.000000,11.000000
+3,20211002-071917,11.000000,11.000000,11.000000
+3,20211002-071316,11.000000,11.000000,11.000000
+3,20211002-071849,11.000000,11.000000,11.000000
+3,20211002-071606,11.000000,11.000000,11.000000
+3,20211002-071236,11.000000,11.000000,11.000000
+3,20211002-071546,11.000000,11.000000,11.000000
+2,20211002-071926,11.000000,11.000000,11.000000
+2,20211002-071848,11.000000,11.000000,11.000000
+2,20211002-071628,11.000000,11.000000,11.000000
+2,20211002-071419,11.000000,11.000000,11.000000
+2,20211002-071409,11.000000,11.000000,11.000000
+2,20211002-071728,11.000000,11.000000,11.000000
+2,20211002-071327,11.000000,11.000000,11.000000
+2,20211002-071249,11.000000,11.000000,11.000000
+2,20211002-071339,11.000000,11.000000,11.000000
+2,20211002-071616,11.000000,11.000000,11.000000
+2,20211002-071838,11.000000,11.000000,11.000000
+2,20211002-071818,11.000000,11.000000,11.000000
+2,20211002-071309,11.000000,11.000000,11.000000
+2,20211002-071826,11.000000,11.000000,11.000000
+2,20211002-071507,11.000000,11.000000,11.000000
+2,20211002-071247,11.000000,11.000000,11.000000
+2,20211002-071706,11.000000,11.000000,11.000000
+2,20211002-071459,11.000000,11.000000,11.000000
+2,20211002-071756,11.000000,11.000000,11.000000
+2,20211002-071656,11.000000,11.000000,11.000000
+2,20211002-071227,11.000000,11.000000,11.000000
+2,20211002-071858,11.000000,11.000000,11.000000
+2,20211002-071936,11.000000,11.000000,11.000000
+2,20211002-071537,11.000000,11.000000,11.000000
+2,20211002-071527,11.000000,11.000000,11.000000
+2,20211002-071518,11.000000,11.000000,11.000000
+2,20211002-071906,11.000000,11.000000,11.000000
+2,20211002-071429,11.000000,11.000000,11.000000
+2,20211002-071509,11.000000,11.000000,11.000000
+2,20211002-071726,11.000000,11.000000,11.000000
+2,20211002-071646,11.000000,11.000000,11.000000
+2,20211002-071407,11.000000,11.000000,11.000000
+2,20211002-071457,11.000000,11.000000,11.000000
+2,20211002-071207,11.000000,11.000000,11.000000
+2,20211002-071329,11.000000,11.000000,11.000000
+2,20211002-071748,11.000000,11.000000,11.000000
+2,20211002-071916,11.000000,11.000000,11.000000
+2,20211002-071758,11.000000,11.000000,11.000000
+2,20211002-071149,11.000000,11.000000,11.000000
+2,20211002-071636,11.000000,11.000000,11.000000
+2,20211002-071158,11.000000,11.000000,11.000000
+2,20211002-071638,11.000000,11.000000,11.000000
+2,20211002-071557,11.000000,11.000000,11.000000
+2,20211002-071626,11.000000,11.000000,11.000000
+2,20211002-071558,11.000000,11.000000,11.000000
+2,20211002-071217,11.000000,11.000000,11.000000
+2,20211002-071528,11.000000,11.000000,11.000000
+2,20211002-071209,11.000000,11.000000,11.000000
+2,20211002-071317,11.000000,11.000000,11.000000
+2,20211002-071607,11.000000,11.000000,11.000000
+2,20211002-071417,11.000000,11.000000,11.000000
+2,20211002-071828,11.000000,11.000000,11.000000
+2,20211002-071908,11.000000,11.000000,11.000000
+2,20211002-071856,11.000000,11.000000,11.000000
+2,20211002-071159,11.000000,11.000000,11.000000
+2,20211002-071257,11.000000,11.000000,11.000000
+2,20211002-071547,11.000000,11.000000,11.000000
+2,20211002-071219,11.000000,11.000000,11.000000
+2,20211002-071449,11.000000,11.000000,11.000000
+2,20211002-071608,11.000000,11.000000,11.000000
+2,20211002-071439,11.000000,11.000000,11.000000
+2,20211002-071347,11.000000,11.000000,11.000000
+2,20211002-071148,11.000000,11.000000,11.000000
+2,20211002-071437,11.000000,11.000000,11.000000
+2,20211002-071237,11.000000,11.000000,11.000000
+2,20211002-071806,11.000000,11.000000,11.000000
+2,20211002-071716,11.000000,11.000000,11.000000
+2,20211002-071259,11.000000,11.000000,11.000000
+2,20211002-071349,11.000000,11.000000,11.000000
+2,20211002-071307,11.000000,11.000000,11.000000
+2,20211002-071359,11.000000,11.000000,11.000000
+2,20211002-071229,11.000000,11.000000,11.000000
+2,20211002-071946,11.000000,11.000000,11.000000
+2,20211002-071548,11.000000,11.000000,11.000000
+2,20211002-071447,11.000000,11.000000,11.000000
+2,20211002-071648,11.000000,11.000000,11.000000
+2,20211002-071808,11.000000,11.000000,11.000000
+2,20211002-071746,11.000000,11.000000,11.000000
+2,20211002-071816,11.000000,11.000000,11.000000
+2,20211002-071846,11.000000,11.000000,11.000000
+2,20211002-071357,11.000000,11.000000,11.000000
+2,20211002-071517,11.000000,11.000000,11.000000
+2,20211002-071928,11.000000,11.000000,11.000000
+2,20211002-071239,11.000000,11.000000,11.000000
+2,20211002-071718,11.000000,11.000000,11.000000
+2,20211002-071658,11.000000,11.000000,11.000000
+2,20211002-071938,11.000000,11.000000,11.000000
+2,20211002-071836,11.000000,11.000000,11.000000
+2,20211002-071319,11.000000,11.000000,11.000000
+2,20211002-071337,11.000000,11.000000,11.000000
+2,20211002-071738,11.000000,11.000000,11.000000
+2,20211002-071618,11.000000,11.000000,11.000000
+2,20211002-071708,11.000000,11.000000,11.000000
+2,20211002-071538,11.000000,11.000000,11.000000
+2,20211002-071918,11.000000,11.000000,11.000000
+2,20211002-071427,11.000000,11.000000,11.000000
+2,20211002-071736,11.000000,11.000000,11.000000
+1,20211002-071949,15.000000,15.000000,15.000000
diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
new file mode 100644
index 0000000..bfca039
--- /dev/null
+++ b/integrationtests/dmap_test.go
@@ -0,0 +1,51 @@
+package integrationtests
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func TestDMap(t *testing.T) {
+	inFile := "mapr_testdata.log"
+	stdoutFile := "dmap.stdout.tmp"
+	csvFile := "dmap.csv.tmp"
+	expectedCsvFile := "dmap.csv.expected"
+
+	query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile)
+
+	if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := compareFiles(t, csvFile, expectedCsvFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(stdoutFile)
+	os.Remove(csvFile)
+}
+
+func TestDMap2(t *testing.T) {
+	inFile := "mapr_testdata.log"
+	stdoutFile := "dmap2.stdout.tmp"
+	csvFile := "dmap2.csv.tmp"
+	expectedCsvFile := "dmap2.csv.expected"
+
+	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile)
+
+	if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(stdoutFile)
+	os.Remove(csvFile)
+}
diff --git a/integrationtests/mapr_testdata.log b/integrationtests/mapr_testdata.log
index d81ad79..fc3fe42 100644
--- a/integrationtests/mapr_testdata.log
+++ b/integrationtests/mapr_testdata.log
@@ -594,4 +594,4 @@ INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|current
 INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
 INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
 INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
-INFO|20211002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
+INFO|20211002-071949|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6