From b0973dd7d17269ad9d4ebfd4842072e739c59eca Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 3 Oct 2021 13:09:32 +0300
Subject: add dmap tests

---
 integrationtests/dmap_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 integrationtests/dmap_test.go

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
new file mode 100644
index 0000000..bfca039
--- /dev/null
+++ b/integrationtests/dmap_test.go
@@ -0,0 +1,51 @@
+package integrationtests
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func TestDMap(t *testing.T) {
+	inFile := "mapr_testdata.log"
+	stdoutFile := "dmap.stdout.tmp"
+	csvFile := "dmap.csv.tmp"
+	expectedCsvFile := "dmap.csv.expected"
+
+	query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile)
+
+	if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := compareFiles(t, csvFile, expectedCsvFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(stdoutFile)
+	os.Remove(csvFile)
+}
+
+func TestDMap2(t *testing.T) {
+	inFile := "mapr_testdata.log"
+	stdoutFile := "dmap2.stdout.tmp"
+	csvFile := "dmap2.csv.tmp"
+	expectedCsvFile := "dmap2.csv.expected"
+
+	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile)
+
+	if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(stdoutFile)
+	os.Remove(csvFile)
+}
-- 
cgit v1.2.3


From 117f062a01e9d456e19f738a0b7047d9bb01285f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 3 Oct 2021 13:32:20 +0300
Subject: when a mapreduce outfile is specified also always write a
 outfile.query file

---
 integrationtests/dmap_test.go | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index bfca039..dc508e2 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -11,6 +11,8 @@ func TestDMap(t *testing.T) {
 	stdoutFile := "dmap.stdout.tmp"
 	csvFile := "dmap.csv.tmp"
 	expectedCsvFile := "dmap.csv.expected"
+	queryFile := fmt.Sprintf("%s.query", csvFile)
+	expectedQueryFile := "dmap.csv.query.expected"
 
 	query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile)
 
@@ -18,14 +20,18 @@ func TestDMap(t *testing.T) {
 		t.Error(err)
 		return
 	}
-
 	if err := compareFiles(t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 		return
 	}
+	if err := compareFiles(t, queryFile, expectedQueryFile); err != nil {
+		t.Error(err)
+		return
+	}
 
 	os.Remove(stdoutFile)
 	os.Remove(csvFile)
+	os.Remove(queryFile)
 }
 
 func TestDMap2(t *testing.T) {
@@ -33,6 +39,8 @@ func TestDMap2(t *testing.T) {
 	stdoutFile := "dmap2.stdout.tmp"
 	csvFile := "dmap2.csv.tmp"
 	expectedCsvFile := "dmap2.csv.expected"
+	queryFile := fmt.Sprintf("%s.query", csvFile)
+	expectedQueryFile := "dmap2.csv.query.expected"
 
 	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile)
 
@@ -40,12 +48,16 @@ func TestDMap2(t *testing.T) {
 		t.Error(err)
 		return
 	}
-
 	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 		return
 	}
+	if err := compareFiles(t, queryFile, expectedQueryFile); err != nil {
+		t.Error(err)
+		return
+	}
 
 	os.Remove(stdoutFile)
 	os.Remove(csvFile)
+	os.Remove(queryFile)
 }
-- 
cgit v1.2.3


From 7563abe9d5beaa18fa1eab0f65668f5dfcf79052 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 9 Oct 2021 16:44:28 +0300
Subject: add dtail health check unit test.

---
 integrationtests/dmap_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index dc508e2..b512985 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -16,7 +16,7 @@ func TestDMap(t *testing.T) {
 
 	query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile)
 
-	if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+	if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
@@ -44,7 +44,7 @@ func TestDMap2(t *testing.T) {
 
 	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile)
 
-	if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+	if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
-- 
cgit v1.2.3


From ea1de3044e129d419f4e807f2624a009343a128f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 9 Oct 2021 21:10:29 +0300
Subject: vetting and linting and some code restyling

---
 integrationtests/dmap_test.go | 50 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 4 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index b512985..f5c78e0 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -14,9 +14,12 @@ func TestDMap(t *testing.T) {
 	queryFile := fmt.Sprintf("%s.query", csvFile)
 	expectedQueryFile := "dmap.csv.query.expected"
 
-	query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile)
+	query := fmt.Sprintf("from STATS select count($line),last($time),"+
+		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+
+		"group by $hostname outfile %s", csvFile)
+	args := []string{"-query", query, inFile}
 
-	if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+	if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
@@ -42,9 +45,48 @@ func TestDMap2(t *testing.T) {
 	queryFile := fmt.Sprintf("%s.query", csvFile)
 	expectedQueryFile := "dmap2.csv.query.expected"
 
-	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile)
+	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+
+		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
+		"outfile %s", csvFile)
 
-	if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+	args := []string{"-query", query, inFile}
+	if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil {
+		t.Error(err)
+		return
+	}
+	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+		t.Error(err)
+		return
+	}
+	if err := compareFiles(t, queryFile, expectedQueryFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(stdoutFile)
+	os.Remove(csvFile)
+	os.Remove(queryFile)
+}
+
+func TestDMap3(t *testing.T) {
+	inFile := "mapr_testdata.log"
+	stdoutFile := "dmap3.stdout.tmp"
+	csvFile := "dmap3.csv.tmp"
+	expectedCsvFile := "dmap3.csv.expected"
+	queryFile := fmt.Sprintf("%s.query", csvFile)
+	expectedQueryFile := "dmap3.csv.query.expected"
+
+	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+
+		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
+		"outfile %s", csvFile)
+
+	// Read many input files at once.
+	args := []string{"-query", query}
+	for i := 0; i < 100; i++ {
+		args = append(args, inFile)
+	}
+
+	if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
-- 
cgit v1.2.3


From c9b7064a0cc4273e4e44c1a545f4d2a207a31b55 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 10 Oct 2021 19:42:48 +0300
Subject: refactor

---
 integrationtests/dmap_test.go | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index f5c78e0..966944a 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -1,6 +1,7 @@
 package integrationtests
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"testing"
@@ -17,12 +18,15 @@ func TestDMap(t *testing.T) {
 	query := fmt.Sprintf("from STATS select count($line),last($time),"+
 		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+
 		"group by $hostname outfile %s", csvFile)
-	args := []string{"-query", query, inFile}
 
-	if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil {
+	_, err := runCommand(context.TODO(), stdoutFile,
+		"../dmap", "--query", query, inFile)
+
+	if err != nil {
 		t.Error(err)
 		return
 	}
+
 	if err := compareFiles(t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 		return
@@ -49,11 +53,13 @@ func TestDMap2(t *testing.T) {
 		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
 		"outfile %s", csvFile)
 
-	args := []string{"-query", query, inFile}
-	if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil {
+	_, err := runCommand(context.TODO(), stdoutFile,
+		"../dmap", "--query", query, inFile)
+	if err != nil {
 		t.Error(err)
 		return
 	}
+
 	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 		return
@@ -86,7 +92,7 @@ func TestDMap3(t *testing.T) {
 		args = append(args, inFile)
 	}
 
-	if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil {
+	if _, err := runCommand(context.TODO(), stdoutFile, "../dmap", args...); err != nil {
 		t.Error(err)
 		return
 	}
-- 
cgit v1.2.3


From 67af621613e8794d47f0265287b7b9d82e19c6bb Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 13 Oct 2021 09:00:03 +0300
Subject: add another dcat integration test - catting 100 files at once

---
 integrationtests/dmap_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 966944a..59be3f4 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -19,7 +19,7 @@ func TestDMap(t *testing.T) {
 		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+
 		"group by $hostname outfile %s", csvFile)
 
-	_, err := runCommand(context.TODO(), stdoutFile,
+	_, err := runCommand(context.TODO(), t, stdoutFile,
 		"../dmap", "--query", query, inFile)
 
 	if err != nil {
@@ -53,7 +53,7 @@ func TestDMap2(t *testing.T) {
 		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
 		"outfile %s", csvFile)
 
-	_, err := runCommand(context.TODO(), stdoutFile,
+	_, err := runCommand(context.TODO(), t, stdoutFile,
 		"../dmap", "--query", query, inFile)
 	if err != nil {
 		t.Error(err)
@@ -92,7 +92,7 @@ func TestDMap3(t *testing.T) {
 		args = append(args, inFile)
 	}
 
-	if _, err := runCommand(context.TODO(), stdoutFile, "../dmap", args...); err != nil {
+	if _, err := runCommand(context.TODO(), t, stdoutFile, "../dmap", args...); err != nil {
 		t.Error(err)
 		return
 	}
-- 
cgit v1.2.3


From 2c8beeabd3c1814ede5d1d460cbafb316f21485e Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Fri, 15 Oct 2021 13:20:48 +0300
Subject: Can configure DTail client not to mess with ~/.ssh/known_hosts via
 env var - this is useful for running unit and integration tests in jenkins

---
 integrationtests/dmap_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 59be3f4..a112456 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -87,8 +87,8 @@ func TestDMap3(t *testing.T) {
 		"outfile %s", csvFile)
 
 	// Read many input files at once.
-	args := []string{"-query", query}
-	for i := 0; i < 100; i++ {
+	args := []string{"--logLevel", "trace", "--query", query}
+	for i := 0; i < 1; i++ {
 		args = append(args, inFile)
 	}
 
-- 
cgit v1.2.3


From deeed697f9bb11db70a9e67f30332bab5029ffbf Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 19 Oct 2021 19:01:12 +0300
Subject: Set DTAIL_RUN_INTEGRATIONT_TEST to yes for integration tests

---
 integrationtests/dmap_test.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index a112456..b3c048a 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -5,9 +5,15 @@ import (
 	"fmt"
 	"os"
 	"testing"
+
+	"github.com/mimecast/dtail/internal/config"
 )
 
 func TestDMap(t *testing.T) {
+	if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") {
+		t.Log("Skipping")
+		return
+	}
 	inFile := "mapr_testdata.log"
 	stdoutFile := "dmap.stdout.tmp"
 	csvFile := "dmap.csv.tmp"
@@ -42,6 +48,10 @@ func TestDMap(t *testing.T) {
 }
 
 func TestDMap2(t *testing.T) {
+	if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") {
+		t.Log("Skipping")
+		return
+	}
 	inFile := "mapr_testdata.log"
 	stdoutFile := "dmap2.stdout.tmp"
 	csvFile := "dmap2.csv.tmp"
@@ -75,6 +85,10 @@ func TestDMap2(t *testing.T) {
 }
 
 func TestDMap3(t *testing.T) {
+	if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") {
+		t.Log("Skipping")
+		return
+	}
 	inFile := "mapr_testdata.log"
 	stdoutFile := "dmap3.stdout.tmp"
 	csvFile := "dmap3.csv.tmp"
-- 
cgit v1.2.3


From 6ff466ddbac457ba469b65319622fac48cc46300 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Wed, 20 Oct 2021 06:41:08 +0300
Subject: make pprof bind address configurable

---
 integrationtests/dmap_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index b3c048a..350ac6a 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -102,7 +102,7 @@ func TestDMap3(t *testing.T) {
 
 	// Read many input files at once.
 	args := []string{"--logLevel", "trace", "--query", query}
-	for i := 0; i < 1; i++ {
+	for i := 0; i < 100; i++ {
 		args = append(args, inFile)
 	}
 
-- 
cgit v1.2.3


From 706dad5ec669b186de32bcd2bdc8033cd9184d33 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Thu, 21 Oct 2021 20:53:30 +0300
Subject: add inventory

---
 integrationtests/dmap_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 350ac6a..7d8b8b5 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -101,7 +101,7 @@ func TestDMap3(t *testing.T) {
 		"outfile %s", csvFile)
 
 	// Read many input files at once.
-	args := []string{"--logLevel", "trace", "--query", query}
+	args := []string{"--logLevel", "trace", "--pprof", "localhost:8080", "--query", query}
 	for i := 0; i < 100; i++ {
 		args = append(args, inFile)
 	}
-- 
cgit v1.2.3


From 995c850d1f07f6221558d1c01924f2da6294f4ec Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 24 Oct 2021 12:59:08 +0300
Subject: Fix deadlock around aggregating data + server max concurrent file
 read limiter

---
 integrationtests/dmap_test.go | 42 ++++++++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 10 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 7d8b8b5..2de679b 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -15,7 +15,6 @@ func TestDMap(t *testing.T) {
 		return
 	}
 	inFile := "mapr_testdata.log"
-	stdoutFile := "dmap.stdout.tmp"
 	csvFile := "dmap.csv.tmp"
 	expectedCsvFile := "dmap.csv.expected"
 	queryFile := fmt.Sprintf("%s.query", csvFile)
@@ -25,14 +24,23 @@ func TestDMap(t *testing.T) {
 		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+
 		"group by $hostname outfile %s", csvFile)
 
-	_, err := runCommand(context.TODO(), t, stdoutFile,
-		"../dmap", "--query", query, inFile)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t, "../dmap",
+		"--query", query,
+		"--logger", "stdout",
+		"--logLevel", "error",
+		"--noColor",
+		inFile)
 
 	if err != nil {
 		t.Error(err)
 		return
 	}
 
+	waitForCommand(ctx, t, stdoutCh, stderrCh, cmdErrCh)
+
 	if err := compareFiles(t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 		return
@@ -42,7 +50,6 @@ func TestDMap(t *testing.T) {
 		return
 	}
 
-	os.Remove(stdoutFile)
 	os.Remove(csvFile)
 	os.Remove(queryFile)
 }
@@ -100,16 +107,31 @@ func TestDMap3(t *testing.T) {
 		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
 		"outfile %s", csvFile)
 
-	// Read many input files at once.
-	args := []string{"--logLevel", "trace", "--pprof", "localhost:8080", "--query", query}
-	for i := 0; i < 100; i++ {
-		args = append(args, inFile)
-	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t, "../dmap",
+		"--query", query,
+		"--logger", "stdout",
+		"--logLevel", "info",
+		"--noColor",
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile,
+		inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile, inFile)
 
-	if _, err := runCommand(context.TODO(), t, stdoutFile, "../dmap", args...); err != nil {
+	if err != nil {
 		t.Error(err)
 		return
 	}
+	waitForCommand(ctx, t, stdoutCh, stderrCh, cmdErrCh)
+
 	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 		return
-- 
cgit v1.2.3


From b166c3a9900a75f7e817cd40e3b09df6c53d09c4 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 24 Oct 2021 13:44:06 +0300
Subject: integration tests use a different known_hosts path and also dont read
 any external config files

---
 integrationtests/dmap_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 2de679b..53b8574 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -28,6 +28,7 @@ func TestDMap(t *testing.T) {
 	defer cancel()
 
 	stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t, "../dmap",
+		"--cfg", "none",
 		"--query", query,
 		"--logger", "stdout",
 		"--logLevel", "error",
@@ -71,7 +72,7 @@ func TestDMap2(t *testing.T) {
 		"outfile %s", csvFile)
 
 	_, err := runCommand(context.TODO(), t, stdoutFile,
-		"../dmap", "--query", query, inFile)
+		"../dmap", "--query", query, "--cfg", "none", inFile)
 	if err != nil {
 		t.Error(err)
 		return
@@ -112,6 +113,7 @@ func TestDMap3(t *testing.T) {
 
 	stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t, "../dmap",
 		"--query", query,
+		"--cfg", "none",
 		"--logger", "stdout",
 		"--logLevel", "info",
 		"--noColor",
-- 
cgit v1.2.3


From bf8bb8edf117c5b55cdd692a4e98ce6bcdad0295 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Fri, 29 Oct 2021 07:50:36 +0300
Subject: Dont use relaxed SSH Auth mode anymore for integration tests

---
 integrationtests/dmap_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 53b8574..6a93b7b 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -10,7 +10,7 @@ import (
 )
 
 func TestDMap(t *testing.T) {
-	if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") {
+	if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		t.Log("Skipping")
 		return
 	}
@@ -56,7 +56,7 @@ func TestDMap(t *testing.T) {
 }
 
 func TestDMap2(t *testing.T) {
-	if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") {
+	if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		t.Log("Skipping")
 		return
 	}
@@ -93,7 +93,7 @@ func TestDMap2(t *testing.T) {
 }
 
 func TestDMap3(t *testing.T) {
-	if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") {
+	if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		t.Log("Skipping")
 		return
 	}
-- 
cgit v1.2.3


From 2dbfcfc56ecf564cf301e1479522f83d6ca0c966 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 31 Oct 2021 17:23:34 +0200
Subject: add dmap integration test with stdin input pipe

---
 integrationtests/dmap_test.go | 56 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 14 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 6a93b7b..c60a828 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -14,6 +14,20 @@ func TestDMap(t *testing.T) {
 		t.Log("Skipping")
 		return
 	}
+
+	t.Log("Testing dmap with input file")
+	if err := testDmap(t, false); err != nil {
+		t.Log(err)
+		return
+	}
+	t.Log("Testing dmap with stdin input pipe")
+	if err := testDmap(t, true); err != nil {
+		t.Log(err)
+		return
+	}
+}
+
+func testDmap(t *testing.T, usePipe bool) error {
 	inFile := "mapr_testdata.log"
 	csvFile := "dmap.csv.tmp"
 	expectedCsvFile := "dmap.csv.expected"
@@ -27,32 +41,45 @@ func TestDMap(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t, "../dmap",
-		"--cfg", "none",
-		"--query", query,
-		"--logger", "stdout",
-		"--logLevel", "error",
-		"--noColor",
-		inFile)
+	var stdoutCh, stderrCh <-chan string
+	var cmdErrCh <-chan error
+	var err error
+
+	if usePipe {
+		stdoutCh, stderrCh, cmdErrCh, err = startCommand(ctx, t,
+			inFile, "../dmap",
+			"--cfg", "none",
+			"--query", query,
+			"--logger", "stdout",
+			"--logLevel", "error",
+			"--noColor")
+	} else {
+		stdoutCh, stderrCh, cmdErrCh, err = startCommand(ctx, t,
+			"", "../dmap",
+			"--cfg", "none",
+			"--query", query,
+			"--logger", "stdout",
+			"--logLevel", "error",
+			"--noColor",
+			inFile)
+	}
 
 	if err != nil {
-		t.Error(err)
-		return
+		return err
 	}
 
 	waitForCommand(ctx, t, stdoutCh, stderrCh, cmdErrCh)
 
 	if err := compareFiles(t, csvFile, expectedCsvFile); err != nil {
-		t.Error(err)
-		return
+		return err
 	}
 	if err := compareFiles(t, queryFile, expectedQueryFile); err != nil {
-		t.Error(err)
-		return
+		return err
 	}
 
 	os.Remove(csvFile)
 	os.Remove(queryFile)
+	return nil
 }
 
 func TestDMap2(t *testing.T) {
@@ -111,7 +138,8 @@ func TestDMap3(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t, "../dmap",
+	stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t,
+		"", "../dmap",
 		"--query", query,
 		"--cfg", "none",
 		"--logger", "stdout",
-- 
cgit v1.2.3


From ebbddd454714670affaf7a64c5ace2de0173268a Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 2 Nov 2021 09:15:43 +0200
Subject: Refactor integration tests. Also fix the dmap1 PIPE test

---
 integrationtests/dmap_test.go | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index c60a828..5e5f4d3 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -9,30 +9,30 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 )
 
-func TestDMap(t *testing.T) {
+func TestDMap1(t *testing.T) {
 	if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		t.Log("Skipping")
 		return
 	}
 
 	t.Log("Testing dmap with input file")
-	if err := testDmap(t, false); err != nil {
-		t.Log(err)
+	if err := testDmap1(t, false); err != nil {
+		t.Error(err)
 		return
 	}
 	t.Log("Testing dmap with stdin input pipe")
-	if err := testDmap(t, true); err != nil {
-		t.Log(err)
+	if err := testDmap1(t, true); err != nil {
+		t.Error(err)
 		return
 	}
 }
 
-func testDmap(t *testing.T, usePipe bool) error {
+func testDmap1(t *testing.T, usePipe bool) error {
 	inFile := "mapr_testdata.log"
-	csvFile := "dmap.csv.tmp"
-	expectedCsvFile := "dmap.csv.expected"
+	csvFile := "dmap1.csv.tmp"
+	expectedCsvFile := "dmap1.csv.expected"
 	queryFile := fmt.Sprintf("%s.query", csvFile)
-	expectedQueryFile := "dmap.csv.query.expected"
+	expectedQueryFile := "dmap1.csv.query.expected"
 
 	query := fmt.Sprintf("from STATS select count($line),last($time),"+
 		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+
@@ -88,7 +88,7 @@ func TestDMap2(t *testing.T) {
 		return
 	}
 	inFile := "mapr_testdata.log"
-	stdoutFile := "dmap2.stdout.tmp"
+	outFile := "dmap2.stdout.tmp"
 	csvFile := "dmap2.csv.tmp"
 	expectedCsvFile := "dmap2.csv.expected"
 	queryFile := fmt.Sprintf("%s.query", csvFile)
@@ -98,7 +98,7 @@ func TestDMap2(t *testing.T) {
 		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
 		"outfile %s", csvFile)
 
-	_, err := runCommand(context.TODO(), t, stdoutFile,
+	_, err := runCommand(context.TODO(), t, outFile,
 		"../dmap", "--query", query, "--cfg", "none", inFile)
 	if err != nil {
 		t.Error(err)
@@ -114,7 +114,7 @@ func TestDMap2(t *testing.T) {
 		return
 	}
 
-	os.Remove(stdoutFile)
+	os.Remove(outFile)
 	os.Remove(csvFile)
 	os.Remove(queryFile)
 }
@@ -125,7 +125,7 @@ func TestDMap3(t *testing.T) {
 		return
 	}
 	inFile := "mapr_testdata.log"
-	stdoutFile := "dmap3.stdout.tmp"
+	outFile := "dmap3.stdout.tmp"
 	csvFile := "dmap3.csv.tmp"
 	expectedCsvFile := "dmap3.csv.expected"
 	queryFile := fmt.Sprintf("%s.query", csvFile)
@@ -171,7 +171,7 @@ func TestDMap3(t *testing.T) {
 		return
 	}
 
-	os.Remove(stdoutFile)
+	os.Remove(outFile)
 	os.Remove(csvFile)
 	os.Remove(queryFile)
 }
-- 
cgit v1.2.3


From fa581afacc4bae7d86e001bb8686caba39ad8802 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Wed, 8 Dec 2021 09:52:21 +0000
Subject: refactor dmap1 unit test to be more modular

---
 integrationtests/dmap_test.go | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 5e5f4d3..ec14cfb 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -15,28 +15,29 @@ func TestDMap1(t *testing.T) {
 		return
 	}
 
+	query := fmt.Sprintf("from STATS select count($line),last($time)," +
+		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) " +
+		"group by $hostname")
+
 	t.Log("Testing dmap with input file")
-	if err := testDmap1(t, false); err != nil {
+	if err := testDmap1(t, query, "a", false); err != nil {
 		t.Error(err)
 		return
 	}
 	t.Log("Testing dmap with stdin input pipe")
-	if err := testDmap1(t, true); err != nil {
+	if err := testDmap1(t, query, "a", true); err != nil {
 		t.Error(err)
 		return
 	}
 }
 
-func testDmap1(t *testing.T, usePipe bool) error {
+func testDmap1(t *testing.T, query, subtestName string, usePipe bool) error {
 	inFile := "mapr_testdata.log"
-	csvFile := "dmap1.csv.tmp"
-	expectedCsvFile := "dmap1.csv.expected"
+	csvFile := fmt.Sprintf("dmap1%s.csv.tmp", subtestName)
+	expectedCsvFile := fmt.Sprintf("dmap1%s.csv.expected", subtestName)
 	queryFile := fmt.Sprintf("%s.query", csvFile)
-	expectedQueryFile := "dmap1.csv.query.expected"
-
-	query := fmt.Sprintf("from STATS select count($line),last($time),"+
-		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+
-		"group by $hostname outfile %s", csvFile)
+	expectedQueryFile := fmt.Sprintf("dmap1%s.csv.query.expected", subtestName)
+	query = fmt.Sprintf("%s outfile %s", query, csvFile)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-- 
cgit v1.2.3


From 0e3de397d8f36c150b8e8fc32a6ee38fa671bac4 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Wed, 8 Dec 2021 10:19:42 +0000
Subject: add where clause integration test to dmap1, all mapreduce token
 fields are lower case

---
 integrationtests/dmap_test.go | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index ec14cfb..e408b74 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -15,19 +15,26 @@ func TestDMap1(t *testing.T) {
 		return
 	}
 
-	query := fmt.Sprintf("from STATS select count($line),last($time)," +
-		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) " +
-		"group by $hostname")
-
-	t.Log("Testing dmap with input file")
-	if err := testDmap1(t, query, "a", false); err != nil {
-		t.Error(err)
-		return
+	testTable := map[string]string{
+		"a": "from STATS select count($line),last($time)," +
+			"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) " +
+			"group by $hostname",
+		"b": "from STATS select count($line),last($time)," +
+			"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) " +
+			"group by $hostname where lifetimeConnections >= 3",
 	}
-	t.Log("Testing dmap with stdin input pipe")
-	if err := testDmap1(t, query, "a", true); err != nil {
-		t.Error(err)
-		return
+
+	for subtestName, query := range testTable {
+		t.Log("Testing dmap with input file")
+		if err := testDmap1(t, query, subtestName, false); err != nil {
+			t.Error(err)
+			return
+		}
+		t.Log("Testing dmap with stdin input pipe")
+		if err := testDmap1(t, query, subtestName, true); err != nil {
+			t.Error(err)
+			return
+		}
 	}
 }
 
-- 
cgit v1.2.3


From ad071fb221d5318433c938d3fbeb6f4666b85631 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Wed, 8 Dec 2021 10:29:39 +0000
Subject: add string based mapreduce where clause integration test

---
 integrationtests/dmap_test.go | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index e408b74..4321f70 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -22,6 +22,9 @@ func TestDMap1(t *testing.T) {
 		"b": "from STATS select count($line),last($time)," +
 			"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) " +
 			"group by $hostname where lifetimeConnections >= 3",
+		"c": "from STATS select count($line),last($time)," +
+			"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) " +
+			"group by $hostname where $time eq \"20211002-071949\"",
 	}
 
 	for subtestName, query := range testTable {
-- 
cgit v1.2.3


From 6d4e9538748da280bf48aaf3d3130369240c361b Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 12 Dec 2021 20:37:32 +0000
Subject: add set condition integraion test

---
 integrationtests/dmap_test.go | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 4321f70..34dab24 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -25,6 +25,9 @@ func TestDMap1(t *testing.T) {
 		"c": "from STATS select count($line),last($time)," +
 			"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) " +
 			"group by $hostname where $time eq \"20211002-071949\"",
+		"d": "from STATS select $mask,$md5,$foo,$bar,$baz,last($time)," +
+			" set $mask = maskdigits($time), $md5 = md5sum($time), " +
+			"$foo = 42, $bar = \"baz\", $baz = $time group by $hostname",
 	}
 
 	for subtestName, query := range testTable {
-- 
cgit v1.2.3


From 4d4b8a9fe3bd420e6f88573eab9a61592e3cf4ea Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 21 Dec 2021 15:14:02 +0000
Subject: adjust integration tests to reflect the last datetime string change

---
 integrationtests/dmap_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'integrationtests/dmap_test.go')

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 34dab24..d466d9f 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -24,7 +24,7 @@ func TestDMap1(t *testing.T) {
 			"group by $hostname where lifetimeConnections >= 3",
 		"c": "from STATS select count($line),last($time)," +
 			"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) " +
-			"group by $hostname where $time eq \"20211002-071949\"",
+			"group by $hostname where $time eq \"1002-071949\"",
 		"d": "from STATS select $mask,$md5,$foo,$bar,$baz,last($time)," +
 			" set $mask = maskdigits($time), $md5 = md5sum($time), " +
 			"$foo = 42, $bar = \"baz\", $baz = $time group by $hostname",
-- 
cgit v1.2.3