Merge pull request #24 from snonux/develop

Bugfixes around integration tests

27 files changed, 3059 insertions, 2927 deletions

diff --git a/Makefile b/Makefile
index 0e66ac4..a3b299c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,4 @@
 GO ?= go
-# This is so that all the tests don't manipulate ~/.ssh/known_hosts
-DTAIL_SSH_DONT_ADD_HOSTS_TO_KNOWNHOSTS_FILE = yes
 all: build
 build: dserver dcat dgrep dmap dtail dtailhealth
 dserver:
diff --git a/integrationtests/commandutils.go b/integrationtests/commandutils.go
index 23b9c37..d5b5987 100644
--- a/integrationtests/commandutils.go
+++ b/integrationtests/commandutils.go
@@ -12,9 +12,6 @@ import (
 	"time"
 )
 
-// The exit code and the Go error of the command terminated.
-type exitPromise func() (int, error)
-
 func runCommand(ctx context.Context, t *testing.T, stdoutFile, cmdStr string,
 	args ...string) (int, error) {
 
@@ -51,7 +48,7 @@ func runCommandRetry(ctx context.Context, t *testing.T, retries int, stdoutFile,
 }
 
 func startCommand(ctx context.Context, t *testing.T, cmdStr string,
-	args ...string) (<-chan string, <-chan string, exitPromise, error) {
+	args ...string) (<-chan string, <-chan string, <-chan error, error) {
 
 	stdoutCh := make(chan string)
 	stderrCh := make(chan string)
@@ -90,10 +87,33 @@ func startCommand(ctx context.Context, t *testing.T, cmdStr string,
 		close(stderrCh)
 	}()
 
-	return stdoutCh, stderrCh, func() (int, error) {
-		err := cmd.Wait()
-		return exitCodeFromError(err), err
-	}, nil
+	cmdErrCh := make(chan error)
+	go func() {
+		cmdErrCh <- cmd.Wait()
+	}()
+
+	return stdoutCh, stderrCh, cmdErrCh, nil
+}
+
+func waitForCommand(ctx context.Context, t *testing.T,
+	stdoutCh, stderrCh <-chan string, cmdErrCh <-chan error) {
+
+	for {
+		select {
+		case line, ok := <-stdoutCh:
+			if ok {
+				t.Log(line)
+			}
+		case line, ok := <-stderrCh:
+			if ok {
+				t.Log(line)
+			}
+		case cmdErr := <-cmdErrCh:
+			t.Log(fmt.Sprintf("Command finished with with exit code %d: %v",
+				exitCodeFromError(cmdErr), cmdErr))
+			return
+		}
+	}
 }
 
 func exitCodeFromError(err error) int {
diff --git a/integrationtests/dcat_test.go b/integrationtests/dcat_test.go
index 2516867..777e835 100644
--- a/integrationtests/dcat_test.go
+++ b/integrationtests/dcat_test.go
@@ -17,7 +17,7 @@ func TestDCat(t *testing.T) {
 	stdoutFile := "dcat.out"
 
 	_, err := runCommand(context.TODO(), t, stdoutFile,
-		"../dcat", "--spartan", testdataFile)
+		"../dcat", "--spartan", "--cfg", "none", testdataFile)
 
 	if err != nil {
 		t.Error(err)
@@ -40,14 +40,15 @@ func TestDCat2(t *testing.T) {
 	expectedFile := "dcat2.txt.expected"
 	stdoutFile := "dcat2.out"
 
-	args := []string{"--spartan", "--logLevel", "error"}
+	args := []string{"--spartan", "--logLevel", "error", "--cfg", "none"}
 
 	// Cat file 100 times in one session.
 	for i := 0; i < 100; i++ {
 		args = append(args, testdataFile)
 	}
 
-	if _, err := runCommand(context.TODO(), t, stdoutFile, "../dcat", args...); err != nil {
+	_, err := runCommand(context.TODO(), t, stdoutFile, "../dcat", args...)
+	if err != nil {
 		t.Error(err)
 		return
 	}
@@ -60,19 +61,17 @@ func TestDCat2(t *testing.T) {
 	os.Remove(stdoutFile)
 }
 
-/*
-// TODO: The test currently fails as there is a hostname in the output. What needs
-// to be done is to ignore the hostnames in the output (which is field 2 of the output)
 func TestDCatColors(t *testing.T) {
 	if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") {
 		return
 	}
+
 	testdataFile := "dcatcolors.txt"
 	stdoutFile := "dcatcolors.out"
 	expectedFile := "dcatcolors.expected"
 
 	_, err := runCommand(context.TODO(), t, stdoutFile,
-		"../dcat", "--logLevel", "error", testdataFile)
+		"../dcat", "--logLevel", "error", "--cfg", "none", testdataFile)
 
 	if err != nil {
 		t.Error(err)
@@ -86,4 +85,3 @@ func TestDCatColors(t *testing.T) {
 
 	os.Remove(stdoutFile)
 }
-*/
diff --git a/integrationtests/dcatcolors.expected b/integrationtests/dcatcolors.expected
index 3ccdeff..fcd8dbf 100644
--- a/integrationtests/dcatcolors.expected
+++ b/integrationtests/dcatcolors.expected
@@ -1,2754 +1,2754 @@
-REMOTE|earth|100|1|dcatcolors.txt|FATAL|20211015-053919|SSH relaxed-auth mode enabled
-REMOTE|earth|100|2|dcatcolors.txt|INFO|20211015-053919|Creating server|DTail 4.0.0-RC1 Protocol 4 Have a lot of fun!
-REMOTE|earth|100|3|dcatcolors.txt|INFO|20211015-053919|Generating private server RSA host key
-REMOTE|earth|100|4|dcatcolors.txt|ERROR|20211015-053919|Unable to write private server RSA host key to file|cache/ssh_host_key|open cache/ssh_host_key: no such file or directory
-REMOTE|earth|100|5|dcatcolors.txt|INFO|20211015-053919|Starting server
-REMOTE|earth|100|6|dcatcolors.txt|INFO|20211015-053919|Binding server|0.0.0.0:2222
-REMOTE|earth|100|7|dcatcolors.txt|DEBUG|20211015-053919|Starting listener loop
-REMOTE|earth|100|8|dcatcolors.txt|INFO|20211015-053919|Starting continuous job runner after 10s
-REMOTE|earth|100|9|dcatcolors.txt|INFO|20211015-053919|Starting scheduled job runner after 10s
-REMOTE|earth|100|10|dcatcolors.txt|INFO|20211015-053926|Handling connection
-REMOTE|earth|100|11|dcatcolors.txt|INFO|20211015-053928|paul@172.17.0.1:33710|Incoming authorization
-REMOTE|earth|100|12|dcatcolors.txt|FATAL|20211015-053928|paul@172.17.0.1:33710|Granting permissions via relaxed-auth
-REMOTE|earth|100|13|dcatcolors.txt|INFO|20211015-053928|1|stats.go:53|8|16|7|1.34|781h28m6s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=1
-REMOTE|earth|100|14|dcatcolors.txt|INFO|20211015-053928|paul@172.17.0.1:33710|Invoking channel handler
-REMOTE|earth|100|15|dcatcolors.txt|INFO|20211015-053928|paul@172.17.0.1:33710|Invoking request handler
-REMOTE|earth|100|16|dcatcolors.txt|DEBUG|20211015-053928|paul@172.17.0.1:33710|Creating new server handler
-REMOTE|earth|100|17|dcatcolors.txt|DEBUG|20211015-053928|paul@172.17.0.1:33710|protocol 4 base64 dGFpbDogL3Zhci9sb2cvZHNlcnZlci8qIHJlZ2V4Om5vb3Ag
-REMOTE|earth|100|18|dcatcolors.txt|TRACE|20211015-053928|paul@172.17.0.1:33710|Base64 decoded received command|tail: /var/log/dserver/* regex:noop |36|[tail: /var/log/dserver/* regex:noop ]|at /home/paul/git/dtail/internal/server/handlers/basehandler.go:225
-REMOTE|earth|100|19|dcatcolors.txt|DEBUG|20211015-053928|paul@172.17.0.1:33710|Handling user command|36|[tail: /var/log/dserver/* regex:noop ]
-REMOTE|earth|100|20|dcatcolors.txt|DEBUG|20211015-053928|paul@172.17.0.1:33710|/var/log/dserver/dserver.log|readfiles|Checking config permissions
-REMOTE|earth|100|21|dcatcolors.txt|FATAL|20211015-053928|paul@172.17.0.1:33710|/var/log/dserver/dserver.log|readfiles|Server releaxed auth enabled
-REMOTE|earth|100|22|dcatcolors.txt|INFO|20211015-053928|paul@172.17.0.1:33710|Start reading file|/var/log/dserver/dserver.log|dserver.log
-REMOTE|earth|100|23|dcatcolors.txt|DEBUG|20211015-053928|readFile|readFile(filePath:/var/log/dserver/dserver.log,globID:dserver.log,retry:true,canSkipLines:true,seekEOF:true)
-REMOTE|earth|100|24|dcatcolors.txt|INFO|20211015-053929|1|stats.go:53|8|26|7|1.34|781h28m7s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=1
-REMOTE|earth|100|25|dcatcolors.txt|DEBUG|20211015-053931|/var/log/dserver/dserver.log|File truncation check
-REMOTE|earth|100|26|dcatcolors.txt|DEBUG|20211015-053934|/var/log/dserver/dserver.log|File truncation check
-REMOTE|earth|100|27|dcatcolors.txt|DEBUG|20211015-053937|/var/log/dserver/dserver.log|File truncation check
-REMOTE|earth|100|28|dcatcolors.txt|INFO|20211015-053939|1|stats.go:53|8|16|7|1.21|781h28m16s|MAPREDUCE:STATS|lifetimeConnections=1|currentConnections=0
-REMOTE|earth|100|29|dcatcolors.txt|INFO|20211015-053939|paul@172.17.0.1:33710|Good bye Mister!
-REMOTE|earth|100|30|dcatcolors.txt|DEBUG|20211015-053939|paul@172.17.0.1:33710|shutdown()
-REMOTE|earth|100|31|dcatcolors.txt|TRACE|20211015-053939|paul@172.17.0.1:33710|flush()|at /home/paul/git/dtail/internal/server/handlers/basehandler.go:278
-REMOTE|earth|100|32|dcatcolors.txt|DEBUG|20211015-053939|paul@172.17.0.1:33710|ALL lines sent|0xc0002aa000
-REMOTE|earth|100|33|dcatcolors.txt|INFO|20211015-053939|1|stats.go:53|8|11|7|1.21|781h28m17s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=1
-REMOTE|earth|100|34|dcatcolors.txt|INFO|20211015-053942|Handling connection
-REMOTE|earth|100|35|dcatcolors.txt|INFO|20211015-053942|paul@172.17.0.1:33712|Incoming authorization
-REMOTE|earth|100|36|dcatcolors.txt|FATAL|20211015-053942|paul@172.17.0.1:33712|Granting permissions via relaxed-auth
-REMOTE|earth|100|37|dcatcolors.txt|INFO|20211015-053942|1|stats.go:53|8|15|7|1.11|781h28m19s|MAPREDUCE:STATS|lifetimeConnections=2|currentConnections=1
-REMOTE|earth|100|38|dcatcolors.txt|INFO|20211015-053942|paul@172.17.0.1:33712|Invoking channel handler
-REMOTE|earth|100|39|dcatcolors.txt|INFO|20211015-053942|paul@172.17.0.1:33712|Invoking request handler
-REMOTE|earth|100|40|dcatcolors.txt|DEBUG|20211015-053942|paul@172.17.0.1:33712|Creating new server handler
-REMOTE|earth|100|41|dcatcolors.txt|DEBUG|20211015-053942|paul@172.17.0.1:33712|protocol 4 base64 Y2F0OiAvZXRjL3Bhc3N3ZCByZWdleDpub29wIA==
-REMOTE|earth|100|42|dcatcolors.txt|TRACE|20211015-053942|paul@172.17.0.1:33712|Base64 decoded received command|cat: /etc/passwd regex:noop |28|[cat: /etc/passwd regex:noop ]|at /home/paul/git/dtail/internal/server/handlers/basehandler.go:225
-REMOTE|earth|100|43|dcatcolors.txt|DEBUG|20211015-053942|paul@172.17.0.1:33712|Handling user command|28|[cat: /etc/passwd regex:noop ]
-REMOTE|earth|100|44|dcatcolors.txt|DEBUG|20211015-053942|paul@172.17.0.1:33712|/etc/passwd|readfiles|Checking config permissions
-REMOTE|earth|100|45|dcatcolors.txt|FATAL|20211015-053942|paul@172.17.0.1:33712|/etc/passwd|readfiles|Server releaxed auth enabled
-REMOTE|earth|100|46|dcatcolors.txt|INFO|20211015-053942|paul@172.17.0.1:33712|Start reading file|/etc/passwd|passwd
-REMOTE|earth|100|47|dcatcolors.txt|DEBUG|20211015-053942|readFile|readFile(filePath:/etc/passwd,globID:passwd,retry:false,canSkipLines:false,seekEOF:false)
-REMOTE|earth|100|48|dcatcolors.txt|INFO|20211015-053942|/etc/passwd|End of file reached
-REMOTE|earth|100|49|dcatcolors.txt|DEBUG|20211015-053942|paul@172.17.0.1:33712|shutdown()
-REMOTE|earth|100|50|dcatcolors.txt|TRACE|20211015-053942|paul@172.17.0.1:33712|flush()|at /home/paul/git/dtail/internal/server/handlers/basehandler.go:278
-REMOTE|earth|100|51|dcatcolors.txt|DEBUG|20211015-053942|paul@172.17.0.1:33712|Still lines to be sent
-REMOTE|earth|100|52|dcatcolors.txt|DEBUG|20211015-053942|paul@172.17.0.1:33712|ALL lines sent|0xc0002aa0e0
-REMOTE|earth|100|53|dcatcolors.txt|INFO|20211015-053942|1|stats.go:53|8|21|7|1.11|781h28m19s|MAPREDUCE:STATS|lifetimeConnections=2|currentConnections=0
-REMOTE|earth|100|54|dcatcolors.txt|INFO|20211015-053942|paul@172.17.0.1:33712|Good bye Mister!
-REMOTE|earth|100|55|dcatcolors.txt|INFO|20211015-053949|Handling connection
-REMOTE|earth|100|56|dcatcolors.txt|INFO|20211015-053949|paul@172.17.0.1:33714|Incoming authorization
-REMOTE|earth|100|57|dcatcolors.txt|FATAL|20211015-053949|paul@172.17.0.1:33714|Granting permissions via relaxed-auth
-REMOTE|earth|100|58|dcatcolors.txt|INFO|20211015-053949|1|stats.go:53|8|15|7|1.10|781h28m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=3
-REMOTE|earth|100|59|dcatcolors.txt|INFO|20211015-053949|paul@172.17.0.1:33714|Invoking channel handler
-REMOTE|earth|100|60|dcatcolors.txt|INFO|20211015-053949|paul@172.17.0.1:33714|Invoking request handler
-REMOTE|earth|100|61|dcatcolors.txt|DEBUG|20211015-053949|paul@172.17.0.1:33714|Creating new server handler
-REMOTE|earth|100|62|dcatcolors.txt|DEBUG|20211015-053949|paul@172.17.0.1:33714|protocol 4 base64 bWFwIGZyb20gc3RhdHMgc2VsZWN0IGF2ZygkZ29yb3V0aW5lcyksbWF4KCRnb3JvdXRpbmVzKSxtaW4oJGdvcm91dGluZXMpLGxhc3QoJGdvcm91dGluZXMpLGNvdW50KCRob3N0bmFtZSksJGhvc3RuYW1lIGdyb3VwIGJ5ICRob3N0bmFtZSBvcmRlciBieSBhdmcoJGdvcm91dGluZXMp
-REMOTE|earth|100|63|dcatcolors.txt|TRACE|20211015-053949|paul@172.17.0.1:33714|Base64 decoded received command|map from stats select avg($goroutines),max($goroutines),min($goroutines),last($goroutines),count($hostname),$hostname group by $hostname order by avg($goroutines)|162|[map from stats select avg($goroutines),max($goroutines),min($goroutines),last($goroutines),count($hostname),$hostname group by $hostname order by avg($goroutines)]|at /home/paul/git/dtail/internal/server/handlers/basehandler.go:225
-REMOTE|earth|100|64|dcatcolors.txt|DEBUG|20211015-053949|paul@172.17.0.1:33714|Handling user command|162|[map from stats select avg($goroutines),max($goroutines),min($goroutines),last($goroutines),count($hostname),$hostname group by $hostname order by avg($goroutines)]
-REMOTE|earth|100|65|dcatcolors.txt|INFO|20211015-053949|Creating log format parser|default
-REMOTE|earth|100|66|dcatcolors.txt|DEBUG|20211015-053949|paul@172.17.0.1:33714|protocol 4 base64 Y2F0OiAvdmFyL2xvZy9kc2VydmVyLyogcmVnZXg6ZGVmYXVsdCBcfE1BUFJFRFVDRTpTVEFUU1x8
-REMOTE|earth|100|67|dcatcolors.txt|TRACE|20211015-053949|paul@172.17.0.1:33714|Base64 decoded received command|cat: /var/log/dserver/* regex:default \|MAPREDUCE:STATS\||57|[cat: /var/log/dserver/* regex:default \|MAPREDUCE:STATS\|]|at /home/paul/git/dtail/internal/server/handlers/basehandler.go:225
-REMOTE|earth|100|68|dcatcolors.txt|DEBUG|20211015-053949|paul@172.17.0.1:33714|Handling user command|57|[cat: /var/log/dserver/* regex:default \|MAPREDUCE:STATS\|]
-REMOTE|earth|100|69|dcatcolors.txt|DEBUG|20211015-053949|paul@172.17.0.1:33714|/var/log/dserver/dserver.log|readfiles|Checking config permissions
-REMOTE|earth|100|70|dcatcolors.txt|FATAL|20211015-053949|paul@172.17.0.1:33714|/var/log/dserver/dserver.log|readfiles|Server releaxed auth enabled
-REMOTE|earth|100|71|dcatcolors.txt|INFO|20211015-053949|paul@172.17.0.1:33714|Start reading file|/var/log/dserver/dserver.log|dserver.log
-REMOTE|earth|100|72|dcatcolors.txt|DEBUG|20211015-053949|readFile|readFile(filePath:/var/log/dserver/dserver.log,globID:dserver.log,retry:false,canSkipLines:false,seekEOF:false)
-REMOTE|earth|100|73|dcatcolors.txt|INFO|20211015-053949|/var/log/dserver/dserver.log|End of file reached
-REMOTE|earth|100|74|dcatcolors.txt|INFO|20211015-053949|Serializing mapreduce result
-REMOTE|earth|100|75|dcatcolors.txt|TRACE|20211015-053949|Serialising mapr.AggregateSet|AggregateSet(Samples:7,FValues:map[avg($goroutines):120 count($hostname):7 max($goroutines):26 min($goroutines):11],SValues:map[$hostname:serv6 last($goroutines):15])|at /home/paul/git/dtail/internal/mapr/aggregateset.go:72
-REMOTE|earth|100|76|dcatcolors.txt|DEBUG|20211015-053949|paul@172.17.0.1:33714|shutdown()
-REMOTE|earth|100|77|dcatcolors.txt|TRACE|20211015-053949|paul@172.17.0.1:33714|flush()|at /home/paul/git/dtail/internal/server/handlers/basehandler.go:278
-REMOTE|earth|100|78|dcatcolors.txt|DEBUG|20211015-053949|paul@172.17.0.1:33714|ALL lines sent|0xc0004f0000
-REMOTE|earth|100|79|dcatcolors.txt|INFO|20211015-053949|1|stats.go:53|8|13|7|1.10|781h28m26s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=3
-REMOTE|earth|100|80|dcatcolors.txt|INFO|20211015-053949|paul@172.17.0.1:33714|Good bye Mister!
-REMOTE|earth|100|81|dcatcolors.txt|INFO|20211015-053949|1|stats.go:53|8|12|7|1.10|781h28m27s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=3
-REMOTE|earth|100|82|dcatcolors.txt|INFO|20211015-053959|1|stats.go:53|8|11|7|1.01|781h28m37s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=3
-REMOTE|earth|100|83|dcatcolors.txt|INFO|20211015-054002|Handling connection
-REMOTE|earth|100|84|dcatcolors.txt|INFO|20211015-054002|paul@172.17.0.1:33716|Incoming authorization
-REMOTE|earth|100|85|dcatcolors.txt|FATAL|20211015-054002|paul@172.17.0.1:33716|Granting permissions via relaxed-auth
-REMOTE|earth|100|86|dcatcolors.txt|INFO|20211015-054002|1|stats.go:53|8|15|7|1.09|781h28m39s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=4
-REMOTE|earth|100|87|dcatcolors.txt|INFO|20211015-054002|paul@172.17.0.1:33716|Invoking channel handler
-REMOTE|earth|100|88|dcatcolors.txt|INFO|20211015-054002|paul@172.17.0.1:33716|Invoking request handler
-REMOTE|earth|100|89|dcatcolors.txt|DEBUG|20211015-054002|paul@172.17.0.1:33716|Creating new server handler
-REMOTE|earth|100|90|dcatcolors.txt|DEBUG|20211015-054002|paul@172.17.0.1:33716|protocol 4 base64 Z3JlcDogL3Zhci9sb2cvZHNlcnZlci8qIHJlZ2V4OmRlZmF1bHQgTUFQUkVEVUNF
-REMOTE|earth|100|91|dcatcolors.txt|TRACE|20211015-054002|paul@172.17.0.1:33716|Base64 decoded received command|grep: /var/log/dserver/* regex:default MAPREDUCE|48|[grep: /var/log/dserver/* regex:default MAPREDUCE]|at /home/paul/git/dtail/in