vetting and linting and some code restyling

93 files changed, 670 insertions, 581 deletions

diff --git a/Makefile b/Makefile
index 6a0f828..543b29e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 GO ?= go
-all: build test
+all: build
 build: dserver dcat dgrep dmap dtail dtailhealthcheck
 dserver:
 ifndef USE_ACL
@@ -43,7 +43,7 @@ lint:
 	find . -type d | while read dir; do \
 	  echo golint $$dir; \
 	  golint $$dir; \
-	done
+	done | grep -F .go:
 test:
 	${GO} clean -testcache
 ifndef USE_ACL
diff --git a/TODO.md b/TODO.md
index 45ed4e3..72eb0f3 100644
--- a/TODO.md
+++ b/TODO.md
@@ -15,5 +15,5 @@ This is a loose list of what to do. Maybe for the next releae or maybe for a lat
 [ ] Manual test/adjust dtail colors
 [ ] More integration test colors (via dcat?)
 [ ] Integration test for dtail in serverless mode
-[ ] Integration test for health check serverless mode
-[ ] Rewrite + test health client (copy catclient)
+[ ] Go through the whole source and change indentation (try not to exceed 80char line lengths by too much)
+[ ] Fix the sync.Pools (they aren't concurrent as it seems and can cause a panic)
diff --git a/cmd/dcat/main.go b/cmd/dcat/main.go
index 662a50d..87ece9d 100644
--- a/cmd/dcat/main.go
+++ b/cmd/dcat/main.go
@@ -33,7 +33,8 @@ func main() {
 	flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode")
 	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
-	flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU, "How many connections established per CPU core concurrently")
+	flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU,
+		"How many connections established per CPU core concurrently")
 	flag.IntVar(&args.SSHPort, "port", config.DefaultSSHPort, "SSH server port")
 	flag.IntVar(&pprof, "pprof", -1, "Start PProf server this port")
 	flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path")
diff --git a/cmd/dgrep/main.go b/cmd/dgrep/main.go
index 529331d..576e22b 100644
--- a/cmd/dgrep/main.go
+++ b/cmd/dgrep/main.go
@@ -20,7 +20,6 @@ func main() {
 	var args config.Args
 	var displayVersion bool
 	var grep string
-
 	userName := user.Name()
 
 	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
@@ -29,7 +28,8 @@ func main() {
 	flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode")
 	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
-	flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU, "How many connections established per CPU core concurrently")
+	flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU,
+		"How many connections established per CPU core concurrently")
 	flag.IntVar(&args.SSHPort, "port", config.DefaultSSHPort, "SSH server port")
 	flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path")
 	flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method")
diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go
index acc1dc6..1f44076 100644
--- a/cmd/dmap/main.go
+++ b/cmd/dmap/main.go
@@ -19,11 +19,9 @@ import (
 // The evil begins here.
 func main() {
 	var displayVersion bool
-
 	args := config.Args{
 		Mode: omode.MapClient,
 	}
-
 	userName := user.Name()
 
 	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
@@ -31,7 +29,8 @@ func main() {
 	flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode")
 	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
-	flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU, "How many connections established per CPU core concurrently")
+	flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU,
+		"How many connections established per CPU core concurrently")
 	flag.IntVar(&args.SSHPort, "port", config.DefaultSSHPort, "SSH server port")
 	flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection")
 	flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path")
diff --git a/cmd/dserver/main.go b/cmd/dserver/main.go
index b4fc873..cf726cf 100644
--- a/cmd/dserver/main.go
+++ b/cmd/dserver/main.go
@@ -32,7 +32,8 @@ func main() {
 	user.NoRootCheck()
 
 	flag.BoolVar(&color, "color", false, "Enable ANSII terminal colors")
-	flag.BoolVar(&config.ServerRelaxedAuthEnable, "relaxedAuth", false, "Enable relaxced SSH auth mode (don't use in production!)")
+	flag.BoolVar(&config.ServerRelaxedAuthEnable, "relaxedAuth", false,
+		"Enable relaxced SSH auth mode (don't use in production!)")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
 	flag.IntVar(&args.SSHPort, "port", config.DefaultSSHPort, "SSH server port")
 	flag.IntVar(&pprof, "pprof", -1, "Start PProf server this port")
diff --git a/cmd/dtail/main.go b/cmd/dtail/main.go
index adfeaa5..2863370 100644
--- a/cmd/dtail/main.go
+++ b/cmd/dtail/main.go
@@ -44,7 +44,8 @@ func main() {
 	flag.BoolVar(&displayColorTable, "colorTable", false, "Show color table")
 	flag.BoolVar(&displayWideColorTable, "wideColorTable", false, "Show a large color table")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
-	flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU, "How many connections established per CPU core concurrently")
+	flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU,
+		"How many connections established per CPU core concurrently")
 	flag.IntVar(&args.SSHPort, "port", config.DefaultSSHPort, "SSH server port")
 	flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection")
 	flag.IntVar(&pprof, "pprof", -1, "Start PProf server this port")
@@ -93,7 +94,8 @@ func main() {
 	dlog.Start(ctx, &wg, source.Client)
 
 	if checkHealth {
-		fmt.Println("WARN: DTail health check has moved to separate binary dtailhealtcheck - please adjust the monitoring scripts!")
+		fmt.Println("WARN: DTail health check has moved to separate binary dtailhealtcheck" +
+			" - please adjust the monitoring scripts!")
 		cancel()
 		os.Exit(1)
 	}
diff --git a/cmd/dtailhealthcheck/main.go b/cmd/dtailhealthcheck/main.go
index 0f37f8a..b0ba4cd 100644
--- a/cmd/dtailhealthcheck/main.go
+++ b/cmd/dtailhealthcheck/main.go
@@ -35,7 +35,6 @@ func main() {
 	defer cancel()
 	var wg sync.WaitGroup
 	wg.Add(1)
-
 	dlog.Start(ctx, &wg, source.HealthCheck)
 
 	if pprof > -1 {
diff --git a/integrationtests/commons.go b/integrationtests/commons.go
index f96b532..2fdbfc3 100644
--- a/integrationtests/commons.go
+++ b/integrationtests/commons.go
@@ -15,10 +15,12 @@ import (
 )
 
 func runCommand(t *testing.T, cmd string, args []string, stdoutFile string) (int, error) {
-	return runCommandContext(t, context.TODO(), cmd, args, stdoutFile)
+	return runCommandContext(context.TODO(), t, cmd, args, stdoutFile)
 }
 
-func runCommandContext(t *testing.T, ctx context.Context, cmd string, args []string, stdoutFile string) (int, error) {
+func runCommandContext(ctx context.Context, t *testing.T, cmd string, args []string,
+	stdoutFile string) (int, error) {
+
 	if _, err := os.Stat(cmd); err != nil {
 		return -1, fmt.Errorf("No such binary %s, please compile first (%v)", cmd, err)
 	}
@@ -76,7 +78,8 @@ func compareFilesContents(t *testing.T, fileA, fileB string) error {
 				return fmt.Errorf("Files differ, line '%s' is missing in one of them", line)
 			}
 			if countA != countB {
-				return fmt.Errorf("Files differ, count of line '%s' is %d in one but %d in another", line, countA, countB)
+				return fmt.Errorf("Files differ, count of line '%s' is %d in one but %d in another",
+					line, countA, countB)
 			}
 		}
 		return nil
@@ -92,11 +95,13 @@ func compareFilesContents(t *testing.T, fileA, fileB string) error {
 	}
 
 	// The mapreduce result can be in a different order each time (Golang maps are not sorted).
-	t.Log(fmt.Sprintf("Checking whether %s has same lines as file %s (ignoring line order)", fileA, fileB))
+	t.Log(fmt.Sprintf("Checking whether %s has same lines as file %s (ignoring line order)",
+		fileA, fileB))
 	if err := compareMaps(a, b); err != nil {
 		return err
 	}
-	t.Log(fmt.Sprintf("Checking whether %s has same lines as file %s (ignoring line order)", fileB, fileA))
+	t.Log(fmt.Sprintf("Checking whether %s has same lines as file %s (ignoring line order)",
+		fileB, fileA))
 	if err := compareMaps(b, a); err != nil {
 		return err
 	}
diff --git a/integrationtests/dcat_test.go b/integrationtests/dcat_test.go
index a164960..342ebd0 100644
--- a/integrationtests/dcat_test.go
+++ b/integrationtests/dcat_test.go
@@ -8,12 +8,12 @@ import (
 func TestDCat(t *testing.T) {
 	testdataFile := "dcat.txt.expected"
 	stdoutFile := "dcat.out"
+	args := []string{"-spartan", testdataFile}
 
-	if _, err := runCommand(t, "../dcat", []string{"-spartan", testdataFile}, stdoutFile); err != nil {
+	if _, err := runCommand(t, "../dcat", args, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
-
 	if err := compareFiles(t, stdoutFile, testdataFile); err != nil {
 		t.Error(err)
 		return
diff --git a/integrationtests/dgrep_test.go b/integrationtests/dgrep_test.go
index 6a15ebd..4d54a2d 100644
--- a/integrationtests/dgrep_test.go
+++ b/integrationtests/dgrep_test.go
@@ -9,12 +9,12 @@ func TestDGrep(t *testing.T) {
 	inFile := "mapr_testdata.log"
 	stdoutFile := "dgrep.stdout.tmp"
 	expectedStdoutFile := "dgrep.txt.expected"
+	args := []string{"-spartan", "--grep", "20211002-071947", inFile}
 
-	if _, err := runCommand(t, "../dgrep", []string{"-spartan", "--grep", "20211002-071947", inFile}, stdoutFile); err != nil {
+	if _, err := runCommand(t, "../dgrep", args, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
-
 	if err := compareFiles(t, stdoutFile, expectedStdoutFile); err != nil {
 		t.Error(err)
 		return
@@ -26,12 +26,12 @@ func TestDGrep2(t *testing.T) {
 	inFile := "mapr_testdata.log"
 	stdoutFile := "dgrep2.stdout.tmp"
 	expectedStdoutFile := "dgrep2.txt.expected"
+	args := []string{"-spartan", "--grep", "20211002-071947", "--invert", inFile}
 
-	if _, err := runCommand(t, "../dgrep", []string{"-spartan", "--grep", "20211002-071947", "--invert", inFile}, stdoutFile); err != nil {
+	if _, err := runCommand(t, "../dgrep", args, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
-
 	if err := compareFiles(t, stdoutFile, expectedStdoutFile); err != nil {
 		t.Error(err)
 		return
diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index b512985..f5c78e0 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -14,9 +14,12 @@ func TestDMap(t *testing.T) {
 	queryFile := fmt.Sprintf("%s.query", csvFile)
 	expectedQueryFile := "dmap.csv.query.expected"
 
-	query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile)
+	query := fmt.Sprintf("from STATS select count($line),last($time),"+
+		"avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+
+		"group by $hostname outfile %s", csvFile)
+	args := []string{"-query", query, inFile}
 
-	if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+	if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil {
 		t.Error(err)
 		return
 	}
@@ -42,9 +45,48 @@ func TestDMap2(t *testing.T) {
 	queryFile := fmt.Sprintf("%s.query", csvFile)
 	expectedQueryFile := "dmap2.csv.query.expected"
 
-	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile)
+	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+
+		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
+		"outfile %s", csvFile)
 
-	if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil {
+	args := []string{"-query", query, inFile}
+	if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil {
+		t.Error(err)
+		return
+	}
+	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+		t.Error(err)
+		return
+	}
+	if err := compareFiles(t, queryFile, expectedQueryFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(stdoutFile)
+	os.Remove(csvFile)
+	os.Remove(queryFile)
+}
+
+func TestDMap3(t *testing.T) {
+	inFile := "mapr_testdata.log"
+	stdoutFile := "dmap3.stdout.tmp"
+	csvFile := "dmap3.csv.tmp"
+	expectedCsvFile := "dmap3.csv.expected"
+	queryFile := fmt.Sprintf("%s.query", csvFile)
+	expectedQueryFile := "dmap3.csv.query.expected"
+
+	query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+
+		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
+		"outfile %s", csvFile)
+
+	// Read many input files at once.
+	args := []string{"-query", query}
+	for i := 0; i < 100; i++ {
+		args = append(arg