From 07e1470892beacf0722276f94bacbd822b002540 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Oct 2021 13:09:32 +0300 Subject: add dmap tests --- integrationtests/dmap_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 integrationtests/dmap_test.go (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go new file mode 100644 index 0000000..bfca039 --- /dev/null +++ b/integrationtests/dmap_test.go @@ -0,0 +1,51 @@ +package integrationtests + +import ( + "fmt" + "os" + "testing" +) + +func TestDMap(t *testing.T) { + inFile := "mapr_testdata.log" + stdoutFile := "dmap.stdout.tmp" + csvFile := "dmap.csv.tmp" + expectedCsvFile := "dmap.csv.expected" + + query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile) + + if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + t.Error(err) + return + } + + if err := compareFiles(t, csvFile, expectedCsvFile); err != nil { + t.Error(err) + return + } + + os.Remove(stdoutFile) + os.Remove(csvFile) +} + +func TestDMap2(t *testing.T) { + inFile := "mapr_testdata.log" + stdoutFile := "dmap2.stdout.tmp" + csvFile := "dmap2.csv.tmp" + expectedCsvFile := "dmap2.csv.expected" + + query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile) + + if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + t.Error(err) + return + } + + if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { + t.Error(err) + return + } + + os.Remove(stdoutFile) + os.Remove(csvFile) +} -- cgit v1.2.3 From b2dbe133347ef220ff781ffeb1f8137245f5235f Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Oct 2021 13:32:20 +0300 Subject: when a mapreduce outfile is specified also always write a outfile.query file --- integrationtests/dmap_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index bfca039..dc508e2 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -11,6 +11,8 @@ func TestDMap(t *testing.T) { stdoutFile := "dmap.stdout.tmp" csvFile := "dmap.csv.tmp" expectedCsvFile := "dmap.csv.expected" + queryFile := fmt.Sprintf("%s.query", csvFile) + expectedQueryFile := "dmap.csv.query.expected" query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile) @@ -18,14 +20,18 @@ func TestDMap(t *testing.T) { t.Error(err) return } - if err := compareFiles(t, csvFile, expectedCsvFile); err != nil { t.Error(err) return } + if err := compareFiles(t, queryFile, expectedQueryFile); err != nil { + t.Error(err) + return + } os.Remove(stdoutFile) os.Remove(csvFile) + os.Remove(queryFile) } func TestDMap2(t *testing.T) { @@ -33,6 +39,8 @@ func TestDMap2(t *testing.T) { stdoutFile := "dmap2.stdout.tmp" csvFile := "dmap2.csv.tmp" expectedCsvFile := "dmap2.csv.expected" + queryFile := fmt.Sprintf("%s.query", csvFile) + expectedQueryFile := "dmap2.csv.query.expected" query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile) @@ -40,12 +48,16 @@ func TestDMap2(t *testing.T) { t.Error(err) return } - if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { t.Error(err) return } + if err := compareFiles(t, queryFile, expectedQueryFile); err != nil { + t.Error(err) + return + } os.Remove(stdoutFile) os.Remove(csvFile) + os.Remove(queryFile) } -- cgit v1.2.3 From 7a7169791a64190e1002e38bc9c04ad0d5c1ce1f Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 16:44:28 +0300 Subject: add dtail health check unit test. --- integrationtests/dmap_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index dc508e2..b512985 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -16,7 +16,7 @@ func TestDMap(t *testing.T) { query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile) - if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { t.Error(err) return } @@ -44,7 +44,7 @@ func TestDMap2(t *testing.T) { query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile) - if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { t.Error(err) return } -- cgit v1.2.3 From 97747ea0f3178f7f5890512d483fdccaa82846b0 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 21:10:29 +0300 Subject: vetting and linting and some code restyling --- integrationtests/dmap_test.go | 50 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index b512985..f5c78e0 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -14,9 +14,12 @@ func TestDMap(t *testing.T) { queryFile := fmt.Sprintf("%s.query", csvFile) expectedQueryFile := "dmap.csv.query.expected" - query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile) + query := fmt.Sprintf("from STATS select count($line),last($time),"+ + "avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+ + "group by $hostname outfile %s", csvFile) + args := []string{"-query", query, inFile} - if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { t.Error(err) return } @@ -42,9 +45,48 @@ func TestDMap2(t *testing.T) { queryFile := fmt.Sprintf("%s.query", csvFile) expectedQueryFile := "dmap2.csv.query.expected" - query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile) + query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+ + "avg($goroutines),min($goroutines) group by $time order by count($time) "+ + "outfile %s", csvFile) - if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + args := []string{"-query", query, inFile} + if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { + t.Error(err) + return + } + if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { + t.Error(err) + return + } + if err := compareFiles(t, queryFile, expectedQueryFile); err != nil { + t.Error(err) + return + } + + os.Remove(stdoutFile) + os.Remove(csvFile) + os.Remove(queryFile) +} + +func TestDMap3(t *testing.T) { + inFile := "mapr_testdata.log" + stdoutFile := "dmap3.stdout.tmp" + csvFile := "dmap3.csv.tmp" + expectedCsvFile := "dmap3.csv.expected" + queryFile := fmt.Sprintf("%s.query", csvFile) + expectedQueryFile := "dmap3.csv.query.expected" + + query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+ + "avg($goroutines),min($goroutines) group by $time order by count($time) "+ + "outfile %s", csvFile) + + // Read many input files at once. + args := []string{"-query", query} + for i := 0; i < 100; i++ { + args = append(args, inFile) + } + + if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { t.Error(err) return } -- cgit v1.2.3 From 71f89dc7ec7cf993d1eca98771212afe6310e9c8 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 10 Oct 2021 19:42:48 +0300 Subject: refactor --- integrationtests/dmap_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index f5c78e0..966944a 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -1,6 +1,7 @@ package integrationtests import ( + "context" "fmt" "os" "testing" @@ -17,12 +18,15 @@ func TestDMap(t *testing.T) { query := fmt.Sprintf("from STATS select count($line),last($time),"+ "avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+ "group by $hostname outfile %s", csvFile) - args := []string{"-query", query, inFile} - if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { + _, err := runCommand(context.TODO(), stdoutFile, + "../dmap", "--query", query, inFile) + + if err != nil { t.Error(err) return } + if err := compareFiles(t, csvFile, expectedCsvFile); err != nil { t.Error(err) return @@ -49,11 +53,13 @@ func TestDMap2(t *testing.T) { "avg($goroutines),min($goroutines) group by $time order by count($time) "+ "outfile %s", csvFile) - args := []string{"-query", query, inFile} - if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { + _, err := runCommand(context.TODO(), stdoutFile, + "../dmap", "--query", query, inFile) + if err != nil { t.Error(err) return } + if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { t.Error(err) return @@ -86,7 +92,7 @@ func TestDMap3(t *testing.T) { args = append(args, inFile) } - if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { + if _, err := runCommand(context.TODO(), stdoutFile, "../dmap", args...); err != nil { t.Error(err) return } -- cgit v1.2.3 From 7b873100d94ddc3c698a620cb83b61dcb2074303 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 13 Oct 2021 09:00:03 +0300 Subject: add another dcat integration test - catting 100 files at once --- integrationtests/dmap_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index 966944a..59be3f4 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -19,7 +19,7 @@ func TestDMap(t *testing.T) { "avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+ "group by $hostname outfile %s", csvFile) - _, err := runCommand(context.TODO(), stdoutFile, + _, err := runCommand(context.TODO(), t, stdoutFile, "../dmap", "--query", query, inFile) if err != nil { @@ -53,7 +53,7 @@ func TestDMap2(t *testing.T) { "avg($goroutines),min($goroutines) group by $time order by count($time) "+ "outfile %s", csvFile) - _, err := runCommand(context.TODO(), stdoutFile, + _, err := runCommand(context.TODO(), t, stdoutFile, "../dmap", "--query", query, inFile) if err != nil { t.Error(err) @@ -92,7 +92,7 @@ func TestDMap3(t *testing.T) { args = append(args, inFile) } - if _, err := runCommand(context.TODO(), stdoutFile, "../dmap", args...); err != nil { + if _, err := runCommand(context.TODO(), t, stdoutFile, "../dmap", args...); err != nil { t.Error(err) return } -- cgit v1.2.3 From 10314cef906fd9b73e003be69c2f6b7b3d66570c Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Oct 2021 13:20:48 +0300 Subject: Can configure DTail client not to mess with ~/.ssh/known_hosts via env var - this is useful for running unit and integration tests in jenkins --- integrationtests/dmap_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index 59be3f4..a112456 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -87,8 +87,8 @@ func TestDMap3(t *testing.T) { "outfile %s", csvFile) // Read many input files at once. - args := []string{"-query", query} - for i := 0; i < 100; i++ { + args := []string{"--logLevel", "trace", "--query", query} + for i := 0; i < 1; i++ { args = append(args, inFile) } -- cgit v1.2.3 From 860c41441b3bcf542b5701ae2257812879ce47b4 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 19 Oct 2021 19:01:12 +0300 Subject: Set DTAIL_RUN_INTEGRATIONT_TEST to yes for integration tests --- integrationtests/dmap_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index a112456..b3c048a 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -5,9 +5,15 @@ import ( "fmt" "os" "testing" + + "github.com/mimecast/dtail/internal/config" ) func TestDMap(t *testing.T) { + if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + t.Log("Skipping") + return + } inFile := "mapr_testdata.log" stdoutFile := "dmap.stdout.tmp" csvFile := "dmap.csv.tmp" @@ -42,6 +48,10 @@ func TestDMap(t *testing.T) { } func TestDMap2(t *testing.T) { + if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + t.Log("Skipping") + return + } inFile := "mapr_testdata.log" stdoutFile := "dmap2.stdout.tmp" csvFile := "dmap2.csv.tmp" @@ -75,6 +85,10 @@ func TestDMap2(t *testing.T) { } func TestDMap3(t *testing.T) { + if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + t.Log("Skipping") + return + } inFile := "mapr_testdata.log" stdoutFile := "dmap3.stdout.tmp" csvFile := "dmap3.csv.tmp" -- cgit v1.2.3 From b679462fa687b375ccefd3c11fec3fd96b9b4d60 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 20 Oct 2021 06:41:08 +0300 Subject: make pprof bind address configurable --- integrationtests/dmap_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index b3c048a..350ac6a 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -102,7 +102,7 @@ func TestDMap3(t *testing.T) { // Read many input files at once. args := []string{"--logLevel", "trace", "--query", query} - for i := 0; i < 1; i++ { + for i := 0; i < 100; i++ { args = append(args, inFile) } -- cgit v1.2.3 From 3f5672291c14da1dd3d5699867a286e2d53d26b9 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 21 Oct 2021 20:53:30 +0300 Subject: add inventory --- integrationtests/dmap_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index 350ac6a..7d8b8b5 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -101,7 +101,7 @@ func TestDMap3(t *testing.T) { "outfile %s", csvFile) // Read many input files at once. - args := []string{"--logLevel", "trace", "--query", query} + args := []string{"--logLevel", "trace", "--pprof", "localhost:8080", "--query", query} for i := 0; i < 100; i++ { args = append(args, inFile) } -- cgit v1.2.3