From b0973dd7d17269ad9d4ebfd4842072e739c59eca Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Oct 2021 13:09:32 +0300 Subject: add dmap tests --- integrationtests/dmap_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 integrationtests/dmap_test.go (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go new file mode 100644 index 0000000..bfca039 --- /dev/null +++ b/integrationtests/dmap_test.go @@ -0,0 +1,51 @@ +package integrationtests + +import ( + "fmt" + "os" + "testing" +) + +func TestDMap(t *testing.T) { + inFile := "mapr_testdata.log" + stdoutFile := "dmap.stdout.tmp" + csvFile := "dmap.csv.tmp" + expectedCsvFile := "dmap.csv.expected" + + query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile) + + if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + t.Error(err) + return + } + + if err := compareFiles(t, csvFile, expectedCsvFile); err != nil { + t.Error(err) + return + } + + os.Remove(stdoutFile) + os.Remove(csvFile) +} + +func TestDMap2(t *testing.T) { + inFile := "mapr_testdata.log" + stdoutFile := "dmap2.stdout.tmp" + csvFile := "dmap2.csv.tmp" + expectedCsvFile := "dmap2.csv.expected" + + query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile) + + if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + t.Error(err) + return + } + + if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { + t.Error(err) + return + } + + os.Remove(stdoutFile) + os.Remove(csvFile) +} -- cgit v1.2.3 From 117f062a01e9d456e19f738a0b7047d9bb01285f Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Oct 2021 13:32:20 +0300 Subject: when a mapreduce outfile is specified also always write a outfile.query file --- integrationtests/dmap_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index bfca039..dc508e2 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -11,6 +11,8 @@ func TestDMap(t *testing.T) { stdoutFile := "dmap.stdout.tmp" csvFile := "dmap.csv.tmp" expectedCsvFile := "dmap.csv.expected" + queryFile := fmt.Sprintf("%s.query", csvFile) + expectedQueryFile := "dmap.csv.query.expected" query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile) @@ -18,14 +20,18 @@ func TestDMap(t *testing.T) { t.Error(err) return } - if err := compareFiles(t, csvFile, expectedCsvFile); err != nil { t.Error(err) return } + if err := compareFiles(t, queryFile, expectedQueryFile); err != nil { + t.Error(err) + return + } os.Remove(stdoutFile) os.Remove(csvFile) + os.Remove(queryFile) } func TestDMap2(t *testing.T) { @@ -33,6 +39,8 @@ func TestDMap2(t *testing.T) { stdoutFile := "dmap2.stdout.tmp" csvFile := "dmap2.csv.tmp" expectedCsvFile := "dmap2.csv.expected" + queryFile := fmt.Sprintf("%s.query", csvFile) + expectedQueryFile := "dmap2.csv.query.expected" query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile) @@ -40,12 +48,16 @@ func TestDMap2(t *testing.T) { t.Error(err) return } - if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { t.Error(err) return } + if err := compareFiles(t, queryFile, expectedQueryFile); err != nil { + t.Error(err) + return + } os.Remove(stdoutFile) os.Remove(csvFile) + os.Remove(queryFile) } -- cgit v1.2.3 From 7563abe9d5beaa18fa1eab0f65668f5dfcf79052 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 16:44:28 +0300 Subject: add dtail health check unit test. --- integrationtests/dmap_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index dc508e2..b512985 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -16,7 +16,7 @@ func TestDMap(t *testing.T) { query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile) - if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { t.Error(err) return } @@ -44,7 +44,7 @@ func TestDMap2(t *testing.T) { query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile) - if err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { t.Error(err) return } -- cgit v1.2.3 From ea1de3044e129d419f4e807f2624a009343a128f Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 21:10:29 +0300 Subject: vetting and linting and some code restyling --- integrationtests/dmap_test.go | 50 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index b512985..f5c78e0 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -14,9 +14,12 @@ func TestDMap(t *testing.T) { queryFile := fmt.Sprintf("%s.query", csvFile) expectedQueryFile := "dmap.csv.query.expected" - query := fmt.Sprintf("from STATS select count($line),last($time),avg($goroutines),min(concurrentConnections),max(lifetimeConnections) group by $hostname outfile %s", csvFile) + query := fmt.Sprintf("from STATS select count($line),last($time),"+ + "avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+ + "group by $hostname outfile %s", csvFile) + args := []string{"-query", query, inFile} - if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { t.Error(err) return } @@ -42,9 +45,48 @@ func TestDMap2(t *testing.T) { queryFile := fmt.Sprintf("%s.query", csvFile) expectedQueryFile := "dmap2.csv.query.expected" - query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile %s", csvFile) + query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+ + "avg($goroutines),min($goroutines) group by $time order by count($time) "+ + "outfile %s", csvFile) - if _, err := runCommand(t, "../dmap", []string{"-query", query, inFile}, stdoutFile); err != nil { + args := []string{"-query", query, inFile} + if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { + t.Error(err) + return + } + if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { + t.Error(err) + return + } + if err := compareFiles(t, queryFile, expectedQueryFile); err != nil { + t.Error(err) + return + } + + os.Remove(stdoutFile) + os.Remove(csvFile) + os.Remove(queryFile) +} + +func TestDMap3(t *testing.T) { + inFile := "mapr_testdata.log" + stdoutFile := "dmap3.stdout.tmp" + csvFile := "dmap3.csv.tmp" + expectedCsvFile := "dmap3.csv.expected" + queryFile := fmt.Sprintf("%s.query", csvFile) + expectedQueryFile := "dmap3.csv.query.expected" + + query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+ + "avg($goroutines),min($goroutines) group by $time order by count($time) "+ + "outfile %s", csvFile) + + // Read many input files at once. + args := []string{"-query", query} + for i := 0; i < 100; i++ { + args = append(args, inFile) + } + + if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { t.Error(err) return } -- cgit v1.2.3 From c9b7064a0cc4273e4e44c1a545f4d2a207a31b55 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 10 Oct 2021 19:42:48 +0300 Subject: refactor --- integrationtests/dmap_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index f5c78e0..966944a 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -1,6 +1,7 @@ package integrationtests import ( + "context" "fmt" "os" "testing" @@ -17,12 +18,15 @@ func TestDMap(t *testing.T) { query := fmt.Sprintf("from STATS select count($line),last($time),"+ "avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+ "group by $hostname outfile %s", csvFile) - args := []string{"-query", query, inFile} - if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { + _, err := runCommand(context.TODO(), stdoutFile, + "../dmap", "--query", query, inFile) + + if err != nil { t.Error(err) return } + if err := compareFiles(t, csvFile, expectedCsvFile); err != nil { t.Error(err) return @@ -49,11 +53,13 @@ func TestDMap2(t *testing.T) { "avg($goroutines),min($goroutines) group by $time order by count($time) "+ "outfile %s", csvFile) - args := []string{"-query", query, inFile} - if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { + _, err := runCommand(context.TODO(), stdoutFile, + "../dmap", "--query", query, inFile) + if err != nil { t.Error(err) return } + if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { t.Error(err) return @@ -86,7 +92,7 @@ func TestDMap3(t *testing.T) { args = append(args, inFile) } - if _, err := runCommand(t, "../dmap", args, stdoutFile); err != nil { + if _, err := runCommand(context.TODO(), stdoutFile, "../dmap", args...); err != nil { t.Error(err) return } -- cgit v1.2.3 From 67af621613e8794d47f0265287b7b9d82e19c6bb Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 13 Oct 2021 09:00:03 +0300 Subject: add another dcat integration test - catting 100 files at once --- integrationtests/dmap_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'integrationtests/dmap_test.go') diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index 966944a..59be3f4 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -19,7 +19,7 @@ func TestDMap(t *testing.T) { "avg($goroutines),min(concurrentConnections),max(lifetimeConnections) "+ "group by $hostname outfile %s", csvFile) - _, err := runCommand(context.TODO(), stdoutFile, + _, err := runCommand(context.TODO(), t, stdoutFile, "../dmap", "--query", query, inFile) if err != nil { @@ -53,7 +53,7 @@ func TestDMap2(t *testing.T) { "avg($goroutines),min($goroutines) group by $time order by count($time) "+ "outfile %s", csvFile) - _, err := runCommand(context.TODO(), stdoutFile, + _, err := runCommand(context.TODO(), t, stdoutFile, "../dmap", "--query", query, inFile) if err != nil { t.Error(err) @@ -92,7 +92,7 @@ func TestDMap3(t *testing.T) { args = append(args, inFile) } - if _, err := runCommand(context.TODO(), stdoutFile, "../dmap", args...); err != nil { + if _, err := runCommand(context.TODO(), t, stdoutFile, "../dmap", args...); err != nil { t.Error(err) return } -- cgit v1.2.3