From 8a84f9a9f3b48a894ea4e6227d879e701817b883 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 31 Jul 2021 17:53:03 +0300 Subject: Fix Docker tests - also add dcat example --- docker/Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 029adf6..eef32d7 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,10 +1,14 @@ -all: +all: build +testrun: build spinup dcat spindown +build: cp ../dserver . docker build . -t dserver:develop rm ./dserver -run: - docker run -p 2222:2222 dserver:develop spinup: ./spinup.sh 10 spindown: ./spindown.sh 10 +dcat: + ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts +spinup1: + docker run -p 2222:2222 dserver:develop -- cgit v1.2.3 From 9883a190109623b64e6d311dc2b462a6eae68003 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 22 Aug 2021 10:07:00 +0300 Subject: introduces the protocol package --- docker/Makefile | 2 ++ 1 file changed, 2 insertions(+) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index eef32d7..5013c28 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -8,6 +8,8 @@ spinup: ./spinup.sh 10 spindown: ./spindown.sh 10 +dtail: + ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts dcat: ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts spinup1: -- cgit v1.2.3 From 2c1c70313bb03cf2b2d7e7afadb07a48ff6bb690 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 6 Sep 2021 09:22:21 +0300 Subject: REMOTE and CLIENT colors are brushed correctly too now --- docker/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 5013c28..a4ffa19 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -9,8 +9,8 @@ spinup: spindown: ./spindown.sh 10 dtail: - ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts + ../dtail --servers serverlist.txt --files '/var/log/dserver/*,/does/not/exist' --trustAllHosts --debug dcat: - ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts + ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts --debug spinup1: docker run -p 2222:2222 dserver:develop -- cgit v1.2.3 From f74a9e4b35feb8c07d8a70b5a581088a0a59889d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 7 Sep 2021 10:01:32 +0300 Subject: Produce MAPREDUCE lines, can aggregate these via default log format --- docker/Makefile | 1 + 1 file changed, 1 insertion(+) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index a4ffa19..910b23c 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,5 +1,6 @@ all: build testrun: build spinup dcat spindown +serverfarm: spindown build spinup build: cp ../dserver . docker build . -t dserver:develop -- cgit v1.2.3 From 16dc57e1e1c28e9d762424e596223a980770e059 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 8 Sep 2021 19:10:50 +0300 Subject: mapreduce tables are in colors now too --- docker/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 910b23c..28d458a 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -10,8 +10,12 @@ spinup: spindown: ./spindown.sh 10 dtail: - ../dtail --servers serverlist.txt --files '/var/log/dserver/*,/does/not/exist' --trustAllHosts --debug + ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --debug +dgrep: + ../dgrep --servers serverlist.txt --files '/var/log/dserver/*' --regex MAPREDUCE --trustAllHosts dcat: ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts --debug +dmap: + ../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg(goroutines),max(goroutines),min(goroutines),last(goroutines),count($$hostname),$$hostname group by $$hostname order by avg(goroutines)' spinup1: docker run -p 2222:2222 dserver:develop -- cgit v1.2.3 From 2ebe7e9d63ba62c6f19749c39fe0a577d86ca775 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 12 Sep 2021 19:04:42 +0300 Subject: bugfix: dmap skipped the last couple of mapreduce lines --- docker/Makefile | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 28d458a..5cdb931 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,6 +2,7 @@ all: build testrun: build spinup dcat spindown serverfarm: spindown build spinup build: + cp ../testdata/mapr_testdata.log . cp ../dserver . docker build . -t dserver:develop rm ./dserver @@ -17,5 +18,10 @@ dcat: ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts --debug dmap: ../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg(goroutines),max(goroutines),min(goroutines),last(goroutines),count($$hostname),$$hostname group by $$hostname order by avg(goroutines)' +dmap2: + ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' + ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-B.csv' + @echo Expecting zero diff! + diff -u <(sort dmap2-A.csv) <(sort dmap2-B.csv) spinup1: docker run -p 2222:2222 dserver:develop -- cgit v1.2.3 From 5e717af91e8012c72ec7dc0204420dea46f187db Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 18 Sep 2021 11:57:09 +0300 Subject: new docker test cases - also change default FATAL bg color to magenta --- docker/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 5cdb931..75aed79 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -12,10 +12,12 @@ spindown: ./spindown.sh 10 dtail: ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --debug +dtail2: + ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --debug --query 'from stats select max(goroutines),count($$hostname),$$hostname,last($$time) group by $$hostname order by max(goroutines)' dgrep: ../dgrep --servers serverlist.txt --files '/var/log/dserver/*' --regex MAPREDUCE --trustAllHosts dcat: - ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts --debug + ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts dmap: ../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg(goroutines),max(goroutines),min(goroutines),last(goroutines),count($$hostname),$$hostname group by $$hostname order by avg(goroutines)' dmap2: -- cgit v1.2.3 From abeac87aec44249bf67f1b0eca471a31086265ca Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 18 Sep 2021 19:27:50 +0300 Subject: fix auto reconnect --- docker/Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 75aed79..71fd249 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -18,12 +18,21 @@ dgrep: ../dgrep --servers serverlist.txt --files '/var/log/dserver/*' --regex MAPREDUCE --trustAllHosts dcat: ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts +dcat2: + ../dcat /etc/passwd dmap: ../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg(goroutines),max(goroutines),min(goroutines),last(goroutines),count($$hostname),$$hostname group by $$hostname order by avg(goroutines)' dmap2: ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-B.csv' + ../dmap --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-serverless.csv' ./mapr_testdata.log @echo Expecting zero diff! diff -u <(sort dmap2-A.csv) <(sort dmap2-B.csv) + diff -u <(sort dmap2-A.csv) <(sort dmap2-serverless.csv) +dmap3: + ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' + ../dmap --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-serverless.csv' ./mapr_testdata.log + @echo Expecting zero diff! + diff -u <(sort dmap2-A.csv) <(sort dmap2-serverless.csv) spinup1: docker run -p 2222:2222 dserver:develop -- cgit v1.2.3 From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 19 Sep 2021 13:22:59 +0300 Subject: move args to config package logger package rewrite as dlog --- docker/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 71fd249..c89467c 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -18,7 +18,10 @@ dgrep: ../dgrep --servers serverlist.txt --files '/var/log/dserver/*' --regex MAPREDUCE --trustAllHosts dcat: ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts +dcat_notrust: + ../dcat --servers serverlist.txt --files '/etc/passwd' dcat2: + # TODO: All serverless tests in this Makefile have to move to actual unit tests ../dcat /etc/passwd dmap: ../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg(goroutines),max(goroutines),min(goroutines),last(goroutines),count($$hostname),$$hostname group by $$hostname order by avg(goroutines)' @@ -30,7 +33,7 @@ dmap2: diff -u <(sort dmap2-A.csv) <(sort dmap2-B.csv) diff -u <(sort dmap2-A.csv) <(sort dmap2-serverless.csv) dmap3: - ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' + ../dmap --servers <(head -n 1 serverlist.txt) --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' ../dmap --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-serverless.csv' ./mapr_testdata.log @echo Expecting zero diff! diff -u <(sort dmap2-A.csv) <(sort dmap2-serverless.csv) -- cgit v1.2.3 From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 26 Sep 2021 16:42:47 +0300 Subject: refactor config reader - also looks in additional search paths for config file unless NONE is specified --- docker/Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index c89467c..68e7ad8 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -11,9 +11,9 @@ spinup: spindown: ./spindown.sh 10 dtail: - ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --debug + ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG dtail2: - ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --debug --query 'from stats select max(goroutines),count($$hostname),$$hostname,last($$time) group by $$hostname order by max(goroutines)' + ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG --query 'from stats select max(goroutines),count($$hostname),$$hostname,last($$time) group by $$hostname order by max(goroutines)' dgrep: ../dgrep --servers serverlist.txt --files '/var/log/dserver/*' --regex MAPREDUCE --trustAllHosts dcat: @@ -28,10 +28,8 @@ dmap: dmap2: ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-B.csv' - ../dmap --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-serverless.csv' ./mapr_testdata.log @echo Expecting zero diff! diff -u <(sort dmap2-A.csv) <(sort dmap2-B.csv) - diff -u <(sort dmap2-A.csv) <(sort dmap2-serverless.csv) dmap3: ../dmap --servers <(head -n 1 serverlist.txt) --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' ../dmap --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-serverless.csv' ./mapr_testdata.log -- cgit v1.2.3 From 764ef99a3d779a0db1fb60679292af52425ba2f6 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 10:46:47 +0300 Subject: add more default fields to MAPREDUCE --- docker/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 68e7ad8..3b98e74 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -13,7 +13,7 @@ spindown: dtail: ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG dtail2: - ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG --query 'from stats select max(goroutines),count($$hostname),$$hostname,last($$time) group by $$hostname order by max(goroutines)' + ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG --query 'from stats select max($$goroutines),count($$hostname),$$hostname,last($$time) group by $$hostname order by max($$goroutines)' dgrep: ../dgrep --servers serverlist.txt --files '/var/log/dserver/*' --regex MAPREDUCE --trustAllHosts dcat: @@ -24,7 +24,7 @@ dcat2: # TODO: All serverless tests in this Makefile have to move to actual unit tests ../dcat /etc/passwd dmap: - ../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg(goroutines),max(goroutines),min(goroutines),last(goroutines),count($$hostname),$$hostname group by $$hostname order by avg(goroutines)' + ../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg($$goroutines),max($$goroutines),min($$goroutines),last($$goroutines),count($$hostname),$$hostname group by $$hostname order by avg($$goroutines)' dmap2: ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-B.csv' -- cgit v1.2.3 From 86ec83754e0ee7153ad55091f7b6da448bc529c5 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 13:44:27 +0300 Subject: add dcat test --- docker/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 3b98e74..921ad28 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,7 +2,7 @@ all: build testrun: build spinup dcat spindown serverfarm: spindown build spinup build: - cp ../testdata/mapr_testdata.log . + cp ../integrationtests/mapr_testdata.log . cp ../dserver . docker build . -t dserver:develop rm ./dserver -- cgit v1.2.3 From b2dbe133347ef220ff781ffeb1f8137245f5235f Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Oct 2021 13:32:20 +0300 Subject: when a mapreduce outfile is specified also always write a outfile.query file --- docker/Makefile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 921ad28..4ffa423 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -6,10 +6,13 @@ build: cp ../dserver . docker build . -t dserver:develop rm ./dserver + rm ./mapr_testdata.log spinup: ./spinup.sh 10 spindown: ./spindown.sh 10 +spinup1: + docker run -p 2222:2222 dserver:develop dtail: ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG dtail2: @@ -20,20 +23,16 @@ dcat: ../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts dcat_notrust: ../dcat --servers serverlist.txt --files '/etc/passwd' -dcat2: - # TODO: All serverless tests in this Makefile have to move to actual unit tests - ../dcat /etc/passwd dmap: ../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg($$goroutines),max($$goroutines),min($$goroutines),last($$goroutines),count($$hostname),$$hostname group by $$hostname order by avg($$goroutines)' -dmap2: +test: dmap_test dmap2_test +dmap_test: ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' ../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-B.csv' @echo Expecting zero diff! diff -u <(sort dmap2-A.csv) <(sort dmap2-B.csv) -dmap3: +dmap2_test: ../dmap --servers <(head -n 1 serverlist.txt) --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' ../dmap --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-serverless.csv' ./mapr_testdata.log @echo Expecting zero diff! diff -u <(sort dmap2-A.csv) <(sort dmap2-serverless.csv) -spinup1: - docker run -p 2222:2222 dserver:develop -- cgit v1.2.3 From f44792c9102488774c9993b080f35c65287a64b1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 10 Oct 2021 14:02:12 +0300 Subject: add another dmap test - reading 100 source files at once fix a data race when reading multiple files on one server from the same session at once --- docker/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docker/Makefile') diff --git a/docker/Makefile b/docker/Makefile index 4ffa423..fdd6c5b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -12,7 +12,7 @@ spinup: spindown: ./spindown.sh 10 spinup1: - docker run -p 2222:2222 dserver:develop + docker run -p 2222:2222 dserver:develop dtail: ../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG dtail2: -- cgit v1.2.3