merge develop

7 files changed, 60 insertions, 39 deletions

diff --git a/docker/.gitignore b/docker/.gitignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docker/.gitignore
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 61a1f7d..da3b0db 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,11 +2,12 @@
 # The container can be used for developing and testing
 # Purposes
 
-FROM fedora:33
+FROM fedora:34
 RUN mkdir -p /etc/dserver /var/run/dserver/ /var/log/dserver
 
 ADD ./dtail.json /etc/dserver/dtail.json
 ADD ./dserver /usr/local/bin/dserver
+ADD ./mapr_testdata.log /var/log/mapr_testdata.log
 
 RUN useradd dserver
 RUN chown -R dserver /var/run/dserver /var/log/dserver
@@ -14,6 +15,5 @@ USER dserver
 
 WORKDIR /var/run/dserver
 EXPOSE 2222/tcp
-EXPOSE 8080/tcp
 
-CMD ["/usr/local/bin/dserver", "-relaxedAuth", "-cfg", "/etc/dserver/dtail.json", "-pprof", "8080"]
+CMD ["/usr/local/bin/dserver", "-relaxedAuth", "-cfg", "/etc/dserver/dtail.json"]
diff --git a/docker/Makefile b/docker/Makefile
index f09d9e0..fdd6c5b 100644
--- a/docker/Makefile
+++ b/docker/Makefile
@@ -1,11 +1,38 @@
-all:
+all: build
+testrun: build spinup dcat spindown
+serverfarm: spindown build spinup
+build:
+	cp ../integrationtests/mapr_testdata.log .
 	cp ../dserver .
 	docker build . -t dserver:develop
 	rm ./dserver
-run:
-	# http://localhost:8080/debug/pprof/goroutines?debug=1
-	docker run -p 2222:2222 -p 8080:8080 dserver:develop
+	rm ./mapr_testdata.log
 spinup:
 	./spinup.sh 10
 spindown:
 	./spindown.sh 10
+spinup1:
+	docker run -p 2222:2222 dserver:develop
+dtail:
+	../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG
+dtail2:
+	../dtail --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG --query 'from stats select max($$goroutines),count($$hostname),$$hostname,last($$time) group by $$hostname order by max($$goroutines)'
+dgrep:
+	../dgrep --servers serverlist.txt --files '/var/log/dserver/*' --regex MAPREDUCE --trustAllHosts
+dcat:
+	../dcat --servers serverlist.txt --files '/etc/passwd' --trustAllHosts
+dcat_notrust:
+	../dcat --servers serverlist.txt --files '/etc/passwd'
+dmap:
+	../dmap --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg($$goroutines),max($$goroutines),min($$goroutines),last($$goroutines),count($$hostname),$$hostname group by $$hostname order by avg($$goroutines)'
+test: dmap_test dmap2_test
+dmap_test:
+	../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv'
+	../dmap --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-B.csv'
+	@echo Expecting zero diff!
+	diff -u <(sort dmap2-A.csv) <(sort dmap2-B.csv)
+dmap2_test:
+	../dmap --servers <(head -n 1 serverlist.txt) --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv'
+	../dmap --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-serverless.csv' ./mapr_testdata.log
+	@echo Expecting zero diff!
+	diff -u <(sort dmap2-A.csv) <(sort dmap2-serverless.csv)
diff --git a/docker/dtail.json b/docker/dtail.json
index d923101..acef22a 100644
--- a/docker/dtail.json
+++ b/docker/dtail.json
@@ -7,33 +7,15 @@
     "MaxConnections": 50,
     "MapreduceLogFormat" : "default",
     "HostKeyFile" : "cache/ssh_host_key",
-    "HostKeyBits" : 2048,
-    "Permissions": {
-      "Default": [
-        "readfiles:^/.*$",
-        "runcommands:^/.*$"
-      ],
-      "Users": {
-        "pbuetow": [
-          "readfiles:^/.*$",
-          "runcommands:^/.*$"
-        ],
-        "jblake": [
-          "readfiles:^/tmp/foo.log$",
-          "readfiles:^/.*$",
-          "readfiles:!^/tmp/bar.log$",
-          "runcommands:!^/.*$"
-        ]
-      }
-    }
+    "HostKeyBits" : 2048
   },
   "Common": {
-    "LogDir" : "/var/log/dserver",
-    "CacheDir" : "cache",
-    "TmpDir" : "tmp",
-    "LogStrategy": "daily",
+    "LogDir": "/var/log/dserver",
+    "Logger": "Fout",
+    "LogLevel": "trace",
+    "LogRotation": "Daily",
+    "CacheDir": "cache",
     "SSHPort": 2222,
-    "DebugEnable": true,
     "ExperimentalFeaturesEnable": false
   }
 }
diff --git a/docker/serverlist.txt b/docker/serverlist.txt
new file mode 100644
index 0000000..62a6ea6
--- /dev/null
+++ b/docker/serverlist.txt
@@ -0,0 +1,10 @@
+localhost:2223
+localhost:2224
+localhost:2225
+localhost:2226
+localhost:2227
+localhost:2228
+localhost:2229
+localhost:2230
+localhost:2231
+localhost:2232
diff --git a/docker/spindown.sh b/docker/spindown.sh
index 1b171f3..7cf9cc6 100755
--- a/docker/spindown.sh
+++ b/docker/spindown.sh
@@ -3,10 +3,13 @@
 declare -i NUM_INSTANCES=$1
 declare -i BASE_PORT=2222
 
-rm serverlist.txt &>/dev/null
-
 for (( i=0; i < $NUM_INSTANCES; i++ )); do
     port=$[ BASE_PORT + i + 1 ]
-	docker stop dserver-serv$i 
-	docker rm dserver-serv$i 
+    name=dserver-serv$i
+    echo Stopping $name
+    docker stop $name
+    echo Removing $name
+    docker rm $name
 done
+
+exit 0
diff --git a/docker/spinup.sh b/docker/spinup.sh
index 0625967..399d781 100755
--- a/docker/spinup.sh
+++ b/docker/spinup.sh
@@ -3,10 +3,9 @@
 declare -i NUM_INSTANCES=$1
 declare -i BASE_PORT=2222
 
-echo > serverlist.txt
-
 for (( i=0; i < $NUM_INSTANCES; i++ )); do
     port=$[ BASE_PORT + i + 1 ]
-	docker run -d --name dserver-serv$i --hostname serv$i -p $port:2222 dserver:develop
-	echo localhost:$port >> serverlist.txt
+    name=dserver-serv$i
+    echo Creating $name
+    docker run -d --name $name --hostname serv$i -p $port:2222 dserver:develop
 done