diff options
Diffstat (limited to 'docker')
| -rw-r--r-- | docker/.gitignore | 3 | ||||
| -rw-r--r-- | docker/Dockerfile | 14 | ||||
| -rw-r--r-- | docker/Makefile | 36 | ||||
| -rw-r--r-- | docker/dtail.json | 30 | ||||
| -rw-r--r-- | docker/serverlist.txt | 50 | ||||
| -rwxr-xr-x | docker/spindown.sh | 11 | ||||
| -rwxr-xr-x | docker/spinup.sh | 11 |
7 files changed, 113 insertions, 42 deletions
diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 0000000..fb43ecf --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,3 @@ +id_rsa_docker* +dmap2-A.csv.query +dmap2-B.csv.query diff --git a/docker/Dockerfile b/docker/Dockerfile index 61a1f7d..c3b0092 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,18 +2,24 @@ # The container can be used for developing and testing # Purposes -FROM fedora:33 -RUN mkdir -p /etc/dserver /var/run/dserver/ /var/log/dserver +FROM fedora:35 +RUN mkdir -p /etc/dserver /var/run/dserver/cache /var/log/dserver ADD ./dtail.json /etc/dserver/dtail.json +# NEXT: Compile dserver in a container as well, as otherwise might have glibc errors. ADD ./dserver /usr/local/bin/dserver +ADD ./mapr_testdata.log /var/log/mapr_testdata.log +# Normal Linux user (simulates someone who want's to use DTail) +RUN useradd paul +ADD ./id_rsa_docker.pub /var/run/dserver/cache/paul.authorized_keys + +# DTail server user RUN useradd dserver RUN chown -R dserver /var/run/dserver /var/log/dserver USER dserver WORKDIR /var/run/dserver EXPOSE 2222/tcp -EXPOSE 8080/tcp -CMD ["/usr/local/bin/dserver", "-relaxedAuth", "-cfg", "/etc/dserver/dtail.json", "-pprof", "8080"] +CMD ["/usr/local/bin/dserver", "-cfg", "/etc/dserver/dtail.json"] diff --git a/docker/Makefile b/docker/Makefile index f09d9e0..57240e2 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,11 +1,35 @@ -all: +NUM_INSTANCES ?= 10 +all: build +testrun: build spinup dcat spindown +serverfarm: spindown build spinup +build: + sh -c 'yes | ssh-keygen -t rsa -f id_rsa_docker' + cp ../integrationtests/mapr_testdata.log . cp ../dserver . docker build . -t dserver:develop rm ./dserver -run: - # http://localhost:8080/debug/pprof/goroutines?debug=1 - docker run -p 2222:2222 -p 8080:8080 dserver:develop + rm ./mapr_testdata.log spinup: - ./spinup.sh 10 + ./spinup.sh ${NUM_INSTANCES} spindown: - ./spindown.sh 10 + ./spindown.sh ${NUM_INSTANCES} +spinup1: + docker run -p 2222:2222 dserver:develop +dtail: + ../dtail --user paul --key id_rsa_docker --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG +dtail2: + ../dtail --user paul --key id_rsa_docker --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --logLevel DEBUG --query 'from stats select max($$goroutines),count($$hostname),$$hostname,last($$time) group by $$hostname order by max($$goroutines)' +dgrep: + ../dgrep --user paul --key id_rsa_docker --servers serverlist.txt --files '/var/log/dserver/*' --regex MAPREDUCE --trustAllHosts +dcat: + ../dcat --user paul --key id_rsa_docker --servers serverlist.txt --files '/etc/passwd' --trustAllHosts +dcat_notrust: + ../dcat --user paul --key id_rsa_docker --servers serverlist.txt --files '/etc/passwd' +dmap: + ../dmap --user paul --key id_rsa_docker --servers serverlist.txt --files '/var/log/dserver/*' --trustAllHosts --query 'from stats select avg($$goroutines),max($$goroutines),min($$goroutines),last($$goroutines),count($$hostname),$$hostname group by $$hostname order by avg($$goroutines)' +test: dmap_test dmap2_test +dmap_test: + ../dmap --user paul --key id_rsa_docker --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-A.csv' + ../dmap --user paul --key id_rsa_docker --servers serverlist.txt --files '/var/log/mapr_testdata.log' --trustAllHosts --query 'from stats select count($$time),last($$time) group by $$time order by count($$time) outfile dmap2-B.csv' + @echo Expecting zero diff! + diff -u <(sort dmap2-A.csv) <(sort dmap2-B.csv) diff --git a/docker/dtail.json b/docker/dtail.json index d923101..98c4409 100644 --- a/docker/dtail.json +++ b/docker/dtail.json @@ -7,33 +7,15 @@ "MaxConnections": 50, "MapreduceLogFormat" : "default", "HostKeyFile" : "cache/ssh_host_key", - "HostKeyBits" : 2048, - "Permissions": { - "Default": [ - "readfiles:^/.*$", - "runcommands:^/.*$" - ], - "Users": { - "pbuetow": [ - "readfiles:^/.*$", - "runcommands:^/.*$" - ], - "jblake": [ - "readfiles:^/tmp/foo.log$", - "readfiles:^/.*$", - "readfiles:!^/tmp/bar.log$", - "runcommands:!^/.*$" - ] - } - } + "HostKeyBits" : 2048 }, "Common": { - "LogDir" : "/var/log/dserver", - "CacheDir" : "cache", - "TmpDir" : "tmp", - "LogStrategy": "daily", + "LogDir": "/var/log/dserver", + "Logger": "Fout", + "LogLevel": "Trace", + "LogRotation": "Daily", + "CacheDir": "cache", "SSHPort": 2222, - "DebugEnable": true, "ExperimentalFeaturesEnable": false } } diff --git a/docker/serverlist.txt b/docker/serverlist.txt new file mode 100644 index 0000000..8a5d389 --- /dev/null +++ b/docker/serverlist.txt @@ -0,0 +1,50 @@ +localhost:2223 +localhost:2224 +localhost:2225 +localhost:2226 +localhost:2227 +localhost:2228 +localhost:2229 +localhost:2230 +localhost:2231 +localhost:2232 +localhost:2233 +localhost:2234 +localhost:2235 +localhost:2236 +localhost:2237 +localhost:2238 +localhost:2239 +localhost:2240 +localhost:2241 +localhost:2242 +localhost:2243 +localhost:2244 +localhost:2245 +localhost:2246 +localhost:2247 +localhost:2248 +localhost:2249 +localhost:2250 +localhost:2251 +localhost:2252 +localhost:2253 +localhost:2254 +localhost:2255 +localhost:2256 +localhost:2257 +localhost:2258 +localhost:2259 +localhost:2260 +localhost:2261 +localhost:2262 +localhost:2263 +localhost:2264 +localhost:2265 +localhost:2266 +localhost:2267 +localhost:2268 +localhost:2269 +localhost:2270 +localhost:2271 +localhost:2272 diff --git a/docker/spindown.sh b/docker/spindown.sh index 1b171f3..7cf9cc6 100755 --- a/docker/spindown.sh +++ b/docker/spindown.sh @@ -3,10 +3,13 @@ declare -i NUM_INSTANCES=$1 declare -i BASE_PORT=2222 -rm serverlist.txt &>/dev/null - for (( i=0; i < $NUM_INSTANCES; i++ )); do port=$[ BASE_PORT + i + 1 ] - docker stop dserver-serv$i - docker rm dserver-serv$i + name=dserver-serv$i + echo Stopping $name + docker stop $name + echo Removing $name + docker rm $name done + +exit 0 diff --git a/docker/spinup.sh b/docker/spinup.sh index 0625967..d9e5ac3 100755 --- a/docker/spinup.sh +++ b/docker/spinup.sh @@ -2,11 +2,14 @@ declare -i NUM_INSTANCES=$1 declare -i BASE_PORT=2222 - -echo > serverlist.txt +declare -r SERVERLIST=serverlist.txt for (( i=0; i < $NUM_INSTANCES; i++ )); do port=$[ BASE_PORT + i + 1 ] - docker run -d --name dserver-serv$i --hostname serv$i -p $port:2222 dserver:develop - echo localhost:$port >> serverlist.txt + name=dserver-serv$i + echo Creating $name + docker run -d --name $name --hostname serv$i -p $port:2222 dserver:develop + echo localhost:$port >> $SERVERLIST.tmp done + +mv $SERVERLIST.tmp $SERVERLIST |