Move commands to cmd/ and move internal dependencies to internal/

77 files changed, 705 insertions, 449 deletions

diff --git a/Makefile b/Makefile
index 3d24800..456987d 100644
--- a/Makefile
+++ b/Makefile
@@ -2,19 +2,19 @@ GO ?= go
 all: build
 build:
 	${GO} version
-	${GO} build
-	cp -pv ./dtail ./dcat
-	cp -pv ./dtail ./dgrep
-	cp -pv ./dtail ./dmap
-	cp -pv ./dtail ./dserver
+	${GO} build -o dtail ./cmd/dtail/main.go
+	${GO} build -o dcat ./cmd/dcat/main.go
+	${GO} build -o dgrep ./cmd/dgrep/main.go
+	${GO} build -o dmap ./cmd/dmap/main.go
+	${GO} build -o dserver ./cmd/dserver/main.go
 clean:
 	rm -v dtail dgrep dcat dmap dserver 2>/dev/null
-install:
-	${GO} install
-	cp -pv ${GOPATH}/bin/dtail ${GOPATH}/bin/dcat
-	cp -pv ${GOPATH}/bin/dtail ${GOPATH}/bin/dgrep
-	cp -pv ${GOPATH}/bin/dtail ${GOPATH}/bin/dmap
-	cp -pv ${GOPATH}/bin/dtail ${GOPATH}/bin/dserver
+install: build
+	cp -pv dtail ${GOPATH}/bin/dtail
+	cp -pv dcat ${GOPATH}/bin/dcat
+	cp -pv dgrep ${GOPATH}/bin/dgrep
+	cp -pv dmap ${GOPATH}/bin/dmap
+	cp -pv dserver ${GOPATH}/bin/dserver
 vet:
 	find . -type d | while read dir; do \
 	  echo ${GO} vet $$dir; \
diff --git a/clients/args.go b/clients/args.go
deleted file mode 100644
index 4d5a029..0000000
--- a/clients/args.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package clients
-
-import (
-	"dtail/omode"
-)
-
-// Args is a helper struct to summarize common client arguments.
-type Args struct {
-	// The operating mode (tail, grep, ...)
-	Mode omode.Mode
-	// The raw server string
-	ServersStr string
-	// SSH user name (e.g. 'pbuetow')
-	UserName string
-	// The files to follow.
-	Files string
-	// Regex for filtering.
-	Regex string
-	// Trust all unknown host keys?
-	TrustAllHosts bool
-	// Server discovery method
-	Discovery          string
-	MaxInitConnections int
-	// Server ping timeout (0 means pings disabled)
-	PingTimeout int
-}
diff --git a/cmd/dcat/main.go b/cmd/dcat/main.go
new file mode 100644
index 0000000..b02d369
--- /dev/null
+++ b/cmd/dcat/main.go
@@ -0,0 +1,80 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/mimecast/dtail/internal/clients"
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/logger"
+	"github.com/mimecast/dtail/internal/pprof"
+	"github.com/mimecast/dtail/internal/user"
+	"github.com/mimecast/dtail/internal/version"
+)
+
+// The evil begins here.
+func main() {
+	var cfgFile string
+	var connectionsPerCPU int
+	var debugEnable bool
+	var discovery string
+	var displayVersion bool
+	var files string
+	var noColor bool
+	var pprofEnable bool
+	var serversStr string
+	var silentEnable bool
+	var sshPort int
+	var trustAllHosts bool
+
+	pingTimeoutS := 60
+	userName := user.Name()
+
+	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
+	flag.BoolVar(&displayVersion, "version", false, "Display version")
+	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&pprofEnable, "pprofEnable", false, "Enable pprof server")
+	flag.BoolVar(&silentEnable, "silent", false, "Reduce output")
+	flag.BoolVar(&trustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys")
+	flag.IntVar(&connectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
+	flag.IntVar(&pingTimeoutS, "pingTimeout", 10, "The server ping timeout (0 means disable pings)")
+	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
+	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
+	flag.StringVar(&discovery, "discovery", "", "Server discovery method")
+	flag.StringVar(&files, "files", "", "File(s) to read")
+	flag.StringVar(&serversStr, "servers", "", "Remote servers to connect")
+	flag.StringVar(&userName, "user", userName, "Your system user name")
+
+	flag.Parse()
+
+	config.Read(cfgFile, sshPort)
+	color.Colored = !noColor
+
+	if displayVersion {
+		version.PrintAndExit()
+	}
+
+	serverEnable := false
+	logger.Start(serverEnable, debugEnable, silentEnable, silentEnable)
+	defer logger.Stop()
+
+	if pprofEnable || config.Common.PProfEnable {
+		pprof.Start()
+	}
+
+	args := clients.Args{
+		ConnectionsPerCPU: connectionsPerCPU,
+		ServersStr:        serversStr,
+		Discovery:         discovery,
+		UserName:          userName,
+		Files:             files,
+		TrustAllHosts:     trustAllHosts,
+		PingTimeout:       pingTimeoutS,
+	}
+
+	client, err := clients.NewCatClient(args)
+	if err != nil {
+		panic(err)
+	}
+	client.Start()
+}
diff --git a/cmd/dgrep/main.go b/cmd/dgrep/main.go
new file mode 100644
index 0000000..d1a7d52
--- /dev/null
+++ b/cmd/dgrep/main.go
@@ -0,0 +1,83 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/mimecast/dtail/internal/clients"
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/logger"
+	"github.com/mimecast/dtail/internal/pprof"
+	"github.com/mimecast/dtail/internal/user"
+	"github.com/mimecast/dtail/internal/version"
+)
+
+// The evil begins here.
+func main() {
+	var cfgFile string
+	var connectionsPerCPU int
+	var debugEnable bool
+	var discovery string
+	var displayVersion bool
+	var files string
+	var noColor bool
+	var pprofEnable bool
+	var regex string
+	var serversStr string
+	var silentEnable bool
+	var sshPort int
+	var trustAllHosts bool
+
+	pingTimeoutS := 60
+	userName := user.Name()
+
+	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
+	flag.BoolVar(&displayVersion, "version", false, "Display version")
+	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&pprofEnable, "pprofEnable", false, "Enable pprof server")
+	flag.BoolVar(&silentEnable, "silent", false, "Reduce output")
+	flag.BoolVar(&trustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys")
+	flag.IntVar(&connectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
+	flag.IntVar(&pingTimeoutS, "pingTimeout", 10, "The server ping timeout (0 means disable pings)")
+	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
+	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
+	flag.StringVar(&discovery, "discovery", "", "Server discovery method")
+	flag.StringVar(&files, "files", "", "File(s) to read")
+	flag.StringVar(&regex, "regex", ".", "Regular expression")
+	flag.StringVar(&serversStr, "servers", "", "Remote servers to connect")
+	flag.StringVar(&userName, "user", userName, "Your system user name")
+
+	flag.Parse()
+
+	config.Read(cfgFile, sshPort)
+	color.Colored = !noColor
+
+	if displayVersion {
+		version.PrintAndExit()
+	}
+
+	serverEnable := false
+	logger.Start(serverEnable, debugEnable, silentEnable, silentEnable)
+	defer logger.Stop()
+
+	if pprofEnable || config.Common.PProfEnable {
+		pprof.Start()
+	}
+
+	args := clients.Args{
+		ConnectionsPerCPU: connectionsPerCPU,
+		ServersStr:        serversStr,
+		Discovery:         discovery,
+		UserName:          userName,
+		Files:             files,
+		TrustAllHosts:     trustAllHosts,
+		PingTimeout:       pingTimeoutS,
+		Regex:             regex,
+	}
+
+	client, err := clients.NewGrepClient(args)
+	if err != nil {
+		panic(err)
+	}
+	client.Start()
+}
diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go
new file mode 100644
index 0000000..83dad50
--- /dev/null
+++ b/cmd/dmap/main.go
@@ -0,0 +1,85 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/mimecast/dtail/internal/omode"
+	"github.com/mimecast/dtail/internal/clients"
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/logger"
+	"github.com/mimecast/dtail/internal/pprof"
+	"github.com/mimecast/dtail/internal/user"
+	"github.com/mimecast/dtail/internal/version"
+)
+
+// The evil begins here.
+func main() {
+	var cfgFile string
+	var connectionsPerCPU int
+	var debugEnable bool
+	var discovery string
+	var displayVersion bool
+	var files string
+	var noColor bool
+	var pprofEnable bool
+	var queryStr string
+	var serversStr string
+	var silentEnable bool
+	var sshPort int
+	var trustAllHosts bool
+
+	pingTimeoutS := 900
+	userName := user.Name()
+
+	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
+	flag.BoolVar(&displayVersion, "version", false, "Display version")
+	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&pprofEnable, "pprofEnable", false, "Enable pprof server")
+	flag.BoolVar(&silentEnable, "silent", false, "Reduce output")
+	flag.BoolVar(&trustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys")
+	flag.IntVar(&connectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
+	flag.IntVar(&pingTimeoutS, "pingTimeout", 10, "The server ping timeout (0 means disable pings)")
+	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
+	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
+	flag.StringVar(&discovery, "discovery", "", "Server discovery method")
+	flag.StringVar(&files, "files", "", "File(s) to read")
+	flag.StringVar(&queryStr, "query", "", "Map reduce query")
+	flag.StringVar(&serversStr, "servers", "", "Remote servers to connect")
+	flag.StringVar(&userName, "user", userName, "Your system user name")
+
+	flag.Parse()
+
+	config.Read(cfgFile, sshPort)
+	color.Colored = !noColor
+
+	if displayVersion {
+		version.PrintAndExit()
+	}
+
+	serverEnable := false
+	logger.Start(serverEnable, debugEnable, silentEnable, silentEnable)
+	defer logger.Stop()
+
+	if pprofEnable || config.Common.PProfEnable {
+		pprof.Start()
+	}
+
+	args := clients.Args{
+		ConnectionsPerCPU: connectionsPerCPU,
+		ServersStr:        serversStr,
+		Discovery:         discovery,
+		UserName:          userName,
+		Files:             files,
+		TrustAllHosts:     trustAllHosts,
+		PingTimeout:       pingTimeoutS,
+		Mode:              omode.MapClient,
+	}
+
+	client, err := clients.NewMaprClient(args, queryStr)
+	if err != nil {
+		panic(err)
+	}
+
+	client.Start()
+}
diff --git a/cmd/dserver/main.go b/cmd/dserver/main.go
new file mode 100644
index 0000000..489910b
--- /dev/null
+++ b/cmd/dserver/main.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"time"
+
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/logger"
+	"github.com/mimecast/dtail/internal/pprof"
+	"github.com/mimecast/dtail/internal/server"
+	"github.com/mimecast/dtail/internal/user"
+	"github.com/mimecast/dtail/internal/version"
+)
+
+// The evil begins here.
+func main() {
+	var cfgFile string
+	var debugEnable bool
+	var displayVersion bool
+	var noColor bool
+	var pprofEnable bool
+	var shutdownAfter int
+	var sshPort int
+
+	userName := user.Name()
+
+	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
+	flag.BoolVar(&displayVersion, "version", false, "Display version")
+	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&pprofEnable, "pprofEnable", false, "Enable pprof server")
+	flag.IntVar(&shutdownAfter, "shutdownAfter", 0, "Automatically shutdown after so many seconds")
+	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
+	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
+
+	flag.Parse()
+
+	config.Read(cfgFile, sshPort)
+	color.Colored = !noColor
+
+	if displayVersion {
+		version.PrintAndExit()
+	}
+
+	serverEnable := true
+	silentEnable := false
+	nothingEnable := false
+	logger.Start(serverEnable, debugEnable, silentEnable, nothingEnable)
+	defer logger.Stop()
+
+	if shutdownAfter > 0 {
+		go func() {
+			defer os.Exit(1)
+
+			logger.Info("Enabling auto shutdown timer", shutdownAfter)
+			time.Sleep(time.Duration(shutdownAfter) * time.Second)
+			logger.Info("Auto shutdown timer reached, shutting down now")
+		}()
+	}
+
+	if pprofEnable || config.Common.PProfEnable {
+		pprof.Start()
+	}
+
+	logger.Info("Launching server", version.String(), userName)
+	sshServer := server.New()
+	sshServer.Start()
+}
diff --git a/cmd/dtail/main.go b/cmd/dtail/main.go
new file mode 100644
index 0000000..1bf77c7
--- /dev/null
+++ b/cmd/dtail/main.go
@@ -0,0 +1,108 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/mimecast/dtail/internal/clients"
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/logger"
+	"github.com/mimecast/dtail/internal/omode"
+	"github.com/mimecast/dtail/internal/pprof"
+	"github.com/mimecast/dtail/internal/user"
+	"github.com/mimecast/dtail/internal/version"
+)
+
+// The evil begins here.
+func main() {
+	var cfgFile string
+	var checkHealth bool
+	var connectionsPerCPU int
+	var debugEnable bool
+	var discovery string
+	var displayVersion bool
+	var files string
+	var noColor bool
+	var pprofEnable bool
+	var queryStr string
+	var regex string
+	var serversStr string
+	var silentEnable bool
+	var sshPort int
+	var trustAllHosts bool
+
+	pingTimeoutS := 5
+	userName := user.Name()
+
+	flag.BoolVar(&checkHealth, "checkHealth", false, "Only check for server health")
+	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
+	flag.BoolVar(&displayVersion, "version", false, "Display version")
+	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&pprofEnable, "pprofEnable", false, "Enable pprof server")
+	flag.BoolVar(&silentEnable, "silent", false, "Reduce output")
+	flag.BoolVar(&trustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys")
+	flag.IntVar(&connectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
+	flag.IntVar(&pingTimeoutS, "pingTimeout", 10, "The server ping timeout (0 means disable pings)")
+	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
+	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
+	flag.StringVar(&discovery, "discovery", "", "Server discovery method")
+	flag.StringVar(&files, "files", "", "File(s) to read")
+	flag.StringVar(&queryStr, "query", "", "Map reduce query")
+	flag.StringVar(&regex, "regex", ".", "Regular expression")
+	flag.StringVar(&serversStr, "servers", "", "Remote servers to connect")
+	flag.StringVar(&userName, "user", userName, "Your system user name")
+
+	flag.Parse()
+
+	config.Read(cfgFile, sshPort)
+	color.Colored = !noColor
+
+	if displayVersion {
+		version.PrintAndExit()
+	}
+
+	if checkHealth {
+		healthClient, _ := clients.NewHealthClient(omode.HealthClient)
+		os.Exit(healthClient.Start())
+	}
+
+	serverEnable := false
+	if checkHealth {
+		silentEnable = true
+	}
+	logger.Start(serverEnable, debugEnable, silentEnable, silentEnable)
+	defer logger.Stop()
+
+	if pprofEnable || config.Common.PProfEnable {
+		pprof.Start()
+	}
+
+	args := clients.Args{
+		ConnectionsPerCPU: connect