move args to config package

logger package rewrite as dlog

77 files changed, 1350 insertions, 548 deletions

diff --git a/.gitignore b/.gitignore
index c9e8333..1f158b6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,9 @@
 *_proprietary.go
-<<<<<<< HEAD
+*.csv
+*.tmp
 cache/
 log/
 tags
-=======
 /cache/
 /log/
 /dtail
@@ -11,4 +11,3 @@ tags
 /dcat
 /dmap
 /dserver
->>>>>>> 7ee0121afed3e7cab6457142f70e411020ab2b21
diff --git a/Makefile b/Makefile
index 2109e56..b544388 100644
--- a/Makefile
+++ b/Makefile
@@ -3,18 +3,18 @@ all: test build
 build: dserver dcat dgrep dmap dtail
 dserver:
 ifndef USE_ACL
-	${GO} build -o dserver ./cmd/dserver/main.go
+	${GO} build ${GO_FLAGS} -o dserver ./cmd/dserver/main.go
 else
-	${GO} build -tags linuxacl -o dserver ./cmd/dserver/main.go
+	${GO} build ${GO_FLAGS} -tags linuxacl -o dserver ./cmd/dserver/main.go
 endif
 dcat:
-	${GO} build -o dcat ./cmd/dcat/main.go
+	${GO} build ${GO_FLAGS} -o dcat ./cmd/dcat/main.go
 dgrep:
-	${GO} build -o dgrep ./cmd/dgrep/main.go
+	${GO} build ${GO_FLAGS} -o dgrep ./cmd/dgrep/main.go
 dmap:
-	${GO} build -o dmap ./cmd/dmap/main.go
+	${GO} build ${GO_FLAGS} -o dmap ./cmd/dmap/main.go
 dtail:
-	${GO} build -o dtail ./cmd/dtail/main.go
+	${GO} build ${GO_FLAGS} -o dtail ./cmd/dtail/main.go
 install:
 ifndef USE_ACL
 	${GO} install ./cmd/dserver/main.go
diff --git a/TODO.md b/TODO.md
index 156c745..49fc9b9 100644
--- a/TODO.md
+++ b/TODO.md
@@ -15,20 +15,17 @@ This is a loose list of what to do. Maybe for the next releae or maybe for a lat
 [?] Client 4.x should print a warning when trying to connect to a 3.x server.
 [ ] Update docs for color configuration
 [ ] Update animated gifs
-[x] Fix dmap so that it always reads to the end of file
 [ ] Add more default fields to the default MAPREDUCE format.
 [x] By default connect to localhost
 [x] Can use additional args as file lists
 [ ] Document the two things above
 [x] Implement spartan mode
-[x] Make sure that diff is the same (plain file fs dcatted-file) in spartan mode
-[ ] Document servless mode
+[ ] Document serverless mode
 [x] Implement serverless mode
-[ ] Fix serverless mode (e.g. dmap doesn't aggregate all lines)
 [ ] document spartan mode
 [ ] Default client log dir is ~/log not ./log
-  [ ] Make sure dmap results aren't in color in local log file
-[ ] Unit test for dcat in serverless mode
-[ ] Unit test for dgrep in serverless mode
-[ ] Unit test for dmap in serverless mode
+[ ] Integration test for dcat in serverless mode
+[ ] Integration test for dgrep in serverless mode
+[ ] Integration test for dmap in serverless mode
 [ ] Separate logger into server logger and client logger for serverless operation (e.g. server info logs are all Debug)
+[ ] In serverless, use prefix LOCAL and not REMOTE. And also use another color schema (magenta?)
diff --git a/cmd/dcat/main.go b/cmd/dcat/main.go
index 63b4b61..d5dfba4 100644
--- a/cmd/dcat/main.go
+++ b/cmd/dcat/main.go
@@ -4,10 +4,11 @@ import (
 	"context"
 	"flag"
 	"os"
+	"sync"
 
 	"github.com/mimecast/dtail/internal/clients"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/signal"
 	"github.com/mimecast/dtail/internal/user"
 	"github.com/mimecast/dtail/internal/version"
@@ -15,33 +16,28 @@ import (
 
 // The evil begins here.
 func main() {
-	var args clients.Args
-	var cfgFile string
-	var debugEnable bool
+	var args config.Args
 	var displayVersion bool
-	var sshPort int
 
 	userName := user.Name()
 
+	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
 	flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode")
 	flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode")
-	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys")
-	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
+	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
-	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
 	flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
-	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
+	flag.IntVar(&args.SSHPort, "port", 2222, "SSH server port")
+	flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path")
 	flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method")
+	flag.StringVar(&args.LogLevel, "logLevel", "", "Log level")
 	flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key")
 	flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect")
 	flag.StringVar(&args.UserName, "user", userName, "Your system user name")
 	flag.StringVar(&args.What, "files", "", "File(s) to read")
-	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
 
 	flag.Parse()
-	args.Transform(flag.Args())
-	config.Read(cfgFile, sshPort, args.NoColor)
-	args.TransformAfterConfigFile()
+	config.Setup(&args, flag.Args())
 
 	if displayVersion {
 		version.PrintAndExit()
@@ -50,11 +46,10 @@ func main() {
 		version.Print()
 	}
 
-	ctx := context.TODO()
-	logger.Start(ctx, logger.Modes{
-		Debug: debugEnable || config.Common.DebugEnable,
-		Quiet: args.Quiet,
-	})
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	dlog.Start(ctx, &wg, dlog.CLIENT, config.Common.LogLevel)
 
 	client, err := clients.NewCatClient(args)
 	if err != nil {
@@ -62,6 +57,8 @@ func main() {
 	}
 
 	status := client.Start(ctx, signal.InterruptCh(ctx))
-	logger.Flush()
+	cancel()
+
+	wg.Wait()
 	os.Exit(status)
 }
diff --git a/cmd/dgrep/main.go b/cmd/dgrep/main.go
index 7b96472..c6bece0 100644
--- a/cmd/dgrep/main.go
+++ b/cmd/dgrep/main.go
@@ -4,10 +4,11 @@ import (
 	"context"
 	"flag"
 	"os"
+	"sync"
 
 	"github.com/mimecast/dtail/internal/clients"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/signal"
 	"github.com/mimecast/dtail/internal/user"
 	"github.com/mimecast/dtail/internal/version"
@@ -15,37 +16,32 @@ import (
 
 // The evil begins here.
 func main() {
-	var args clients.Args
-	var cfgFile string
-	var debugEnable bool
+	var args config.Args
 	var displayVersion bool
 	var grep string
-	var sshPort int
 
 	userName := user.Name()
 
+	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
 	flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode")
 	flag.BoolVar(&args.RegexInvert, "invert", false, "Invert regex")
 	flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode")
-	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys")
-	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
+	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
-	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
 	flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
-	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
+	flag.IntVar(&args.SSHPort, "port", 2222, "SSH server port")
+	flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path")
 	flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method")
+	flag.StringVar(&args.LogLevel, "logLevel", "", "Log level")
 	flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key")
 	flag.StringVar(&args.RegexStr, "regex", ".", "Regular expression")
 	flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect")
 	flag.StringVar(&args.UserName, "user", userName, "Your system user name")
 	flag.StringVar(&args.What, "files", "", "File(s) to read")
-	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
 	flag.StringVar(&grep, "grep", "", "Alias for -regex")
 
 	flag.Parse()
-	args.Transform(flag.Args())
-	config.Read(cfgFile, sshPort, args.NoColor)
-	args.TransformAfterConfigFile()
+	config.Setup(&args, flag.Args())
 
 	if displayVersion {
 		version.PrintAndExit()
@@ -54,11 +50,10 @@ func main() {
 		version.Print()
 	}
 
-	ctx := context.TODO()
-	logger.Start(ctx, logger.Modes{
-		Debug: debugEnable || config.Common.DebugEnable,
-		Quiet: args.Quiet,
-	})
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	dlog.Start(ctx, &wg, dlog.CLIENT, args.LogLevel)
 
 	if grep != "" {
 		args.RegexStr = grep
@@ -70,6 +65,8 @@ func main() {
 	}
 
 	status := client.Start(ctx, signal.InterruptCh(ctx))
-	logger.Flush()
+	cancel()
+
+	wg.Wait()
 	os.Exit(status)
 }
diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go
index 4525503..061d859 100644
--- a/cmd/dmap/main.go
+++ b/cmd/dmap/main.go
@@ -4,10 +4,11 @@ import (
 	"context"
 	"flag"
 	"os"
+	"sync"
 
 	"github.com/mimecast/dtail/internal/clients"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/signal"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/user"
@@ -16,39 +17,36 @@ import (
 
 // The evil begins here.
 func main() {
-	var cfgFile string
-	var debugEnable bool
 	var displayVersion bool
 	var queryStr string
-	var sshPort int
 
-	args := clients.Args{
+	args := config.Args{
 		Mode: omode.MapClient,
 	}
 
 	userName := user.Name()
 
+	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
 	flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode")
 	flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode")
-	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys")
-	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
+	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
-	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
+	// TODO: Make ConnectionsPerCPU default value as a constant in config package.
 	flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
+	// TODO: make default ssh port a constant in the config package.
+	flag.IntVar(&args.SSHPort, "port", 2222, "SSH server port")
 	flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection")
-	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
+	flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path")
 	flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method")
+	flag.StringVar(&args.LogLevel, "logLevel", "", "Log level")
 	flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key")
 	flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect")
 	flag.StringVar(&args.UserName, "user", userName, "Your system user name")
 	flag.StringVar(&args.What, "files", "", "File(s) to read")
-	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
 	flag.StringVar(&queryStr, "query", "", "Map reduce query")
 
 	flag.Parse()
-	args.Transform(flag.Args())
-	config.Read(cfgFile, sshPort, args.NoColor)
-	args.TransformAfterConfigFile()
+	config.Setup(&args, flag.Args())
 
 	if displayVersion {
 		version.PrintAndExit()
@@ -57,18 +55,19 @@ func main() {
 		version.Print()
 	}
 
-	ctx := context.TODO()
-	logger.Start(ctx, logger.Modes{
-		Debug: debugEnable || config.Common.DebugEnable,
-		Quiet: args.Quiet,
-	})
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	dlog.Start(ctx, &wg, dlog.CLIENT, config.Common.LogLevel)
 
 	client, err := clients.NewMaprClient(args, queryStr, clients.DefaultMode)
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Client.FatalPanic(err)
 	}
 
 	status := client.Start(ctx, signal.InterruptCh(ctx))
-	logger.Flush()
+	cancel()
+
+	wg.Wait()
 	os.Exit(status)
 }
diff --git a/cmd/dserver/main.go b/cmd/dserver/main.go
index bc3cb91..5788a87 100644
--- a/cmd/dserver/main.go
+++ b/cmd/dserver/main.go
@@ -9,11 +9,12 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 	"time"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/server"
 	"github.com/mimecast/dtail/internal/user"
 	"github.com/mimecast/dtail/internal/version"
@@ -21,31 +22,31 @@ import (
 
 // The evil begins here.
 func main() {
-	var cfgFile string
+	var args config.Args
 	var color bool
-	var debugEnable bool
 	var displayVersion bool
 	var logDir string
 	var pprof int
 	var shutdownAfter int
-	var sshPort int
 
 	user.NoRootCheck()
 
 	flag.BoolVar(&color, "color", false, "Enable ANSII terminal colors")
 	flag.BoolVar(&config.ServerRelaxedAuthEnable, "relaxedAuth", false, "Enable relaxced SSH auth mode (don't use in production!)")
-	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
+	flag.IntVar(&args.SSHPort, "port", 2222, "SSH server port")
 	flag.IntVar(&pprof, "pprof", -1, "Start PProf server this port")
-	flag.IntVar(&shutdownAfter, "shutdownAfter", 0, "Automatically shutdown after so many seconds")
-	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
-	flag.StringVar(&cfgFile, "cfg", "", "Config file path")
+	flag.IntVar(&shutdownAfter, "shutdownAfter", 0, "Shutdown after so many seconds")
+	flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path")
+	flag.StringVar(&args.LogLevel, "logLevel", "", "Log level")
 	flag.StringVar(&logDir, "logDir", "", "Log dir path")
 
 	flag.Parse()
-	config.Read(cfgFile, sshPort, !color)
+	args.NoColor = !color
+	config.Setup(&args, flag.Args())
 
 	if logDir != "" {
+		// TODO: Re-Implement log strategy support.
 		config.Common.LogDir = logDir
 		if config.Common.LogStrategy == "" {
 			config.Common.LogStrategy = "daily"
@@ -73,25 +74,25 @@ func main() {
 		}
 	}()
 
-	if debugEnable {
-		config.Common.DebugEnable = true
-	}
-
-	logger.Start(ctx, logger.Modes{Server: true, Debug: debugEnable || config.Common.DebugEnable})
+	var wg sync.WaitGroup
+	wg.Add(1)
+	dlog.Start(ctx, &wg, dlog.SERVER, config.Common.LogLevel)
 
 	if config.ServerRelaxedAuthEnable {
-		logger.Fatal("SSH relaxed-auth mode enabled")
+		dlog.Server.Fatal("SSH relaxed-auth mode enabled")
 	}
 
 	if pprof > -1 {
-		// For debugging purposes only
+		// Start of pprof server for development purposes only.
 		pprofArgs := fmt.Sprintf("0.0.0.0:%d", pprof)
-		logger.Info("Starting PProf", pprofArgs)
+		dlog.Server.Info("Starting PProf", pprofArgs)
 		go http.ListenAndServe(pprofArgs, nil)
 	}
 
 	serv := server.New()
 	status := serv.Start(ctx)
-	logger.Flush()
+	cancel()
+
+	wg.Wait()
 	os.Exit(status)
 }
diff --git a/cmd/dtail/main.go b/cmd/dtail/main.go
index 5c2d393..dcf1fab 100644
--- a/cmd/dtail/main.go
+++ b/cmd/dtail/main.go
@@ -8,12 +8,13 @@ import (
 	_ "net/http"
 	_ "net/http/pprof"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/mimecast/dtail/internal/clients"
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/signal"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/user"
@@ -22,41 +23,41 @@ import (
 
 // The evil begins here.
 func main() {
-	var args clients.Args
-	var cfgFile string
+	var args config.Args
 	var checkHealth bool
-	var debugEnable bool
 	var displayColorTable bool
+	var displayWideColorTable bool
 	var displayVersion bool
 	var grep string
 	var pprof int
 	var queryStr string
 	var shutdownAfter int
-	var sshPort int
 
 	userName := user.Name()
 
+	flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors")
 	flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode")
 	flag.BoolVar(&args.RegexInvert, "invert", false, "Invert regex")
 	flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode")
-	flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys")
+	flag.BoolVar(&args.Tr