From 92f2ac65f3fd2fc6b086d23447676aaf5549ad04 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 8 Dec 2020 14:49:41 +0000
Subject: merge develop

---
 internal/clients/baseclient.go             |   4 +-
 internal/clients/client.go                 |   2 +-
 internal/clients/handlers/basehandler.go   |  23 +++-
 internal/clients/handlers/clienthandler.go |   5 +-
 internal/clients/handlers/handler.go       |   3 +-
 internal/clients/handlers/healthhandler.go |  19 ++-
 internal/clients/handlers/maprhandler.go   |   5 +-
 internal/clients/healthclient.go           |   2 +-
 internal/clients/maprclient.go             |   4 +-
 internal/clients/remote/connection.go      |   7 +-
 internal/clients/stats.go                  |  42 +++++--
 internal/config/config.go                  |   3 +
 internal/config/server.go                  |   1 +
 internal/io/logger/logger.go               |   2 +-
 internal/io/signal/signal.go               |  29 +++--
 internal/mapr/server/aggregate.go          |  81 +++++++------
 internal/regex/flag.go                     |   2 +
 internal/regex/regex.go                    |  11 ++
 internal/server/continuous.go              |   2 +-
 internal/server/handlers/controlhandler.go |  28 +++--
 internal/server/handlers/handler.go        |   2 +
 internal/server/handlers/serverhandler.go  | 182 +++++++----------------------
 internal/server/scheduler.go               |   2 +-
 internal/server/server.go                  |  44 +++----
 internal/version/version.go                |   2 +-
 25 files changed, 241 insertions(+), 266 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 008a01e..69055a3 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -66,7 +66,7 @@ func (c *baseClient) makeConnections(maker maker) {
 	c.stats = newTailStats(len(c.connections))
 }
 
-func (c *baseClient) Start(ctx context.Context, statsCh <-chan struct{}) (status int) {
+func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status int) {
 	// Periodically check for unknown hosts, and ask the user whether to trust them or not.
 	go c.hostKeyCallback.PromptAddHosts(ctx)
 	// Print client stats every time something on statsCh is recieved.
@@ -99,7 +99,7 @@ func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, con
 	defer func() { <-active }()
 
 	for {
-		connCtx, cancel := conn.Handler.WithCancel(ctx)
+		connCtx, cancel := context.WithCancel(ctx)
 		defer cancel()
 
 		conn.Start(connCtx, cancel, c.throttleCh, c.stats.connectionsEstCh)
diff --git a/internal/clients/client.go b/internal/clients/client.go
index eb8452d..4a547e8 100644
--- a/internal/clients/client.go
+++ b/internal/clients/client.go
@@ -4,5 +4,5 @@ import "context"
 
 // Client is the interface for the end user command line client.
 type Client interface {
-	Start(ctx context.Context, statsCh <-chan struct{}) int
+	Start(ctx context.Context, statsCh <-chan string) int
 }
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 65bbfd7..b5045e2 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -8,12 +8,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/version"
 )
 
 type baseHandler struct {
-	withCancel
+	done         *internal.Done
 	server       string
 	shellStarted bool
 	commands     chan string
@@ -29,6 +30,14 @@ func (h *baseHandler) Status() int {
 	return h.status
 }
 
+func (h *baseHandler) Done() <-chan struct{} {
+	return h.done.Done()
+}
+
+func (h *baseHandler) Shutdown() {
+	h.done.Shutdown()
+}
+
 // SendMessage to the server.
 func (h *baseHandler) SendMessage(command string) error {
 	encoded := base64.StdEncoding.EncodeToString([]byte(command))
@@ -38,7 +47,8 @@ func (h *baseHandler) SendMessage(command string) error {
 	case h.commands <- fmt.Sprintf("protocol %s base64 %v;", version.ProtocolCompat, encoded):
 	case <-time.After(time.Second * 5):
 		return fmt.Errorf("Timed out sending command '%s' (base64: '%s')", command, encoded)
-	case <-h.ctx.Done():
+	case <-h.Done():
+		return nil
 	}
 
 	return nil
@@ -65,7 +75,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	select {
 	case command := <-h.commands:
 		n = copy(p, []byte(command))
-	case <-h.ctx.Done():
+	case <-h.Done():
 		return 0, io.EOF
 	}
 	return
@@ -95,10 +105,11 @@ func (h *baseHandler) handleHiddenMessage(message string) {
 	case strings.HasPrefix(message, ".syn close connection"):
 		h.SendMessage(".ack close connection")
 		select {
-		case <-time.After(time.Second * 1):
+		case <-time.After(time.Second * 5):
 			logger.Debug("Shutting down client after timeout and sending ack to server")
-			h.withCancel.shutdown()
-		case <-h.ctx.Done():
+			h.Shutdown()
+		case <-h.Done():
+			return
 		}
 
 	case strings.HasPrefix(message, ".run exitstatus"):
diff --git a/internal/clients/handlers/clienthandler.go b/internal/clients/handlers/clienthandler.go
index fcd8052..2bcb038 100644
--- a/internal/clients/handlers/clienthandler.go
+++ b/internal/clients/handlers/clienthandler.go
@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
 )
 
@@ -19,9 +20,7 @@ func NewClientHandler(server string) *ClientHandler {
 			shellStarted: false,
 			commands:     make(chan string),
 			status:       -1,
-			withCancel: withCancel{
-				done: make(chan struct{}),
-			},
+			done:         internal.NewDone(),
 		},
 	}
 }
diff --git a/internal/clients/handlers/handler.go b/internal/clients/handlers/handler.go
index c53ca34..afa87e2 100644
--- a/internal/clients/handlers/handler.go
+++ b/internal/clients/handlers/handler.go
@@ -1,7 +1,6 @@
 package handlers
 
 import (
-	"context"
 	"io"
 )
 
@@ -11,6 +10,6 @@ type Handler interface {
 	SendMessage(command string) error
 	Server() string
 	Status() int
-	WithCancel(ctx context.Context) (context.Context, context.CancelFunc)
+	Shutdown()
 	Done() <-chan struct{}
 }
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 9051015..08ed137 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -4,11 +4,13 @@ import (
 	"errors"
 	"fmt"
 	"time"
+
+	"github.com/mimecast/dtail/internal"
 )
 
 // HealthHandler implements the handler required for health checks.
 type HealthHandler struct {
-	withCancel
+	done *internal.Done
 	// Buffer of incoming data from server.
 	receiveBuf []byte
 	// To send commands to the server.
@@ -27,9 +29,7 @@ func NewHealthHandler(server string, receive chan<- string) *HealthHandler {
 		receive:  receive,
 		commands: make(chan string),
 		status:   -1,
-		withCancel: withCancel{
-			done: make(chan struct{}),
-		},
+		done:     internal.NewDone(),
 	}
 
 	return &h
@@ -45,12 +45,23 @@ func (h *HealthHandler) Status() int {
 	return h.status
 }
 
+// Done returns done channel of the handler.
+func (h *HealthHandler) Done() <-chan struct{} {
+	return h.done.Done()
+}
+
+// Shutdown the handler.
+func (h *HealthHandler) Shutdown() {
+	h.done.Shutdown()
+}
+
 // SendMessage sends a DTail command to the server.
 func (h *HealthHandler) SendMessage(command string) error {
 	select {
 	case h.commands <- fmt.Sprintf("%s;", command):
 	case <-time.NewTimer(time.Second * 10).C:
 		return errors.New("Timed out sending command " + command)
+	case <-h.Done():
 	}
 
 	return nil
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index b908f3b..fb71c8f 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -3,6 +3,7 @@ package handlers
 import (
 	"strings"
 
+	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/client"
@@ -24,9 +25,7 @@ func NewMaprHandler(server string, query *mapr.Query, globalGroup *mapr.GlobalGr
 			shellStarted: false,
 			commands:     make(chan string),
 			status:       -1,
-			withCancel: withCancel{
-				done: make(chan struct{}),
-			},
+			done:         internal.NewDone(),
 		},
 		query:     query,
 		aggregate: client.NewAggregate(server, query, globalGroup),
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index 7313583..e93f6be 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -50,7 +50,7 @@ func (c *HealthClient) Start(ctx context.Context) (status int) {
 	conn.Handler = handlers.NewHealthHandler(c.server, receive)
 	conn.Commands = []string{c.mode.String()}
 
-	connCtx, cancel := conn.Handler.WithCancel(ctx)
+	connCtx, cancel := context.WithCancel(ctx)
 	go conn.Start(connCtx, cancel, throttleCh, statsCh)
 
 	for {
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 581db44..6aadd6b 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -94,7 +94,7 @@ func NewMaprClient(args Args, queryStr string, maprClientMode MaprClientMode) (*
 }
 
 // Start starts the mapreduce client.
-func (c *MaprClient) Start(ctx context.Context, statsCh <-chan struct{}) (status int) {
+func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status int) {
 	go c.periodicReportResults(ctx)
 
 	status = c.baseClient.Start(ctx, statsCh)
@@ -123,7 +123,7 @@ func (c MaprClient) makeCommands() (commands []string) {
 			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout, modeStr, file, c.Regex.Serialize()))
 			continue
 		}
-		commands = append(commands, fmt.Sprintf("%s %s regex %s", modeStr, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s %s %s", modeStr, file, c.Regex.Serialize()))
 	}
 
 	return
diff --git a/internal/clients/remote/connection.go b/internal/clients/remote/connection.go
index 2d97d14..b29ffed 100644
--- a/internal/clients/remote/connection.go
+++ b/internal/clients/remote/connection.go
@@ -177,21 +177,21 @@ func (c *Connection) handle(ctx context.Context, cancel context.CancelFunc, sess
 	}
 
 	go func() {
-		defer cancel()
 		io.Copy(stdinPipe, c.Handler)
+		cancel()
 	}()
 
 	go func() {
-		defer cancel()
 		io.Copy(c.Handler, stdoutPipe)
+		cancel()
 	}()
 
 	go func() {
-		defer cancel()
 		select {
 		case <-c.Handler.Done():
 		case <-ctx.Done():
 		}
+		cancel()
 	}()
 
 	// Send all commands to client.
@@ -207,5 +207,6 @@ func (c *Connection) handle(ctx context.Context, cancel context.CancelFunc, sess
 	}
 
 	<-ctx.Done()
+	c.Handler.Shutdown()
 	return nil
 }
diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index a6ac0c5..17343b5 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -4,9 +4,11 @@ import (
 	"context"
 	"fmt"
 	"runtime"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
 )
 
@@ -32,16 +34,18 @@ func newTailStats(connectionsTotal int) *stats {
 
 // Start starts printing client connection stats every time a signal is recieved or
 // connection count has changed.
-func (s *stats) Start(ctx context.Context, throttleCh, statsCh <-chan struct{}) {
+func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string) {
 	var connectedLast int
 
 	for {
 		var force bool
+		var messages []string
 
 		select {
-		case <-statsCh:
+		case message := <-statsCh:
+			messages = append(messages, message)
 			force = true
-		case <-time.After(time.Second * 2):
+		case <-time.After(time.Second * 10):
 		case <-ctx.Done():
 			return
 		}
@@ -54,7 +58,15 @@ func (s *stats) Start(ctx context.Context, throttleCh, statsCh <-chan struct{})
 		if connected == connectedLast && !force {
 			continue
 		}
-		s.log(connected, newConnections, throttle)
+
+		stats := s.statsLine(connected, newConnections, throttle)
+		switch force {
+		case true:
+			messages = append(messages, fmt.Sprintf("Connection stats: %s", stats))
+			s.printStatsOnInterrupt(messages)
+		default:
+			logger.Info(stats)
+		}
 
 		connectedLast = connected
 		s.mutex.Lock()
@@ -63,15 +75,25 @@ func (s *stats) Start(ctx context.Context, throttleCh, statsCh <-chan struct{})
 	}
 }
 
-func (s *stats) log(connected, newConnections int, throttle int) {
+func (s *stats) printStatsOnInterrupt(messages []string) {
+	logger.Pause()
+	for _, message := range messages {
+		fmt.Println(fmt.Sprintf(" %s", message))
+	}
+	time.Sleep(time.Second * time.Duration(config.InterruptTimeoutS))
+	logger.Resume()
+}
+
+func (s *stats) statsLine(connected, newConnections int, throttle int) string {
 	percConnected := percentOf(float64(s.connectionsTotal), float64(connected))
 
-	connectedStr := fmt.Sprintf("connected=%d/%d(%d%%)", connected, s.connectionsTotal, int(percConnected))
-	newConnStr := fmt.Sprintf("new=%d", newConnections)
-	throttleStr := fmt.Sprintf("throttle=%d", throttle)
-	cpusGoroutinesStr := fmt.Sprintf("cpus/goroutines=%d/%d", runtime.NumCPU(), runtime.NumGoroutine())
+	var stats []string
+	stats = append(stats, fmt.Sprintf("connected=%d/%d(%d%%)", connected, s.connectionsTotal, int(percConnected)))
+	stats = append(stats, fmt.Sprintf("new=%d", newConnections))
+	stats = append(stats, fmt.Sprintf("throttle=%d", throttle))
+	stats = append(stats, fmt.Sprintf("cpus/goroutines=%d/%d", runtime.NumCPU(), runtime.NumGoroutine()))
 
-	logger.Info("stats", connectedStr, newConnStr, throttleStr, cpusGoroutinesStr)
+	return strings.Join(stats, "|")
 }
 
 func (s *stats) numConnected() int {
diff --git a/internal/config/config.go b/internal/config/config.go
index dc96d6b..276ddcf 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -15,6 +15,9 @@ const ScheduleUser string = "DTAIL-SCHEDULE"
 // ContinuousUser is used for non-interactive continuous mapreduce queries.
 const ContinuousUser string = "DTAIL-CONTINUOUS"
 
+// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
+const InterruptTimeoutS int = 3
+
 // Client holds a DTail client configuration.
 var Client *ClientConfig
 
diff --git a/internal/config/server.go b/internal/config/server.go
index db12cec..dc0d587 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -61,6 +61,7 @@ type ServerConfig struct {
 	Continuous []Continuous `json:",omitempty"`
 }
 
+// ServerRelaxedAuthEnable should be used for development and testing purposes only.
 var ServerRelaxedAuthEnable bool
 
 // Create a new default server configuration.
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index d059cbb..b7db0a7 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -224,7 +224,7 @@ func log(what string, severity string, args []interface{}) string {
 		return ""
 	}
 
-	messages := []string{severity}
+	messages := []string{}
 
 	for _, arg := range args {
 		switch v := arg.(type) {
diff --git a/internal/io/signal/signal.go b/internal/io/signal/signal.go
index bca7e6e..500c530 100644
--- a/internal/io/signal/signal.go
+++ b/internal/io/signal/signal.go
@@ -5,24 +5,37 @@ import (
 	"os"
 	gosignal "os/signal"
 	"syscall"
+	"time"
+
+	"github.com/mimecast/dtail/internal/config"
 )
 
-// StatsCh returns a channel for "please print stats" signalling.
-func StatsCh(ctx context.Context) <-chan struct{} {
-	sigCh := make(chan os.Signal)
-	gosignal.Notify(sigCh, syscall.SIGINFO, syscall.SIGUSR1)
+// InterruptCh returns a channel for "please print stats" signalling.
+func InterruptCh(ctx context.Context) <-chan string {
+	sigIntCh := make(chan os.Signal)
+	gosignal.Notify(sigIntCh, os.Interrupt)
+
+	sigOtherCh := make(chan os.Signal)
+	gosignal.Notify(sigOtherCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT)
 
-	statsCh := make(chan struct{})
+	statsCh := make(chan string)
 
 	go func() {
 		for {
 			select {
-			case <-sigCh:
+			case <-sigIntCh:
 				select {
-				case statsCh <- struct{}{}:
+				case statsCh <- "Hint: Hit Ctrl+C again to exit":
+					select {
+					case <-sigIntCh:
+						os.Exit(0)
+					case <-time.After(time.Second * time.Duration(config.InterruptTimeoutS)):
+					}
 				default:
-					// Stats currently already printed.
+					// Stats already printed.
 				}
+			case <-sigOtherCh:
+				os.Exit(0)
 			case <-ctx.Done():
 				return
 			}
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 1028943..28bb074 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
@@ -15,6 +16,7 @@ import (
 
 // Aggregate is for aggregating mapreduce data on the DTail server side.
 type Aggregate struct {
+	done *internal.Done
 	// Log lines to process (parsing MAPREDUCE lines).
 	Lines chan line.Line
 	// Hostname of the current server (used to populate $hostname field).
@@ -23,12 +25,12 @@ type Aggregate struct {
 	serialize chan struct{}
 	// Signals to flush data.
 	flush chan struct{}
+	// Signals that data has been flushed
+	flushed chan struct{}
 	// The mapr query
 	query *mapr.Query
 	// The mapr log format parser
 	parser *logformat.Parser
-	cancel context.CancelFunc
-	ctx    context.Context
 }
 
 // NewAggregate return a new server side aggregator.
@@ -64,56 +66,64 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 		}
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
-
 	a := Aggregate{
+		done:      internal.NewDone(),
 		Lines:     make(chan line.Line, 100),
 		serialize: make(chan struct{}),
 		flush:     make(chan struct{}),
+		flushed:   make(chan struct{}),
 		hostname:  s[0],
 		query:     query,
 		parser:    logParser,
-		ctx:       ctx,
-		cancel:    cancel,
 	}
 
 	return &a, nil
 }
 
+// Shutdown the aggregation engine.
+func (a *Aggregate) Shutdown() {
+	a.Flush()
+	a.done.Shutdown()
+}
+
 // Start an aggregation.
 func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
-	defer a.cancel()
 
-	fieldsCh := a.linesToFields(ctx)
+	myCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	go func() {
+		select {
+		case <-myCtx.Done():
+			a.done.Shutdown()
+		case <-a.done.Done():
+			cancel()
+		}
+	}()
+
+	fieldsCh := a.makeFields(myCtx)
 
 	// Add fields (e.g. via 'set' clause)
 	if len(a.query.Set) > 0 {
-		fieldsCh = a.addMoreFields(ctx, fieldsCh)
+		fieldsCh = a.addFields(myCtx, fieldsCh)
 	}
 
-	go a.fieldsToMaprLines(ctx, fieldsCh, maprLines)
-	a.periodicAggregateTimer(ctx)
+	go a.aggregateTimer(myCtx)
+	a.makeMaprLines(myCtx, fieldsCh, maprLines)
 }
 
-// Cancel the aggregation.
-func (a *Aggregate) Cancel() {
-	a.cancel()
-}
-
-func (a *Aggregate) periodicAggregateTimer(ctx context.Context) {
+func (a *Aggregate) aggregateTimer(ctx context.Context) {
 	for {
 		select {
 		case <-time.After(a.query.Interval):
 			a.Serialize(ctx)
 		case <-ctx.Done():
 			return
-		case <-a.ctx.Done():
-			return
 		}
 	}
 }
 
-func (a *Aggregate) linesToFields(ctx context.Context) <-chan map[string]string {
+func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 	ch := make(chan map[string]string)
 
 	go func() {
@@ -144,8 +154,6 @@ func (a *Aggregate) linesToFields(ctx context.Context) <-chan map[string]string
 				}
 			case <-ctx.Done():
 				return
-			case <-a.ctx.Done():
-				return
 			}
 		}
 	}()
@@ -153,14 +161,14 @@ func (a *Aggregate) linesToFields(ctx context.Context) <-chan map[string]string
 	return ch
 }
 
-func (a *Aggregate) addMoreFields(ctx context.Context, fieldsCh <-chan map[string]string) <-chan map[string]string {
+func (a *Aggregate) addFields(ctx context.Context, fieldsCh <-chan map[string]string) <-chan map[string]string {
 	ch := make(chan map[string]string)
 
 	go func() {
 		defer close(ch)
 
 		for {
-			// fieldsCh will be closed via 'linesToFields' if ctx is done
+			// fieldsCh will be closed via 'makeFields' if ctx is done
 			fields, ok := <-fieldsCh
 			if !ok {
 				return
@@ -179,7 +187,7 @@ func (a *Aggregate) addMoreFields(ctx context.Context, fieldsCh <-chan map[strin
 	return ch
 }
 
-func (a *Aggregate) fieldsToMaprLines(ctx context.Context, fieldsCh <-chan map[string]string, maprLines chan<- string) {
+func (a *Aggregate) makeMaprLines(ctx context.Context, fieldsCh <-chan map[string]string, maprLines chan<- string) {
 	group := mapr.NewGroupSet()
 
 	serialize := func() {
@@ -200,18 +208,10 @@ func (a *Aggregate) fieldsToMaprLines(ctx context.Context, fieldsCh <-chan map[s
 		case <-a.serialize:
 			serialize()
 		case <-a.flush:
-			logger.Info("Flushing mapreduce result")
 			serialize()
-			a.flush <- struct{}{}
-			logger.Info("Done flushing mapreduce result")
+			a.flushed <- struct{}{}
 		case <-ctx.Done():
 			return
-		case <-a.ctx.Done():
-			logger.Info("Flushing mapreduce result")
-			serialize()
-			a.flush <- struct{}{}
-			logger.Info("Done flushing mapreduce result")
-			return
 		}
 	}
 }
@@ -254,6 +254,8 @@ func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 func (a *Aggregate) Serialize(ctx context.Context) {
 	select {
 	case a.serialize <- struct{}{}:
+	case <-time.After(time.Minute):
+		logger.Warn("Starting to serialize mapredice data takes over a minute")
 	case <-ctx.Done():
 	}
 }
@@ -261,15 +263,20 @@ func (a *Aggregate) Serialize(ctx context.Context) {
 // Flush all data.
 func (a *Aggregate) Flush() {
 	select {
-	case <-a.ctx.Done():
-		return
 	case a.flush <- struct{}{}:
+		logger.Info("Flushing mapreduce data")
 	case <-time.After(time.Minute):
+		logger.Warn("Starting to flush mapreduce data takes over a minute")
+		return
+	case <-a.done.Done():
 		return
 	}
 
 	select {
-	case <-a.flush:
+	case <-a.flushed:
+		logger.Info("Done flushing")
 	case <-time.After(time.Minute):
+		logger.Warn("Waiting for data to be flushed takes over a minute")
+	case <-a.done.Done():
 	}
 }
diff --git a/internal/regex/flag.go b/internal/regex/flag.go
index d3ff712..396bda0 100644
--- a/internal/regex/flag.go
+++ b/internal/regex/flag.go
@@ -2,6 +2,7 @@ package regex
 
 import "fmt"
 
+// Flag for regex.
 type Flag int
 
 const (
@@ -15,6 +16,7 @@ const (
 	Noop Flag = iota
 )
 
+// NewFlag returns a new regex flag.
 func NewFlag(str string) (Flag, error) {
 	switch str {
 	case "default":
diff --git a/internal/regex/regex.go b/internal/regex/regex.go
index 707cb48..2561659 100644
--- a/internal/regex/regex.go
+++ b/internal/regex/regex.go
@@ -8,6 +8,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/logger"
 )
 
+// Regex for filtering lines.
 type Regex struct {
 	// The original regex string
 	regexStr string
@@ -24,6 +25,7 @@ func (r Regex) String() string {
 		r.regexStr, r.flags, r.initialized, r.re == nil)
 }
 
+// NewNoop is a noop regex (doing nothing).
 func NewNoop() Regex {
 	return Regex{
 		flags:       []Flag{Noop},
@@ -31,6 +33,7 @@ func NewNoop() Regex {
 	}
 }
 
+// New returns a new regex object.
 func New(regexStr string, flag Flag) (Regex, error) {
 	if regexStr == "" || regexStr == "." || regexStr == ".*" {
 		return NewNoop(), nil
@@ -39,6 +42,10 @@ func New(regexStr string, flag Flag) (Regex, error) {
 }
 
 func new(regexStr string, flags []Flag) (Regex, error) {
+	if len(flags) == 0 {
+		flags = append(flags, Default)
+	}
+
 	r := Regex{
 		regexStr: regexStr,
 		flags:    flags,
@@ -55,6 +62,7 @@ func new(regexStr string, flags []Flag) (Regex, error) {
 	return r, nil
 }
 
+// Match a byte string.
 func (r Regex) Match(bytes []byte) bool {
 	switch r.flags[0] {
 	case Default:
@@ -68,6 +76,7 @@ func (r Regex) Match(bytes []byte) bool {
 	}
 }
 
+// MatchString matches a string.
 func (r Regex) MatchString(str string) bool {
 	switch r.flags[0] {
 	case Default:
@@ -81,6 +90,7 @@ func (r Regex) MatchString(str string) bool {
 	}
 }
 
+// Serialize the regex.
 func (r Regex) Serialize() string {
 	var flags []string
 	for _, flag := range r.flags {
@@ -94,6 +104,7 @@ func (r Regex) Serialize() string {
 	return fmt.Sprintf("regex:%s %s", strings.Join(flags, ","), r.regexStr)
 }
 
+// Deserialize the regex.
 func Deserialize(str string) (Regex, error) {
 	// Get regex string
 	s := strings.SplitN(str, " ", 2)
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index 583d136..f75c732 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -92,7 +92,7 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	}
 
 	logger.Info(fmt.Sprintf("Starting job %s", job.Name))
-	status := client.Start(jobCtx, make(chan struct{}))
+	status := client.Start(jobCtx, make(chan string))
 	logMessage := fmt.Sprintf("Job exited with status %d", status)
 
 	if status != 0 {
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
index daa9835..8cc5a40 100644
--- a/internal/server/handlers/controlhandler.go
+++ b/internal/server/handlers/controlhandler.go
@@ -1,20 +1,19 @@
 package handlers
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
 	"strings"
 
+	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
 // ControlHandler is used for control functions and health monitoring.
 type ControlHandler struct {
-	ctx            context.Context
-	done           chan struct{}
+	done           *internal.Done
 	hostname       string
 	payload        []byte
 	serverMessages chan string
@@ -22,12 +21,11 @@ type ControlHandler struct {
 }
 
 // NewControlHandler returns a new control handler.
-func NewControlHandler(ctx context.Context, user *user.User) (*ControlHandler, <-chan struct{}) {
+func NewControlHandler(user *user.User) *ControlHandler {
 	logger.Debug(user, "Creating control handler")
 
 	h := ControlHandler{
-		ctx:            ctx,
-		done:           make(chan struct{}),
+		done:           internal.NewDone(),
 		serverMessages: make(chan string, 10),
 		user:           user,
 	}
@@ -40,7 +38,17 @@ func NewControlHandler(ctx context.Context, user *user.User) (*ControlHandler, <
 	s := strings.Split(fqdn, ".")
 	h.hostname = s[0]
 
-	return &h, h.done
+	return &h
+}
+
+// Shutdown the handler.
+func (h *ControlHandler) Shutdown() {
+	h.done.Shutdown()
+}
+
+// Done channel of the handler.
+func (h *ControlHandler) Done() <-chan struct{} {
+	return h.done.Done()
 }
 
 // Read is to send data to the client via the Reader interface.
@@ -51,7 +59,7 @@ func (h *ControlHandler) Read(p []byte) (n int, err error) {
 			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
 			n = copy(p, wholePayload)
 			return
-		case <-h.ctx.Done():
+		case <-h.done.Done():
 			return 0, io.EOF
 		}
 	}
@@ -63,7 +71,7 @@ func (h *ControlHandler) Write(p []byte) (n int, err error) {
 		switch c {
 		case ';':
 			wholePayload := strings.TrimSpace(string(h.payload))
-			h.handleCommand(h.ctx, wholePayload)
+			h.handleCommand(wholePayload)
 			h.payload = nil
 
 		default:
@@ -75,7 +83,7 @@ func (h *ControlHandler) Write(p []byte) (n int, err error) {
 	return
 }
 
-func (h *ControlHandler) handleCommand(ctx context.Context, command string) {
+func (h *ControlHandler) handleCommand(command string) {
 	logger.Info(h.user, command)
 	s := strings.Split(command, " ")
 	logger.Debug(h.user, "Receiving command", command, s)
diff --git a/internal/server/handlers/handler.go b/internal/server/handlers/handler.go
index c42ceb9..b04e854 100644
--- a/internal/server/handlers/handler.go
+++ b/internal/server/handlers/handler.go
@@ -5,4 +5,6 @@ import "io"
 // Handler interface for server side functionality.
 type Handler interface {
 	io.ReadWriter
+	Shutdown()
+	Done() <-chan struct{}
 }
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 7017f3e..5cf8041 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -7,18 +7,16 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
-	"github.com/mimecast/dtail/internal/server/background"
 	user "github.com/mimecast/dtail/internal/user/server"
 	"github.com/mimecast/dtail/internal/version"
 )
@@ -31,33 +29,27 @@ const (
 // the Bi-directional communication between SSH client and server.
 // This handler implements the handler of the SSH server.
 type ServerHandler struct {
-	lines              chan line.Line
-	regex              string
-	aggregate          *server.Aggregate
-	aggregatedMessages chan string
-	serverMessages     chan string
-	payload            []byte
-	hostname           string
-	user               *user.User
-	// TODO: Move all these channels into a separate struct for readability!
+	done                *internal.Done
+	lines               chan line.Line
+	regex               string
+	aggregate           *server.Aggregate
+	aggregatedMessages  chan string
+	serverMessages      chan string
+	payload             []byte
+	hostname            string
+	user                *user.User
 	catLimiter          chan struct{}
 	tailLimiter         chan struct{}
 	globalServerWaitFor chan struct{}
 	ackCloseReceived    chan struct{}
-	serverCtx           context.Context
-	handlerCtx          context.Context
-	done                chan struct{}
 	activeCommands      int32
 	activeReaders       int32
-	background          background.Background
 }
 
 // NewServerHandler returns the server handler.
-func NewServerHandler(handlerCtx, serverCtx context.Context, user *user.User, catLimiter, tailLimiter, globalServerWaitFor chan struct{}, background background.Background) (*ServerHandler, <-chan struct{}) {
+func NewServerHandler(user *user.User, catLimiter, tailLimiter, globalServerWaitFor chan struct{}) *ServerHandler {
 	h := ServerHandler{
-		serverCtx:           serverCtx,
-		handlerCtx:          handlerCtx,
-		done:                make(chan struct{}),
+		done:                internal.NewDone(),
 		lines:               make(chan line.Line, 100),
 		serverMessages:      make(chan string, 10),
 		aggregatedMessages:  make(chan string, 10),
@@ -67,7 +59,6 @@ func NewServerHandler(handlerCtx, serverCtx context.Context, user *user.User, ca
 		globalServerWaitFor: globalServerWaitFor,
 		regex:               ".",
 		user:                user,
-		background:          background,
 	}
 
 	fqdn, err := os.Hostname()
@@ -78,7 +69,17 @@ func NewServerHandler(handlerCtx, serverCtx context.Context, user *user.User, ca
 	s := strings.Split(fqdn, ".")
 	h.hostname = s[0]
 
-	return &h, h.done
+	return &h
+}
+
+// Shutdown the handler.
+func (h *ServerHandler) Shutdown() {
+	h.done.Shutdown()
+}
+
+// Done channel of the handler.
+func (h *ServerHandler) Done() <-chan struct{} {
+	return h.done.Done()
 }
 
 // Read is to send data to the dtail client via Reader interface.
@@ -120,7 +121,7 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		case <-time.After(time.Second):
 			// Once in a while check whether we are done.
 			select {
-			case <-h.handlerCtx.Done():
+			case <-h.done.Done():
 				return 0, io.EOF
 			default:
 			}
@@ -134,7 +135,7 @@ func (h *ServerHandler) Write(p []byte) (n int, err error) {
 		switch c {
 		case ';':
 			commandStr := strings.TrimSpace(string(h.payload))
-			h.handleCommand(h.handlerCtx, commandStr)
+			h.handleCommand(commandStr)
 			h.payload = nil
 		default:
 			h.payload = append(h.payload, c)
@@ -145,9 +146,9 @@ func (h *ServerHandler) Write(p []byte) (n int, err error) {
 	return
 }
 
-func (h *ServerHandler) handleCommand(ctx context.Context, commandStr string) {
+func (h *ServerHandler) handleCommand(commandStr string) {
 	logger.Debug(h.user, commandStr)
-	var timeout time.Duration
+	ctx := context.Background()
 
 	args, argc, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
 	if err != nil {
@@ -161,30 +162,18 @@ func (h *ServerHandler) handleCommand(ctx context.Context, commandStr string) {
 		return
 	}
 
-	args, argc, timeout, err = h.handleTimeout(args, argc)
-	if err != nil {
-		h.send(h.serverMessages, logger.Error(h.user, err))
-		return
-	}
-
 	if h.user.Name == config.ControlUser {
 		h.handleControlCommand(argc, args)
 		return
 	}
 
-	if timeout > 0 {
-		logger.Info(h.user, "Command with timeout context", argc, args, timeout)
-		commandCtx, cancel := context.WithTimeout(ctx, timeout)
-		go func() {
-			<-commandCtx.Done()
-			logger.Info(h.user, "Command timed out, canceling it", args, args, timeout)
-			cancel()
-		}()
-		h.handleUserCommand(commandCtx, argc, args, timeout)
-		return
-	}
+	ctx, cancel := context.WithCancel(ctx)
+	go func() {
+		<-h.done.Done()
+		cancel()
+	}()
 
-	h.handleUserCommand(ctx, argc, args, timeout)
+	h.handleUserCommand(ctx, argc, args)
 }
 
 func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, error) {
@@ -222,16 +211,6 @@ func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, er
 	return args, argc, nil
 }
 
-func (h *ServerHandler) handleTimeout(args []string, argc int) ([]string, int, time.Duration, error) {
-	if argc <= 2 || args[0] != "timeout" {
-		// No timeout specified
-		return args, argc, time.Duration(0) * time.Second, nil
-	}
-
-	timeout, err := strconv.Atoi(args[1])
-	return args[2:], argc - 2, time.Duration(timeout) * time.Second, err
-}
-
 func (h *ServerHandler) handleControlCommand(argc int, args []string) {
 	switch args[0] {
 	case "debug":
@@ -241,7 +220,7 @@ func (h *ServerHandler) handleControlCommand(argc int, args []string) {
 	}
 }
 
-func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string, timeout time.Duration) {
+func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string) {
 	logger.Debug(h.user, "handleUserCommand", argc, args)
 
 	h.incrementActiveCommands()
@@ -255,7 +234,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			if h.aggregate == nil {
 				return
 			}
-			h.aggregate.Cancel()
+			h.aggregate.Shutdown()
 		}
 	}
 
@@ -303,86 +282,6 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			commandFinished()
 		}()
 
-	case "run":
-		// TODO: Refactor this "run" case, move code to runcommand.go
-		command := newRunCommand(h)
-
-		jobName, _ := options["jobName"]
-		logger.Debug(h.user, "run", options)
-
-		if val, ok := options["background"]; ok && (val == "cancel" || val == "stop") {
-			if err := h.background.Cancel(h.user.Name, jobName); err != nil {
-				h.sendServerMessage(logger.Error(h.user, err, jobName, args))
-			} else {
-				h.sendServerMessage(logger.Info(h.user, "job cancelled", jobName))
-			}
-			commandFinished()
-			return
-		}
-
-		if val, ok := options["background"]; ok && val == "list" {
-			h.sendServerMessage("Listing jobs")
-			count := 0
-			for jobName := range h.background.ListJobsC(h.user.Name) {
-				h.sendServerMessage(jobName)
-				count++
-			}
-			h.sendServerMessage(fmt.Sprintf("Found %d jobs", count))
-			commandFinished()
-			return
-		}
-
-		str, _ := options["outerArgs"]
-		outerArgs := strings.Split(str, " ")
-
-		var background bool
-		if val, ok := options["background"]; ok && val == "start" {
-			background = true
-		}
-
-		var wg sync.WaitGroup
-		wg.Add(1)
-
-		if background {
-			if timeout == 0 {
-				// Set default background timeout.
-				timeout = time.Hour * 1
-			}
-			// Use a new context based on the server context, so that background job does not get
-			// terminated when handler/SSH connection terminates.
-			commandCtx, cancel := context.WithTimeout(h.serverCtx, timeout)
-
-			if err := h.background.Add(h.user.Name, jobName, cancel, &wg); err != nil {
-				h.sendServerMessage(logger.Error(h.user, err, jobName, args))
-				commandFinished()
-				return
-			}
-			ctx = commandCtx
-		}
-
-		if err := command.StartBackground(ctx, &wg, argc, args, outerArgs); err != nil {
-			h.sendServerMessage(logger.Error(h.user, "Unable to execute command", argc, args, err))
-			commandFinished()
-			return
-		}
-
-		// Make sure that server waits for all sub-processes to finish on shutdown
-		go func() { h.globalServerWaitFor <- struct{}{} }()
-		go func() {
-			wg.Wait()
-			<-h.globalServerWaitFor
-		}()
-
-		if background {
-			h.sendServerMessage(logger.Info(h.user, jobName, "job started in background"))
-			commandFinished()
-			return
-		}
-
-		// Command run in foreground, wait for it to complete before finishing the connection.
-		wg.Wait()
-		commandFinished()
-
 	case "ack", ".ack":
 		h.handleAckCommand(argc, args)
 		commandFinished()
@@ -406,7 +305,7 @@ func (h *ServerHandler) handleAckCommand(argc int, args []string) {
 func (h *ServerHandler) send(ch chan<- string, message string) {
 	select {
 	case ch <- message:
-	case <-h.handlerCtx.Done():
+	case <-h.done.Done():
 	}
 }
 
@@ -447,7 +346,7 @@ func (h *ServerHandler) shutdown() {
 	go func() {
 		select {
 		case h.serverMessageC() <- ".syn close connection":
-		case <-h.handlerCtx.Done():
+		case <-h.done.Done():
 		}
 	}()
 
@@ -455,13 +354,10 @@ func (h *ServerHandler) shutdown() {
 	case <-h.ackCloseReceived:
 	case <-time.After(time.Second * 5):
 		logger.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
-	case <-h.handlerCtx.Done():
+	case <-h.done.Done():
 	}
 
-	select {
-	case h.done <- struct{}{}:
-	default:
-	}
+	h.done.Shutdown()
 }
 
 func (h *ServerHandler) incrementActiveCommands() {
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index 9d76a3b..a1e9e36 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -93,7 +93,7 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		defer cancel()
 
 		logger.Info(fmt.Sprintf("Starting job %s", job.Name))
-		status := client.Start(jobCtx, make(chan struct{}))
+		status := client.Start(jobCtx, make(chan string))
 		logMessage := fmt.Sprintf("Job exited with status %d", status)
 
 		if status != 0 {
diff --git a/internal/server/server.go b/internal/server/server.go
index a446738..31fa85d 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -11,7 +11,6 @@ import (
 
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/server/background"
 	"github.com/mimecast/dtail/internal/server/handlers"
 	"github.com/mimecast/dtail/internal/ssh/server"
 	user "github.com/mimecast/dtail/internal/user/server"
@@ -35,9 +34,8 @@ type Server struct {
 	// Mointor log files for pattern (if configured)
 	cont *continuous
 	// Wait counter, e.g. there might be still subprocesses (forked by drun) to be killed.
+	// TODO: Remove this counter.
 	shutdownWaitFor chan struct{}
-	// Background jobs
-	background background.Background
 }
 
 // New returns a new server.
@@ -51,7 +49,6 @@ func New() *Server {
 		shutdownWaitFor: make(chan struct{}, 1000),
 		sched:           newScheduler(),
 		cont:            newContinuous(),
-		background:      background.New(),
 	}
 
 	s.sshServerConfig.PasswordCallback = s.Callback
@@ -178,53 +175,46 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 
 		switch req.Type {
 		case "shell":
-			handlerCtx, cancel := context.WithCancel(ctx)
-
 			var handler handlers.Handler
-			var done <-chan struct{}
-
 			switch user.Name {
 			case config.ControlUser:
-				handler, done = handlers.NewControlHandler(handlerCtx, user)
+				handler = handlers.NewControlHandler(user)
 			default:
-				handler, done = handlers.NewServerHandler(handlerCtx, ctx, user, s.catLimiter, s.tailLimiter, s.shutdownWaitFor, s.background)
+				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter, s.shutdownWaitFor)
 			}
 
-			go func() {
-				// Handler finished work, cancel all remaining routines
-				defer cancel()
-
-				<-done
-			}()
+			terminate := func() {
+				handler.Shutdown()
+				sshConn.Close()
+			}
 
 			go func() {
 				// Broken pipe, cancel
-				defer cancel()
-
 				io.Copy(channel, handler)
+				terminate()
 			}()
 
 			go func() {
 				// Broken pipe, cancel
-				defer cancel()
-
 				io.Copy(handler, channel)
+				terminate()
 			}()
 
 			go func() {
-				defer cancel()
+				select {
+				case <-ctx.Done():
+				case <-handler.Done():
+				}
+				terminate()
+			}()
 
+			go func() {
 				if err := sshConn.Wait(); err != nil && err != io.EOF {
 					logger.Error(user, err)
 				}
 				s.stats.decrementConnections()
 				logger.Info(user, "Good bye Mister!")
-			}()
-
-			go func() {
-				<-handlerCtx.Done()
-				sshConn.Close()
-				logger.Info(user, "Closed SSH connection")
+				terminate()
 			}()
 
 			// Only serving shell type
diff --git a/internal/version/version.go b/internal/version/version.go
index 36ef62c..b513b40 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,7 +13,7 @@ const (
 	// Version of DTail.
 	Version string = "3.1.0"
 	// Additional information for DTail
-	Additional string = "develop"
+	Additional string = ""
 	// ProtocolCompat -ibility version.
 	ProtocolCompat string = "3"
 )
-- 
cgit v1.2.3


From 2240769b76c654404ac3e942da73bdcfab94a510 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 8 Dec 2020 15:16:12 +0000
Subject: add done

---
 internal/done.go | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 internal/done.go

(limited to 'internal')

diff --git a/internal/done.go b/internal/done.go
new file mode 100644
index 0000000..54e5e8e
--- /dev/null
+++ b/internal/done.go
@@ -0,0 +1,36 @@
+package internal
+
+import (
+	"sync"
+)
+
+// Done is a cleanup/shutdown helper.
+type Done struct {
+	ch    chan struct{}
+	mutex sync.Mutex
+}
+
+// NewDone returns a new cleanup/shutdown helper.
+func NewDone() *Done {
+	return &Done{
+		ch: make(chan struct{}),
+	}
+}
+
+// Done returns the done channel (closed when done)
+func (d *Done) Done() <-chan struct{} {
+	return d.ch
+}
+
+// Shutdown closes the done channel. It can be called multiple times.
+func (d *Done) Shutdown() {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+
+	select {
+	case <-d.ch:
+		return
+	default:
+		close(d.ch)
+	}
+}
-- 
cgit v1.2.3


From 404cc052f028b6e2ba98c791553e7a561fecf626 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Thu, 10 Dec 2020 14:37:19 +0000
Subject: add hasprefix, nhasprefix, hassuffix, nhassuffix operation support to
 where clause

---
 internal/mapr/wherecondition.go | 22 ++++++++++++++++++++++
 internal/version/version.go     |  4 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)

(limited to 'internal')

diff --git a/internal/mapr/wherecondition.go b/internal/mapr/wherecondition.go
index ff1b489..70e9c32 100644
--- a/internal/mapr/wherecondition.go
+++ b/internal/mapr/wherecondition.go
@@ -19,6 +19,10 @@ const (
 	StringNe            QueryOperation = iota
 	StringContains      QueryOperation = iota
 	StringNotContains   QueryOperation = iota
+	StringHasPrefix     QueryOperation = iota
+	StringNotHasPrefix  QueryOperation = iota
+	StringHasSuffix     QueryOperation = iota
+	StringNotHasSuffix  QueryOperation = iota
 	FloatOperation      QueryOperation = iota
 	FloatEq             QueryOperation = iota
 	FloatNe             QueryOperation = iota
@@ -78,7 +82,17 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 		case "contains":
 			wc.Operation = StringContains
 		case "lacks":
+			fallthrough
+		case "ncontains":
 			wc.Operation = StringNotContains
+		case "hasprefix":
+			wc.Operation = StringHasPrefix
+		case "nhasprefix":
+			wc.Operation = StringNotHasPrefix
+		case "hassuffix":
+			wc.Operation = StringHasSuffix
+		case "nhassuffix":
+			wc.Operation = StringNotHasSuffix
 		default:
 			return wc, nil, errors.New(invalidQuery + "Unknown operation in 'where' clause: " + whereOp)
 		}
@@ -169,6 +183,14 @@ func (wc *whereCondition) stringClause(lValue string, rValue string) bool {
 		return strings.Contains(lValue, rValue)
 	case StringNotContains:
 		return !strings.Contains(lValue, rValue)
+	case StringHasPrefix:
+		return strings.HasPrefix(lValue, rValue)
+	case StringNotHasPrefix:
+		return !strings.HasPrefix(lValue, rValue)
+	case StringHasSuffix:
+		return strings.HasSuffix(lValue, rValue)
+	case StringNotHasSuffix:
+		return !strings.HasSuffix(lValue, rValue)
 	default:
 		logger.Error("Unknown string operation", lValue, wc.Operation, rValue)
 	}
diff --git a/internal/version/version.go b/internal/version/version.go
index b513b40..0c21e26 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -11,9 +11,9 @@ const (
 	// Name of DTail.
 	Name string = "DTail"
 	// Version of DTail.
-	Version string = "3.1.0"
+	Version string = "3.2.0"
 	// Additional information for DTail
-	Additional string = ""
+	Additional string = "develop"
 	// ProtocolCompat -ibility version.
 	ProtocolCompat string = "3"
 )
-- 
cgit v1.2.3


From 629e929c1fca32681c307cea12d0e7c7668add93 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Thu, 10 Dec 2020 15:08:56 +0000
Subject: bump verison

---
 internal/version/version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'internal')

diff --git a/internal/version/version.go b/internal/version/version.go
index 0c21e26..4fa9890 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,7 +13,7 @@ const (
 	// Version of DTail.
 	Version string = "3.2.0"
 	// Additional information for DTail
-	Additional string = "develop"
+	Additional string = "develop2"
 	// ProtocolCompat -ibility version.
 	ProtocolCompat string = "3"
 )
-- 
cgit v1.2.3


From a8058d2a2702e2dcb2cb418fcc7053aca8a1a046 Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Sat, 26 Dec 2020 10:48:51 +0000
Subject: code cleanup and minor refactorings

---
 internal/clients/grepclient.go             |   1 +
 internal/clients/handlers/basehandler.go   |  18 +---
 internal/clients/handlers/healthhandler.go |   1 +
 internal/clients/maker.go                  |   3 +
 internal/clients/runclient.go              |  87 -----------------
 internal/clients/stats.go                  |   7 +-
 internal/clients/tailclient.go             |   1 +
 internal/color/colorfy.go                  |  12 +--
 internal/config/common.go                  |  10 +-
 internal/discovery/comma.go                |   3 +-
 internal/discovery/file.go                 |   3 +-
 internal/io/line/line.go                   |   6 +-
 internal/io/prompt/prompt.go               |   6 +-
 internal/io/run/run.go                     | 150 -----------------------------
 internal/mapr/funcs/function.go            |   1 +
 internal/mapr/query.go                     |   6 --
 internal/mapr/selectcondition.go           |   1 +
 internal/mapr/token.go                     |   3 +
 internal/mapr/wherecondition.go            |   2 +
 internal/omode/mode.go                     |   3 -
 internal/server/handlers/runcommand.go     | 111 ---------------------
 internal/server/handlers/serverhandler.go  |  50 +++++-----
 internal/server/server.go                  |  24 +----
 23 files changed, 68 insertions(+), 441 deletions(-)
 delete mode 100644 internal/clients/runclient.go
 delete mode 100644 internal/io/run/run.go
 delete mode 100644 internal/server/handlers/runcommand.go

(limited to 'internal')

diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index 4024083..e6fc94a 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -44,5 +44,6 @@ func (c GrepClient) makeCommands() (commands []string) {
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s %s %s", c.Mode.String(), file, c.Regex.Serialize()))
 	}
+
 	return
 }
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index b5045e2..f07fd90 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
-	"strconv"
 	"strings"
 	"time"
 
@@ -78,6 +77,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	case <-h.Done():
 		return 0, io.EOF
 	}
+
 	return
 }
 
@@ -111,21 +111,5 @@ func (h *baseHandler) handleHiddenMessage(message string) {
 		case <-h.Done():
 			return
 		}
-
-	case strings.HasPrefix(message, ".run exitstatus"):
-		splitted := strings.Split(strings.TrimSuffix(message, "\n"), " ")
-		if len(splitted) != 3 {
-			logger.Error("Unable to retrieve exitstatus", message)
-			return
-		}
-		i, err := strconv.Atoi(splitted[2])
-		if err != nil {
-			logger.Error("Unable to retrieve exitstatus", message, err)
-			return
-		}
-		logger.Debug("Retrieved exitstatus", h.status)
-		if i > h.status {
-			h.status = i
-		}
 	}
 }
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 08ed137..0440706 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -19,6 +19,7 @@ type HealthHandler struct {
 	receive chan<- string
 	// The remote server address
 	server string
+	// The return status.
 	status int
 }
 
diff --git a/internal/clients/maker.go b/internal/clients/maker.go
index 1ba6482..d5ffd8b 100644
--- a/internal/clients/maker.go
+++ b/internal/clients/maker.go
@@ -4,6 +4,9 @@ import (
 	"github.com/mimecast/dtail/internal/clients/handlers"
 )
 
+// maker interface helps to re-use code in all DTail client implementations.
+// All clients share the baseClient but have different connection handlers
+// and send different commands to the DTail server.
 type maker interface {
 	makeHandler(server string) handlers.Handler
 	makeCommands() (commands []string)
diff --git a/internal/clients/runclient.go b/internal/clients/runclient.go
deleted file mode 100644
index 5464d54..0000000
--- a/internal/clients/runclient.go
+++ /dev/null
@@ -1,87 +0,0 @@
-package clients
-
-import (
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/hex"
-	"fmt"
-	"runtime"
-	"strings"
-
-	"github.com/mimecast/dtail/internal/clients/handlers"
-	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/omode"
-)
-
-// RunClient is a client to run various commands on the server.
-type RunClient struct {
-	baseClient
-	jobName    string
-	background string
-}
-
-// NewRunClient returns a new run client to execute commands on the remote server.
-func NewRunClient(args Args, background, jobName string) (*RunClient, error) {
-	args.Mode = omode.RunClient
-
-	if jobName == "" {
-		jobName = hash(strings.Join(args.Arguments, " "))
-	}
-
-	c := RunClient{
-		baseClient: baseClient{
-			Args:       args,
-			throttleCh: make(chan struct{}, args.ConnectionsPerCPU*runtime.NumCPU()),
-			retry:      false,
-		},
-		jobName:    jobName,
-		background: background,
-	}
-
-	c.init()
-	c.makeConnections(c)
-
-	return &c, nil
-}
-
-func (c RunClient) makeHandler(server string) handlers.Handler {
-	return handlers.NewClientHandler(server)
-}
-
-func (c RunClient) makeCommands() (commands []string) {
-	if c.Timeout > 0 {
-		commands = append(commands, fmt.Sprintf("timeout %d run%s %s", c.Timeout, c.options(), c.What))
-		return
-	}
-
-	commands = append(commands, fmt.Sprintf("run%s %s", c.options(), c.What))
-	return
-}
-
-func (c RunClient) options() string {
-	var sb strings.Builder
-
-	logger.Debug("options", fmt.Sprintf(":background=%s", c.background))
-	sb.WriteString(fmt.Sprintf(":background=%s", c.background))
-
-	logger.Debug("options", fmt.Sprintf(":jobName=%s", c.jobName))
-	sb.WriteString(fmt.Sprintf(":jobName=%s", c.jobName))
-
-	if len(c.Arguments) > 0 {
-		logger.Debug("options", fmt.Sprintf(":outerArgs=base64%%%s", strings.Join(c.Arguments, " ")))
-		sb.WriteString(fmt.Sprintf(":outerArgs=base64%%%s", encode64(strings.Join(c.Arguments, " "))))
-	}
-
-	return sb.String()
-}
-
-func encode64(str string) string {
-	return base64.StdEncoding.EncodeToString([]byte(str))
-}
-
-func hash(str string) string {
-	h := sha256.New()
-	h.Write([]byte(str))
-
-	return hex.EncodeToString(h.Sum(nil))
-}
diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index 17343b5..2ec6f22 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -45,7 +45,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 		case message := <-statsCh:
 			messages = append(messages, message)
 			force = true
-		case <-time.After(time.Second * 10):
+		case <-time.After(time.Second * 3):
 		case <-ctx.Done():
 			return
 		}
@@ -63,7 +63,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 		switch force {
 		case true:
 			messages = append(messages, fmt.Sprintf("Connection stats: %s", stats))
-			s.printStatsOnInterrupt(messages)
+			s.printStatsDueInterrupt(messages)
 		default:
 			logger.Info(stats)
 		}
@@ -75,7 +75,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 	}
 }
 
-func (s *stats) printStatsOnInterrupt(messages []string) {
+func (s *stats) printStatsDueInterrupt(messages []string) {
 	logger.Pause()
 	for _, message := range messages {
 		fmt.Println(fmt.Sprintf(" %s", message))
@@ -107,5 +107,6 @@ func percentOf(total float64, value float64) float64 {
 	if total == 0 || total == value {
 		return 100
 	}
+
 	return value / (total / 100.0)
 }
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index 53b5ba4..ff2f46e 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -29,6 +29,7 @@ func NewTailClient(args Args) (*TailClient, error) {
 
 	c.init()
 	c.makeConnections(c)
+
 	return &c, nil
 }
 
diff --git a/internal/color/colorfy.go b/internal/color/colorfy.go
index 9ae46f5..a2beb7a 100644
--- a/internal/color/colorfy.go
+++ b/internal/color/colorfy.go
@@ -41,16 +41,14 @@ func paintClientStats(line string) string {
 
 // Colorfy a given line based on the line's content.
 func Colorfy(line string) string {
-	if strings.HasPrefix(line, "REMOTE") {
+	switch {
+	case strings.HasPrefix(line, "REMOTE"):
 		return paintRemote(line)
-	}
-	if strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|") {
+	case strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|"):
 		return paintClientStats(line)
-	}
-	if strings.Contains(line, "ERROR") {
+	case strings.Contains(line, "ERROR"):
 		return Paint(Magenta, line)
-	}
-	if strings.Contains(line, "WARN") {
+	case strings.Contains(line, "WARN"):
 		return Paint(Magenta, line)
 	}
 
diff --git a/internal/config/common.go b/internal/config/common.go
index 103b390..c3e203e 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -2,10 +2,14 @@ package config
 
 // CommonConfig stores configuration keys shared by DTail server and client.
 type CommonConfig struct {
-	SSHPort                    int
+	// The SSH port number
+	SSHPort int
+	// Enable experimental features (mainly for dev purposes)
 	ExperimentalFeaturesEnable bool `json:",omitempty"`
-	DebugEnable                bool `json:",omitempty"`
-	TraceEnable                bool `json:",omitempty"`
+	// Enable debug logging. Don't enable in production.
+	DebugEnable bool `json:",omitempty"`
+	// Enable trace logging. Don't enable in production.
+	TraceEnable bool `json:",omitempty"`
 	// The log strategy to use, one of
 	//   stdout: only log to stdout (useful when used with systemd)
 	//   daily: create a log file for every day
diff --git a/internal/discovery/comma.go b/internal/discovery/comma.go
index 94276c7..4344240 100644
--- a/internal/discovery/comma.go
+++ b/internal/discovery/comma.go
@@ -1,8 +1,9 @@
 package discovery
 
 import (
-	"github.com/mimecast/dtail/internal/io/logger"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/io/logger"
 )
 
 // ServerListFromCOMMA retrieves a list of servers from comma separated input list.
diff --git a/internal/discovery/file.go b/internal/discovery/file.go
index c04173e..1250755 100644
--- a/internal/discovery/file.go
+++ b/internal/discovery/file.go
@@ -2,8 +2,9 @@ package discovery
 
 import (
 	"bufio"
-	"github.com/mimecast/dtail/internal/io/logger"
 	"os"
+
+	"github.com/mimecast/dtail/internal/io/logger"
 )
 
 // ServerListFromFILE retrieves a list of servers from a file.
diff --git a/internal/io/line/line.go b/internal/io/line/line.go
index 9db93c0..715be34 100644
--- a/internal/io/line/line.go
+++ b/internal/io/line/line.go
@@ -15,7 +15,11 @@ type Line struct {
 	// lines if that happens but it will signal to the client how
 	// many log lines in % could be transmitted to the client.
 	TransmittedPerc int
-	SourceID        string
+	// Contains the unique identifier of the source log file.
+	// It could be the name of the log or it could be one of the parent
+	// directories in case multiple log files with the same basename are
+	// followed.
+	SourceID string
 }
 
 // Return a human readable representation of the followed line.
diff --git a/internal/io/prompt/prompt.go b/internal/io/prompt/prompt.go
index a438d33..36ebdb5 100644
--- a/internal/io/prompt/prompt.go
+++ b/internal/io/prompt/prompt.go
@@ -3,9 +3,10 @@ package prompt
 import (
 	"bufio"
 	"fmt"
-	"github.com/mimecast/dtail/internal/io/logger"
 	"os"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/io/logger"
 )
 
 // Answer is a user input of a prompt question.
@@ -18,8 +19,7 @@ type Answer struct {
 	Callback func()
 	// Runs after Callback and after logging resumes
 	EndCallback func()
-
-	AskAgain bool
+	AskAgain    bool
 }
 
 // Prompt used for interactive user input.
diff --git a/internal/io/run/run.go b/internal/io/run/run.go
deleted file mode 100644
index 2bb3756..0000000
--- a/internal/io/run/run.go
+++ /dev/null
@@ -1,150 +0,0 @@
-package run
-
-import (
-	"bufio"
-	"context"
-	"io"
-	"os/exec"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
-)
-
-// Run is for execute a command.
-type Run struct {
-	command      string
-	args         []string
-	cmd          *exec.Cmd
-	pgroupKilled chan struct{}
-}
-
-// New returns a new command runner.
-func New(command string, args []string) Run {
-	return Run{
-		command:      command,
-		args:         args,
-		pgroupKilled: make(chan struct{}),
-	}
-}
-
-// StartBackground starts running the command in background.
-func (r Run) StartBackground(ctx context.Context, wg *sync.WaitGroup, ec chan<- int, lines chan<- line.Line) (pid int, err error) {
-	pid = -1
-
-	if len(r.args) > 0 {
-		logger.Debug(r.command, r.args, " ")
-		r.cmd = exec.CommandContext(ctx, r.command, r.args...)
-	} else {
-		logger.Debug(r.command)
-		r.cmd = exec.CommandContext(ctx, r.command)
-	}
-
-	// Create a new process group, so that kill() will only kill this command + pgroup.
-	r.cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
-
-	stdoutPipe, myErr := r.cmd.StdoutPipe()
-	if err != nil {
-		wg.Done()
-		err = myErr
-		return
-	}
-
-	stderrPipe, myErr := r.cmd.StderrPipe()
-	if myErr != nil {
-		wg.Done()
-		err = myErr
-		return
-	}
-
-	if myErr := r.cmd.Start(); err != nil {
-		wg.Done()
-		err = myErr
-		return
-	}
-
-	if r.cmd.Process != nil {
-		pid = r.cmd.Process.Pid
-	}
-
-	commandExited := make(chan struct{})
-
-	var pipeWg sync.WaitGroup
-	pipeWg.Add(2)
-
-	go r.killPgroup(ctx, commandExited, pid)
-	go r.pipeToLines(commandExited, &pipeWg, pid, stdoutPipe, "STDOUT", lines)
-	go r.pipeToLines(commandExited, &pipeWg, pid, stderrPipe, "STDERR", lines)
-
-	go func() {
-		exitCode := 255
-		if waitErr := r.cmd.Wait(); waitErr != nil {
-			if exitError, ok := waitErr.(*exec.ExitError); ok {
-				exitCode = exitError.ExitCode()
-			}
-		}
-		ec <- exitCode
-
-		// Tell pipes we are done
-		close(commandExited)
-		// Wait for process group to be killed
-		<-r.pgroupKilled
-		// Wait for the pipes to flush the contents
-		pipeWg.Wait()
-		// Now the job is truly done
-		wg.Done()
-	}()
-
-	return
-}
-
-func (r Run) pipeToLines(commandExited chan struct{}, wg *sync.WaitGroup, pid int, reader io.Reader, what string, lines chan<- line.Line) {
-	defer wg.Done()
-	bufReader := bufio.NewReader(reader)
-
-	for {
-		time.Sleep(time.Millisecond * 10)
-		lineStr, err := bufReader.ReadString('\n')
-
-		if err != nil {
-			select {
-			case <-commandExited:
-				return
-			default:
-			}
-			continue
-		}
-
-		newLine := line.Line{
-			Content:         []byte(lineStr),
-			Count:           uint64(pid),
-			TransmittedPerc: 100,
-			SourceID:        what,
-		}
-
-		select {
-		case lines <- newLine:
-		case <-commandExited:
-			return
-		}
-	}
-}
-
-func (r Run) killPgroup(ctx context.Context, commandExited chan struct{}, pid int) {
-	if pid == -1 {
-		close(r.pgroupKilled)
-		return
-	}
-
-	if pgid, err := syscall.Getpgid(pid); err == nil {
-		// Kill process group when done
-		select {
-		case <-ctx.Done():
-		case <-commandExited:
-		}
-		syscall.Kill(-pgid, syscall.SIGKILL)
-		close(r.pgroupKilled)
-	}
-}
diff --git a/internal/mapr/funcs/function.go b/internal/mapr/funcs/function.go
index 52aaa98..1a89c3a 100644
--- a/internal/mapr/funcs/function.go
+++ b/internal/mapr/funcs/function.go
@@ -12,6 +12,7 @@ type CallbackFunc func(text string) string
 type Function struct {
 	// Name of the callback function
 	Name string
+	// The Go-callback function to call for this DTail function.
 	call CallbackFunc
 }
 
diff --git a/internal/mapr/query.go b/internal/mapr/query.go
index 7f6b63c..01852da 100644
--- a/internal/mapr/query.go
+++ b/internal/mapr/query.go
@@ -177,12 +177,6 @@ func (q *Query) parse(tokens []token) error {
 		}
 	}
 
-	// Comment out for empty table support, which is "all" log lines.
-	/*
-		if q.Table == "" {
-			return errors.New(invalidQuery + "Empty table specified in 'from' clause")
-		}
-	*/
 	if len(q.Select) < 1 {
 		return errors.New(invalidQuery + "Expected at least one field in 'select' clause but got none")
 	}
diff --git a/internal/mapr/selectcondition.go b/internal/mapr/selectcondition.go
index 1882b7e..d6aa0d4 100644
--- a/internal/mapr/selectcondition.go
+++ b/internal/mapr/selectcondition.go
@@ -92,5 +92,6 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 		}
 		sel = append(sel, sc)
 	}
+
 	return sel, nil
 }
diff --git a/internal/mapr/token.go b/internal/mapr/token.go
index d337bd2..8972188 100644
--- a/internal/mapr/token.go
+++ b/internal/mapr/token.go
@@ -22,6 +22,7 @@ func (t token) isKeyword() bool {
 			return true
 		}
 	}
+
 	return false
 }
 
@@ -94,6 +95,7 @@ func tokensConsumeStr(tokens []token) ([]token, []string) {
 	for _, token := range found {
 		strings = append(strings, token.str)
 	}
+
 	return tokens, strings
 }
 
@@ -104,5 +106,6 @@ func tokensConsumeOptional(tokens []token, optional string) []token {
 	if strings.ToLower(tokens[0].str) == strings.ToLower(optional) {
 		return tokens[1:]
 	}
+
 	return tokens
 }
diff --git a/internal/mapr/wherecondition.go b/internal/mapr/wherecondition.go
index 70e9c32..7a60dba 100644
--- a/internal/mapr/wherecondition.go
+++ b/internal/mapr/wherecondition.go
@@ -170,6 +170,7 @@ func (wc *whereCondition) floatClause(lValue float64, rValue float64) bool {
 	default:
 		logger.Error("Unknown float operation", lValue, wc.Operation, rValue)
 	}
+
 	return false
 }
 
@@ -194,5 +195,6 @@ func (wc *whereCondition) stringClause(lValue string, rValue string) bool {
 	default:
 		logger.Error("Unknown string operation", lValue, wc.Operation, rValue)
 	}
+
 	return false
 }
diff --git a/internal/omode/mode.go b/internal/omode/mode.go
index e29aacc..1aafcfc 100644
--- a/internal/omode/mode.go
+++ b/internal/omode/mode.go
@@ -12,7 +12,6 @@ const (
 	GrepClient   Mode = iota
 	MapClient    Mode = iota
 	HealthClient Mode = iota
-	RunClient    Mode = iota
 )
 
 func (m Mode) String() string {
@@ -29,8 +28,6 @@ func (m Mode) String() string {
 		return "map"
 	case HealthClient:
 		return "health"
-	case RunClient:
-		return "run"
 	default:
 		return "unknown"
 	}
diff --git a/internal/server/handlers/runcommand.go b/internal/server/handlers/runcommand.go
deleted file mode 100644
index 8e5895b..0000000
--- a/internal/server/handlers/runcommand.go
+++ /dev/null
@@ -1,111 +0,0 @@
-package handlers
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/io/run"
-)
-
-type runCommand struct {
-	server *ServerHandler
-	run    run.Run
-}
-
-func newRunCommand(server *ServerHandler) runCommand {
-	return runCommand{
-		server: server,
-	}
-}
-
-func (r runCommand) StartBackground(ctx context.Context, wg *sync.WaitGroup, argc int, args, outerArgs []string) error {
-	if argc < 2 {
-		return fmt.Errorf("%s: args:%v argc:%d", commandParseWarning, args, argc)
-	}
-
-	ec := make(chan int, 1)
-	var pid int
-	var err error
-
-	command := strings.Join(args[1:], " ")
-	if strings.Contains(command, ";") || strings.Contains(command, "\n") {
-		if pid, err = r.startScript(ctx, wg, ec, command, outerArgs); err != nil {
-			r.server.sendServerMessage(".run exitstatus 255")
-			return err
-		}
-		return nil
-	}
-
-	if pid, err = r.start(ctx, wg, ec, strings.TrimSpace(command), outerArgs); err != nil {
-		r.server.sendServerMessage(".run exitstatus 255")
-		return err
-	}
-
-	exitCode := <-ec
-	r.server.sendServerMessage(fmt.Sprintf(".run exitstatus %d", exitCode))
-	r.server.sendServerMessage(logger.Info(fmt.Sprintf("Process %d exited with status %d", pid, exitCode)))
-
-	return nil
-}
-
-func (r runCommand) startScript(ctx context.Context, wg *sync.WaitGroup, ec chan<- int, script string, outerArgs []string) (int, error) {
-	if _, err := os.Stat(config.Common.TmpDir); os.IsNotExist(err) {
-		return -1, err
-	}
-
-	timestamp := time.Now().UnixNano()
-	scriptPath := fmt.Sprintf("%s/%s_%v.sh", config.Common.TmpDir, r.server.user.Name, timestamp)
-
-	// TODO: On dserver startup delete all previously written scripts (there might be left overs due to a crash or so)
-	logger.Debug(r.server.user, "Writing temp script", scriptPath)
-
-	script = fmt.Sprintf("#!/bin/sh\n%s", script)
-	if err := ioutil.WriteFile(scriptPath, []byte(script), 0700); err != nil {
-		return -1, err
-	}
-
-	pid, err := r.start(ctx, wg, ec, scriptPath, outerArgs)
-	go func() {
-		wg.Wait()
-		logger.Debug("Deleting script", scriptPath)
-		os.Remove(scriptPath)
-	}()
-
-	return pid, err
-}
-
-func (r runCommand) start(ctx context.Context, wg *sync.WaitGroup, ec chan<- int, command string, outerArgs []string) (int, error) {
-	if len(command) == 0 {
-		return -1, errors.New("Empty command provided")
-	}
-
-	splitted := strings.Split(command, " ")
-	path := splitted[0]
-	args := splitted[1:]
-	args = append(args, outerArgs...)
-
-	qualifiedPath, err := exec.LookPath(path)
-	if err != nil {
-		return -1, err
-	}
-
-	if !r.server.user.HasFilePermission(qualifiedPath, "runcommands") {
-		return -1, fmt.Errorf("No permission to execute path: %s", qualifiedPath)
-	}
-
-	r.run = run.New(qualifiedPath, args)
-	pid, err := r.run.StartBackground(ctx, wg, ec, r.server.lines)
-	if err != nil {
-		return pid, err
-	}
-	return pid, nil
-}
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 5cf8041..3d1a53d 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -29,36 +29,34 @@ const (
 // the Bi-directional communication between SSH client and server.
 // This handler implements the handler of the SSH server.
 type ServerHandler struct {
-	done                *internal.Done
-	lines               chan line.Line
-	regex               string
-	aggregate           *server.Aggregate
-	aggregatedMessages  chan string
-	serverMessages      chan string
-	payload             []byte
-	hostname            string
-	user                *user.User
-	catLimiter          chan struct{}
-	tailLimiter         chan struct{}
-	globalServerWaitFor chan struct{}
-	ackCloseReceived    chan struct{}
-	activeCommands      int32
-	activeReaders       int32
+	done               *internal.Done
+	lines              chan line.Line
+	regex              string
+	aggregate          *server.Aggregate
+	aggregatedMessages chan string
+	serverMessages     chan string
+	payload            []byte
+	hostname           string
+	user               *user.User
+	catLimiter         chan struct{}
+	tailLimiter        chan struct{}
+	ackCloseReceived   chan struct{}
+	activeCommands     int32
+	activeReaders      int32
 }
 
 // NewServerHandler returns the server handler.
-func NewServerHandler(user *user.User, catLimiter, tailLimiter, globalServerWaitFor chan struct{}) *ServerHandler {
+func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
 	h := ServerHandler{
-		done:                internal.NewDone(),
-		lines:               make(chan line.Line, 100),
-		serverMessages:      make(chan string, 10),
-		aggregatedMessages:  make(chan string, 10),
-		ackCloseReceived:    make(chan struct{}),
-		catLimiter:          catLimiter,
-		tailLimiter:         tailLimiter,
-		globalServerWaitFor: globalServerWaitFor,
-		regex:               ".",
-		user:                user,
+		done:               internal.NewDone(),
+		lines:              make(chan line.Line, 100),
+		serverMessages:     make(chan string, 10),
+		aggregatedMessages: make(chan string, 10),
+		ackCloseReceived:   make(chan struct{}),
+		catLimiter:         catLimiter,
+		tailLimiter:        tailLimiter,
+		regex:              ".",
+		user:               user,
 	}
 
 	fqdn, err := os.Hostname()
diff --git a/internal/server/server.go b/internal/server/server.go
index 31fa85d..a20737e 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -7,7 +7,6 @@ import (
 	"io"
 	"net"
 	"strings"
-	"time"
 
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
@@ -33,9 +32,6 @@ type Server struct {
 	sched *scheduler
 	// Mointor log files for pattern (if configured)
 	cont *continuous
-	// Wait counter, e.g. there might be still subprocesses (forked by drun) to be killed.
-	// TODO: Remove this counter.
-	shutdownWaitFor chan struct{}
 }
 
 // New returns a new server.
@@ -46,7 +42,6 @@ func New() *Server {
 		sshServerConfig: &gossh.ServerConfig{},
 		catLimiter:      make(chan struct{}, config.Server.MaxConcurrentCats),
 		tailLimiter:     make(chan struct{}, config.Server.MaxConcurrentTails),
-		shutdownWaitFor: make(chan struct{}, 1000),
 		sched:           newScheduler(),
 		cont:            newContinuous(),
 	}
@@ -80,27 +75,12 @@ func (s *Server) Start(ctx context.Context) int {
 	go s.cont.start(ctx)
 	go s.listenerLoop(ctx, listener)
 
-	select {
-	case <-ctx.Done():
-		// Wait until all commands/jobs/children are no more!
-		s.wait()
-	}
+	<-ctx.Done()
 
 	// For future use.
 	return 0
 }
 
-func (s *Server) wait() {
-	for {
-		num := len(s.shutdownWaitFor)
-		logger.Debug("Waiting for stuff to finish", num)
-		if num <= 0 {
-			return
-		}
-		time.Sleep(time.Second)
-	}
-}
-
 func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 	logger.Debug("Starting listener loop")
 
@@ -180,7 +160,7 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 			case config.ControlUser:
 				handler = handlers.NewControlHandler(user)
 			default:
-				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter, s.shutdownWaitFor)
+				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter)
 			}
 
 			terminate := func() {
-- 
cgit v1.2.3


From 4b4971ebe2ff7e202666f7c90d882db6a6b21836 Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Sat, 26 Dec 2020 11:44:31 +0000
Subject: initial spartan mode support

---
 internal/clients/args.go                  | 1 +
 internal/clients/baseclient.go            | 2 +-
 internal/clients/catclient.go             | 3 ++-
 internal/clients/grepclient.go            | 3 ++-
 internal/clients/maprclient.go            | 3 ++-
 internal/clients/stats.go                 | 4 ++--
 internal/clients/tailclient.go            | 3 ++-
 internal/server/handlers/serverhandler.go | 7 +++++++
 internal/user/server/user.go              | 4 ++++
 9 files changed, 23 insertions(+), 7 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
index 34fcfa2..cd0f174 100644
--- a/internal/clients/args.go
+++ b/internal/clients/args.go
@@ -22,4 +22,5 @@ type Args struct {
 	SSHAuthMethods     []gossh.AuthMethod
 	SSHHostKeyCallback gossh.HostKeyCallback
 	PrivateKeyPathFile string
+	Spartan            bool
 }
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 69055a3..1e40ff2 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -70,7 +70,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	// Periodically check for unknown hosts, and ask the user whether to trust them or not.
 	go c.hostKeyCallback.PromptAddHosts(ctx)
 	// Print client stats every time something on statsCh is recieved.
-	go c.stats.Start(ctx, c.throttleCh, statsCh)
+	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Spartan)
 	// Keep count of active connections
 	active := make(chan struct{}, len(c.connections))
 
diff --git a/internal/clients/catclient.go b/internal/clients/catclient.go
index d8e9196..50a8d18 100644
--- a/internal/clients/catclient.go
+++ b/internal/clients/catclient.go
@@ -42,8 +42,9 @@ func (c CatClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c CatClient) makeCommands() (commands []string) {
+	options := fmt.Sprintf("spartan=%v", c.Args.Spartan)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s %s %s", c.Mode.String(), file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
 	return
 }
diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index e6fc94a..5fa0ae0 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -41,8 +41,9 @@ func (c GrepClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c GrepClient) makeCommands() (commands []string) {
+	options := fmt.Sprintf("spartan=%v", c.Args.Spartan)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s %s %s", c.Mode.String(), file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
 
 	return
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 6aadd6b..d201d40 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -112,6 +112,7 @@ func (c MaprClient) makeHandler(server string) handlers.Handler {
 
 func (c MaprClient) makeCommands() (commands []string) {
 	commands = append(commands, fmt.Sprintf("map %s", c.query.RawQuery))
+	options := fmt.Sprintf("spartan=%v", c.Args.Spartan)
 
 	modeStr := "cat"
 	if c.Mode == omode.TailClient {
@@ -123,7 +124,7 @@ func (c MaprClient) makeCommands() (commands []string) {
 			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout, modeStr, file, c.Regex.Serialize()))
 			continue
 		}
-		commands = append(commands, fmt.Sprintf("%s %s %s", modeStr, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s", modeStr, options, file, c.Regex.Serialize()))
 	}
 
 	return
diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index 2ec6f22..3ea59b7 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -34,7 +34,7 @@ func newTailStats(connectionsTotal int) *stats {
 
 // Start starts printing client connection stats every time a signal is recieved or
 // connection count has changed.
-func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string) {
+func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string, spartan bool) {
 	var connectedLast int
 
 	for {
@@ -55,7 +55,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 
 		newConnections := connected - connectedLast
 
-		if connected == connectedLast && !force {
+		if (connected == connectedLast || spartan) && !force {
 			continue
 		}
 
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index ff2f46e..853bdf1 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -38,8 +38,9 @@ func (c TailClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c TailClient) makeCommands() (commands []string) {
+	options := fmt.Sprintf("spartan=%v", c.Args.Spartan)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s %s %s", c.Mode.String(), file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
 	logger.Debug(commands)
 
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 3d1a53d..5b948b3 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -43,6 +43,7 @@ type ServerHandler struct {
 	ackCloseReceived   chan struct{}
 	activeCommands     int32
 	activeReaders      int32
+	spartan            bool
 }
 
 // NewServerHandler returns the server handler.
@@ -245,6 +246,12 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		commandFinished()
 		return
 	}
+	if spartan, ok := options["spartan"]; ok {
+		if spartan == "true" {
+			logger.Debug(h.user, "Enabling spartan mode")
+			h.spartan = true
+		}
+	}
 
 	switch commandName {
 	case "grep", "cat":
diff --git a/internal/user/server/user.go b/internal/user/server/user.go
index c4e8b7b..637945c 100644
--- a/internal/user/server/user.go
+++ b/internal/user/server/user.go
@@ -40,6 +40,10 @@ func (u *User) String() string {
 // HasFilePermission is used to determine whether user is alowed to read a file.
 func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission bool) {
 	logger.Debug(u, filePath, permissionType, "Checking config permissions")
+	if config.ServerRelaxedAuthEnable {
+		logger.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
+		return true
+	}
 
 	if u.Name == config.ScheduleUser || u.Name == config.ContinuousUser {
 		// Background user has same permissions as dtail process itself.
-- 
cgit v1.2.3


From fb6fe55a5c97848360dab8bf5bb42be724b21442 Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Sat, 26 Dec 2020 14:00:41 +0000
Subject: rename spartan to quiet

---
 internal/clients/args.go                  |  2 +-
 internal/clients/baseclient.go            |  2 +-
 internal/clients/catclient.go             |  2 +-
 internal/clients/grepclient.go            |  2 +-
 internal/clients/maprclient.go            |  2 +-
 internal/clients/stats.go                 |  4 ++--
 internal/clients/tailclient.go            |  2 +-
 internal/server/handlers/serverhandler.go | 10 +++++-----
 8 files changed, 13 insertions(+), 13 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
index cd0f174..7f782f1 100644
--- a/internal/clients/args.go
+++ b/internal/clients/args.go
@@ -22,5 +22,5 @@ type Args struct {
 	SSHAuthMethods     []gossh.AuthMethod
 	SSHHostKeyCallback gossh.HostKeyCallback
 	PrivateKeyPathFile string
-	Spartan            bool
+	Quiet            bool
 }
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 1e40ff2..572680c 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -70,7 +70,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	// Periodically check for unknown hosts, and ask the user whether to trust them or not.
 	go c.hostKeyCallback.PromptAddHosts(ctx)
 	// Print client stats every time something on statsCh is recieved.
-	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Spartan)
+	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet)
 	// Keep count of active connections
 	active := make(chan struct{}, len(c.connections))
 
diff --git a/internal/clients/catclient.go b/internal/clients/catclient.go
index 50a8d18..b7b6131 100644
--- a/internal/clients/catclient.go
+++ b/internal/clients/catclient.go
@@ -42,7 +42,7 @@ func (c CatClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c CatClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("spartan=%v", c.Args.Spartan)
+	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index 5fa0ae0..652c31b 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -41,7 +41,7 @@ func (c GrepClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c GrepClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("spartan=%v", c.Args.Spartan)
+	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index d201d40..9beea13 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -112,7 +112,7 @@ func (c MaprClient) makeHandler(server string) handlers.Handler {
 
 func (c MaprClient) makeCommands() (commands []string) {
 	commands = append(commands, fmt.Sprintf("map %s", c.query.RawQuery))
-	options := fmt.Sprintf("spartan=%v", c.Args.Spartan)
+	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 
 	modeStr := "cat"
 	if c.Mode == omode.TailClient {
diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index 3ea59b7..d8163d4 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -34,7 +34,7 @@ func newTailStats(connectionsTotal int) *stats {
 
 // Start starts printing client connection stats every time a signal is recieved or
 // connection count has changed.
-func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string, spartan bool) {
+func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string, quiet bool) {
 	var connectedLast int
 
 	for {
@@ -55,7 +55,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 
 		newConnections := connected - connectedLast
 
-		if (connected == connectedLast || spartan) && !force {
+		if (connected == connectedLast || quiet) && !force {
 			continue
 		}
 
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index 853bdf1..cefbaa7 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -38,7 +38,7 @@ func (c TailClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c TailClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("spartan=%v", c.Args.Spartan)
+	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 5b948b3..3212ee1 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -43,7 +43,7 @@ type ServerHandler struct {
 	ackCloseReceived   chan struct{}
 	activeCommands     int32
 	activeReaders      int32
-	spartan            bool
+	quiet            bool
 }
 
 // NewServerHandler returns the server handler.
@@ -246,10 +246,10 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		commandFinished()
 		return
 	}
-	if spartan, ok := options["spartan"]; ok {
-		if spartan == "true" {
-			logger.Debug(h.user, "Enabling spartan mode")
-			h.spartan = true
+	if quiet, ok := options["quiet"]; ok {
+		if quiet == "true" {
+			logger.Debug(h.user, "Enabling quiet mode")
+			h.quiet = true
 		}
 	}
 
-- 
cgit v1.2.3


From 619f6a1d54c6455087b12a4d7522a691c64bb4a3 Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Sat, 26 Dec 2020 14:28:31 +0000
Subject: initial quiet switch

---
 internal/clients/baseclient.go             |  4 ++--
 internal/clients/maprclient.go             |  6 +++---
 internal/io/logger/logger.go               | 22 ++++++++++++++--------
 internal/io/logger/modes.go                |  1 +
 internal/server/handlers/controlhandler.go |  4 +---
 internal/server/handlers/readcommand.go    | 11 +++++------
 internal/server/handlers/serverhandler.go  | 11 +++++++++--
 7 files changed, 35 insertions(+), 24 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 572680c..f20156f 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -39,7 +39,7 @@ type baseClient struct {
 }
 
 func (c *baseClient) init() {
-	logger.Info("Initiating base client")
+	logger.Debug("Initiating base client")
 
 	flag := regex.Default
 	if c.Args.RegexInvert {
@@ -127,7 +127,7 @@ func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMe
 }
 
 func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
-	defer logger.Info("Terminated connection")
+	defer logger.Debug("Terminated connection")
 
 	// We want to have at least one active connection
 	<-active
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 9beea13..1c0c2cc 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -99,7 +99,7 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 
 	status = c.baseClient.Start(ctx, statsCh)
 	if c.cumulative {
-		logger.Info("Received final mapreduce result")
+		logger.Debug("Received final mapreduce result")
 		c.reportResults()
 	}
 
@@ -134,7 +134,7 @@ func (c *MaprClient) periodicReportResults(ctx context.Context) {
 	for {
 		select {
 		case <-time.After(c.query.Interval):
-			logger.Info("Gathering interim mapreduce result")
+			logger.Debug("Gathering interim mapreduce result")
 			c.reportResults()
 		case <-ctx.Done():
 			return
@@ -166,7 +166,7 @@ func (c *MaprClient) printResults() {
 	}
 
 	if numLines == 0 {
-		logger.Info("Empty result set this time...")
+		logger.Warn("Empty result set this time...")
 		return
 	}
 
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index b7db0a7..7674dd1 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -72,12 +72,15 @@ type buf struct {
 func Start(ctx context.Context, mode Modes) {
 	Mode = mode
 
-	if Mode.Nothing {
+	switch {
+	case Mode.Nothing:
 		return
-	}
-
-	if Mode.Trace {
+	case Mode.Quiet:
+		Mode.Trace = false
+		Mode.Debug = false
+	case Mode.Trace:
 		Mode.Debug = true
+	default:
 	}
 
 	strategy := logStrategy()
@@ -87,7 +90,7 @@ func Start(ctx context.Context, mode Modes) {
 	case DailyStrategy:
 		_, err := os.Stat(config.Common.LogDir)
 		Mode.logToFile = !os.IsNotExist(err)
-		Mode.logToStdout = !Mode.Server || Mode.Debug || Mode.Trace
+		Mode.logToStdout = !Mode.Server || Mode.Debug || Mode.Trace || Mode.Quiet
 	case StdoutStrategy:
 		fallthrough
 	default:
@@ -131,11 +134,14 @@ func Info(args ...interface{}) string {
 
 // Warn message logging.
 func Warn(args ...interface{}) string {
-	if Mode.Server {
-		return log(serverStr, warnStr, args)
+	if !Mode.Quiet {
+		if Mode.Server {
+			return log(serverStr, warnStr, args)
+		}
+		return log(clientStr, warnStr, args)
 	}
 
-	return log(clientStr, warnStr, args)
+	return ""
 }
 
 // Error message logging.
diff --git a/internal/io/logger/modes.go b/internal/io/logger/modes.go
index 47dadfe..8864179 100644
--- a/internal/io/logger/modes.go
+++ b/internal/io/logger/modes.go
@@ -6,6 +6,7 @@ type Modes struct {
 	Trace       bool
 	Debug       bool
 	Nothing     bool
+	Quiet       bool
 	logToStdout bool
 	logToFile   bool
 }
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
index 8cc5a40..1e17c78 100644
--- a/internal/server/handlers/controlhandler.go
+++ b/internal/server/handlers/controlhandler.go
@@ -92,9 +92,7 @@ func (h *ControlHandler) handleCommand(command string) {
 	case "health":
 		h.serverMessages <- "OK: DTail SSH Server seems fine"
 		h.serverMessages <- "done;"
-	case "debug":
-		h.serverMessages <- logger.Debug(h.user, "Receiving debug command", command, s)
 	default:
-		h.serverMessages <- logger.Warn(h.user, "Received unknown control command", command, s)
+		h.serverMessages <- logger.Error(h.user, "Received unknown control command", command, s)
 	}
 }
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 0f9207d..5b8ce3a 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -31,14 +31,13 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string) {
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			logger.Error(err)
 			r.server.sendServerMessage(logger.Error(r.server.user, commandParseWarning, err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.sendServerMessage(logger.Warn(r.server.user, commandParseWarning, args, argc))
+		r.server.sendServerWarnMessage(logger.Warn(r.server.user, commandParseWarning, args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re)
@@ -52,7 +51,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex)
 	for {
 		maxRetries--
 		if maxRetries < 0 {
-			r.server.sendServerMessage(logger.Warn(r.server.user, "Giving up to read file(s)"))
+			r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Giving up to read file(s)"))
 			return
 		}
 
@@ -65,7 +64,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex)
 
 		if numPaths := len(paths); numPaths == 0 {
 			logger.Error(r.server.user, "No such file(s) to read", glob)
-			r.server.sendServerMessage(logger.Warn(r.server.user, "Unable to read file(s), check server logs"))
+			r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Unable to read file(s), check server logs"))
 			select {
 			case <-ctx.Done():
 				return
@@ -97,7 +96,7 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGr
 
 	if !r.server.user.HasFilePermission(path, "readfiles") {
 		logger.Error(r.server.user, "No permission to read file", path, globID)
-		r.server.sendServerMessage(logger.Warn(r.server.user, "Unable to read file(s), check server logs"))
+		r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Unable to read file(s), check server logs"))
 		return
 	}
 
@@ -161,6 +160,6 @@ func (r *readCommand) makeGlobID(path, glob string) string {
 		return pathParts[len(pathParts)-1]
 	}
 
-	r.server.sendServerMessage(logger.Error("Empty file path given?", path, glob))
+	r.server.sendServerWarnMessage(logger.Warn("Empty file path given?", path, glob))
 	return ""
 }
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 3212ee1..681598c 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -43,7 +43,7 @@ type ServerHandler struct {
 	ackCloseReceived   chan struct{}
 	activeCommands     int32
 	activeReaders      int32
-	quiet            bool
+	quiet              bool
 }
 
 // NewServerHandler returns the server handler.
@@ -299,7 +299,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 
 func (h *ServerHandler) handleAckCommand(argc int, args []string) {
 	if argc < 3 {
-		h.sendServerMessage(logger.Warn(h.user, commandParseWarning, args, argc))
+		h.sendServerWarnMessage(logger.Warn(h.user, commandParseWarning, args, argc))
 		return
 	}
 	if args[1] == "close" && args[2] == "connection" {
@@ -318,6 +318,13 @@ func (h *ServerHandler) sendServerMessage(message string) {
 	h.send(h.serverMessageC(), message)
 }
 
+func (h *ServerHandler) sendServerWarnMessage(message string) {
+	if h.quiet {
+		return
+	}
+	h.send(h.serverMessageC(), message)
+}
+
 func (h *ServerHandler) serverMessageC() chan<- string {
 	return h.serverMessages
 }
-- 
cgit v1.2.3


From ce1f090d1390cb0bff19a0b5d6ddeeede20bd3ab Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Sat, 26 Dec 2020 14:34:32 +0000
Subject: correctly format server messages

---
 internal/io/logger/logger.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'internal')

diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 7674dd1..4254eef 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -248,7 +248,7 @@ func log(what string, severity string, args []interface{}) string {
 	message := strings.Join(messages, "|")
 	write(what, severity, message)
 
-	return message
+	return fmt.Sprintf("%s|%s", severity, message)
 }
 
 // Raw message logging.
-- 
cgit v1.2.3


From 9396089b1f941ce2c5801f0a9ca1d82fad4d6ade Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Sun, 27 Dec 2020 16:31:03 +0000
Subject: only try to read a file once in cat and grep mode but 10 times in
 tail mode

---
 internal/server/handlers/readcommand.go   | 20 ++++++++------------
 internal/server/handlers/serverhandler.go |  4 ++--
 internal/version/version.go               |  2 +-
 3 files changed, 11 insertions(+), 15 deletions(-)

(limited to 'internal')

diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 5b8ce3a..5bab26f 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -25,7 +25,7 @@ func newReadCommand(server *ServerHandler, mode omode.Mode) *readCommand {
 	}
 }
 
-func (r *readCommand) Start(ctx context.Context, argc int, args []string) {
+func (r *readCommand) Start(ctx context.Context, argc int, args []string, retries int) {
 	re := regex.NewNoop()
 
 	if argc >= 4 {
@@ -40,21 +40,14 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string) {
 		r.server.sendServerWarnMessage(logger.Warn(r.server.user, commandParseWarning, args, argc))
 		return
 	}
-	r.readGlob(ctx, args[1], re)
+	r.readGlob(ctx, args[1], re, retries)
 }
 
-func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex) {
+func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex, retries int) {
 	retryInterval := time.Second * 5
 	glob = filepath.Clean(glob)
 
-	maxRetries := 10
-	for {
-		maxRetries--
-		if maxRetries < 0 {
-			r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Giving up to read file(s)"))
-			return
-		}
-
+	for retryCount := 0; retryCount < retries; retryCount++ {
 		paths, err := filepath.Glob(glob)
 		if err != nil {
 			logger.Warn(r.server.user, glob, err)
@@ -75,8 +68,11 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex)
 		}
 
 		r.readFiles(ctx, paths, glob, re, retryInterval)
-		break
+		return
 	}
+
+	r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Giving up to read file(s)"))
+	return
 }
 
 func (r *readCommand) readFiles(ctx context.Context, paths []string, glob string, re regex.Regex, retryInterval time.Duration) {
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 681598c..185e7c2 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -258,7 +258,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		command := newReadCommand(h, omode.CatClient)
 		go func() {
 			h.incrementActiveReaders()
-			command.Start(ctx, argc, args)
+			command.Start(ctx, argc, args, 1)
 			readerFinished()
 			commandFinished()
 		}()
@@ -267,7 +267,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		command := newReadCommand(h, omode.TailClient)
 		go func() {
 			h.incrementActiveReaders()
-			command.Start(ctx, argc, args)
+			command.Start(ctx, argc, args, 10)
 			readerFinished()
 			commandFinished()
 		}()
diff --git a/internal/version/version.go b/internal/version/version.go
index 4fa9890..a64417f 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,7 +13,7 @@ const (
 	// Version of DTail.
 	Version string = "3.2.0"
 	// Additional information for DTail
-	Additional string = "develop2"
+	Additional string = "develop-3"
 	// ProtocolCompat -ibility version.
 	ProtocolCompat string = "3"
 )
-- 
cgit v1.2.3


From 6c7c70f02d556afebc9f995518e75ceeb68c606f Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Sun, 27 Dec 2020 16:42:03 +0000
Subject: make lint happy

---
 internal/io/fs/permissions/permission_linux.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'internal')

diff --git a/internal/io/fs/permissions/permission_linux.go b/internal/io/fs/permissions/permission_linux.go
index feae729..bbc039b 100644
--- a/internal/io/fs/permissions/permission_linux.go
+++ b/internal/io/fs/permissions/permission_linux.go
@@ -11,7 +11,7 @@ import (
 	"unsafe"
 )
 
-// To check whether user has Linux file system permissions to read a given file.
+// ToRead checks whether user has Linux file system permissions to read a given file.
 func ToRead(user, filePath string) (bool, error) {
 	cUser := C.CString(user)
 	cFilePath := C.CString(filePath)
-- 
cgit v1.2.3


From 40b2bf2856fd1af8bbf8aa29b51603e853071a94 Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Mon, 28 Dec 2020 09:40:38 +0000
Subject: refactor

---
 internal/server/handlers/serverhandler.go | 50 +++++++++++++++----------------
 internal/server/server.go                 | 20 +------------
 2 files changed, 25 insertions(+), 45 deletions(-)

(limited to 'internal')

diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 843eabc..db917bd 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -29,36 +29,34 @@ const (
 // the Bi-directional communication between SSH client and server.
 // This handler implements the handler of the SSH server.
 type ServerHandler struct {
-	done                *internal.Done
-	lines               chan line.Line
-	regex               string
-	aggregate           *server.Aggregate
-	aggregatedMessages  chan string
-	serverMessages      chan string
-	payload             []byte
-	hostname            string
-	user                *user.User
-	catLimiter          chan struct{}
-	tailLimiter         chan struct{}
-	globalServerWaitFor chan struct{}
-	ackCloseReceived    chan struct{}
-	activeCommands      int32
-	activeReaders       int32
+	done               *internal.Done
+	lines              chan line.Line
+	regex              string
+	aggregate          *server.Aggregate
+	aggregatedMessages chan string
+	serverMessages     chan string
+	payload            []byte
+	hostname           string
+	user               *user.User
+	catLimiter         chan struct{}
+	tailLimiter        chan struct{}
+	ackCloseReceived   chan struct{}
+	activeCommands     int32
+	activeReaders      int32
 }
 
 // NewServerHandler returns the server handler.
-func NewServerHandler(user *user.User, catLimiter, tailLimiter, globalServerWaitFor chan struct{}) *ServerHandler {
+func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
 	h := ServerHandler{
-		done:                internal.NewDone(),
-		lines:               make(chan line.Line, 100),
-		serverMessages:      make(chan string, 10),
-		aggregatedMessages:  make(chan string, 10),
-		ackCloseReceived:    make(chan struct{}),
-		catLimiter:          catLimiter,
-		tailLimiter:         tailLimiter,
-		globalServerWaitFor: globalServerWaitFor,
-		regex:               ".",
-		user:                user,
+		done:               internal.NewDone(),
+		lines:              make(chan line.Line, 100),
+		serverMessages:     make(chan string, 10),
+		aggregatedMessages: make(chan string, 10),
+		ackCloseReceived:   make(chan struct{}),
+		catLimiter:         catLimiter,
+		tailLimiter:        tailLimiter,
+		regex:              ".",
+		user:               user,
 	}
 
 	fqdn, err := os.Hostname()
diff --git a/internal/server/server.go b/internal/server/server.go
index 31fa85d..d8d43c9 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -7,7 +7,6 @@ import (
 	"io"
 	"net"
 	"strings"
-	"time"
 
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
@@ -33,9 +32,6 @@ type Server struct {
 	sched *scheduler
 	// Mointor log files for pattern (if configured)
 	cont *continuous
-	// Wait counter, e.g. there might be still subprocesses (forked by drun) to be killed.
-	// TODO: Remove this counter.
-	shutdownWaitFor chan struct{}
 }
 
 // New returns a new server.
@@ -46,7 +42,6 @@ func New() *Server {
 		sshServerConfig: &gossh.ServerConfig{},
 		catLimiter:      make(chan struct{}, config.Server.MaxConcurrentCats),
 		tailLimiter:     make(chan struct{}, config.Server.MaxConcurrentTails),
-		shutdownWaitFor: make(chan struct{}, 1000),
 		sched:           newScheduler(),
 		cont:            newContinuous(),
 	}
@@ -82,25 +77,12 @@ func (s *Server) Start(ctx context.Context) int {
 
 	select {
 	case <-ctx.Done():
-		// Wait until all commands/jobs/children are no more!
-		s.wait()
 	}
 
 	// For future use.
 	return 0
 }
 
-func (s *Server) wait() {
-	for {
-		num := len(s.shutdownWaitFor)
-		logger.Debug("Waiting for stuff to finish", num)
-		if num <= 0 {
-			return
-		}
-		time.Sleep(time.Second)
-	}
-}
-
 func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 	logger.Debug("Starting listener loop")
 
@@ -180,7 +162,7 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 			case config.ControlUser:
 				handler = handlers.NewControlHandler(user)
 			default:
-				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter, s.shutdownWaitFor)
+				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter)
 			}
 
 			terminate := func() {
-- 
cgit v1.2.3


From 8b244ec577f0eb6f34dcf12688ba0e26e2e714a5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <git@mx.buetow.org>
Date: Tue, 29 Dec 2020 08:34:04 +0000
Subject: Make Linux ACL support optional, as it requires CGo and makes the
 binary less portable

---
 internal/io/fs/permissions/permission.go          |   2 +-
 internal/io/fs/permissions/permission_linux.c     | 395 ---------------------
 internal/io/fs/permissions/permission_linux.go    |  33 --
 internal/io/fs/permissions/permission_linux.h     |  60 ----
 internal/io/fs/permissions/permission_linuxacl.c  | 397 ++++++++++++++++++++++
 internal/io/fs/permissions/permission_linuxacl.go |  35 ++
 internal/io/fs/permissions/permission_linuxacl.h  |  62 ++++
 internal/io/fs/permissions/permission_test.go     |   2 +-
 8 files changed, 496 insertions(+), 490 deletions(-)
 delete mode 100644 internal/io/fs/permissions/permission_linux.c
 delete mode 100644 internal/io/fs/permissions/permission_linux.go
 delete mode 100644 internal/io/fs/permissions/permission_linux.h
 create mode 100644 internal/io/fs/permissions/permission_linuxacl.c
 create mode 100644 internal/io/fs/permissions/permission_linuxacl.go
 create mode 100644 internal/io/fs/permissions/permission_linuxacl.h

(limited to 'internal')

diff --git a/internal/io/fs/permissions/permission.go b/internal/io/fs/permissions/permission.go
index 0ed4f17..cc5dd9b 100644
--- a/internal/io/fs/permissions/permission.go
+++ b/internal/io/fs/permissions/permission.go
@@ -1,4 +1,4 @@
-// +build !linux
+// +build !linuxacl
 
 package permissions
 
diff --git a/internal/io/fs/permissions/permission_linux.c b/internal/io/fs/permissions/permission_linux.c
deleted file mode 100644
index cd10525..0000000
--- a/internal/io/fs/permissions/permission_linux.c
+++ /dev/null
@@ -1,395 +0,0 @@
-#include "permission_linux.h"
-
-#ifdef DEBUG
-void debug_print_checker(struct permission_checker *pc) {
-    fprintf(stderr, "DEBUG: user_name:%s (%d)\n",
-        pc->user_name, pc->uid);
-
-    fprintf(stderr, "DEBUG: ngids:%d\n", pc->ngids);
-    int j;
-    for (j = 0; j < pc->ngids; j++) {
-        fprintf(stderr, "DEBUG: %d", pc->gids[j]);
-        struct group *gr = getgrgid(pc->gids[j]);
-        if (gr != NULL)
-            fprintf(stderr, " (%s)", gr->gr_name);
-        fprintf(stderr, "\n");
-    }
-
-    fprintf(stderr, "DEBUG: file_path:%s (%d:%d)\n",
-            pc->file_path, pc->file_stat.st_uid, pc->file_stat.st_gid);
-}
-#endif // DEBUG
-
-int stat_file(struct permission_checker *pc) {
-    if (stat(pc->file_path, &pc->file_stat) != 0)
-      return -1;
-
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: File'%s' is owned by '%d:%d'\n",
-                pc->file_path, pc->file_stat.st_uid, pc->file_stat.st_gid);
-#endif
-
-    return 0;
-}
-
-int get_user_uid(struct permission_checker *pc) {
-    struct passwd *result = NULL;
-
-    size_t bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
-    if (bufsize == -1)
-        bufsize = 16384;
-
-    char *buf = malloc(bufsize);
-    if (buf == NULL) {
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Unabel to allocate bufer while retrieving user '%s'\n", pc->user_name);
-#endif
-        return -1;
-    }
-
-    int rc = getpwnam_r(pc->user_name, &pc->pw, buf, bufsize, &result);
-
-    if (result == NULL) {
-#ifdef DEBUG
-        if (rc == 0) {
-            fprintf(stderr, "DEBUG: No user '%s' found\n", pc->user_name);
-        } else {
-            fprintf(stderr, "DEBUG: Unknown error while retrieving user '%s'\n", pc->user_name);
-        }
-#endif
-
-        free(buf);
-        return -1;
-    }
-
-    pc->uid = pc->pw.pw_uid;
-
-    free(buf);
-    return 0;
-}
-
-int get_user_groups(struct permission_checker *pc) {
-    // First assume we are in 10 groups max
-    pc->ngids = 10;
-    pc->gids = malloc(pc->ngids * sizeof(gid_t));
-
-    if (pc->gids == NULL) {
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Unable to allocate space for gids.");
-#endif
-        return -1;
-    }
-
-    // Try so many times to load group list until it fits into group array.
-    while (getgrouplist(pc->user_name, pc->pw.pw_gid, pc->gids, &pc->ngids) == -1) {
-        // Too many groups, enlarge group array and try again
-        int newngids = pc->ngids + 100;
-        size_t newsize = newngids * sizeof(gid_t);
-
-        if (SIZE_MAX / newngids < sizeof(gid_t)) {
-            // Overflow
-#ifdef DEBUG
-            fprintf(stderr, "DEBUG: Overflow detected.");
-#endif
-            return -1;
-        }
-
-        gid_t *newgids = realloc(pc->gids, newsize);
-        if (newgids == NULL) {
-#ifdef DEBUG
-            fprintf(stderr, "DEBUG: Unable to allocate space for gids.");
-#endif
-            free(pc->gids);
-            return -1;
-        }
-
-        pc->gids = newgids;
-        pc->ngids = newngids;
-    }
-
-    return 0;
-}
-
-int is_member_of_group(struct permission_checker *pc, gid_t gid) {
-    int j;
-    for (j = 0; j < pc->ngids; j++)
-        if (pc->gids[j] == gid)
-            return 1;
-    return 0;
-}
-
-int check_acl_uid_matches(uid_t uid, acl_entry_t entry) {
-    int ret = -1;
-    uid_t *acl_uid = acl_get_qualifier(entry);
-    if (acl_uid == NULL) {
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Unable to retrieve user uid from ACL entry");
-#endif
-        return -1;
-    }
-
-    ret = *acl_uid == uid ? 0 : -1;
-#ifdef DEBUG
-    fprintf(stderr, "DEBUG: ACL user match?: %d <=> %d: %d\n", *acl_uid, uid, ret);
-#endif
-    acl_free(acl_uid);
-    return ret;
-}
-
-int check_acl_gid_matches(gid_t *gids, int ngids, acl_entry_t entry) {
-    int ret = -1;
-    gid_t *acl_gid = acl_get_qualifier(entry);
-    if (acl_gid == NULL) {
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Unable to retrieve user uid from ACL entry");
-#endif
-        return -1;
-    }
-
-    int j;
-    for (j = 0; j < ngids; j++) {
-        if (*acl_gid == gids[j]) {
-#ifdef DEBUG
-            fprintf(stderr, "DEBUG: User is in group %d", *acl_gid);
-#endif
-            ret = 0;
-            break;
-        }
-    }
-
-#ifdef DEBUG
-    fprintf(stderr, "DEBUG: ACL group match?: %d <=> ...: %d\n", *acl_gid, ret);
-#endif
-    acl_free(acl_gid);
-    return ret;
-}
-
-int check_acl(struct permission_checker *pc, const int flag) {
-    // By default user has no read perm.
-    int has_read_perm = 0;
-
-    // By default mask tells that there are read perm. However in order to have
-    // read permissions both, has_read_perm and mask_allows_read_access must be 1!
-    int mask_allows_read_access = 1;
-
-    acl_type_t type = ACL_TYPE_ACCESS;
-    acl_t acl = acl_get_file(pc->file_path, type);
-
-    if (acl == NULL)
-        // Unable to retrieve ACL.
-        return -1;
-
-    // Walk through each entry of this ACL.
-    int id;
-    for (id = ACL_FIRST_ENTRY; ; id = ACL_NEXT_ENTRY) {
-        acl_entry_t entry;
-        if (acl_get_entry(acl, id, &entry) != 1)
-            // No more ACL entries.
-            break;
-
-        acl_tag_t tag;
-        if (acl_get_tag_type(entry, &tag) == -1)
-            // Unable to retrieve ACL tag.
-            return -1;
-
-        switch (tag) {
-            case ACL_USER_OBJ:
-                if (flag == GROUP_CHECK)
-                    continue;
-#ifdef DEBUG
-                fprintf(stderr, "DEBUG: ACL_USER_OBJ\n");
-#endif
-                // Ignore this ACL entry if user is not owner of file.
-                if (pc->uid != pc->file_stat.st_uid)
-                    continue;
-                break;
-            case ACL_USER:
-                if (flag == GROUP_CHECK)
-                    continue;
-#ifdef DEBUG
-                fprintf(stderr, "DEBUG: ACL_USER\n");
-#endif
-                // Ignore this ACL entry if uid does not match.
-                if (check_acl_uid_matches(pc->uid, entry) != 0)
-                    continue;
-                break;
-            case ACL_GROUP_OBJ:
-                if (flag == USER_CHECK)
-                    continue;
-#ifdef DEBUG
-                fprintf(stderr, "DEBUG: ACL_GROUP_OBJ\n");
-#endif
-                // Ignore ACL entry if user is not in group of file.
-                if (!is_member_of_group(pc, pc->file_stat.st_gid))
-                    continue;
-                break;
-            case ACL_GROUP:
-                if (flag == USER_CHECK)
-                    continue;
-#ifdef DEBUG
-                fprintf(stderr, "DEBUG: ACL_GROUP\n");
-#endif
-                // Ignore ACL entry if user is not in group of entry.
-                if (check_acl_gid_matches(pc->gids, pc->ngids, entry) != 0)
-                    continue;
-                break;
-            case ACL_OTHER:
-                if (flag == GROUP_CHECK)
-                    continue;
-#ifdef DEBUG
-                fprintf(stderr, "DEBUG: ACL_OTHER\n");
-#endif
-                break;
-            case ACL_MASK:
-#ifdef DEBUG
-                fprintf(stderr, "DEBUG: ACL_MASK\n");
-#endif
-                break;
-            default:
-#ifdef DEBUG
-                fprintf(stderr, "DEBUG: Unknown ACL tag\n");
-#endif
-                return -1;
-        }
-
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Retrieving permset\n");
-#endif
-        acl_permset_t permset;
-        int permission;
-        if (acl_get_permset(entry, &permset) == -1)
-            // Unable to retrieve permset.
-            return -1;
-
-        if ((permission = acl_get_perm(permset, ACL_READ)) == -1)
-            // Unable to retrieve permset value.
-            return -1;
-
-        if (permission == 1 && tag != ACL_MASK) {
-#ifdef DEBUG
-            fprintf(stderr, "DEBUG: ACL says user has permission to read file.\n");
-#endif
-            has_read_perm = 1;
-        } else if (permission == 0 && tag == ACL_MASK)  {
-            // Mask says that there are no permissions to read.
-            mask_allows_read_access = 0;
-#ifdef DEBUG
-            fprintf(stderr, "DEBUG: ACL mask says no permission to read file.\n");
-#endif
-        }
-    }
-
-    if (has_read_perm && mask_allows_read_access) {
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: ACL end result: User has permission to read file.\n");
-#endif
-        return 1;
-    }
-
-#ifdef DEBUG
-    fprintf(stderr, "DEBUG: ACL end result: User has no permission to read file.\n");
-#endif
-    return 0;
-}
-
-int check_traditional(struct permission_checker *pc, const int flag) {
-    mode_t mode = pc->file_stat.st_mode;
-    uid_t uid = pc->file_stat.st_uid;
-    gid_t gid = pc->file_stat.st_gid;
-
-    if (flag == USER_CHECK && (mode & S_IROTH)) {
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Others can read file '%s'\n",
-                pc->file_path);
-#endif
-        return 1;
-
-    } else if (flag == USER_CHECK && (mode & S_IRUSR) && uid == pc->uid) {
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: User '%s' can read file '%s'\n",
-                pc->user_name, pc->file_path);
-#endif
-        return 1;
-
-    } else if (flag == GROUP_CHECK && (mode & S_IRGRP) && is_member_of_group(pc, gid)) {
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: User's '%s' group can read file '%s'\n",
-                pc->user_name, pc->file_path);
-#endif
-        return 1;
-    }
-
-    return 0;
-}
-
-int permission_to_read(char* user_name, char *file_path) {
-    int rc = -1;
-
-#ifdef DEBUG
-    fprintf(stderr, "DEBUG: User check '%s' for file '%s'\n", user_name, file_path);
-#endif
-    struct permission_checker pc = {
-        .user_name = user_name,
-        .gids = NULL,
-        .ngids = 0,
-        .file_path = file_path,
-    };
-
-    // Gather user's UID.
-    if ((rc = get_user_uid(&pc)) == -1)
-        // Could not retrieve UID.
-        goto cleanup;
-
-    // Gather file owner (user and group).
-    if ((rc = stat_file(&pc)) == -1)
-        // Could not stat file.
-        goto cleanup;
-
-    // Check whether there is an ACL entry which would allow the user
-    // to read the file. Don't check for any groups yet. The issue with
-    // groups is that it can be very slow to retrieve the list of groups
-    // of a specific user when done via a remote LDAP server!
-    if ((rc = check_acl(&pc, USER_CHECK)) == 1)
-        // Yes, has permissions.
-        goto cleanup;
-
-    // Check whether ACLs of file could be retrieved.
-    if (rc == -1) {
-        if (errno != ENOTSUP)
-            // Unknown error.
-            goto cleanup;
-
-        // File system does not support ACLs.
-        // Fallback to traditional permissions.
-        if ((rc = check_traditional(&pc, USER_CHECK)) == 1)
-            // Yes, has traditional permissions.
-            goto cleanup;
-
-        if ((rc = get_user_groups(&pc)) == -1)
-            // Can not retrieve user's groups.
-            goto cleanup;
-
-        rc = check_traditional(&pc, GROUP_CHECK);
-        goto cleanup;
-    }
-
-    if ((rc = get_user_groups(&pc)) == -1)
-        // Can not retrieve use'r groups.
-        goto cleanup;
-
-    // Check whether there is an ACL entry which would allow any of the
-    // user's groups to read the file.
-    rc = check_acl(&pc, GROUP_CHECK);
-
-cleanup:
-#ifdef DEBUG
-    debug_print_checker(&pc);
-#endif
-
-    if (pc.ngids)
-        free(pc.gids);
-
-    return rc;
-}
-
-// vim: set tabstop=8 softtabstop=0 expandtab shiftwidth=4 smarttab
diff --git a/internal/io/fs/permissions/permission_linux.go b/internal/io/fs/permissions/permission_linux.go
deleted file mode 100644
index bbc039b..0000000
--- a/internal/io/fs/permissions/permission_linux.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package permissions
-
-/*
-#include "permission_linux.h"
-#cgo LDFLAGS: -L. -lacl
-*/
-import "C"
-
-import (
-	"errors"
-	"unsafe"
-)
-
-// ToRead checks whether user has Linux file system permissions to read a given file.
-func ToRead(user, filePath string) (bool, error) {
-	cUser := C.CString(user)
-	cFilePath := C.CString(filePath)
-
-	defer C.free(unsafe.Pointer(cUser))
-	defer C.free(unsafe.Pointer(cFilePath))
-
-	cOk, err := C.permission_to_read(cUser, cFilePath)
-	if cOk == 1 {
-		return true, nil
-	}
-
-	if err != nil {
-		// err contains errno message
-		return false, err
-	}
-
-	return false, errors.New("User without permission to read file")
-}
diff --git a/internal/io/fs/permissions/permission_linux.h b/internal/io/fs/permissions/permission_linux.h
deleted file mode 100644
index a2c266e..0000000
--- a/internal/io/fs/permissions/permission_linux.h
+++ /dev/null
@@ -1,60 +0,0 @@
-#ifndef PERMISSION_LINUX_H
-#define PERMISSION_LINUX_H
-
-#include <acl/libacl.h>
-#include <errno.h>
-#include <grp.h>
-#include <pwd.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <sys/acl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-//#define DEBUG
-#define USER_CHECK 0
-#define GROUP_CHECK 1
-
-struct permission_checker {
-  char *user_name;
-  uid_t uid;
-  gid_t *gids;
-  int ngids;
-  char *file_path;
-  struct stat file_stat;
-  struct passwd pw;
-};
-
-
-#ifdef DEBUG
-// Print out permission_checker struct.
-void debug_print_checker(struct permission_checker *pc);
-#endif
-
-// Stat a given file to retrieve traditional UNIX permissions.
-int stat_file(struct permission_checker *pc);
-
-// Retrieve UID of user.
-int get_user_uid(struct permission_checker *pc);
-
-// Retrieve all groups of the user.
-int get_user_groups(struct permission_checker *pc);
-
-// Check whether user is member of a group or not.
-int is_member_of_group(struct permission_checker *pc, gid_t gid);
-
-// Check whether user can read file according Linux ACLs.
-// As flag use either USER_CHECK or GROUP_CHECK.
-int check_acl(struct permission_checker *pc, const int flag);
-
-// Check whether user has permissions to read file according traditional
-// UNIX permissions. As flag use either USER_CHECK or GROUP_CHECK.
-int check_traditional(struct permission_checker *pc, const int flag);
-
-// Returns 1 if user has permission to read file.
-// Returns <0 on error and returns 0 if no permissions.
-int permission_to_read(char* user, char *file_path);
-
-#endif // PERMISSION_LINUX_H
diff --git a/internal/io/fs/permissions/permission_linuxacl.c b/internal/io/fs/permissions/permission_linuxacl.c
new file mode 100644
index 0000000..86b1185
--- /dev/null
+++ b/internal/io/fs/permissions/permission_linuxacl.c
@@ -0,0 +1,397 @@
+// +build linuxacl
+
+#include "permission_linuxacl.h"
+
+#ifdef DEBUG
+void debug_print_checker(struct permission_checker *pc) {
+    fprintf(stderr, "DEBUG: user_name:%s (%d)\n",
+        pc->user_name, pc->uid);
+
+    fprintf(stderr, "DEBUG: ngids:%d\n", pc->ngids);
+    int j;
+    for (j = 0; j < pc->ngids; j++) {
+        fprintf(stderr, "DEBUG: %d", pc->gids[j]);
+        struct group *gr = getgrgid(pc->gids[j]);
+        if (gr != NULL)
+            fprintf(stderr, " (%s)", gr->gr_name);
+        fprintf(stderr, "\n");
+    }
+
+    fprintf(stderr, "DEBUG: file_path:%s (%d:%d)\n",
+            pc->file_path, pc->file_stat.st_uid, pc->file_stat.st_gid);
+}
+#endif // DEBUG
+
+int stat_file(struct permission_checker *pc) {
+    if (stat(pc->file_path, &pc->file_stat) != 0)
+      return -1;
+
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: File'%s' is owned by '%d:%d'\n",
+                pc->file_path, pc->file_stat.st_uid, pc->file_stat.st_gid);
+#endif
+
+    return 0;
+}
+
+int get_user_uid(struct permission_checker *pc) {
+    struct passwd *result = NULL;
+
+    size_t bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
+    if (bufsize == -1)
+        bufsize = 16384;
+
+    char *buf = malloc(bufsize);
+    if (buf == NULL) {
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: Unabel to allocate bufer while retrieving user '%s'\n", pc->user_name);
+#endif
+        return -1;
+    }
+
+    int rc = getpwnam_r(pc->user_name, &pc->pw, buf, bufsize, &result);
+
+    if (result == NULL) {
+#ifdef DEBUG
+        if (rc == 0) {
+            fprintf(stderr, "DEBUG: No user '%s' found\n", pc->user_name);
+        } else {
+            fprintf(stderr, "DEBUG: Unknown error while retrieving user '%s'\n", pc->user_name);
+        }
+#endif
+
+        free(buf);
+        return -1;
+    }
+
+    pc->uid = pc->pw.pw_uid;
+
+    free(buf);
+    return 0;
+}
+
+int get_user_groups(struct permission_checker *pc) {
+    // First assume we are in 10 groups max
+    pc->ngids = 10;
+    pc->gids = malloc(pc->ngids * sizeof(gid_t));
+
+    if (pc->gids == NULL) {
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: Unable to allocate space for gids.");
+#endif
+        return -1;
+    }
+
+    // Try so many times to load group list until it fits into group array.
+    while (getgrouplist(pc->user_name, pc->pw.pw_gid, pc->gids, &pc->ngids) == -1) {
+        // Too many groups, enlarge group array and try again
+        int newngids = pc->ngids + 100;
+        size_t newsize = newngids * sizeof(gid_t);
+
+        if (SIZE_MAX / newngids < sizeof(gid_t)) {
+            // Overflow
+#ifdef DEBUG
+            fprintf(stderr, "DEBUG: Overflow detected.");
+#endif
+            return -1;
+        }
+
+        gid_t *newgids = realloc(pc->gids, newsize);
+        if (newgids == NULL) {
+#ifdef DEBUG
+            fprintf(stderr, "DEBUG: Unable to allocate space for gids.");
+#endif
+            free(pc->gids);
+            return -1;
+        }
+
+        pc->gids = newgids;
+        pc->ngids = newngids;
+    }
+
+    return 0;
+}
+
+int is_member_of_group(struct permission_checker *pc, gid_t gid) {
+    int j;
+    for (j = 0; j < pc->ngids; j++)
+        if (pc->gids[j] == gid)
+            return 1;
+    return 0;
+}
+
+int check_acl_uid_matches(uid_t uid, acl_entry_t entry) {
+    int ret = -1;
+    uid_t *acl_uid = acl_get_qualifier(entry);
+    if (acl_uid == NULL) {
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: Unable to retrieve user uid from ACL entry");
+#endif
+        return -1;
+    }
+
+    ret = *acl_uid == uid ? 0 : -1;
+#ifdef DEBUG
+    fprintf(stderr, "DEBUG: ACL user match?: %d <=> %d: %d\n", *acl_uid, uid, ret);
+#endif
+    acl_free(acl_uid);
+    return ret;
+}
+
+int check_acl_gid_matches(gid_t *gids, int ngids, acl_entry_t entry) {
+    int ret = -1;
+    gid_t *acl_gid = acl_get_qualifier(entry);
+    if (acl_gid == NULL) {
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: Unable to retrieve user uid from ACL entry");
+#endif
+        return -1;
+    }
+
+    int j;
+    for (j = 0; j < ngids; j++) {
+        if (*acl_gid == gids[j]) {
+#ifdef DEBUG
+            fprintf(stderr, "DEBUG: User is in group %d", *acl_gid);
+#endif
+            ret = 0;
+            break;
+        }
+    }
+
+#ifdef DEBUG
+    fprintf(stderr, "DEBUG: ACL group match?: %d <=> ...: %d\n", *acl_gid, ret);
+#endif
+    acl_free(acl_gid);
+    return ret;
+}
+
+int check_acl(struct permission_checker *pc, const int flag) {
+    // By default user has no read perm.
+    int has_read_perm = 0;
+
+    // By default mask tells that there are read perm. However in order to have
+    // read permissions both, has_read_perm and mask_allows_read_access must be 1!
+    int mask_allows_read_access = 1;
+
+    acl_type_t type = ACL_TYPE_ACCESS;
+    acl_t acl = acl_get_file(pc->file_path, type);
+
+    if (acl == NULL)
+        // Unable to retrieve ACL.
+        return -1;
+
+    // Walk through each entry of this ACL.
+    int id;
+    for (id = ACL_FIRST_ENTRY; ; id = ACL_NEXT_ENTRY) {
+        acl_entry_t entry;
+        if (acl_get_entry(acl, id, &entry) != 1)
+            // No more ACL entries.
+            break;
+
+        acl_tag_t tag;
+        if (acl_get_tag_type(entry, &tag) == -1)
+            // Unable to retrieve ACL tag.
+            return -1;
+
+        switch (tag) {
+            case ACL_USER_OBJ:
+                if (flag == GROUP_CHECK)
+                    continue;
+#ifdef DEBUG
+                fprintf(stderr, "DEBUG: ACL_USER_OBJ\n");
+#endif
+                // Ignore this ACL entry if user is not owner of file.
+                if (pc->uid != pc->file_stat.st_uid)
+                    continue;
+                break;
+            case ACL_USER:
+                if (flag == GROUP_CHECK)
+                    continue;
+#ifdef DEBUG
+                fprintf(stderr, "DEBUG: ACL_USER\n");
+#endif
+                // Ignore this ACL entry if uid does not match.
+                if (check_acl_uid_matches(pc->uid, entry) != 0)
+                    continue;
+                break;
+            case ACL_GROUP_OBJ:
+                if (flag == USER_CHECK)
+                    continue;
+#ifdef DEBUG
+                fprintf(stderr, "DEBUG: ACL_GROUP_OBJ\n");
+#endif
+                // Ignore ACL entry if user is not in group of file.
+                if (!is_member_of_group(pc, pc->file_stat.st_gid))
+                    continue;
+                break;
+            case ACL_GROUP:
+                if (flag == USER_CHECK)
+                    continue;
+#ifdef DEBUG
+                fprintf(stderr, "DEBUG: ACL_GROUP\n");
+#endif
+                // Ignore ACL entry if user is not in group of entry.
+                if (check_acl_gid_matches(pc->gids, pc->ngids, entry) != 0)
+                    continue;
+                break;
+            case ACL_OTHER:
+                if (flag == GROUP_CHECK)
+                    continue;
+#ifdef DEBUG
+                fprintf(stderr, "DEBUG: ACL_OTHER\n");
+#endif
+                break;
+            case ACL_MASK:
+#ifdef DEBUG
+                fprintf(stderr, "DEBUG: ACL_MASK\n");
+#endif
+                break;
+            default:
+#ifdef DEBUG
+                fprintf(stderr, "DEBUG: Unknown ACL tag\n");
+#endif
+                return -1;
+        }
+
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: Retrieving permset\n");
+#endif
+        acl_permset_t permset;
+        int permission;
+        if (acl_get_permset(entry, &permset) == -1)
+            // Unable to retrieve permset.
+            return -1;
+
+        if ((permission = acl_get_perm(permset, ACL_READ)) == -1)
+            // Unable to retrieve permset value.
+            return -1;
+
+        if (permission == 1 && tag != ACL_MASK) {
+#ifdef DEBUG
+            fprintf(stderr, "DEBUG: ACL says user has permission to read file.\n");
+#endif
+            has_read_perm = 1;
+        } else if (permission == 0 && tag == ACL_MASK)  {
+            // Mask says that there are no permissions to read.
+            mask_allows_read_access = 0;
+#ifdef DEBUG
+            fprintf(stderr, "DEBUG: ACL mask says no permission to read file.\n");
+#endif
+        }
+    }
+
+    if (has_read_perm && mask_allows_read_access) {
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: ACL end result: User has permission to read file.\n");
+#endif
+        return 1;
+    }
+
+#ifdef DEBUG
+    fprintf(stderr, "DEBUG: ACL end result: User has no permission to read file.\n");
+#endif
+    return 0;
+}
+
+int check_traditional(struct permission_checker *pc, const int flag) {
+    mode_t mode = pc->file_stat.st_mode;
+    uid_t uid = pc->file_stat.st_uid;
+    gid_t gid = pc->file_stat.st_gid;
+
+    if (flag == USER_CHECK && (mode & S_IROTH)) {
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: Others can read file '%s'\n",
+                pc->file_path);
+#endif
+        return 1;
+
+    } else if (flag == USER_CHECK && (mode & S_IRUSR) && uid == pc->uid) {
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: User '%s' can read file '%s'\n",
+                pc->user_name, pc->file_path);
+#endif
+        return 1;
+
+    } else if (flag == GROUP_CHECK && (mode & S_IRGRP) && is_member_of_group(pc, gid)) {
+#ifdef DEBUG
+        fprintf(stderr, "DEBUG: User's '%s' group can read file '%s'\n",
+                pc->user_name, pc->file_path);
+#endif
+        return 1;
+    }
+
+    return 0;
+}
+
+int permission_to_read(char* user_name, char *file_path) {
+    int rc = -1;
+
+#ifdef DEBUG
+    fprintf(stderr, "DEBUG: User check '%s' for file '%s'\n", user_name, file_path);
+#endif
+    struct permission_checker pc = {
+        .user_name = user_name,
+        .gids = NULL,
+        .ngids = 0,
+        .file_path = file_path,
+    };
+
+    // Gather user's UID.
+    if ((rc = get_user_uid(&pc)) == -1)
+        // Could not retrieve UID.
+        goto cleanup;
+
+    // Gather file owner (user and group).
+    if ((rc = stat_file(&pc)) == -1)
+        // Could not stat file.
+        goto cleanup;
+
+    // Check whether there is an ACL entry which would allow the user
+    // to read the file. Don't check for any groups yet. The issue with
+    // groups is that it can be very slow to retrieve the list of groups
+    // of a specific user when done via a remote LDAP server!
+    if ((rc = check_acl(&pc, USER_CHECK)) == 1)
+        // Yes, has permissions.
+        goto cleanup;
+
+    // Check whether ACLs of file could be retrieved.
+    if (rc == -1) {
+        if (errno != ENOTSUP)
+            // Unknown error.
+            goto cleanup;
+
+        // File system does not support ACLs.
+        // Fallback to traditional permissions.
+        if ((rc = check_traditional(&pc, USER_CHECK)) == 1)
+            // Yes, has traditional permissions.
+            goto cleanup;
+
+        if ((rc = get_user_groups(&pc)) == -1)
+            // Can not retrieve user's groups.
+            goto cleanup;
+
+        rc = check_traditional(&pc, GROUP_CHECK);
+        goto cleanup;
+    }
+
+    if ((rc = get_user_groups(&pc)) == -1)
+        // Can not retrieve use'r groups.
+        goto cleanup;
+
+    // Check whether there is an ACL entry which would allow any of the
+    // user's groups to read the file.
+    rc = check_acl(&pc, GROUP_CHECK);
+
+cleanup:
+#ifdef DEBUG
+    debug_print_checker(&pc);
+#endif
+
+    if (pc.ngids)
+        free(pc.gids);
+
+    return rc;
+}
+
+// vim: set tabstop=8 softtabstop=0 expandtab shiftwidth=4 smarttab
diff --git a/internal/io/fs/permissions/permission_linuxacl.go b/internal/io/fs/permissions/permission_linuxacl.go
new file mode 100644
index 0000000..7d2d7ca
--- /dev/null
+++ b/internal/io/fs/permissions/permission_linuxacl.go
@@ -0,0 +1,35 @@
+// +build linuxacl
+
+package permissions
+
+/*
+#include "permission_linuxacl.h"
+#cgo LDFLAGS: -L. -lacl
+*/
+import "C"
+
+import (
+	"errors"
+	"unsafe"
+)
+
+// ToRead checks whether user has Linux file system permissions to read a given file.
+func ToRead(user, filePath string) (bool, error) {
+	cUser := C.CString(user)
+	cFilePath := C.CString(filePath)
+
+	defer C.free(unsafe.Pointer(cUser))
+	defer C.free(unsafe.Pointer(cFilePath))
+
+	cOk, err := C.permission_to_read(cUser, cFilePath)
+	if cOk == 1 {
+		return true, nil
+	}
+
+	if err != nil {
+		// err contains errno message
+		return false, err
+	}
+
+	return false, errors.New("User without permission to read file")
+}
diff --git a/internal/io/fs/permissions/permission_linuxacl.h b/internal/io/fs/permissions/permission_linuxacl.h
new file mode 100644
index 0000000..52dadcf
--- /dev/null
+++ b/internal/io/fs/permissions/permission_linuxacl.h
@@ -0,0 +1,62 @@
+// +build linuxacl
+
+#ifndef PERMISSION_LINUX_H
+#define PERMISSION_LINUX_H
+
+#include <acl/libacl.h>
+#include <errno.h>
+#include <grp.h>
+#include <pwd.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/acl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+//#define DEBUG
+#define USER_CHECK 0
+#define GROUP_CHECK 1
+
+struct permission_checker {
+  char *user_name;
+  uid_t uid;
+  gid_t *gids;
+  int ngids;
+  char *file_path;
+  struct stat file_stat;
+  struct passwd pw;
+};
+
+
+#ifdef DEBUG
+// Print out permission_checker struct.
+void debug_print_checker(struct permission_checker *pc);
+#endif
+
+// Stat a given file to retrieve traditional UNIX permissions.
+int stat_file(struct permission_checker *pc);
+
+// Retrieve UID of user.
+int get_user_uid(struct permission_checker *pc);
+
+// Retrieve all groups of the user.
+int get_user_groups(struct permission_checker *pc);
+
+// Check whether user is member of a group or not.
+int is_member_of_group(struct permission_checker *pc, gid_t gid);
+
+// Check whether user can read file according Linux ACLs.
+// As flag use either USER_CHECK or GROUP_CHECK.
+int check_acl(struct permission_checker *pc, const int flag);
+
+// Check whether user has permissions to read file according traditional
+// UNIX permissions. As flag use either USER_CHECK or GROUP_CHECK.
+int check_traditional(struct permission_checker *pc, const int flag);
+
+// Returns 1 if user has permission to read file.
+// Returns <0 on error and returns 0 if no permissions.
+int permission_to_read(char* user, char *file_path);
+
+#endif // PERMISSION_LINUX_H
diff --git a/internal/io/fs/permissions/permission_test.go b/internal/io/fs/permissions/permission_test.go
index d415ac2..c0ef038 100644
--- a/internal/io/fs/permissions/permission_test.go
+++ b/internal/io/fs/permissions/permission_test.go
@@ -1,4 +1,4 @@
-// +build linux
+// +build linuxacl
 
 package permissions
 
-- 
cgit v1.2.3


From 00f48a94726b8a8989e1d1e953d5409c7a61ae0a Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 29 Dec 2020 17:17:12 +0000
Subject: Quiet mode also affects client side logging

---
 internal/discovery/discovery.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'internal')

diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go
index 1090ea9..3608ce7 100644
--- a/internal/discovery/discovery.go
+++ b/internal/discovery/discovery.go
@@ -160,7 +160,7 @@ func (d *Discovery) dedupList(servers []string) (deduped []string) {
 		}
 	}
 
-	logger.Info("Deduped server list", len(servers), len(deduped))
+	logger.Debug("Deduped server list", len(servers), len(deduped))
 	return
 }
 
-- 
cgit v1.2.3


From 5b077bfcd68f84892a7ed66bbf92ec8d602ae8ec Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Mon, 4 Jan 2021 10:30:18 +0000
Subject: cut new version

---
 internal/version/version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'internal')

diff --git a/internal/version/version.go b/internal/version/version.go
index a64417f..c0f349c 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,7 +13,7 @@ const (
 	// Version of DTail.
 	Version string = "3.2.0"
 	// Additional information for DTail
-	Additional string = "develop-3"
+	Additional string = ""
 	// ProtocolCompat -ibility version.
 	ProtocolCompat string = "3"
 )
-- 
cgit v1.2.3


From cf3308564c7ca46caba4a8a8649743f5cf0319bc Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Fri, 30 Jul 2021 08:51:05 +0300
Subject: initial color config support

---
 internal/color/color.go   |  3 +--
 internal/config/client.go | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index 0736199..7309544 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -26,8 +26,7 @@ const (
 	Yellow    Color = escape + "[36m"
 	LightGray Color = escape + "[37m"
 
-	BgGray      Color = escape + "[40m"
-	BgRed       Color = escape + "[41m"
+	BgGray      Color = escape + "[40m" BgRed       Color = escape + "[41m"
 	BgGreen     Color = escape + "[42m"
 	BgOrange    Color = escape + "[43m"
 	BgBlue      Color = escape + "[44m"
diff --git a/internal/config/client.go b/internal/config/client.go
index 1515aae..3d8c45a 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -1,8 +1,25 @@
 package config
 
+// ClientColorConfig allows to override the default terminal color codes.
+type ClientColorConfig struct {
+	OkBgColor Color
+}
+
+/*
+	splitted := strings.Split(line, "|")
+	if splitted[2] == "100" {
+		splitted[2] = Paint(BgGreen, splitted[2])
+	} else {
+		splitted[2] = Paint(BgRed, splitted[2])
+	}
+	info := strings.Join(splitted[0:5], "|")
+	log := strings.Join(splitted[5:], "|")
+*/
+
 // ClientConfig represents a DTail client configuration (empty as of now as there
 // are no available config options yet, but that may changes in the future).
 type ClientConfig struct {
+	TerminalColors ClientColorConfig `json:",omitempty"`
 }
 
 // Create a new default client configuration.
-- 
cgit v1.2.3


From b526fcff5e490f926832d9aad43015eef67b0ad5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 31 Jul 2021 18:20:03 +0300
Subject: more on configurable colors

---
 internal/color/color.go      | 69 --------------------------------------------
 internal/color/colorfy.go    | 56 -----------------------------------
 internal/config/client.go    | 51 ++++++++++++++++++++++----------
 internal/io/logger/logger.go | 10 +++----
 internal/version/version.go  |  3 +-
 5 files changed, 42 insertions(+), 147 deletions(-)
 delete mode 100644 internal/color/color.go
 delete mode 100644 internal/color/colorfy.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
deleted file mode 100644
index 7309544..0000000
--- a/internal/color/color.go
+++ /dev/null
@@ -1,69 +0,0 @@
-// Package color is used to prettify console output via ANSII terminal colors.
-package color
-
-import (
-	"fmt"
-)
-
-// Color name.
-type Color string
-
-// Attribute of a color.
-type Attribute string
-
-// The possible color variations.
-const (
-	escape        = "\x1b"
-	reset         = escape + "[0m"
-	seq    string = "%s%s%s"
-
-	Gray      Color = escape + "[30m"
-	Red       Color = escape + "[31m"
-	Green     Color = escape + "[32m"
-	Orange    Color = escape + "[33m"
-	Blue      Color = escape + "[34m"
-	Magenta   Color = escape + "[35m"
-	Yellow    Color = escape + "[36m"
-	LightGray Color = escape + "[37m"
-
-	BgGray      Color = escape + "[40m" BgRed       Color = escape + "[41m"
-	BgGreen     Color = escape + "[42m"
-	BgOrange    Color = escape + "[43m"
-	BgBlue      Color = escape + "[44m"
-	BgMagenta   Color = escape + "[45m"
-	BgYellow    Color = escape + "[46m"
-	BgLightGray Color = escape + "[47m"
-
-	Bold         Attribute = escape + "[1m"
-	Italic       Attribute = escape + "[3m"
-	Underline    Attribute = escape + "[4m"
-	ReverseColor Attribute = escape + "[7m"
-
-	resetBold      = escape + "[22m"
-	resetItalic    = escape + "[23m"
-	resetUnderline = escape + "[24m"
-
-	Test     Color     = BgYellow
-	TestAttr Attribute = Bold
-)
-
-// Colored DTail client output enabled.
-var Colored bool
-
-// Paint a given string in a given color.
-func Paint(c Color, s string) string {
-	return fmt.Sprintf(seq, c, s, reset)
-}
-
-// Attr adds a given attribute to a given string, such as "bold" or "italic".
-func Attr(c Attribute, s string) string {
-	switch c {
-	case Bold:
-		return fmt.Sprintf(seq, Bold, s, resetBold)
-	case Italic:
-		return fmt.Sprintf(seq, Italic, s, resetItalic)
-	case Underline:
-		return fmt.Sprintf(seq, Underline, s, resetUnderline)
-	}
-	panic("Unknown attribute")
-}
diff --git a/internal/color/colorfy.go b/internal/color/colorfy.go
deleted file mode 100644
index a2beb7a..0000000
--- a/internal/color/colorfy.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package color
-
-import (
-	"fmt"
-	"strings"
-)
-
-// Add some color to log lines received from remote servers.
-func paintRemote(line string) string {
-	splitted := strings.Split(line, "|")
-	if splitted[2] == "100" {
-		splitted[2] = Paint(BgGreen, splitted[2])
-	} else {
-		splitted[2] = Paint(BgRed, splitted[2])
-	}
-	info := strings.Join(splitted[0:5], "|")
-	log := strings.Join(splitted[5:], "|")
-
-	if strings.HasPrefix(log, "WARN") {
-		log = Paint(BgYellow, log)
-	} else if strings.HasPrefix(log, "ERROR") {
-		log = Paint(BgRed, log)
-	} else if strings.HasPrefix(log, "FATAL") {
-		log = Attr(Bold, Paint(BgRed, log))
-	} else {
-		log = Paint(Blue, log)
-	}
-
-	return fmt.Sprintf("%s|%s", info, log)
-}
-
-// Add some color to stats generated by the client.
-func paintClientStats(line string) string {
-	splitted := strings.Split(line, "|")
-	first := strings.Join(splitted[0:4], "|")
-	connected := Paint(BgBlue, splitted[4])
-	last := strings.Join(splitted[5:], "|")
-
-	return fmt.Sprintf("%s|%s|%s", first, connected, last)
-}
-
-// Colorfy a given line based on the line's content.
-func Colorfy(line string) string {
-	switch {
-	case strings.HasPrefix(line, "REMOTE"):
-		return paintRemote(line)
-	case strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|"):
-		return paintClientStats(line)
-	case strings.Contains(line, "ERROR"):
-		return Paint(Magenta, line)
-	case strings.Contains(line, "WARN"):
-		return Paint(Magenta, line)
-	}
-
-	return line
-}
diff --git a/internal/config/client.go b/internal/config/client.go
index 3d8c45a..d6d106f 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -1,28 +1,47 @@
 package config
 
-// ClientColorConfig allows to override the default terminal color codes.
-type ClientColorConfig struct {
-	OkBgColor Color
-}
+import "github.com/mimecast/dtail/internal/color"
 
-/*
-	splitted := strings.Split(line, "|")
-	if splitted[2] == "100" {
-		splitted[2] = Paint(BgGreen, splitted[2])
-	} else {
-		splitted[2] = Paint(BgRed, splitted[2])
-	}
-	info := strings.Join(splitted[0:5], "|")
-	log := strings.Join(splitted[5:], "|")
-*/
+// ClientColorConfig allows to override the default terminal color color.
+type termColors struct {
+	RemoteTextFg      color.Color
+	RemoteStatsOkBg   color.Color
+	RemoteStatsWarnBg color.Color
+	RemoteTraceBg     color.Color
+	RemoteDebugBg     color.Color
+	RemoteWarnBg      color.Color
+	RemoteErrorBg     color.Color
+	RemoteFatalBg     color.Color
+	RemoteFatalAttr   color.Attribute
+	ClientStatsBg     color.Color
+	ClientWarnFg      color.Color
+	ClientErrorFg     color.Color
+}
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
 // are no available config options yet, but that may changes in the future).
 type ClientConfig struct {
-	TerminalColors ClientColorConfig `json:",omitempty"`
+	TermColorsEnabled bool
+	TermColors        termColors `json:",omitempty"`
 }
 
 // Create a new default client configuration.
 func newDefaultClientConfig() *ClientConfig {
-	return &ClientConfig{}
+	return &ClientConfig{
+		TermColorsEnabled: true,
+		TermColors: termColors{
+			RemoteTextFg:      color.LightGray,
+			RemoteStatsOkBg:   color.BgGreen,
+			RemoteStatsWarnBg: color.BgRed,
+			RemoteTraceBg:     color.BgYellow,
+			RemoteDebugBg:     color.BgYellow,
+			RemoteWarnBg:      color.BgYellow,
+			RemoteErrorBg:     color.BgRed,
+			RemoteFatalBg:     color.BgRed,
+			RemoteFatalAttr:   color.Bold,
+			ClientStatsBg:     color.BgBlue,
+			ClientWarnFg:      color.Purple,
+			ClientErrorFg:     color.BgRed,
+		},
+	}
 }
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 4254eef..bef5293 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -12,7 +12,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/color/brush"
 	"github.com/mimecast/dtail/internal/config"
 )
 
@@ -207,8 +207,8 @@ func write(what, severity, message string) {
 	if Mode.logToStdout {
 		line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message)
 
-		if color.Colored {
-			line = color.Colorfy(line)
+		if config.Client.TermColorsEnabled {
+			line = brush.Colorfy(line)
 		}
 
 		stdoutBufCh <- line
@@ -262,8 +262,8 @@ func Raw(message string) {
 	}
 
 	if Mode.logToStdout {
-		if color.Colored {
-			message = color.Colorfy(message)
+		if config.Client.TermColorsEnabled {
+			message = brush.Colorfy(message)
 		}
 		stdoutBufCh <- message
 	}
diff --git a/internal/version/version.go b/internal/version/version.go
index c0f349c..7c206b4 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
 )
 
 const (
@@ -25,7 +26,7 @@ func String() string {
 
 // PaintedString is a prettier string representation of the DTail version.
 func PaintedString() string {
-	if !color.Colored {
+	if !config.Client.TermColorsEnabled {
 		return String()
 	}
 	name := color.Paint(color.Yellow, Name)
-- 
cgit v1.2.3


From c427aecfa9a01ce63bb8a064c23b5a467de2eadc Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 9 Aug 2021 10:22:54 +0300
Subject: we have all basic ansi text color codes and attributes now. we can
 also convert a string to a corresponding code (e.g. from a config file).

---
 internal/color/color.go | 164 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100644 internal/color/color.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
new file mode 100644
index 0000000..c6760d5
--- /dev/null
+++ b/internal/color/color.go
@@ -0,0 +1,164 @@
+// Package color contains all terminal color codes we know of.
+package color
+
+import (
+	"fmt"
+	"strings"
+)
+
+// FgColor is the text foreground color.
+type FgColor string
+
+// BgColor is the text background color.
+type BgColor string
+
+// Attribute of text.
+type Attribute string
+
+// The possible color variations.
+const (
+	escape        = "\x1b"
+	seq    string = "%s%s%s"
+
+	FgBlack   FgColor = escape + "[30m"
+	FgRed     FgColor = escape + "[31m"
+	FgGreen   FgColor = escape + "[32m"
+	FgYellow  FgColor = escape + "[33m"
+	FgBlue    FgColor = escape + "[34m"
+	FgMagenta FgColor = escape + "[35m"
+	FgCyan    FgColor = escape + "[36m"
+	FgWhite   FgColor = escape + "[37m"
+	FgDefault FgColor = escape + "[39m"
+
+	BgBlack   BgColor = escape + "[40m"
+	BgRed     BgColor = escape + "[41m"
+	BgGreen   BgColor = escape + "[42m"
+	BgYellow  BgColor = escape + "[43m"
+	BgBlue    BgColor = escape + "[44m"
+	BgMagenta BgColor = escape + "[45m"
+	BgCyan    BgColor = escape + "[46m"
+	BgWhite   BgColor = escape + "[47m"
+	BgDefault BgColor = escape + "[49m"
+
+	AttReset      Attribute = escape + "[0m"
+	AttBold       Attribute = escape + "[1m"
+	AttDim        Attribute = escape + "[2m"
+	AttItalic     Attribute = escape + "[3m"
+	AttUnderline  Attribute = escape + "[4m"
+	AttBlink      Attribute = escape + "[5m"
+	AttSlowBlink  Attribute = escape + "[5m"
+	AttRapidBlink Attribute = escape + "[6m"
+	AttReverse    Attribute = escape + "[7m"
+	AttHidden     Attribute = escape + "[8m"
+
+	// Internal (manual) testing.
+	FgTest  FgColor   = FgBlue
+	BgTest  BgColor   = BgYellow
+	AttTest Attribute = AttBold
+)
+
+// Colored DTail client output enabled.
+var Colored bool
+
+// Paint paints a given text in a given foreground/background color combination.
+func Paint(fg FgColor, bg FgColor, text string) string {
+	return fmt.Sprintf(seq, fg, bg, text, BgDefault, FgDefault)
+}
+
+// PaintWithAttr paints a given text in a given foreground/background/attribute combination
+func PaintWithAtt(fg FgColor, bg FgColor, att Attribute, text string) string {
+	return fmt.Sprintf(seq, fg, bg, att, text, AttReset, BgDefault, FgDefault)
+}
+
+// PaintFg paints a given text in a given foreground color.
+func PaintFg(fg FgColor, text string) string {
+	return fmt.Sprintf(seq, fg, text, FgDefault)
+}
+
+// PaintBg paints a given text in a given background color.
+func PaintBg(bg BgColor, text string) string {
+	return fmt.Sprintf(seq, bg, text, BgDefault)
+}
+
+// PaintAtt adds a given attribute to a given text, such as "bold" or "italic".
+func PaintAtt(att Attribute, text string) string {
+	return fmt.Sprintf(seq, att, text, AttReset)
+}
+
+// ToFgColor converts a given string (e.g. from a config file) into a foreground color code.
+func ToFgColor(s string) (FgColor, error) {
+	switch strings.ToLower(s) {
+	case "black":
+		return FgBlack, nil
+	case "red":
+		return FgRed, nil
+	case "green":
+		return FgGreen, nil
+	case "yellow":
+		return FgYellow, nil
+	case "blue":
+		return FgBlue, nil
+	case "magenta":
+		return FgMagenta, nil
+	case "cyan":
+		return FgCyan, nil
+	case "white":
+		return FgWhite, nil
+	case "default":
+		return FgDefault, nil
+	default:
+		return FgDefault, fmt.Errorf("unknown foreground text color '" + s + "'")
+	}
+}
+
+// ToBgColor converts a given string (e.g. from a config file) into a background color code.
+func ToBgColor(s string) (BgColor, error) {
+	switch strings.ToLower(s) {
+	case "black":
+		return BgBlack, nil
+	case "red":
+		return BgRed, nil
+	case "green":
+		return BgGreen, nil
+	case "yellow":
+		return BgYellow, nil
+	case "blue":
+		return BgBlue, nil
+	case "magenta":
+		return BgMagenta, nil
+	case "cyan":
+		return BgCyan, nil
+	case "white":
+		return BgWhite, nil
+	case "default":
+		return BgDefault, nil
+	default:
+		return BgDefault, fmt.Errorf("unknown background text color '" + s + "'")
+	}
+}
+
+// ToAttribute converts a given string (e.g. from a config file) into a text attribute.
+func ToAttColor(s string) (Attribute, error) {
+	switch strings.ToLower(s) {
+	case "bold":
+		return AttBold, nil
+	case "dim":
+		return AttDim, nil
+	case "italic":
+		return AttItalic, nil
+	case "underline":
+		return AttUnderline, nil
+	case "blink":
+		return AttBlink, nil
+	case "slowblink":
+		return AttSlowBlink, nil
+	case "rapidblink":
+		return AttRapidBlink, nil
+	case "reverse":
+		return AttReverse, nil
+	case "hidden":
+		return AttHidden, nil
+	default:
+		return AttReset, fmt.Errorf("unknown text attribute '" + s + "'")
+	}
+}
-- 
cgit v1.2.3


From 3901dff1d36de49c209aa017118bea12c35ba9c7 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 9 Aug 2021 10:33:57 +0300
Subject: change paint API

---
 internal/color/color.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index c6760d5..52a5752 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -61,27 +61,27 @@ const (
 var Colored bool
 
 // Paint paints a given text in a given foreground/background color combination.
-func Paint(fg FgColor, bg FgColor, text string) string {
+func Paint(text string, fg FgColor, bg FgColor) string {
 	return fmt.Sprintf(seq, fg, bg, text, BgDefault, FgDefault)
 }
 
 // PaintWithAttr paints a given text in a given foreground/background/attribute combination
-func PaintWithAtt(fg FgColor, bg FgColor, att Attribute, text string) string {
+func PaintWithAtt(text string, fg FgColor, bg FgColor, att Attribute) string {
 	return fmt.Sprintf(seq, fg, bg, att, text, AttReset, BgDefault, FgDefault)
 }
 
 // PaintFg paints a given text in a given foreground color.
-func PaintFg(fg FgColor, text string) string {
+func PaintFg(text string, fg FgColor) string {
 	return fmt.Sprintf(seq, fg, text, FgDefault)
 }
 
 // PaintBg paints a given text in a given background color.
-func PaintBg(bg BgColor, text string) string {
+func PaintBg(text string, bg BgColor) string {
 	return fmt.Sprintf(seq, bg, text, BgDefault)
 }
 
 // PaintAtt adds a given attribute to a given text, such as "bold" or "italic".
-func PaintAtt(att Attribute, text string) string {
+func PaintAtt(text string, att Attribute) string {
 	return fmt.Sprintf(seq, att, text, AttReset)
 }
 
-- 
cgit v1.2.3


From de541a541857bddce6f7aa89c4e4525ebecc4d39 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 10 Aug 2021 11:23:07 +0300
Subject: add color unit test

---
 internal/color/color.go      | 68 ++++++++++++++++++++------------------------
 internal/color/color_test.go | 57 +++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 37 deletions(-)
 create mode 100644 internal/color/color_test.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index 52a5752..965675f 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -17,8 +17,7 @@ type Attribute string
 
 // The possible color variations.
 const (
-	escape        = "\x1b"
-	seq    string = "%s%s%s"
+	escape = "\x1b"
 
 	FgBlack   FgColor = escape + "[30m"
 	FgRed     FgColor = escape + "[31m"
@@ -40,49 +39,44 @@ const (
 	BgWhite   BgColor = escape + "[47m"
 	BgDefault BgColor = escape + "[49m"
 
-	AttReset      Attribute = escape + "[0m"
-	AttBold       Attribute = escape + "[1m"
-	AttDim        Attribute = escape + "[2m"
-	AttItalic     Attribute = escape + "[3m"
-	AttUnderline  Attribute = escape + "[4m"
-	AttBlink      Attribute = escape + "[5m"
-	AttSlowBlink  Attribute = escape + "[5m"
-	AttRapidBlink Attribute = escape + "[6m"
-	AttReverse    Attribute = escape + "[7m"
-	AttHidden     Attribute = escape + "[8m"
-
-	// Internal (manual) testing.
-	FgTest  FgColor   = FgBlue
-	BgTest  BgColor   = BgYellow
-	AttTest Attribute = AttBold
+	AttrReset      Attribute = escape + "[0m"
+	AttrBold       Attribute = escape + "[1m"
+	AttrDim        Attribute = escape + "[2m"
+	AttrItalic     Attribute = escape + "[3m"
+	AttrUnderline  Attribute = escape + "[4m"
+	AttrBlink      Attribute = escape + "[5m"
+	AttrSlowBlink  Attribute = escape + "[5m"
+	AttrRapidBlink Attribute = escape + "[6m"
+	AttrReverse    Attribute = escape + "[7m"
+	AttrHidden     Attribute = escape + "[8m"
 )
 
 // Colored DTail client output enabled.
 var Colored bool
 
 // Paint paints a given text in a given foreground/background color combination.
-func Paint(text string, fg FgColor, bg FgColor) string {
-	return fmt.Sprintf(seq, fg, bg, text, BgDefault, FgDefault)
+func Paint(text string, fg FgColor, bg BgColor) string {
+	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
 }
 
 // PaintWithAttr paints a given text in a given foreground/background/attribute combination
-func PaintWithAtt(text string, fg FgColor, bg FgColor, att Attribute) string {
-	return fmt.Sprintf(seq, fg, bg, att, text, AttReset, BgDefault, FgDefault)
+func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
+	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
 }
 
 // PaintFg paints a given text in a given foreground color.
 func PaintFg(text string, fg FgColor) string {
-	return fmt.Sprintf(seq, fg, text, FgDefault)
+	return fmt.Sprintf("%s%s%s", fg, text, FgDefault)
 }
 
 // PaintBg paints a given text in a given background color.
 func PaintBg(text string, bg BgColor) string {
-	return fmt.Sprintf(seq, bg, text, BgDefault)
+	return fmt.Sprintf("%s%s%s", bg, text, BgDefault)
 }
 
-// PaintAtt adds a given attribute to a given text, such as "bold" or "italic".
-func PaintAtt(text string, att Attribute) string {
-	return fmt.Sprintf(seq, att, text, AttReset)
+// PaintAttr adds a given attribute to a given text, such as "bold" or "italic".
+func PaintAttr(text string, attr Attribute) string {
+	return fmt.Sprintf("%s%s%s", attr, text, AttrReset)
 }
 
 // ToFgColor converts a given string (e.g. from a config file) into a foreground color code.
@@ -138,27 +132,27 @@ func ToBgColor(s string) (BgColor, error) {
 }
 
 // ToAttribute converts a given string (e.g. from a config file) into a text attribute.
-func ToAttColor(s string) (Attribute, error) {
+func ToAttribute(s string) (Attribute, error) {
 	switch strings.ToLower(s) {
 	case "bold":
-		return AttBold, nil
+		return AttrBold, nil
 	case "dim":
-		return AttDim, nil
+		return AttrDim, nil
 	case "italic":
-		return AttItalic, nil
+		return AttrItalic, nil
 	case "underline":
-		return AttUnderline, nil
+		return AttrUnderline, nil
 	case "blink":
-		return AttBlink, nil
+		return AttrBlink, nil
 	case "slowblink":
-		return AttSlowBlink, nil
+		return AttrSlowBlink, nil
 	case "rapidblink":
-		return AttRapidBlink, nil
+		return AttrRapidBlink, nil
 	case "reverse":
-		return AttReverse, nil
+		return AttrReverse, nil
 	case "hidden":
-		return AttHidden, nil
+		return AttrHidden, nil
 	default:
-		return AttReset, fmt.Errorf("unknown text attribute '" + s + "'")
+		return AttrReset, fmt.Errorf("unknown text attribute '" + s + "'")
 	}
 }
diff --git a/internal/color/color_test.go b/internal/color/color_test.go
new file mode 100644
index 0000000..0024b7f
--- /dev/null
+++ b/internal/color/color_test.go
@@ -0,0 +1,57 @@
+package color
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestColors(t *testing.T) {
+	colors := []string{
+		"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
+	}
+
+	text := " Mimecast "
+	builder := strings.Builder{}
+
+	for _, color := range colors {
+		fgColor, err := ToFgColor(color)
+		if err != nil {
+			t.Errorf("unable to paint foreground : %s\n%v", text, err)
+		}
+		builder.WriteString(PaintFg(text, fgColor))
+
+		bgColor, err := ToBgColor(color)
+		if err != nil {
+			t.Errorf("unable to paint background: %s\n%v", text, err)
+		}
+		builder.WriteString(PaintBg(text, bgColor))
+	}
+
+	for _, color := range colors {
+		fgColor, _ := ToFgColor(color)
+		for _, color := range colors {
+			bgColor, _ := ToBgColor(color)
+			builder.WriteString(Paint(text, fgColor, bgColor))
+		}
+	}
+
+	t.Log(builder.String())
+}
+func TestAttributes(t *testing.T) {
+	attributes := []string{
+		"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "hidden",
+	}
+
+	text := " Mimecast "
+	builder := strings.Builder{}
+
+	for _, attribute := range attributes {
+		att, err := ToAttribute(attribute)
+		if err != nil {
+			t.Errorf("unable to paint attribute: %s\n%v", text, err)
+		}
+		builder.WriteString(PaintWithAttr(text, FgWhite, BgBlue, att))
+	}
+
+	t.Log(builder.String())
+}
-- 
cgit v1.2.3


From 03fdeabe6b1e922d0ad064bdcc892b3ccf5ba551 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 10 Aug 2021 12:05:40 +0300
Subject: can compile with new color codes

---
 internal/color/color.go     |   4 ++
 internal/config/client.go   | 108 ++++++++++++++++++++++++++++++++++----------
 internal/version/version.go |  21 ++++++---
 3 files changed, 104 insertions(+), 29 deletions(-)

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index 965675f..f444294 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -39,6 +39,7 @@ const (
 	BgWhite   BgColor = escape + "[47m"
 	BgDefault BgColor = escape + "[49m"
 
+	AttrNone       Attribute = ""
 	AttrReset      Attribute = escape + "[0m"
 	AttrBold       Attribute = escape + "[1m"
 	AttrDim        Attribute = escape + "[2m"
@@ -61,6 +62,9 @@ func Paint(text string, fg FgColor, bg BgColor) string {
 
 // PaintWithAttr paints a given text in a given foreground/background/attribute combination
 func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
+	if attr == AttrNone {
+		return Paint(text, fg, bg)
+	}
 	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
 }
 
diff --git a/internal/config/client.go b/internal/config/client.go
index d6d106f..84ad037 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -4,18 +4,49 @@ import "github.com/mimecast/dtail/internal/color"
 
 // ClientColorConfig allows to override the default terminal color color.
 type termColors struct {
-	RemoteTextFg      color.Color
-	RemoteStatsOkBg   color.Color
-	RemoteStatsWarnBg color.Color
-	RemoteTraceBg     color.Color
-	RemoteDebugBg     color.Color
-	RemoteWarnBg      color.Color
-	RemoteErrorBg     color.Color
-	RemoteFatalBg     color.Color
-	RemoteFatalAttr   color.Attribute
-	ClientStatsBg     color.Color
-	ClientWarnFg      color.Color
-	ClientErrorFg     color.Color
+	ClientErrorAttr color.Attribute
+	ClientErrorBg   color.BgColor
+	ClientErrorFg   color.FgColor
+
+	ClientStatsAttr color.Attribute
+	ClientStatsBg   color.BgColor
+	ClientStatsFg   color.FgColor
+
+	ClientWarnAttr color.Attribute
+	ClientWarnBg   color.BgColor
+	ClientWarnFg   color.FgColor
+
+	RemoteDebugAttr color.Attribute
+	RemoteDebugBg   color.BgColor
+	RemoteDebugFg   color.FgColor
+
+	RemoteErrorAttr color.Attribute
+	RemoteErrorBg   color.BgColor
+	RemoteErrorFg   color.FgColor
+
+	RemoteFatalAttr color.Attribute
+	RemoteFatalBg   color.BgColor
+	RemoteFatalFg   color.FgColor
+
+	RemoteStatsOkAttr color.Attribute
+	RemoteStatsOkBg   color.BgColor
+	RemoteStatsOkFg   color.FgColor
+
+	RemoteStatsWarnAttr color.Attribute
+	RemoteStatsWarnBg   color.BgColor
+	RemoteStatsWarnFg   color.FgColor
+
+	RemoteTextAttr color.Attribute
+	RemoteTextBg   color.BgColor
+	RemoteTextFg   color.FgColor
+
+	RemoteTraceAttr color.Attribute
+	RemoteTraceBg   color.BgColor
+	RemoteTraceFg   color.FgColor
+
+	RemoteWarnAttr color.Attribute
+	RemoteWarnBg   color.BgColor
+	RemoteWarnFg   color.FgColor
 }
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
@@ -30,18 +61,49 @@ func newDefaultClientConfig() *ClientConfig {
 	return &ClientConfig{
 		TermColorsEnabled: true,
 		TermColors: termColors{
-			RemoteTextFg:      color.LightGray,
+			ClientErrorAttr: color.AttrBold,
+			ClientErrorBg:   color.BgBlack,
+			ClientErrorFg:   color.FgRed,
+
+			ClientStatsAttr: color.AttrDim,
+			ClientStatsBg:   color.BgBlue,
+			ClientStatsFg:   color.FgWhite,
+
+			ClientWarnAttr: color.AttrNone,
+			ClientWarnBg:   color.BgBlack,
+			ClientWarnFg:   color.FgMagenta,
+
+			RemoteDebugAttr: color.AttrNone,
+			RemoteDebugBg:   color.BgGreen,
+			RemoteDebugFg:   color.FgWhite,
+
+			RemoteErrorAttr: color.AttrBold,
+			RemoteErrorBg:   color.BgRed,
+			RemoteErrorFg:   color.FgWhite,
+
+			RemoteFatalAttr: color.AttrBlink,
+			RemoteFatalBg:   color.BgRed,
+			RemoteFatalFg:   color.FgBlue,
+
+			RemoteStatsOkAttr: color.AttrNone,
 			RemoteStatsOkBg:   color.BgGreen,
-			RemoteStatsWarnBg: color.BgRed,
-			RemoteTraceBg:     color.BgYellow,
-			RemoteDebugBg:     color.BgYellow,
-			RemoteWarnBg:      color.BgYellow,
-			RemoteErrorBg:     color.BgRed,
-			RemoteFatalBg:     color.BgRed,
-			RemoteFatalAttr:   color.Bold,
-			ClientStatsBg:     color.BgBlue,
-			ClientWarnFg:      color.Purple,
-			ClientErrorFg:     color.BgRed,
+			RemoteStatsOkFg:   color.FgWhite,
+
+			RemoteStatsWarnAttr: color.AttrNone,
+			RemoteStatsWarnBg:   color.BgRed,
+			RemoteStatsWarnFg:   color.FgWhite,
+
+			RemoteTextAttr: color.AttrNone,
+			RemoteTextBg:   color.BgBlack,
+			RemoteTextFg:   color.FgWhite,
+
+			RemoteTraceAttr: color.AttrBold,
+			RemoteTraceBg:   color.BgGreen,
+			RemoteTraceFg:   color.FgWhite,
+
+			RemoteWarnAttr: color.AttrBold,
+			RemoteWarnBg:   color.BgYellow,
+			RemoteWarnFg:   color.FgWhite,
 		},
 	}
 }
diff --git a/internal/version/version.go b/internal/version/version.go
index 7c206b4..a513fdc 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,10 +13,10 @@ const (
 	Name string = "DTail"
 	// Version of DTail.
 	Version string = "3.2.0"
-	// Additional information for DTail
-	Additional string = ""
 	// ProtocolCompat -ibility version.
 	ProtocolCompat string = "3"
+	// Additional information for DTail
+	Additional string = "Have a lot of fun!"
 )
 
 // String representation of the DTail version.
@@ -29,11 +29,20 @@ func PaintedString() string {
 	if !config.Client.TermColorsEnabled {
 		return String()
 	}
-	name := color.Paint(color.Yellow, Name)
-	version := color.Paint(color.Blue, Version)
-	descr := color.Paint(color.Green, Additional)
 
-	return fmt.Sprintf("%s %v Protocol %s %s", name, version, ProtocolCompat, descr)
+	name := color.PaintWithAttr(Name,
+		color.FgYellow, color.BgBlue, color.AttrBold)
+
+	version := color.PaintWithAttr(fmt.Sprintf(" %s ", Version),
+		color.FgBlue, color.BgYellow, color.AttrBold)
+
+	protocol := color.Paint(fmt.Sprintf(" Protocol %s ", ProtocolCompat),
+		color.FgBlack, color.BgGreen)
+
+	additional := color.PaintWithAttr(fmt.Sprintf(" %s ", Additional),
+		color.FgWhite, color.BgMagenta, color.AttrBlink)
+
+	return fmt.Sprintf("%s%v%s%s", name, version, protocol, additional)
 }
 
 // PrintAndExit prints the program version and exists.
-- 
cgit v1.2.3


From 5fdaa4ef60c134f4cb6cb64159aad59a55e0dd11 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 11 Aug 2021 09:32:13 +0300
Subject: add colorTable option

---
 internal/color/color.go      |  4 ++++
 internal/color/color_test.go | 23 +++++++++--------------
 internal/color/table.go      | 41 +++++++++++++++++++++++++++++++++++++++++
 internal/config/client.go    |  6 +++---
 internal/version/version.go  |  2 +-
 5 files changed, 58 insertions(+), 18 deletions(-)
 create mode 100644 internal/color/table.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index f444294..a2490be 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -156,6 +156,10 @@ func ToAttribute(s string) (Attribute, error) {
 		return AttrReverse, nil
 	case "hidden":
 		return AttrHidden, nil
+	case "none":
+		fallthrough
+	case "":
+		return AttrNone, nil
 	default:
 		return AttrReset, fmt.Errorf("unknown text attribute '" + s + "'")
 	}
diff --git a/internal/color/color_test.go b/internal/color/color_test.go
index 0024b7f..16b2f90 100644
--- a/internal/color/color_test.go
+++ b/internal/color/color_test.go
@@ -6,14 +6,10 @@ import (
 )
 
 func TestColors(t *testing.T) {
-	colors := []string{
-		"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
-	}
-
 	text := " Mimecast "
 	builder := strings.Builder{}
 
-	for _, color := range colors {
+	for _, color := range ColorNames {
 		fgColor, err := ToFgColor(color)
 		if err != nil {
 			t.Errorf("unable to paint foreground : %s\n%v", text, err)
@@ -27,10 +23,13 @@ func TestColors(t *testing.T) {
 		builder.WriteString(PaintBg(text, bgColor))
 	}
 
-	for _, color := range colors {
-		fgColor, _ := ToFgColor(color)
-		for _, color := range colors {
-			bgColor, _ := ToBgColor(color)
+	for _, fg := range ColorNames {
+		fgColor, _ := ToFgColor(fg)
+		for _, bg := range ColorNames {
+			if fg == bg {
+				continue
+			}
+			bgColor, _ := ToBgColor(bg)
 			builder.WriteString(Paint(text, fgColor, bgColor))
 		}
 	}
@@ -38,14 +37,10 @@ func TestColors(t *testing.T) {
 	t.Log(builder.String())
 }
 func TestAttributes(t *testing.T) {
-	attributes := []string{
-		"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "hidden",
-	}
-
 	text := " Mimecast "
 	builder := strings.Builder{}
 
-	for _, attribute := range attributes {
+	for _, attribute := range AttributeNames {
 		att, err := ToAttribute(attribute)
 		if err != nil {
 			t.Errorf("unable to paint attribute: %s\n%v", text, err)
diff --git a/internal/color/table.go b/internal/color/table.go
new file mode 100644
index 0000000..8e40a78
--- /dev/null
+++ b/internal/color/table.go
@@ -0,0 +1,41 @@
+package color
+
+import (
+	"fmt"
+	"os"
+)
+
+var ColorNames = []string{
+	"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
+}
+
+var AttributeNames = []string{
+	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "Hidden", "None",
+}
+
+func TablePrintAndExit() {
+	for _, attr := range AttributeNames {
+		if attr == "Hidden" || attr == "SlowBlink" {
+			continue
+		}
+		printColorTable(attr)
+	}
+	os.Exit(0)
+}
+
+func printColorTable(attr string) {
+	for _, fg := range ColorNames {
+		fgColor, _ := ToFgColor(fg)
+		for _, bg := range ColorNames {
+			if fg == bg {
+				continue
+			}
+			bgColor, _ := ToBgColor(bg)
+			attribute, _ := ToAttribute(attr)
+
+			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
+			fmt.Print(PaintWithAttr(text, fgColor, bgColor, attribute))
+			fmt.Print("\n")
+		}
+	}
+}
diff --git a/internal/config/client.go b/internal/config/client.go
index 84ad037..5386576 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -75,7 +75,7 @@ func newDefaultClientConfig() *ClientConfig {
 
 			RemoteDebugAttr: color.AttrNone,
 			RemoteDebugBg:   color.BgGreen,
-			RemoteDebugFg:   color.FgWhite,
+			RemoteDebugFg:   color.FgBlack,
 
 			RemoteErrorAttr: color.AttrBold,
 			RemoteErrorBg:   color.BgRed,
@@ -83,11 +83,11 @@ func newDefaultClientConfig() *ClientConfig {
 
 			RemoteFatalAttr: color.AttrBlink,
 			RemoteFatalBg:   color.BgRed,
-			RemoteFatalFg:   color.FgBlue,
+			RemoteFatalFg:   color.FgWhite,
 
 			RemoteStatsOkAttr: color.AttrNone,
 			RemoteStatsOkBg:   color.BgGreen,
-			RemoteStatsOkFg:   color.FgWhite,
+			RemoteStatsOkFg:   color.FgBlack,
 
 			RemoteStatsWarnAttr: color.AttrNone,
 			RemoteStatsWarnBg:   color.BgRed,
diff --git a/internal/version/version.go b/internal/version/version.go
index a513fdc..7cfe12c 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -30,7 +30,7 @@ func PaintedString() string {
 		return String()
 	}
 
-	name := color.PaintWithAttr(Name,
+	name := color.PaintWithAttr(fmt.Sprintf(" %s ", Name),
 		color.FgYellow, color.BgBlue, color.AttrBold)
 
 	version := color.PaintWithAttr(fmt.Sprintf(" %s ", Version),
-- 
cgit v1.2.3


From 190922c087bfbe00299d73d1ea74bbb9a41852d5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 11 Aug 2021 09:42:51 +0300
Subject: add paint.go

---
 internal/color/color.go | 31 ++++---------------------------
 internal/color/paint.go | 31 +++++++++++++++++++++++++++++++
 internal/color/table.go |  8 --------
 3 files changed, 35 insertions(+), 35 deletions(-)
 create mode 100644 internal/color/paint.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index a2490be..692de51 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -52,35 +52,12 @@ const (
 	AttrHidden     Attribute = escape + "[8m"
 )
 
-// Colored DTail client output enabled.
-var Colored bool
-
-// Paint paints a given text in a given foreground/background color combination.
-func Paint(text string, fg FgColor, bg BgColor) string {
-	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
-}
-
-// PaintWithAttr paints a given text in a given foreground/background/attribute combination
-func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
-	if attr == AttrNone {
-		return Paint(text, fg, bg)
-	}
-	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
-}
-
-// PaintFg paints a given text in a given foreground color.
-func PaintFg(text string, fg FgColor) string {
-	return fmt.Sprintf("%s%s%s", fg, text, FgDefault)
-}
-
-// PaintBg paints a given text in a given background color.
-func PaintBg(text string, bg BgColor) string {
-	return fmt.Sprintf("%s%s%s", bg, text, BgDefault)
+var ColorNames = []string{
+	"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
 }
 
-// PaintAttr adds a given attribute to a given text, such as "bold" or "italic".
-func PaintAttr(text string, attr Attribute) string {
-	return fmt.Sprintf("%s%s%s", attr, text, AttrReset)
+var AttributeNames = []string{
+	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "Hidden", "None",
 }
 
 // ToFgColor converts a given string (e.g. from a config file) into a foreground color code.
diff --git a/internal/color/paint.go b/internal/color/paint.go
new file mode 100644
index 0000000..7862467
--- /dev/null
+++ b/internal/color/paint.go
@@ -0,0 +1,31 @@
+package color
+
+import "fmt"
+
+// Paint paints a given text in a given foreground/background color combination.
+func Paint(text string, fg FgColor, bg BgColor) string {
+	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
+}
+
+// PaintWithAttr paints a given text in a given foreground/background/attribute combination
+func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
+	if attr == AttrNone {
+		return Paint(text, fg, bg)
+	}
+	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
+}
+
+// PaintFg paints a given text in a given foreground color.
+func PaintFg(text string, fg FgColor) string {
+	return fmt.Sprintf("%s%s%s", fg, text, FgDefault)
+}
+
+// PaintBg paints a given text in a given background color.
+func PaintBg(text string, bg BgColor) string {
+	return fmt.Sprintf("%s%s%s", bg, text, BgDefault)
+}
+
+// PaintAttr adds a given attribute to a given text, such as "bold" or "italic".
+func PaintAttr(text string, attr Attribute) string {
+	return fmt.Sprintf("%s%s%s", attr, text, AttrReset)
+}
diff --git a/internal/color/table.go b/internal/color/table.go
index 8e40a78..8c36047 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -5,14 +5,6 @@ import (
 	"os"
 )
 
-var ColorNames = []string{
-	"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
-}
-
-var AttributeNames = []string{
-	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "Hidden", "None",
-}
-
 func TablePrintAndExit() {
 	for _, attr := range AttributeNames {
 		if attr == "Hidden" || attr == "SlowBlink" {
-- 
cgit v1.2.3


From 1cd0902b40fcddbe35ba9d29daccb9084d19fd6e Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Thu, 12 Aug 2021 10:56:36 +0300
Subject: add missing brush and also add color client configs plus jsonschema

---
 internal/color/brush/brush.go | 104 ++++++++++++++++++++++++++++++++++++++++++
 internal/color/color.go       |   2 +-
 internal/color/color_test.go  |   1 +
 internal/config/client.go     |   6 +--
 internal/io/logger/logger.go  |   4 +-
 internal/version/version.go   |   9 +++-
 6 files changed, 118 insertions(+), 8 deletions(-)
 create mode 100644 internal/color/brush/brush.go

(limited to 'internal')

diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
new file mode 100644
index 0000000..c5efff6
--- /dev/null
+++ b/internal/color/brush/brush.go
@@ -0,0 +1,104 @@
+package brush
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
+)
+
+// Add some color to log lines received from remote servers.
+func paintRemote(line string) string {
+	splitted := strings.Split(line, "|")
+	if splitted[2] == "100" {
+		splitted[2] = color.Paint(splitted[2],
+			config.Client.TermColors.RemoteStatsOkFg,
+			config.Client.TermColors.RemoteStatsOkBg)
+	} else {
+		splitted[2] = color.Paint(splitted[2],
+			config.Client.TermColors.RemoteStatsWarnFg,
+			config.Client.TermColors.RemoteStatsWarnBg)
+	}
+
+	info := strings.Join(splitted[0:5], "|")
+	log := strings.Join(splitted[5:], "|")
+
+	switch {
+	case strings.HasPrefix(log, "WARN"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteWarnFg,
+			config.Client.TermColors.RemoteWarnBg,
+			config.Client.TermColors.RemoteWarnAttr)
+
+	case strings.HasPrefix(log, "ERROR"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteErrorFg,
+			config.Client.TermColors.RemoteErrorBg,
+			config.Client.TermColors.RemoteErrorAttr)
+
+	case strings.HasPrefix(log, "FATAL"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteFatalFg,
+			config.Client.TermColors.RemoteFatalBg,
+			config.Client.TermColors.RemoteFatalAttr)
+
+	case strings.HasPrefix(log, "DEBUG"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteDebugFg,
+			config.Client.TermColors.RemoteDebugBg,
+			config.Client.TermColors.RemoteDebugAttr)
+
+	case strings.HasPrefix(log, "TRACE"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteTraceFg,
+			config.Client.TermColors.RemoteTraceBg,
+			config.Client.TermColors.RemoteTraceAttr)
+
+	default:
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteTextFg,
+			config.Client.TermColors.RemoteTextBg,
+			config.Client.TermColors.RemoteTextAttr)
+	}
+
+	return fmt.Sprintf("%s|%s", info, log)
+}
+
+// Add some color to stats generated by the client.
+func paintClientStats(line string) string {
+	splitted := strings.Split(line, "|")
+	first := strings.Join(splitted[0:4], "|")
+	connected := color.PaintWithAttr(splitted[4],
+		config.Client.TermColors.ClientStatsFg,
+		config.Client.TermColors.ClientStatsBg,
+		config.Client.TermColors.ClientStatsAttr)
+	last := strings.Join(splitted[5:], "|")
+
+	return fmt.Sprintf("%s|%s|%s", first, connected, last)
+}
+
+// Colorfy a given line based on the line's content.
+func Colorfy(line string) string {
+	switch {
+	case strings.HasPrefix(line, "REMOTE"):
+		return paintRemote(line)
+
+	case strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|"):
+		return paintClientStats(line)
+
+	case strings.Contains(line, "ERROR"):
+		return color.PaintWithAttr(line,
+			config.Client.TermColors.ClientErrorFg,
+			config.Client.TermColors.ClientErrorBg,
+			config.Client.TermColors.ClientErrorAttr)
+
+	case strings.Contains(line, "WARN"):
+		return color.PaintWithAttr(line,
+			config.Client.TermColors.ClientWarnFg,
+			config.Client.TermColors.ClientWarnBg,
+			config.Client.TermColors.ClientWarnAttr)
+	}
+
+	return line
+}
diff --git a/internal/color/color.go b/internal/color/color.go
index 692de51..5c25855 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -138,6 +138,6 @@ func ToAttribute(s string) (Attribute, error) {
 	case "":
 		return AttrNone, nil
 	default:
-		return AttrReset, fmt.Errorf("unknown text attribute '" + s + "'")
+		return AttrNone, fmt.Errorf("unknown text attribute '" + s + "'")
 	}
 }
diff --git a/internal/color/color_test.go b/internal/color/color_test.go
index 16b2f90..bfc7c54 100644
--- a/internal/color/color_test.go
+++ b/internal/color/color_test.go
@@ -36,6 +36,7 @@ func TestColors(t *testing.T) {
 
 	t.Log(builder.String())
 }
+
 func TestAttributes(t *testing.T) {
 	text := " Mimecast "
 	builder := strings.Builder{}
diff --git a/internal/config/client.go b/internal/config/client.go
index 5386576..c1b488c 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -52,14 +52,14 @@ type termColors struct {
 // ClientConfig represents a DTail client configuration (empty as of now as there
 // are no available config options yet, but that may changes in the future).
 type ClientConfig struct {
-	TermColorsEnabled bool
-	TermColors        termColors `json:",omitempty"`
+	TermColorsEnable bool
+	TermColors       termColors `json:",omitempty"`
 }
 
 // Create a new default client configuration.
 func newDefaultClientConfig() *ClientConfig {
 	return &ClientConfig{
-		TermColorsEnabled: true,
+		TermColorsEnable: true,
 		TermColors: termColors{
 			ClientErrorAttr: color.AttrBold,
 			ClientErrorBg:   color.BgBlack,
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index bef5293..bb9dc02 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -207,7 +207,7 @@ func write(what, severity, message string) {
 	if Mode.logToStdout {
 		line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message)
 
-		if config.Client.TermColorsEnabled {
+		if config.Client.TermColorsEnable {
 			line = brush.Colorfy(line)
 		}
 
@@ -262,7 +262,7 @@ func Raw(message string) {
 	}
 
 	if Mode.logToStdout {
-		if config.Client.TermColorsEnabled {
+		if config.Client.TermColorsEnable {
 			message = brush.Colorfy(message)
 		}
 		stdoutBufCh <- message
diff --git a/internal/version/version.go b/internal/version/version.go
index 7cfe12c..3e683bd 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -26,7 +26,7 @@ func String() string {
 
 // PaintedString is a prettier string representation of the DTail version.
 func PaintedString() string {
-	if !config.Client.TermColorsEnabled {
+	if !config.Client.TermColorsEnable {
 		return String()
 	}
 
@@ -45,8 +45,13 @@ func PaintedString() string {
 	return fmt.Sprintf("%s%v%s%s", name, version, protocol, additional)
 }
 
+// Print the version.
+func Print() {
+	fmt.Println(PaintedString())
+}
+
 // PrintAndExit prints the program version and exists.
 func PrintAndExit() {
-	fmt.Println(PaintedString())
+	Print()
 	os.Exit(0)
 }
-- 
cgit v1.2.3


From b3c161b7887d98ab7aba2fce90c9b5965991ab62 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 21 Aug 2021 14:54:24 +0300
Subject: read files bytewise for more control of whats happening - change
 transport protocol for more control over newlines

---
 internal/clients/handlers/basehandler.go  |  6 +--
 internal/clients/handlers/maprhandler.go  |  3 +-
 internal/io/fs/readfile.go                | 71 ++++++++++++-------------------
 internal/io/logger/logger.go              |  7 ++-
 internal/mapr/aggregateset.go             | 10 +++--
 internal/protocol/protocol.go             | 10 +++++
 internal/server/handlers/serverhandler.go | 15 ++++---
 internal/version/version.go               |  7 ++-
 8 files changed, 66 insertions(+), 63 deletions(-)
 create mode 100644 internal/protocol/protocol.go

(limited to 'internal')

diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index f07fd90..acafe0e 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/version"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 type baseHandler struct {
@@ -43,7 +43,7 @@ func (h *baseHandler) SendMessage(command string) error {
 	logger.Debug("Sending command", h.server, command, encoded)
 
 	select {
-	case h.commands <- fmt.Sprintf("protocol %s base64 %v;", version.ProtocolCompat, encoded):
+	case h.commands <- fmt.Sprintf("protocol %s base64 %v;", protocol.ProtocolCompat, encoded):
 	case <-time.After(time.Second * 5):
 		return fmt.Errorf("Timed out sending command '%s' (base64: '%s')", command, encoded)
 	case <-h.Done():
@@ -57,7 +57,7 @@ func (h *baseHandler) SendMessage(command string) error {
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		h.receiveBuf = append(h.receiveBuf, b)
-		if b == '\n' {
+		if b == protocol.MessageDelimiter {
 			if len(h.receiveBuf) == 0 {
 				continue
 			}
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index fb71c8f..7ac5895 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -7,6 +7,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/client"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // MaprHandler is the handler used on the client side for running mapreduce aggregations.
@@ -58,7 +59,7 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 // related data.
 func (h *MaprHandler) handleAggregateMessage(message string) {
 	h.count++
-	parts := strings.Split(message, "➔")
+	parts := strings.Split(message, protocol.AggregateDelimiter)
 
 	// Index 0 contains 'AGGREGATE', 1 contains server host.
 	// Aggregation data begins from index 2.
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 6757bd6..8a365a1 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/protocol"
 	"github.com/mimecast/dtail/internal/regex"
 
 	"github.com/DataDog/zstd"
@@ -148,80 +149,64 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan []byte, t
 	if err != nil {
 		return err
 	}
-	rawLine := make([]byte, 0, 512)
 
 	lineLengthThreshold := 1024 * 1024 // 1mb
-	longLineWarning := false
+	warnedAboutLongLine := false
+	message := make([]byte, 0, 512)
 
 	for {
 		select {
 		case <-ctx.Done():
 			return nil
-		default:
-		}
-
-		select {
 		case <-truncate:
 			if isTruncated, err := f.truncated(fd); isTruncated {
 				return err
 			}
-			logger.Info(f.filePath, "Current offset", offset)
 		default:
 		}
 
-		// Read some bytes (max 4k at once as of go 1.12). isPrefix will
-		// be set if line does not fit into 4k buffer.
-		bytes, isPrefix, err := reader.ReadLine()
+		b, err := reader.ReadByte()
 
 		if err != nil {
-			// If EOF, sleep a couple of ms and return with nil error.
-			// If other error, return with non-nil error.
 			if err != io.EOF {
 				return err
 			}
 			if !f.seekEOF {
-				logger.Debug(f.FilePath(), "End of file reached")
+				logger.Info(f.FilePath(), "End of file reached")
 				return nil
 			}
 			time.Sleep(time.Millisecond * 100)
 			continue
 		}
+		offset++
 
-		rawLine = append(rawLine, bytes...)
-		offset += uint64(len(bytes))
-
-		if !isPrefix {
-			// last LineRead call returned contend until end of line.
-			rawLine = append(rawLine, '\n')
-			select {
-			case rawLines <- rawLine:
-			case <-ctx.Done():
-				return nil
+		switch b {
+		case '\n':
+			if len(message) == 0 {
+				time.Sleep(time.Millisecond * 100)
+				continue
 			}
-			rawLine = make([]byte, 0, 512)
-			if longLineWarning {
-				longLineWarning = false
-			}
-			continue
-		}
-
-		// Last LineRead call could not read content until end of line, buffer
-		// was too small. Determine whether we exceed the max line length we
-		// want dtail to send to the client at once. Possibly split up log line
-		// into multiple log lines.
-		if len(rawLine) >= lineLengthThreshold {
-			if !longLineWarning {
-				f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
-				// Only print out one warning per long log line.
-				longLineWarning = true
-			}
-			rawLine = append(rawLine, '\n')
 			select {
-			case rawLines <- rawLine:
+			case rawLines <- append(message, protocol.MessageDelimiter):
+				message = make([]byte, 0, 512)
+				warnedAboutLongLine = false
 			case <-ctx.Done():
 				return nil
 			}
-			rawLine = make([]byte, 0, 512)
+		default:
+			if len(message) >= lineLengthThreshold {
+				if !warnedAboutLongLine {
+					f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
+					warnedAboutLongLine = true
+				}
+				select {
+				case <-ctx.Done():
+					return nil
+				case rawLines <- append(message, protocol.MessageDelimiter):
+					message = make([]byte, 0, 512)
+				}
+			}
+			message = append(message, b)
 		}
 	}
 }
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index bb9dc02..3a3935d 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -205,7 +205,7 @@ func Trace(args ...interface{}) string {
 // Write log line to buffer and/or log file.
 func write(what, severity, message string) {
 	if Mode.logToStdout {
-		line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message)
+		line := fmt.Sprintf("%s|%s|%s|%s", what, hostname, severity, message)
 
 		if config.Client.TermColorsEnable {
 			line = brush.Colorfy(line)
@@ -219,7 +219,7 @@ func write(what, severity, message string) {
 		timeStr := t.Format("20060102-150405")
 		fileLogBufCh <- buf{
 			time:    t,
-			message: fmt.Sprintf("%s|%s|%s|%s\n", severity, timeStr, what, message),
+			message: fmt.Sprintf("%s|%s|%s|%s", severity, timeStr, what, message),
 		}
 	}
 }
@@ -326,6 +326,7 @@ func Flush() {
 		select {
 		case message := <-stdoutBufCh:
 			stdoutWriter.WriteString(message)
+			stdoutWriter.WriteString("\n")
 		default:
 			stdoutWriter.Flush()
 			return
@@ -338,6 +339,7 @@ func writeToStdout(ctx context.Context) {
 		select {
 		case message := <-stdoutBufCh:
 			stdoutWriter.WriteString(message)
+			stdoutWriter.WriteString("\n")
 		case <-time.After(time.Millisecond * 100):
 			stdoutWriter.Flush()
 		case <-pauseCh:
@@ -365,6 +367,7 @@ func writeToFile(ctx context.Context) {
 			dateStr := buf.time.Format("20060102")
 			w := fileWriter(dateStr)
 			w.WriteString(buf.message)
+			w.WriteString("\n")
 		case <-pauseCh:
 		PAUSE:
 			for {
diff --git a/internal/mapr/aggregateset.go b/internal/mapr/aggregateset.go
index a6cc6eb..029d87b 100644
--- a/internal/mapr/aggregateset.go
+++ b/internal/mapr/aggregateset.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // AggregateSet represents aggregated key/value pairs from the
@@ -70,20 +72,20 @@ func (s *AggregateSet) Serialize(ctx context.Context, groupKey string, ch chan<-
 	var sb strings.Builder
 
 	sb.WriteString(groupKey)
-	sb.WriteString("➔")
-	sb.WriteString(fmt.Sprintf("%d➔", s.Samples))
+	sb.WriteString(protocol.AggregateDelimiter)
+	sb.WriteString(fmt.Sprintf("%d%s", s.Samples, protocol.AggregateDelimiter))
 
 	for k, v := range s.FValues {
 		sb.WriteString(k)
 		sb.WriteString("=")
-		sb.WriteString(fmt.Sprintf("%v➔", v))
+		sb.WriteString(fmt.Sprintf("%v%s", v, protocol.AggregateDelimiter))
 	}
 
 	for k, v := range s.SValues {
 		sb.WriteString(k)
 		sb.WriteString("=")
 		sb.WriteString(v)
-		sb.WriteString("➔")
+		sb.WriteString(protocol.AggregateDelimiter)
 	}
 
 	select {
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
new file mode 100644
index 0000000..2a570cd
--- /dev/null
+++ b/internal/protocol/protocol.go
@@ -0,0 +1,10 @@
+package protocol
+
+const (
+	// ProtocolCompat -ibility version
+	ProtocolCompat string = "4"
+	// MessageDelimiter delimits separate messages.
+	MessageDelimiter byte = '¬'
+	// AggregateDelimiter delimits parts of an aggregation message.
+	AggregateDelimiter string = "➔"
+)
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 185e7c2..23e3aeb 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -17,8 +17,8 @@ import (
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
+	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
-	"github.com/mimecast/dtail/internal/version"
 )
 
 const (
@@ -92,24 +92,27 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			}
 			if message[0] == '.' {
 				// Handle hidden message (don't display to the user, interpreted by dtail client)
-				wholePayload := []byte(fmt.Sprintf("%s\n", message))
+				wholePayload := []byte(fmt.Sprintf("%s%b", message, protocol.MessageDelimiter))
 				n = copy(p, wholePayload)
 				return
 			}
 
 			// Handle normal server message (display to the user)
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
+			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%b", h.hostname, message, protocol.MessageDelimiter))
 			n = copy(p, wholePayload)
 			return
 
 		case message := <-h.aggregatedMessages:
 			// Send mapreduce-aggregated data as a message.
-			data := fmt.Sprintf("AGGREGATE➔%s➔%s\n", h.hostname, message)
+			data := fmt.Sprintf("AGGREGATE%s%s%s%s%b",
+				protocol.AggregateDelimiter, h.hostname,
+				protocol.AggregateDelimiter, message, protocol.MessageDelimiter)
 			wholePayload := []byte(data)
 			n = copy(p, wholePayload)
 			return
 
 		case line := <-h.lines:
+			//fmt.Printf("<<<%d,%s>>>\n", len(line.Content), line.Content)
 			// Send normal file content data as a message.
 			serverInfo := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|",
 				h.hostname, line.TransmittedPerc, line.Count, line.SourceID))
@@ -182,8 +185,8 @@ func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, err
 		return args, argc, errors.New("unable to determine protocol version")
 	}
 
-	if args[1] != version.ProtocolCompat {
-		err := fmt.Errorf("server with protocol version '%s' but client with '%s', please update DTail", version.ProtocolCompat, args[1])
+	if args[1] != protocol.ProtocolCompat {
+		err := fmt.Errorf("server with protocol version '%s' but client with '%s', please update DTail", protocol.ProtocolCompat, args[1])
 		return args, argc, err
 	}
 
diff --git a/internal/version/version.go b/internal/version/version.go
index 3e683bd..7f07c83 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 const (
@@ -13,15 +14,13 @@ const (
 	Name string = "DTail"
 	// Version of DTail.
 	Version string = "3.2.0"
-	// ProtocolCompat -ibility version.
-	ProtocolCompat string = "3"
 	// Additional information for DTail
 	Additional string = "Have a lot of fun!"
 )
 
 // String representation of the DTail version.
 func String() string {
-	return fmt.Sprintf("%s %v Protocol %s %s", Name, Version, ProtocolCompat, Additional)
+	return fmt.Sprintf("%s %v Protocol %s %s", Name, Version, protocol.ProtocolCompat, Additional)
 }
 
 // PaintedString is a prettier string representation of the DTail version.
@@ -36,7 +35,7 @@ func PaintedString() string {
 	version := color.PaintWithAttr(fmt.Sprintf(" %s ", Version),
 		color.FgBlue, color.BgYellow, color.AttrBold)
 
-	protocol := color.Paint(fmt.Sprintf(" Protocol %s ", ProtocolCompat),
+	protocol := color.Paint(fmt.Sprintf(" Protocol %s ", protocol.ProtocolCompat),
 		color.FgBlack, color.BgGreen)
 
 	additional := color.PaintWithAttr(fmt.Sprintf(" %s ", Additional),
-- 
cgit v1.2.3


From 1eed2b211044415f5f3da0f1b154a8ac9f1967c0 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 22 Aug 2021 10:07:00 +0300
Subject: introduces the protocol package

---
 internal/clients/handlers/basehandler.go   | 3 ++-
 internal/clients/handlers/healthhandler.go | 3 ++-
 internal/clients/handlers/maprhandler.go   | 2 +-
 internal/clients/maprclient.go             | 3 ++-
 internal/server/handlers/controlhandler.go | 3 ++-
 internal/server/handlers/serverhandler.go  | 6 +++---
 internal/ssh/ssh.go                        | 3 ++-
 7 files changed, 14 insertions(+), 9 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index acafe0e..fe83faa 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -56,13 +56,14 @@ func (h *baseHandler) SendMessage(command string) error {
 // Read data from the dtail server via Writer interface.
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		h.receiveBuf = append(h.receiveBuf, b)
 		if b == protocol.MessageDelimiter {
 			if len(h.receiveBuf) == 0 {
 				continue
 			}
 			message := string(h.receiveBuf)
 			h.handleMessageType(message)
+		} else {
+			h.receiveBuf = append(h.receiveBuf, b)
 		}
 	}
 
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 0440706..213748c 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // HealthHandler implements the handler required for health checks.
@@ -72,7 +73,7 @@ func (h *HealthHandler) SendMessage(command string) error {
 func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		h.receiveBuf = append(h.receiveBuf, b)
-		if b == '\n' {
+		if b == protocol.MessageDelimiter { // '\n' {
 			h.receive <- string(h.receiveBuf)
 			h.receiveBuf = h.receiveBuf[:0]
 		}
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index 7ac5895..afad507 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -37,7 +37,7 @@ func NewMaprHandler(server string, query *mapr.Query, globalGroup *mapr.GlobalGr
 func (h *MaprHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		h.baseHandler.receiveBuf = append(h.baseHandler.receiveBuf, b)
-		if b == '\n' {
+		if b == protocol.MessageDelimiter { // '\n' {
 			if len(h.baseHandler.receiveBuf) == 0 {
 				continue
 			}
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 1c0c2cc..77b674b 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -170,7 +170,8 @@ func (c *MaprClient) printResults() {
 		return
 	}
 
-	logger.Raw(fmt.Sprintf("%s\n", c.query.RawQuery))
+	//logger.Raw(fmt.Sprintf("%s\n", c.query.RawQuery))
+	logger.Raw(c.query.RawQuery)
 	logger.Raw(result)
 }
 
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
index 1e17c78..a217b40 100644
--- a/internal/server/handlers/controlhandler.go
+++ b/internal/server/handlers/controlhandler.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
@@ -56,7 +57,7 @@ func (h *ControlHandler) Read(p []byte) (n int, err error) {
 	for {
 		select {
 		case message := <-h.serverMessages:
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
+			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%b", h.hostname, message, protocol.MessageDelimiter))
 			n = copy(p, wholePayload)
 			return
 		case <-h.done.Done():
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 23e3aeb..9541a34 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -92,13 +92,13 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			}
 			if message[0] == '.' {
 				// Handle hidden message (don't display to the user, interpreted by dtail client)
-				wholePayload := []byte(fmt.Sprintf("%s%b", message, protocol.MessageDelimiter))
+				wholePayload := []byte(fmt.Sprintf("%s%s", message, string(protocol.MessageDelimiter)))
 				n = copy(p, wholePayload)
 				return
 			}
 
 			// Handle normal server message (display to the user)
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%b", h.hostname, message, protocol.MessageDelimiter))
+			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%s", h.hostname, message, string(protocol.MessageDelimiter)))
 			n = copy(p, wholePayload)
 			return
 
@@ -112,7 +112,7 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			return
 
 		case line := <-h.lines:
-			//fmt.Printf("<<<%d,%s>>>\n", len(line.Content), line.Content)
+			//fmt.Printf("\t<<<%d,%s>>>\n", len(line.Content), line.Content)
 			// Send normal file content data as a message.
 			serverInfo := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|",
 				h.hostname, line.TransmittedPerc, line.Count, line.SourceID))
diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go
index 3a2e416..78bf99e 100644
--- a/internal/ssh/ssh.go
+++ b/internal/ssh/ssh.go
@@ -6,12 +6,13 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"github.com/mimecast/dtail/internal/io/logger"
 	"io/ioutil"
 	"net"
 	"os"
 	"syscall"
 
+	"github.com/mimecast/dtail/internal/io/logger"
+
 	gossh "golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/crypto/ssh/terminal"
-- 
cgit v1.2.3


From d10426ac1f380e9cfc9571856c4c7f7cedd7307a Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 28 Aug 2021 19:36:46 +0100
Subject: use a byte.Buffer in the file reader

---
 internal/io/fs/readfile.go                | 32 ++++++++++++++++++-------------
 internal/io/line/line.go                  |  5 +++--
 internal/io/pool/bytesbuffer.go           | 19 ++++++++++++++++++
 internal/mapr/server/aggregate.go         |  4 +++-
 internal/server/handlers/serverhandler.go |  9 +++++----
 5 files changed, 49 insertions(+), 20 deletions(-)
 create mode 100644 internal/io/pool/bytesbuffer.go

(limited to 'internal')

diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 8a365a1..e44f30e 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -2,6 +2,7 @@ package fs
 
 import (
 	"bufio"
+	"bytes"
 	"compress/gzip"
 	"context"
 	"errors"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 	"github.com/mimecast/dtail/internal/regex"
 
@@ -90,7 +92,7 @@ func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Re
 		fd.Seek(0, io.SeekEnd)
 	}
 
-	rawLines := make(chan []byte, 100)
+	rawLines := make(chan *bytes.Buffer, 100)
 	truncate := make(chan struct{})
 
 	var wg sync.WaitGroup
@@ -142,7 +144,7 @@ func (f readFile) makeReader(fd *os.File) (reader *bufio.Reader, err error) {
 	return
 }
 
-func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan []byte, truncate <-chan struct{}) error {
+func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Buffer, truncate <-chan struct{}) error {
 	var offset uint64
 
 	reader, err := f.makeReader(fd)
@@ -152,7 +154,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan []byte, t
 
 	lineLengthThreshold := 1024 * 1024 // 1mb
 	warnedAboutLongLine := false
-	message := make([]byte, 0, 512)
+	message := pool.BytesBuffer.Get().(*bytes.Buffer)
 
 	for {
 		select {
@@ -182,37 +184,41 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan []byte, t
 
 		switch b {
 		case '\n':
-			if len(message) == 0 {
+			if message.Len() == 0 {
 				time.Sleep(time.Millisecond * 100)
 				continue
 			}
+			message.WriteByte(protocol.MessageDelimiter)
 			select {
-			case rawLines <- append(message, protocol.MessageDelimiter):
-				message = make([]byte, 0, 512)
+			case rawLines <- message:
+				message = pool.BytesBuffer.Get().(*bytes.Buffer)
+				//fmt.Printf("%d %d %p\n", message.Len(), message.Cap(), message)
 				warnedAboutLongLine = false
 			case <-ctx.Done():
 				return nil
 			}
 		default:
-			if len(message) >= lineLengthThreshold {
+			if message.Len() >= lineLengthThreshold {
 				if !warnedAboutLongLine {
 					f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
+				message.WriteByte(protocol.MessageDelimiter)
 				select {
+				case rawLines <- message:
+					message = pool.BytesBuffer.Get().(*bytes.Buffer)
+					//fmt.Printf("%d %d %p\n", message.Len(), message.Cap(), message)
 				case <-ctx.Done():
 					return nil
-				case rawLines <- append(message, protocol.MessageDelimiter):
-					message = make([]byte, 0, 512)
 				}
 			}
-			message = append(message, b)
+			message.WriteByte(b)
 		}
 	}
 }
 
 // Filter log lines matching a given regular expression.
-func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-chan []byte, lines chan<- line.Line, re regex.Regex) {
+func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
 	defer wg.Done()
 
 	for {
@@ -233,10 +239,10 @@ func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-cha
 	}
 }
 
-func (f readFile) transmittable(lineBytes []byte, length, capacity int, re regex.Regex) (line.Line, bool) {
+func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int, re regex.Regex) (line.Line, bool) {
 	var read line.Line
 
-	if !re.Match(lineBytes) {
+	if !re.Match(lineBytes.Bytes()) {
 		f.updateLineNotMatched()
 		f.updateLineNotTransmitted()
 		return read, false
diff --git a/internal/io/line/line.go b/internal/io/line/line.go
index 715be34..d306c88 100644
--- a/internal/io/line/line.go
+++ b/internal/io/line/line.go
@@ -1,13 +1,14 @@
 package line
 
 import (
+	"bytes"
 	"fmt"
 )
 
 // Line represents a read log line.
 type Line struct {
 	// The content of the log line.
-	Content []byte
+	Content *bytes.Buffer
 	// Until now, how many log lines were processed?
 	Count uint64
 	// Sometimes we produce too many log lines so that the client
@@ -25,7 +26,7 @@ type Line struct {
 // Return a human readable representation of the followed line.
 func (l Line) String() string {
 	return fmt.Sprintf("Line(Content:%s,TransmittedPerc:%v,Count:%v,SourceID:%s)",
-		string(l.Content),
+		l.Content.String(),
 		l.TransmittedPerc,
 		l.Count,
 		l.SourceID)
diff --git a/internal/io/pool/bytesbuffer.go b/internal/io/pool/bytesbuffer.go
new file mode 100644
index 0000000..0a159f5
--- /dev/null
+++ b/internal/io/pool/bytesbuffer.go
@@ -0,0 +1,19 @@
+package pool
+
+import (
+	"bytes"
+	"sync"
+)
+
+var BytesBuffer = sync.Pool{
+	New: func() interface{} {
+		b := bytes.Buffer{}
+		b.Grow(128)
+		return &b
+	},
+}
+
+func RecycleBytesBuffer(b *bytes.Buffer) {
+	b.Reset()
+	BytesBuffer.Put(b)
+}
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 28bb074..9106f52 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -10,6 +10,7 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/logformat"
 )
@@ -136,7 +137,8 @@ func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 					return
 				}
 
-				maprLine := strings.TrimSpace(string(line.Content))
+				maprLine := strings.TrimSpace(line.Content.String())
+				pool.RecycleBytesBuffer(line.Content)
 				fields, err := a.parser.MakeFields(maprLine)
 				logger.Debug(fields, err)
 
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 9541a34..62f3c2b 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -15,6 +15,7 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/protocol"
@@ -114,10 +115,10 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		case line := <-h.lines:
 			//fmt.Printf("\t<<<%d,%s>>>\n", len(line.Content), line.Content)
 			// Send normal file content data as a message.
-			serverInfo := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|",
-				h.hostname, line.TransmittedPerc, line.Count, line.SourceID))
-			wholePayload := append(serverInfo, line.Content[:]...)
-			n = copy(p, wholePayload)
+			payload := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|%s",
+				h.hostname, line.TransmittedPerc, line.Count, line.SourceID, line.Content.String()))
+			n = copy(p, payload)
+			pool.RecycleBytesBuffer(line.Content)
 			return
 
 		case <-time.After(time.Second):
-- 
cgit v1.2.3


From 9ccd8ae93db0d3c6ed204114e1f1ab79f91fadf7 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 28 Aug 2021 20:10:54 +0100
Subject: make use of more buffers on server side

---
 internal/protocol/protocol.go             |  2 ++
 internal/server/handlers/serverhandler.go | 40 +++++++++++++++++++++----------
 2 files changed, 29 insertions(+), 13 deletions(-)

(limited to 'internal')

diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index 2a570cd..43021a2 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -5,6 +5,8 @@ const (
 	ProtocolCompat string = "4"
 	// MessageDelimiter delimits separate messages.
 	MessageDelimiter byte = '¬'
+	// FieldDelimiter delimits aggregation fields.
+	FieldDelimiter byte = '|'
 	// AggregateDelimiter delimits parts of an aggregation message.
 	AggregateDelimiter string = "➔"
 )
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 62f3c2b..f5aefa2 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"bytes"
 	"context"
 	"encoding/base64"
 	"errors"
@@ -93,31 +94,44 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			}
 			if message[0] == '.' {
 				// Handle hidden message (don't display to the user, interpreted by dtail client)
-				wholePayload := []byte(fmt.Sprintf("%s%s", message, string(protocol.MessageDelimiter)))
-				n = copy(p, wholePayload)
+				payload := []byte(fmt.Sprintf("%s%s", message, string(protocol.MessageDelimiter)))
+				n = copy(p, payload)
 				return
 			}
 
 			// Handle normal server message (display to the user)
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%s", h.hostname, message, string(protocol.MessageDelimiter)))
-			n = copy(p, wholePayload)
+			payload := []byte(fmt.Sprintf("SERVER|%s|%s%s", h.hostname, message, string(protocol.MessageDelimiter)))
+			n = copy(p, payload)
 			return
 
 		case message := <-h.aggregatedMessages:
 			// Send mapreduce-aggregated data as a message.
-			data := fmt.Sprintf("AGGREGATE%s%s%s%s%b",
-				protocol.AggregateDelimiter, h.hostname,
-				protocol.AggregateDelimiter, message, protocol.MessageDelimiter)
-			wholePayload := []byte(data)
-			n = copy(p, wholePayload)
+			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
+			buf.WriteString("AGGREGATE")
+			buf.WriteString(protocol.AggregateDelimiter)
+			buf.WriteString(h.hostname)
+			buf.WriteString(protocol.AggregateDelimiter)
+			buf.WriteString(message)
+			buf.WriteByte(protocol.MessageDelimiter)
+			n = copy(p, buf.Bytes())
+			pool.RecycleBytesBuffer(buf)
 			return
 
 		case line := <-h.lines:
-			//fmt.Printf("\t<<<%d,%s>>>\n", len(line.Content), line.Content)
-			// Send normal file content data as a message.
-			payload := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|%s",
-				h.hostname, line.TransmittedPerc, line.Count, line.SourceID, line.Content.String()))
+			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
+			buf.WriteString("REMOTE")
+			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(h.hostname)
+			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
+			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(fmt.Sprintf("%v", line.Count))
+			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(line.SourceID)
+			buf.WriteByte(protocol.FieldDelimiter)
+			payload := append(buf.Bytes(), line.Content.Bytes()...)
 			n = copy(p, payload)
+			pool.RecycleBytesBuffer(buf)
 			pool.RecycleBytesBuffer(line.Content)
 			return
 
-- 
cgit v1.2.3


From f98eab5780d8bc4454cc5293628bad118f424f68 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 28 Aug 2021 20:26:32 +0100
Subject: 1. Major performance gain by not checking for file truncation aftter 
   each bytes read. 2. Introduce field separator to the protocol package.

---
 internal/clients/healthclient.go          |  3 ++-
 internal/color/brush/brush.go             | 11 ++++++-----
 internal/io/fs/readfile.go                | 20 +++++++++-----------
 internal/mapr/logformat/default.go        |  4 +++-
 internal/protocol/protocol.go             |  2 +-
 internal/server/handlers/serverhandler.go | 10 +++++-----
 6 files changed, 26 insertions(+), 24 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index e93f6be..692464c 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -11,6 +11,7 @@ import (
 	"github.com/mimecast/dtail/internal/clients/remote"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/omode"
+	"github.com/mimecast/dtail/internal/protocol"
 
 	gossh "golang.org/x/crypto/ssh"
 )
@@ -57,7 +58,7 @@ func (c *HealthClient) Start(ctx context.Context) (status int) {
 		select {
 		case data := <-receive:
 			// Parse recieved data.
-			s := strings.Split(data, "|")
+			s := strings.Split(data, protocol.FieldDelimiter)
 			message := s[len(s)-1]
 			if strings.HasPrefix(message, "done;") {
 				return
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index c5efff6..87c02d0 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -6,11 +6,12 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // Add some color to log lines received from remote servers.
 func paintRemote(line string) string {
-	splitted := strings.Split(line, "|")
+	splitted := strings.Split(line, protocol.FieldDelimiter)
 	if splitted[2] == "100" {
 		splitted[2] = color.Paint(splitted[2],
 			config.Client.TermColors.RemoteStatsOkFg,
@@ -21,8 +22,8 @@ func paintRemote(line string) string {
 			config.Client.TermColors.RemoteStatsWarnBg)
 	}
 
-	info := strings.Join(splitted[0:5], "|")
-	log := strings.Join(splitted[5:], "|")
+	info := strings.Join(splitted[0:5], protocol.FieldDelimiter)
+	log := strings.Join(splitted[5:], protocol.FieldDelimiter)
 
 	switch {
 	case strings.HasPrefix(log, "WARN"):
@@ -67,8 +68,8 @@ func paintRemote(line string) string {
 
 // Add some color to stats generated by the client.
 func paintClientStats(line string) string {
-	splitted := strings.Split(line, "|")
-	first := strings.Join(splitted[0:4], "|")
+	splitted := strings.Split(line, protocol.FieldDelimiter)
+	first := strings.Join(splitted[0:4], protocol.FieldDelimiter)
 	connected := color.PaintWithAttr(splitted[4],
 		config.Client.TermColors.ClientStatsFg,
 		config.Client.TermColors.ClientStatsBg,
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index e44f30e..f2f672a 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -157,22 +157,21 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 	message := pool.BytesBuffer.Get().(*bytes.Buffer)
 
 	for {
-		select {
-		case <-ctx.Done():
-			return nil
-		case <-truncate:
-			if isTruncated, err := f.truncated(fd); isTruncated {
-				return err
-			}
-		default:
-		}
-
 		b, err := reader.ReadByte()
 
 		if err != nil {
 			if err != io.EOF {
 				return err
 			}
+			select {
+			case <-truncate:
+				if isTruncated, err := f.truncated(fd); isTruncated {
+					return err
+				}
+			case <-ctx.Done():
+				return nil
+			default:
+			}
 			if !f.seekEOF {
 				logger.Info(f.FilePath(), "End of file reached")
 				return nil
@@ -207,7 +206,6 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 				select {
 				case rawLines <- message:
 					message = pool.BytesBuffer.Get().(*bytes.Buffer)
-					//fmt.Printf("%d %d %p\n", message.Len(), message.Cap(), message)
 				case <-ctx.Done():
 					return nil
 				}
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 44bf558..32a34bd 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -3,12 +3,14 @@ package logformat
 import (
 	"errors"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // MakeFieldsDEFAULT is the default log file mapreduce parser.
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields := make(map[string]string, 20)
-	splitted := strings.Split(maprLine, "|")
+	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
 
 	fields["*"] = "*"
 	fields["$line"] = maprLine
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index 43021a2..d3035e4 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -6,7 +6,7 @@ const (
 	// MessageDelimiter delimits separate messages.
 	MessageDelimiter byte = '¬'
 	// FieldDelimiter delimits aggregation fields.
-	FieldDelimiter byte = '|'
+	FieldDelimiter string = "|"
 	// AggregateDelimiter delimits parts of an aggregation message.
 	AggregateDelimiter string = "➔"
 )
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index f5aefa2..14fc5d0 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -120,15 +120,15 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		case line := <-h.lines:
 			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
 			buf.WriteString("REMOTE")
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			buf.WriteString(h.hostname)
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			buf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			buf.WriteString(fmt.Sprintf("%v", line.Count))
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			buf.WriteString(line.SourceID)
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			payload := append(buf.Bytes(), line.Content.Bytes()...)
 			n = copy(p, payload)
 			pool.RecycleBytesBuffer(buf)
-- 
cgit v1.2.3


From 314a542b3c7a4f2bf082389e1233cf257730f857 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 5 Sep 2021 15:51:26 +0300
Subject: finalize new default color schema

---
 internal/color/brush/brush.go | 110 ++++++++++++++++++----------
 internal/color/paint.go       |  59 +++++++++++----
 internal/color/table.go       |   2 +-
 internal/config/client.go     | 164 +++++++++++++++++++++---------------------
 internal/protocol/protocol.go |   4 +-
 internal/version/version.go   |   8 +--
 6 files changed, 212 insertions(+), 135 deletions(-)

(limited to 'internal')

diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 87c02d0..415028a 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -1,7 +1,6 @@
 package brush
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/mimecast/dtail/internal/color"
@@ -10,96 +9,133 @@ import (
 )
 
 // Add some color to log lines received from remote servers.
-func paintRemote(line string) string {
-	splitted := strings.Split(line, protocol.FieldDelimiter)
+func paintRemote(sb *strings.Builder, line string) {
+	splitted := strings.SplitN(line, protocol.FieldDelimiter, 6)
+
+	color.PaintWithAttr(sb, splitted[0],
+		config.Client.TermColors.RemoteStrFg,
+		config.Client.TermColors.RemoteStrBg,
+		config.Client.TermColors.RemoteStrAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[1],
+		config.Client.TermColors.RemoteServerFg,
+		config.Client.TermColors.RemoteServerBg,
+		config.Client.TermColors.RemoteServerAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
 	if splitted[2] == "100" {
-		splitted[2] = color.Paint(splitted[2],
+		color.PaintWithAttr(sb, splitted[2],
 			config.Client.TermColors.RemoteStatsOkFg,
-			config.Client.TermColors.RemoteStatsOkBg)
+			config.Client.TermColors.RemoteStatsOkBg,
+			config.Client.TermColors.RemoteStatsOkAttr)
 	} else {
-		splitted[2] = color.Paint(splitted[2],
+		color.PaintWithAttr(sb, splitted[2],
 			config.Client.TermColors.RemoteStatsWarnFg,
-			config.Client.TermColors.RemoteStatsWarnBg)
+			config.Client.TermColors.RemoteStatsWarnBg,
+			config.Client.TermColors.RemoteStatsWarnAttr)
 	}
 
-	info := strings.Join(splitted[0:5], protocol.FieldDelimiter)
-	log := strings.Join(splitted[5:], protocol.FieldDelimiter)
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[3],
+		config.Client.TermColors.RemoteCountFg,
+		config.Client.TermColors.RemoteCountBg,
+		config.Client.TermColors.RemoteCountAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[4],
+		config.Client.TermColors.RemoteIdFg,
+		config.Client.TermColors.RemoteIdBg,
+		config.Client.TermColors.RemoteIdAttr)
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
+	log := splitted[5]
 
 	switch {
 	case strings.HasPrefix(log, "WARN"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteWarnFg,
 			config.Client.TermColors.RemoteWarnBg,
 			config.Client.TermColors.RemoteWarnAttr)
 
 	case strings.HasPrefix(log, "ERROR"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteErrorFg,
 			config.Client.TermColors.RemoteErrorBg,
 			config.Client.TermColors.RemoteErrorAttr)
 
 	case strings.HasPrefix(log, "FATAL"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteFatalFg,
 			config.Client.TermColors.RemoteFatalBg,
 			config.Client.TermColors.RemoteFatalAttr)
 
 	case strings.HasPrefix(log, "DEBUG"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteDebugFg,
 			config.Client.TermColors.RemoteDebugBg,
 			config.Client.TermColors.RemoteDebugAttr)
 
 	case strings.HasPrefix(log, "TRACE"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteTraceFg,
 			config.Client.TermColors.RemoteTraceBg,
 			config.Client.TermColors.RemoteTraceAttr)
 
 	default:
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteTextFg,
 			config.Client.TermColors.RemoteTextBg,
 			config.Client.TermColors.RemoteTextAttr)
 	}
-
-	return fmt.Sprintf("%s|%s", info, log)
-}
-
-// Add some color to stats generated by the client.
-func paintClientStats(line string) string {
-	splitted := strings.Split(line, protocol.FieldDelimiter)
-	first := strings.Join(splitted[0:4], protocol.FieldDelimiter)
-	connected := color.PaintWithAttr(splitted[4],
-		config.Client.TermColors.ClientStatsFg,
-		config.Client.TermColors.ClientStatsBg,
-		config.Client.TermColors.ClientStatsAttr)
-	last := strings.Join(splitted[5:], "|")
-
-	return fmt.Sprintf("%s|%s|%s", first, connected, last)
 }
 
 // Colorfy a given line based on the line's content.
 func Colorfy(line string) string {
+	sb := strings.Builder{}
+
 	switch {
 	case strings.HasPrefix(line, "REMOTE"):
-		return paintRemote(line)
-
-	case strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|"):
-		return paintClientStats(line)
+		paintRemote(&sb, line)
 
 	case strings.Contains(line, "ERROR"):
-		return color.PaintWithAttr(line,
+		color.PaintWithAttr(&sb, line,
 			config.Client.TermColors.ClientErrorFg,
 			config.Client.TermColors.ClientErrorBg,
 			config.Client.TermColors.ClientErrorAttr)
 
 	case strings.Contains(line, "WARN"):
-		return color.PaintWithAttr(line,
+		color.PaintWithAttr(&sb, line,
 			config.Client.TermColors.ClientWarnFg,
 			config.Client.TermColors.ClientWarnBg,
 			config.Client.TermColors.ClientWarnAttr)
+
+	default:
+		color.PaintWithAttr(&sb, line,
+			color.FgDefault,
+			color.BgDefault,
+			color.AttrNone)
 	}
 
-	return line
+	color.ResetWithAttr(&sb)
+	return sb.String()
 }
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 7862467..2798ff7 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -1,31 +1,66 @@
 package color
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
-// Paint paints a given text in a given foreground/background color combination.
-func Paint(text string, fg FgColor, bg BgColor) string {
+// PaintStr paints a given text in a given foreground/background color combination.
+func PaintStr(text string, fg FgColor, bg BgColor) string {
 	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
 }
 
-// PaintWithAttr paints a given text in a given foreground/background/attribute combination
-func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
+// PaintStrWithAttr paints a given text in a given foreground/background/attribute combination
+func PaintStrWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
 	if attr == AttrNone {
-		return Paint(text, fg, bg)
+		return PaintStr(text, fg, bg)
 	}
 	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
 }
 
-// PaintFg paints a given text in a given foreground color.
-func PaintFg(text string, fg FgColor) string {
+// PaintStrFg paints a given text in a given foreground color.
+func PaintStrFg(text string, fg FgColor) string {
 	return fmt.Sprintf("%s%s%s", fg, text, FgDefault)
 }
 
-// PaintBg paints a given text in a given background color.
-func PaintBg(text string, bg BgColor) string {
+// PaintStrBg paints a given text in a given background color.
+func PaintStrBg(text string, bg BgColor) string {
 	return fmt.Sprintf("%s%s%s", bg, text, BgDefault)
 }
 
-// PaintAttr adds a given attribute to a given text, such as "bold" or "italic".
-func PaintAttr(text string, attr Attribute) string {
+// PaintStrAttr adds a given attribute to a given text, such as "bold" or "italic".
+func PaintStrAttr(text string, attr Attribute) string {
 	return fmt.Sprintf("%s%s%s", attr, text, AttrReset)
 }
+
+// Paint paints a given text in a given foreground/background color combination.
+func Paint(sb *strings.Builder, text string, fg FgColor, bg BgColor) {
+	sb.WriteString(string(fg))
+	sb.WriteString(string(bg))
+	sb.WriteString(text)
+}
+
+// Reset background and foreground colors.
+func Reset(sb *strings.Builder) {
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
+}
+
+// PaintWithAttr starts painting a given text in a given foreground/background/attribute combination.
+func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, attr Attribute) {
+	if attr == AttrNone {
+		Paint(sb, text, fg, bg)
+		return
+	}
+	sb.WriteString(string(fg))
+	sb.WriteString(string(bg))
+	sb.WriteString(string(attr))
+	sb.WriteString(text)
+}
+
+// ResetWithAttr resets background, foreground and attributes.
+func ResetWithAttr(sb *strings.Builder) {
+	sb.WriteString(string(AttrReset))
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
+}
diff --git a/internal/color/table.go b/internal/color/table.go
index 8c36047..e2265e3 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -26,7 +26,7 @@ func printColorTable(attr string) {
 			attribute, _ := ToAttribute(attr)
 
 			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
-			fmt.Print(PaintWithAttr(text, fgColor, bgColor, attribute))
+			fmt.Print(PaintStrWithAttr(text, fgColor, bgColor, attribute))
 			fmt.Print("\n")
 		}
 	}
diff --git a/internal/config/client.go b/internal/config/client.go
index c1b488c..05e4f93 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -4,49 +4,51 @@ import "github.com/mimecast/dtail/internal/color"
 
 // ClientColorConfig allows to override the default terminal color color.
 type termColors struct {
-	ClientErrorAttr color.Attribute
-	ClientErrorBg   color.BgColor
-	ClientErrorFg   color.FgColor
-
-	ClientStatsAttr color.Attribute
-	ClientStatsBg   color.BgColor
-	ClientStatsFg   color.FgColor
-
-	ClientWarnAttr color.Attribute
-	ClientWarnBg   color.BgColor
-	ClientWarnFg   color.FgColor
-
-	RemoteDebugAttr color.Attribute
-	RemoteDebugBg   color.BgColor
-	RemoteDebugFg   color.FgColor
-
-	RemoteErrorAttr color.Attribute
-	RemoteErrorBg   color.BgColor
-	RemoteErrorFg   color.FgColor
-
-	RemoteFatalAttr color.Attribute
-	RemoteFatalBg   color.BgColor
-	RemoteFatalFg   color.FgColor
-
-	RemoteStatsOkAttr color.Attribute
-	RemoteStatsOkBg   color.BgColor
-	RemoteStatsOkFg   color.FgColor
-
+	ClientErrorAttr     color.Attribute
+	ClientErrorBg       color.BgColor
+	ClientErrorFg       color.FgColor
+	ClientWarnAttr      color.Attribute
+	ClientWarnBg        color.BgColor
+	ClientWarnFg        color.FgColor
+	DelimiterAttr       color.Attribute
+	DelimiterBg         color.BgColor
+	DelimiterFg         color.FgColor
+	RemoteCountAttr     color.Attribute
+	RemoteCountBg       color.BgColor
+	RemoteCountFg       color.FgColor
+	RemoteDebugAttr     color.Attribute
+	RemoteDebugBg       color.BgColor
+	RemoteDebugFg       color.FgColor
+	RemoteErrorAttr     color.Attribute
+	RemoteErrorBg       color.BgColor
+	RemoteErrorFg       color.FgColor
+	RemoteFatalAttr     color.Attribute
+	RemoteFatalBg       color.BgColor
+	RemoteFatalFg       color.FgColor
+	RemoteIdAttr        color.Attribute
+	RemoteIdBg          color.BgColor
+	RemoteIdFg          color.FgColor
+	RemoteServerAttr    color.Attribute
+	RemoteServerBg      color.BgColor
+	RemoteServerFg      color.FgColor
+	RemoteStatsOkAttr   color.Attribute
+	RemoteStatsOkBg     color.BgColor
+	RemoteStatsOkFg     color.FgColor
 	RemoteStatsWarnAttr color.Attribute
 	RemoteStatsWarnBg   color.BgColor
 	RemoteStatsWarnFg   color.FgColor
-
-	RemoteTextAttr color.Attribute
-	RemoteTextBg   color.BgColor
-	RemoteTextFg   color.FgColor
-
-	RemoteTraceAttr color.Attribute
-	RemoteTraceBg   color.BgColor
-	RemoteTraceFg   color.FgColor
-
-	RemoteWarnAttr color.Attribute
-	RemoteWarnBg   color.BgColor
-	RemoteWarnFg   color.FgColor
+	RemoteStrAttr       color.Attribute
+	RemoteStrBg         color.BgColor
+	RemoteStrFg         color.FgColor
+	RemoteTextAttr      color.Attribute
+	RemoteTextBg        color.BgColor
+	RemoteTextFg        color.FgColor
+	RemoteTraceAttr     color.Attribute
+	RemoteTraceBg       color.BgColor
+	RemoteTraceFg       color.FgColor
+	RemoteWarnAttr      color.Attribute
+	RemoteWarnBg        color.BgColor
+	RemoteWarnFg        color.FgColor
 }
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
@@ -61,49 +63,51 @@ func newDefaultClientConfig() *ClientConfig {
 	return &ClientConfig{
 		TermColorsEnable: true,
 		TermColors: termColors{
-			ClientErrorAttr: color.AttrBold,
-			ClientErrorBg:   color.BgBlack,
-			ClientErrorFg:   color.FgRed,
-
-			ClientStatsAttr: color.AttrDim,
-			ClientStatsBg:   color.BgBlue,
-			ClientStatsFg:   color.FgWhite,
-
-			ClientWarnAttr: color.AttrNone,
-			ClientWarnBg:   color.BgBlack,
-			ClientWarnFg:   color.FgMagenta,
-
-			RemoteDebugAttr: color.AttrNone,
-			RemoteDebugBg:   color.BgGreen,
-			RemoteDebugFg:   color.FgBlack,
-
-			RemoteErrorAttr: color.AttrBold,
-			RemoteErrorBg:   color.BgRed,
-			RemoteErrorFg:   color.FgWhite,
-
-			RemoteFatalAttr: color.AttrBlink,
-			RemoteFatalBg:   color.BgRed,
-			RemoteFatalFg:   color.FgWhite,
-
-			RemoteStatsOkAttr: color.AttrNone,
-			RemoteStatsOkBg:   color.BgGreen,
-			RemoteStatsOkFg:   color.FgBlack,
-
+			ClientErrorAttr:     color.AttrBold,
+			ClientErrorBg:       color.BgBlack,
+			ClientErrorFg:       color.FgRed,
+			ClientWarnAttr:      color.AttrNone,
+			ClientWarnBg:        color.BgBlack,
+			ClientWarnFg:        color.FgMagenta,
+			DelimiterAttr:       color.AttrDim,
+			DelimiterBg:         color.BgBlue,
+			DelimiterFg:         color.FgCyan,
+			RemoteCountAttr:     color.AttrDim,
+			RemoteCountBg:       color.BgBlue,
+			RemoteCountFg:       color.FgGreen,
+			RemoteDebugAttr:     color.AttrBold,
+			RemoteDebugBg:       color.BgBlack,
+			RemoteDebugFg:       color.FgGreen,
+			RemoteErrorAttr:     color.AttrBold,
+			RemoteErrorBg:       color.BgRed,
+			RemoteErrorFg:       color.FgWhite,
+			RemoteFatalAttr:     color.AttrBlink,
+			RemoteFatalBg:       color.BgRed,
+			RemoteFatalFg:       color.FgWhite,
+			RemoteIdAttr:        color.AttrDim,
+			RemoteIdBg:          color.BgBlue,
+			RemoteIdFg:          color.FgWhite,
+			RemoteServerAttr:    color.AttrBold,
+			RemoteServerBg:      color.BgBlue,
+			RemoteServerFg:      color.FgWhite,
+			RemoteStatsOkAttr:   color.AttrNone,
+			RemoteStatsOkBg:     color.BgGreen,
+			RemoteStatsOkFg:     color.FgBlue,
 			RemoteStatsWarnAttr: color.AttrNone,
 			RemoteStatsWarnBg:   color.BgRed,
 			RemoteStatsWarnFg:   color.FgWhite,
-
-			RemoteTextAttr: color.AttrNone,
-			RemoteTextBg:   color.BgBlack,
-			RemoteTextFg:   color.FgWhite,
-
-			RemoteTraceAttr: color.AttrBold,
-			RemoteTraceBg:   color.BgGreen,
-			RemoteTraceFg:   color.FgWhite,
-
-			RemoteWarnAttr: color.AttrBold,
-			RemoteWarnBg:   color.BgYellow,
-			RemoteWarnFg:   color.FgWhite,
+			RemoteStrAttr:       color.AttrDim,
+			RemoteStrBg:         color.BgBlue,
+			RemoteStrFg:         color.FgWhite,
+			RemoteTextAttr:      color.AttrNone,
+			RemoteTextBg:        color.BgBlack,
+			RemoteTextFg:        color.FgWhite,
+			RemoteTraceAttr:     color.AttrBold,
+			RemoteTraceBg:       color.BgGreen,
+			RemoteTraceFg:       color.FgWhite,
+			RemoteWarnAttr:      color.AttrBold,
+			RemoteWarnBg:        color.BgYellow,
+			RemoteWarnFg:        color.FgWhite,
 		},
 	}
 }
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index d3035e4..e92ec6a 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -7,6 +7,8 @@ const (
 	MessageDelimiter byte = '¬'
 	// FieldDelimiter delimits aggregation fields.
 	FieldDelimiter string = "|"
+	// Arrow for multiple purposes
+	Arrow string = "➔"
 	// AggregateDelimiter delimits parts of an aggregation message.
-	AggregateDelimiter string = "➔"
+	AggregateDelimiter string = Arrow
 )
diff --git a/internal/version/version.go b/internal/version/version.go
index 7f07c83..693192f 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -29,16 +29,16 @@ func PaintedString() string {
 		return String()
 	}
 
-	name := color.PaintWithAttr(fmt.Sprintf(" %s ", Name),
+	name := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Name),
 		color.FgYellow, color.BgBlue, color.AttrBold)
 
-	version := color.PaintWithAttr(fmt.Sprintf(" %s ", Version),
+	version := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Version),
 		color.FgBlue, color.BgYellow, color.AttrBold)
 
-	protocol := color.Paint(fmt.Sprintf(" Protocol %s ", protocol.ProtocolCompat),
+	protocol := color.PaintStr(fmt.Sprintf(" Protocol %s ", protocol.ProtocolCompat),
 		color.FgBlack, color.BgGreen)
 
-	additional := color.PaintWithAttr(fmt.Sprintf(" %s ", Additional),
+	additional := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Additional),
 		color.FgWhite, color.BgMagenta, color.AttrBlink)
 
 	return fmt.Sprintf("%s%v%s%s", name, version, protocol, additional)
-- 
cgit v1.2.3


From ccb1fea7cad05e2143db91838c687fa80b7ca209 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 5 Sep 2021 15:55:57 +0300
Subject: fix unit test

---
 internal/color/color_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'internal')

diff --git a/internal/color/color_test.go b/internal/color/color_test.go
index bfc7c54..7002052 100644
--- a/internal/color/color_test.go
+++ b/internal/color/color_test.go
@@ -14,13 +14,13 @@ func TestColors(t *testing.T) {
 		if err != nil {
 			t.Errorf("unable to paint foreground : %s\n%v", text, err)
 		}
-		builder.WriteString(PaintFg(text, fgColor))
+		builder.WriteString(PaintStrFg(text, fgColor))
 
 		bgColor, err := ToBgColor(color)
 		if err != nil {
 			t.Errorf("unable to paint background: %s\n%v", text, err)
 		}
-		builder.WriteString(PaintBg(text, bgColor))
+		builder.WriteString(PaintStrBg(text, bgColor))
 	}
 
 	for _, fg := range ColorNames {
@@ -30,7 +30,7 @@ func TestColors(t *testing.T) {
 				continue
 			}
 			bgColor, _ := ToBgColor(bg)
-			builder.WriteString(Paint(text, fgColor, bgColor))
+			builder.WriteString(PaintStr(text, fgColor, bgColor))
 		}
 	}
 
@@ -46,7 +46,7 @@ func TestAttributes(t *testing.T) {
 		if err != nil {
 			t.Errorf("unable to paint attribute: %s\n%v", text, err)
 		}
-		builder.WriteString(PaintWithAttr(text, FgWhite, BgBlue, att))
+		builder.WriteString(PaintStrWithAttr(text, FgWhite, BgBlue, att))
 	}
 
 	t.Log(builder.String())
-- 
cgit v1.2.3


From 8e4c7fd64f4e18126d00d52ae96a2c71c44ef0cb Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 5 Sep 2021 16:04:45 +0300
Subject: -colorTable in combination with -debug prints whole sample paragraphs
 in all color combinations.

---
 internal/color/table.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'internal')

diff --git a/internal/color/table.go b/internal/color/table.go
index e2265e3..7ecfbca 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -5,17 +5,19 @@ import (
 	"os"
 )
 
-func TablePrintAndExit() {
+const sampleParagraph string = "Mimecast is Making Email Safer for Business. We believe that securely operating a business in the cloud requires new levels of IT preparedness, centered around cyber resilience. This is why we unify the delivery and management of security, continuity and data protection for email via one, simple-to-use cloud platform. Thousands of organizations trust us to increase their cyber resilience preparedness, streamline compliance, reduce IT complexity and keep their business running. We give employees fast and secure access to sensitive business information, and ensure email keeps flowing in the event of an outage. Mimecast will remain committed to protecting your IT assets through constant innovation and focus on your success."
+
+func TablePrintAndExit(useSampleParagraph bool) {
 	for _, attr := range AttributeNames {
 		if attr == "Hidden" || attr == "SlowBlink" {
 			continue
 		}
-		printColorTable(attr)
+		printColorTable(attr, useSampleParagraph)
 	}
 	os.Exit(0)
 }
 
-func printColorTable(attr string) {
+func printColorTable(attr string, useSampleParagraph bool) {
 	for _, fg := range ColorNames {
 		fgColor, _ := ToFgColor(fg)
 		for _, bg := range ColorNames {
@@ -27,6 +29,11 @@ func printColorTable(attr string) {
 
 			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
 			fmt.Print(PaintStrWithAttr(text, fgColor, bgColor, attribute))
+			if useSampleParagraph {
+				fmt.Print("\n")
+				fmt.Print(PaintStrWithAttr(sampleParagraph, fgColor, bgColor, attribute))
+				fmt.Print("\n")
+			}
 			fmt.Print("\n")
 		}
 	}
-- 
cgit v1.2.3


From fcd55ba9a5378d9e34c1aa27313a4ce184da5268 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 6 Sep 2021 09:22:21 +0300
Subject: REMOTE and CLIENT colors are brushed correctly too now

---
 internal/clients/stats.go     |  14 ++-
 internal/color/brush/brush.go | 231 +++++++++++++++++++++++++++---------------
 internal/config/client.go     | 229 ++++++++++++++++++++++++-----------------
 3 files changed, 297 insertions(+), 177 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index d8163d4..3f76e0f 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
 )
@@ -54,7 +55,6 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 		throttle := len(throttleCh)
 
 		newConnections := connected - connectedLast
-
 		if (connected == connectedLast || quiet) && !force {
 			continue
 		}
@@ -77,7 +77,15 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 
 func (s *stats) printStatsDueInterrupt(messages []string) {
 	logger.Pause()
-	for _, message := range messages {
+	for i, message := range messages {
+		if i > 0 && config.Client.TermColorsEnable {
+			fmt.Println(color.PaintStrWithAttr(message,
+				config.Client.TermColors.Client.TextFg,
+				config.Client.TermColors.Client.TextBg,
+				config.Client.TermColors.Client.TextAttr,
+			))
+			continue
+		}
 		fmt.Println(fmt.Sprintf(" %s", message))
 	}
 	time.Sleep(time.Second * time.Duration(config.InterruptTimeoutS))
@@ -99,7 +107,6 @@ func (s *stats) statsLine(connected, newConnections int, throttle int) string {
 func (s *stats) numConnected() int {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
-
 	return s.connected
 }
 
@@ -107,6 +114,5 @@ func percentOf(total float64, value float64) float64 {
 	if total == 0 || total == value {
 		return 100
 	}
-
 	return value / (total / 100.0)
 }
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 415028a..785a0d0 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -8,105 +8,165 @@ import (
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// Add some color to log lines received from remote servers.
+func paintSeverity(sb *strings.Builder, text string) bool {
+	switch {
+	case strings.HasPrefix(text, "WARN"):
+		color.PaintWithAttr(sb, text,
+			config.Client.TermColors.Common.SeverityWarnFg,
+			config.Client.TermColors.Common.SeverityWarnBg,
+			config.Client.TermColors.Common.SeverityWarnAttr)
+
+	case strings.HasPrefix(text, "ERROR"):
+		color.PaintWithAttr(sb, text,
+			config.Client.TermColors.Common.SeverityErrorFg,
+			config.Client.TermColors.Common.SeverityErrorBg,
+			config.Client.TermColors.Common.SeverityErrorAttr)
+
+	case strings.HasPrefix(text, "FATAL"):
+		color.PaintWithAttr(sb, text,
+			config.Client.TermColors.Common.SeverityFatalFg,
+			config.Client.TermColors.Common.SeverityFatalBg,
+			config.Client.TermColors.Common.SeverityFatalAttr)
+
+	default:
+		return false
+	}
+
+	return true
+}
+
 func paintRemote(sb *strings.Builder, line string) {
 	splitted := strings.SplitN(line, protocol.FieldDelimiter, 6)
 
 	color.PaintWithAttr(sb, splitted[0],
-		config.Client.TermColors.RemoteStrFg,
-		config.Client.TermColors.RemoteStrBg,
-		config.Client.TermColors.RemoteStrAttr)
+		config.Client.TermColors.Remote.RemoteFg,
+		config.Client.TermColors.Remote.RemoteBg,
+		config.Client.TermColors.Remote.RemoteAttr)
 
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
 	color.PaintWithAttr(sb, splitted[1],
-		config.Client.TermColors.RemoteServerFg,
-		config.Client.TermColors.RemoteServerBg,
-		config.Client.TermColors.RemoteServerAttr)
+		config.Client.TermColors.Remote.HostnameFg,
+		config.Client.TermColors.Remote.HostnameBg,
+		config.Client.TermColors.Remote.HostnameAttr)
 
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
 	if splitted[2] == "100" {
 		color.PaintWithAttr(sb, splitted[2],
-			config.Client.TermColors.RemoteStatsOkFg,
-			config.Client.TermColors.RemoteStatsOkBg,
-			config.Client.TermColors.RemoteStatsOkAttr)
+			config.Client.TermColors.Remote.StatsOkFg,
+			config.Client.TermColors.Remote.StatsOkBg,
+			config.Client.TermColors.Remote.StatsOkAttr)
 	} else {
 		color.PaintWithAttr(sb, splitted[2],
-			config.Client.TermColors.RemoteStatsWarnFg,
-			config.Client.TermColors.RemoteStatsWarnBg,
-			config.Client.TermColors.RemoteStatsWarnAttr)
+			config.Client.TermColors.Remote.StatsWarnFg,
+			config.Client.TermColors.Remote.StatsWarnBg,
+			config.Client.TermColors.Remote.StatsWarnAttr)
 	}
 
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
 	color.PaintWithAttr(sb, splitted[3],
-		config.Client.TermColors.RemoteCountFg,
-		config.Client.TermColors.RemoteCountBg,
-		config.Client.TermColors.RemoteCountAttr)
+		config.Client.TermColors.Remote.CountFg,
+		config.Client.TermColors.Remote.CountBg,
+		config.Client.TermColors.Remote.CountAttr)
 
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
 	color.PaintWithAttr(sb, splitted[4],
-		config.Client.TermColors.RemoteIdFg,
-		config.Client.TermColors.RemoteIdBg,
-		config.Client.TermColors.RemoteIdAttr)
+		config.Client.TermColors.Remote.IdFg,
+		config.Client.TermColors.Remote.IdBg,
+		config.Client.TermColors.Remote.IdAttr)
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
-	log := splitted[5]
+	if paintSeverity(sb, splitted[5]) {
+		return
+	}
+	color.PaintWithAttr(sb, splitted[5],
+		config.Client.TermColors.Remote.TextFg,
+		config.Client.TermColors.Remote.TextBg,
+		config.Client.TermColors.Remote.TextAttr)
+}
 
-	switch {
-	case strings.HasPrefix(log, "WARN"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteWarnFg,
-			config.Client.TermColors.RemoteWarnBg,
-			config.Client.TermColors.RemoteWarnAttr)
-
-	case strings.HasPrefix(log, "ERROR"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteErrorFg,
-			config.Client.TermColors.RemoteErrorBg,
-			config.Client.TermColors.RemoteErrorAttr)
-
-	case strings.HasPrefix(log, "FATAL"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteFatalFg,
-			config.Client.TermColors.RemoteFatalBg,
-			config.Client.TermColors.RemoteFatalAttr)
-
-	case strings.HasPrefix(log, "DEBUG"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteDebugFg,
-			config.Client.TermColors.RemoteDebugBg,
-			config.Client.TermColors.RemoteDebugAttr)
-
-	case strings.HasPrefix(log, "TRACE"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteTraceFg,
-			config.Client.TermColors.RemoteTraceBg,
-			config.Client.TermColors.RemoteTraceAttr)
+func paintClient(sb *strings.Builder, line string) {
+	splitted := strings.SplitN(line, protocol.FieldDelimiter, 3)
 
-	default:
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteTextFg,
-			config.Client.TermColors.RemoteTextBg,
-			config.Client.TermColors.RemoteTextAttr)
+	color.PaintWithAttr(sb, splitted[0],
+		config.Client.TermColors.Client.ClientFg,
+		config.Client.TermColors.Client.ClientBg,
+		config.Client.TermColors.Client.ClientAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.Client.DelimiterFg,
+		config.Client.TermColors.Client.DelimiterBg,
+		config.Client.TermColors.Client.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[1],
+		config.Client.TermColors.Client.HostnameFg,
+		config.Client.TermColors.Client.HostnameBg,
+		config.Client.TermColors.Client.HostnameAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.Client.DelimiterFg,
+		config.Client.TermColors.Client.DelimiterBg,
+		config.Client.TermColors.Client.DelimiterAttr)
+
+	if paintSeverity(sb, splitted[2]) {
+		return
 	}
+
+	color.PaintWithAttr(sb, splitted[2],
+		config.Client.TermColors.Client.TextFg,
+		config.Client.TermColors.Client.TextBg,
+		config.Client.TermColors.Client.TextAttr)
+}
+
+func paintServer(sb *strings.Builder, line string) {
+	splitted := strings.SplitN(line, protocol.FieldDelimiter, 3)
+
+	color.PaintWithAttr(sb, splitted[0],
+		config.Client.TermColors.Server.ServerFg,
+		config.Client.TermColors.Server.ServerBg,
+		config.Client.TermColors.Server.ServerAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.Server.DelimiterFg,
+		config.Client.TermColors.Server.DelimiterBg,
+		config.Client.TermColors.Server.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[1],
+		config.Client.TermColors.Server.HostnameFg,
+		config.Client.TermColors.Server.HostnameBg,
+		config.Client.TermColors.Server.HostnameAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.Server.DelimiterFg,
+		config.Client.TermColors.Server.DelimiterBg,
+		config.Client.TermColors.Server.DelimiterAttr)
+
+	if paintSeverity(sb, splitted[2]) {
+		return
+	}
+
+	color.PaintWithAttr(sb, splitted[2],
+		config.Client.TermColors.Server.TextFg,
+		config.Client.TermColors.Server.TextBg,
+		config.Client.TermColors.Server.TextAttr)
 }
 
 // Colorfy a given line based on the line's content.
@@ -117,18 +177,25 @@ func Colorfy(line string) string {
 	case strings.HasPrefix(line, "REMOTE"):
 		paintRemote(&sb, line)
 
-	case strings.Contains(line, "ERROR"):
-		color.PaintWithAttr(&sb, line,
-			config.Client.TermColors.ClientErrorFg,
-			config.Client.TermColors.ClientErrorBg,
-			config.Client.TermColors.ClientErrorAttr)
-
-	case strings.Contains(line, "WARN"):
-		color.PaintWithAttr(&sb, line,
-			config.Client.TermColors.ClientWarnFg,
-			config.Client.TermColors.ClientWarnBg,
-			config.Client.TermColors.ClientWarnAttr)
-
+	case strings.HasPrefix(line, "CLIENT"):
+		paintClient(&sb, line)
+
+	case strings.HasPrefix(line, "SERVER"):
+		paintServer(&sb, line)
+
+		/*
+			case strings.Contains(line, "ERROR"):
+				color.PaintWithAttr(&sb, line,
+					config.Client.TermColors.ClientErrorFg,
+					config.Client.TermColors.ClientErrorBg,
+					config.Client.TermColors.ClientErrorAttr)
+
+			case strings.Contains(line, "WARN"):
+				color.PaintWithAttr(&sb, line,
+					config.Client.TermColors.ClientWarnFg,
+					config.Client.TermColors.ClientWarnBg,
+					config.Client.TermColors.ClientWarnAttr)
+		*/
 	default:
 		color.PaintWithAttr(&sb, line,
 			color.FgDefault,
diff --git a/internal/config/client.go b/internal/config/client.go
index 05e4f93..837c7d6 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -2,53 +2,80 @@ package config
 
 import "github.com/mimecast/dtail/internal/color"
 
-// ClientColorConfig allows to override the default terminal color color.
+type remoteTermColors struct {
+	DelimiterAttr color.Attribute
+	DelimiterBg   color.BgColor
+	DelimiterFg   color.FgColor
+	RemoteAttr    color.Attribute
+	RemoteBg      color.BgColor
+	RemoteFg      color.FgColor
+	CountAttr     color.Attribute
+	CountBg       color.BgColor
+	CountFg       color.FgColor
+	HostnameAttr  color.Attribute
+	HostnameBg    color.BgColor
+	HostnameFg    color.FgColor
+	IdAttr        color.Attribute
+	IdBg          color.BgColor
+	IdFg          color.FgColor
+	StatsOkAttr   color.Attribute
+	StatsOkBg     color.BgColor
+	StatsOkFg     color.FgColor
+	StatsWarnAttr color.Attribute
+	StatsWarnBg   color.BgColor
+	StatsWarnFg   color.FgColor
+	TextAttr      color.Attribute
+	TextBg        color.BgColor
+	TextFg        color.FgColor
+}
+
+type clientTermColors struct {
+	DelimiterAttr color.Attribute
+	DelimiterBg   color.BgColor
+	DelimiterFg   color.FgColor
+	ClientAttr    color.Attribute
+	ClientBg      color.BgColor
+	ClientFg      color.FgColor
+	HostnameAttr  color.Attribute
+	HostnameBg    color.BgColor
+	HostnameFg    color.FgColor
+	TextAttr      color.Attribute
+	TextBg        color.BgColor
+	TextFg        color.FgColor
+}
+
+type serverTermColors struct {
+	DelimiterAttr color.Attribute
+	DelimiterBg   color.BgColor
+	DelimiterFg   color.FgColor
+	ServerAttr    color.Attribute
+	ServerBg      color.BgColor
+	ServerFg      color.FgColor
+	HostnameAttr  color.Attribute
+	HostnameBg    color.BgColor
+	HostnameFg    color.FgColor
+	TextAttr      color.Attribute
+	TextBg        color.BgColor
+	TextFg        color.FgColor
+}
+
+type commonTermColors struct {
+	SeverityErrorAttr color.Attribute
+	SeverityErrorBg   color.BgColor
+	SeverityErrorFg   color.FgColor
+	SeverityFatalAttr color.Attribute
+	SeverityFatalBg   color.BgColor
+	SeverityFatalFg   color.FgColor
+	SeverityWarnAttr  color.Attribute
+	SeverityWarnBg    color.BgColor
+	SeverityWarnFg    color.FgColor
+}
+
 type termColors struct {
-	ClientErrorAttr     color.Attribute
-	ClientErrorBg       color.BgColor
-	ClientErrorFg       color.FgColor
-	ClientWarnAttr      color.Attribute
-	ClientWarnBg        color.BgColor
-	ClientWarnFg        color.FgColor
-	DelimiterAttr       color.Attribute
-	DelimiterBg         color.BgColor
-	DelimiterFg         color.FgColor
-	RemoteCountAttr     color.Attribute
-	RemoteCountBg       color.BgColor
-	RemoteCountFg       color.FgColor
-	RemoteDebugAttr     color.Attribute
-	RemoteDebugBg       color.BgColor
-	RemoteDebugFg       color.FgColor
-	RemoteErrorAttr     color.Attribute
-	RemoteErrorBg       color.BgColor
-	RemoteErrorFg       color.FgColor
-	RemoteFatalAttr     color.Attribute
-	RemoteFatalBg       color.BgColor
-	RemoteFatalFg       color.FgColor
-	RemoteIdAttr        color.Attribute
-	RemoteIdBg          color.BgColor
-	RemoteIdFg          color.FgColor
-	RemoteServerAttr    color.Attribute
-	RemoteServerBg      color.BgColor
-	RemoteServerFg      color.FgColor
-	RemoteStatsOkAttr   color.Attribute
-	RemoteStatsOkBg     color.BgColor
-	RemoteStatsOkFg     color.FgColor
-	RemoteStatsWarnAttr color.Attribute
-	RemoteStatsWarnBg   color.BgColor
-	RemoteStatsWarnFg   color.FgColor
-	RemoteStrAttr       color.Attribute
-	RemoteStrBg         color.BgColor
-	RemoteStrFg         color.FgColor
-	RemoteTextAttr      color.Attribute
-	RemoteTextBg        color.BgColor
-	RemoteTextFg        color.FgColor
-	RemoteTraceAttr     color.Attribute
-	RemoteTraceBg       color.BgColor
-	RemoteTraceFg       color.FgColor
-	RemoteWarnAttr      color.Attribute
-	RemoteWarnBg        color.BgColor
-	RemoteWarnFg        color.FgColor
+	Remote remoteTermColors
+	Client clientTermColors
+	Server serverTermColors
+	Common commonTermColors
 }
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
@@ -63,51 +90,71 @@ func newDefaultClientConfig() *ClientConfig {
 	return &ClientConfig{
 		TermColorsEnable: true,
 		TermColors: termColors{
-			ClientErrorAttr:     color.AttrBold,
-			ClientErrorBg:       color.BgBlack,
-			ClientErrorFg:       color.FgRed,
-			ClientWarnAttr:      color.AttrNone,
-			ClientWarnBg:        color.BgBlack,
-			ClientWarnFg:        color.FgMagenta,
-			DelimiterAttr:       color.AttrDim,
-			DelimiterBg:         color.BgBlue,
-			DelimiterFg:         color.FgCyan,
-			RemoteCountAttr:     color.AttrDim,
-			RemoteCountBg:       color.BgBlue,
-			RemoteCountFg:       color.FgGreen,
-			RemoteDebugAttr:     color.AttrBold,
-			RemoteDebugBg:       color.BgBlack,
-			RemoteDebugFg:       color.FgGreen,
-			RemoteErrorAttr:     color.AttrBold,
-			RemoteErrorBg:       color.BgRed,
-			RemoteErrorFg:       color.FgWhite,
-			RemoteFatalAttr:     color.AttrBlink,
-			RemoteFatalBg:       color.BgRed,
-			RemoteFatalFg:       color.FgWhite,
-			RemoteIdAttr:        color.AttrDim,
-			RemoteIdBg:          color.BgBlue,
-			RemoteIdFg:          color.FgWhite,
-			RemoteServerAttr:    color.AttrBold,
-			RemoteServerBg:      color.BgBlue,
-			RemoteServerFg:      color.FgWhite,
-			RemoteStatsOkAttr:   color.AttrNone,
-			RemoteStatsOkBg:     color.BgGreen,
-			RemoteStatsOkFg:     color.FgBlue,
-			RemoteStatsWarnAttr: color.AttrNone,
-			RemoteStatsWarnBg:   color.BgRed,
-			RemoteStatsWarnFg:   color.FgWhite,
-			RemoteStrAttr:       color.AttrDim,
-			RemoteStrBg:         color.BgBlue,
-			RemoteStrFg:         color.FgWhite,
-			RemoteTextAttr:      color.AttrNone,
-			RemoteTextBg:        color.BgBlack,
-			RemoteTextFg:        color.FgWhite,
-			RemoteTraceAttr:     color.AttrBold,
-			RemoteTraceBg:       color.BgGreen,
-			RemoteTraceFg:       color.FgWhite,
-			RemoteWarnAttr:      color.AttrBold,
-			RemoteWarnBg:        color.BgYellow,
-			RemoteWarnFg:        color.FgWhite,
+			Remote: remoteTermColors{
+				DelimiterAttr: color.AttrDim,
+				DelimiterBg:   color.BgBlue,
+				DelimiterFg:   color.FgCyan,
+				RemoteAttr:    color.AttrDim,
+				RemoteBg:      color.BgBlue,
+				RemoteFg:      color.FgWhite,
+				CountAttr:     color.AttrDim,
+				CountBg:       color.BgBlue,
+				CountFg:       color.FgGreen,
+				HostnameAttr:  color.AttrBold,
+				HostnameBg:    color.BgBlue,
+				HostnameFg:    color.FgWhite,
+				IdAttr:        color.AttrDim,
+				IdBg:          color.BgBlue,
+				IdFg:          color.FgWhite,
+				StatsOkAttr:   color.AttrNone,
+				StatsOkBg:     color.BgGreen,
+				StatsOkFg:     color.FgBlue,
+				StatsWarnAttr: color.AttrNone,
+				StatsWarnBg:   color.BgRed,
+				StatsWarnFg:   color.FgWhite,
+				TextAttr:      color.AttrNone,
+				TextBg:        color.BgBlack,
+				TextFg:        color.FgWhite,
+			},
+			Client: clientTermColors{
+				DelimiterAttr: color.AttrDim,
+				DelimiterBg:   color.BgYellow,
+				DelimiterFg:   color.FgBlack,
+				ClientAttr:    color.AttrDim,
+				ClientBg:      color.BgYellow,
+				ClientFg:      color.FgBlack,
+				HostnameAttr:  color.AttrDim,
+				HostnameBg:    color.BgYellow,
+				HostnameFg:    color.FgBlack,
+				TextAttr:      color.AttrNone,
+				TextBg:        color.BgYellow,
+				TextFg:        color.FgBlack,
+			},
+			Server: serverTermColors{
+				DelimiterAttr: color.AttrDim,
+				DelimiterBg:   color.BgCyan,
+				DelimiterFg:   color.FgBlack,
+				ServerAttr:    color.AttrDim,
+				ServerBg:      color.BgCyan,
+				ServerFg:      color.FgBlack,
+				HostnameAttr:  color.AttrBold,
+				HostnameBg:    color.BgCyan,
+				HostnameFg:    color.FgBlack,
+				TextAttr:      color.AttrNone,
+				TextBg:        color.BgCyan,
+				TextFg:        color.FgBlack,
+			},
+			Common: commonTermColors{
+				SeverityErrorAttr: color.AttrBold,
+				SeverityErrorBg:   color.BgRed,
+				SeverityErrorFg:   color.FgWhite,
+				SeverityFatalAttr: color.AttrBlink,
+				SeverityFatalBg:   color.BgRed,
+				SeverityFatalFg:   color.FgWhite,
+				SeverityWarnAttr:  color.AttrNone,
+				SeverityWarnBg:    color.BgBlack,
+				SeverityWarnFg:    color.FgRed,
+			},
 		},
 	}
 }
-- 
cgit v1.2.3


From d8c204cd68f21f66ef29bcdeb7decc715af24b6e Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 6 Sep 2021 13:48:55 +0300
Subject: Print out client/server update notice even from dtail server 4 to
 dtail client 3.

---
 internal/clients/handlers/basehandler.go  |  2 +-
 internal/config/client.go                 |  8 ++++----
 internal/protocol/protocol.go             |  4 +---
 internal/server/handlers/serverhandler.go | 30 +++++++++++++++++++++++-------
 4 files changed, 29 insertions(+), 15 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index fe83faa..63ceaac 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -56,7 +56,7 @@ func (h *baseHandler) SendMessage(command string) error {
 // Read data from the dtail server via Writer interface.
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		if b == protocol.MessageDelimiter {
+		if b == protocol.MessageDelimiter || b == '\n' {
 			if len(h.receiveBuf) == 0 {
 				continue
 			}
diff --git a/internal/config/client.go b/internal/config/client.go
index 837c7d6..8bde7a4 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -127,8 +127,8 @@ func newDefaultClientConfig() *ClientConfig {
 				HostnameBg:    color.BgYellow,
 				HostnameFg:    color.FgBlack,
 				TextAttr:      color.AttrNone,
-				TextBg:        color.BgYellow,
-				TextFg:        color.FgBlack,
+				TextBg:        color.BgBlack,
+				TextFg:        color.FgWhite,
 			},
 			Server: serverTermColors{
 				DelimiterAttr: color.AttrDim,
@@ -151,9 +151,9 @@ func newDefaultClientConfig() *ClientConfig {
 				SeverityFatalAttr: color.AttrBlink,
 				SeverityFatalBg:   color.BgRed,
 				SeverityFatalFg:   color.FgWhite,
-				SeverityWarnAttr:  color.AttrNone,
+				SeverityWarnAttr:  color.AttrBold,
 				SeverityWarnBg:    color.BgBlack,
-				SeverityWarnFg:    color.FgRed,
+				SeverityWarnFg:    color.FgWhite,
 			},
 		},
 	}
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index e92ec6a..d3035e4 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -7,8 +7,6 @@ const (
 	MessageDelimiter byte = '¬'
 	// FieldDelimiter delimits aggregation fields.
 	FieldDelimiter string = "|"
-	// Arrow for multiple purposes
-	Arrow string = "➔"
 	// AggregateDelimiter delimits parts of an aggregation message.
-	AggregateDelimiter string = Arrow
+	AggregateDelimiter string = "➔"
 )
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 14fc5d0..14f46a3 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -167,9 +168,9 @@ func (h *ServerHandler) handleCommand(commandStr string) {
 	logger.Debug(h.user, commandStr)
 	ctx := context.Background()
 
-	args, argc, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
+	args, argc, add, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
 	if err != nil {
-		h.send(h.serverMessages, logger.Error(h.user, err))
+		h.send(h.serverMessages, logger.Error(h.user, err)+add)
 		return
 	}
 
@@ -193,19 +194,34 @@ func (h *ServerHandler) handleCommand(commandStr string) {
 	h.handleUserCommand(ctx, argc, args)
 }
 
-func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, error) {
+func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
 	argc := len(args)
+	var add string
 
 	if argc <= 2 || args[0] != "protocol" {
-		return args, argc, errors.New("unable to determine protocol version")
+		return args, argc, add, errors.New("unable to determine protocol version")
 	}
 
 	if args[1] != protocol.ProtocolCompat {
-		err := fmt.Errorf("server with protocol version '%s' but client with '%s', please update DTail", protocol.ProtocolCompat, args[1])
-		return args, argc, err
+		clientCompat, _ := strconv.Atoi(args[1])
+		serverCompat, _ := strconv.Atoi(protocol.ProtocolCompat)
+		if clientCompat <= 3 {
+			// Protocol version 3 or lower expect a newline as message separator
+			// One day (after 2 major versions) this exception may be removed!
+			add = "\n"
+		}
+
+		toUpdate := "client"
+		if clientCompat > serverCompat {
+			toUpdate = "server"
+		}
+
+		err := fmt.Errorf("DTail server protocol version '%s' does not match client protocol version '%s', please update DTail %s!",
+			protocol.ProtocolCompat, args[1], toUpdate)
+		return args, argc, add, err
 	}
 
-	return args[2:], argc - 2, nil
+	return args[2:], argc - 2, add, nil
 }
 
 func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, error) {
-- 
cgit v1.2.3


From 0f6e08da5744d8fd0e916bb371d283985775e20e Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 6 Sep 2021 14:07:07 +0300
Subject: fine tweak color schema

---
 internal/clients/stats.go     |  6 +++---
 internal/color/brush/brush.go | 14 --------------
 internal/color/paint.go       |  5 +++++
 internal/config/client.go     |  4 ++--
 4 files changed, 10 insertions(+), 19 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index 3f76e0f..faeb9fb 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -80,9 +80,9 @@ func (s *stats) printStatsDueInterrupt(messages []string) {
 	for i, message := range messages {
 		if i > 0 && config.Client.TermColorsEnable {
 			fmt.Println(color.PaintStrWithAttr(message,
-				config.Client.TermColors.Client.TextFg,
-				config.Client.TermColors.Client.TextBg,
-				config.Client.TermColors.Client.TextAttr,
+				config.Client.TermColors.Client.ClientFg,
+				config.Client.TermColors.Client.ClientBg,
+				config.Client.TermColors.Client.ClientAttr,
 			))
 			continue
 		}
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 785a0d0..524829b 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -183,19 +183,6 @@ func Colorfy(line string) string {
 	case strings.HasPrefix(line, "SERVER"):
 		paintServer(&sb, line)
 
-		/*
-			case strings.Contains(line, "ERROR"):
-				color.PaintWithAttr(&sb, line,
-					config.Client.TermColors.ClientErrorFg,
-					config.Client.TermColors.ClientErrorBg,
-					config.Client.TermColors.ClientErrorAttr)
-
-			case strings.Contains(line, "WARN"):
-				color.PaintWithAttr(&sb, line,
-					config.Client.TermColors.ClientWarnFg,
-					config.Client.TermColors.ClientWarnBg,
-					config.Client.TermColors.ClientWarnAttr)
-		*/
 	default:
 		color.PaintWithAttr(&sb, line,
 			color.FgDefault,
@@ -203,6 +190,5 @@ func Colorfy(line string) string {
 			color.AttrNone)
 	}
 
-	color.ResetWithAttr(&sb)
 	return sb.String()
 }
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 2798ff7..5430acd 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -38,6 +38,8 @@ func Paint(sb *strings.Builder, text string, fg FgColor, bg BgColor) {
 	sb.WriteString(string(fg))
 	sb.WriteString(string(bg))
 	sb.WriteString(text)
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
 }
 
 // Reset background and foreground colors.
@@ -56,6 +58,9 @@ func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, att
 	sb.WriteString(string(bg))
 	sb.WriteString(string(attr))
 	sb.WriteString(text)
+	sb.WriteString(string(AttrReset))
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
 }
 
 // ResetWithAttr resets background, foreground and attributes.
diff --git a/internal/config/client.go b/internal/config/client.go
index 8bde7a4..3c2f7de 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -99,7 +99,7 @@ func newDefaultClientConfig() *ClientConfig {
 				RemoteFg:      color.FgWhite,
 				CountAttr:     color.AttrDim,
 				CountBg:       color.BgBlue,
-				CountFg:       color.FgGreen,
+				CountFg:       color.FgWhite,
 				HostnameAttr:  color.AttrBold,
 				HostnameBg:    color.BgBlue,
 				HostnameFg:    color.FgWhite,
@@ -108,7 +108,7 @@ func newDefaultClientConfig() *ClientConfig {
 				IdFg:          color.FgWhite,
 				StatsOkAttr:   color.AttrNone,
 				StatsOkBg:     color.BgGreen,
-				StatsOkFg:     color.FgBlue,
+				StatsOkFg:     color.FgBlack,
 				StatsWarnAttr: color.AttrNone,
 				StatsWarnBg:   color.BgRed,
 				StatsWarnFg:   color.FgWhite,
-- 
cgit v1.2.3


From 9d80c6c108d0f77eb6ab86684e2eb7f3d0f31c79 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 7 Sep 2021 10:01:32 +0300
Subject: Produce MAPREDUCE lines, can aggregate these via default log format

---
 internal/clients/stats.go               | 49 ++++++++++++++++++++++++---------
 internal/color/brush/brush.go           | 12 ++++----
 internal/io/logger/logger.go            | 40 +++++++++++++++++++++------
 internal/io/pool/builder.go             | 18 ++++++++++++
 internal/mapr/logformat/default.go      | 15 ++++++++--
 internal/mapr/logformat/default_test.go | 39 +++++++++++++++-----------
 internal/server/stats.go                | 12 +++++---
 internal/version/version.go             |  2 +-
 8 files changed, 137 insertions(+), 50 deletions(-)
 create mode 100644 internal/io/pool/builder.go

(limited to 'internal')

diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index faeb9fb..6da443c 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -11,12 +11,13 @@ import (
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // Used to collect and display various client stats.
 type stats struct {
 	// Total amount servers to connect to.
-	connectionsTotal int
+	servers int
 	// To keep track of what connected and disconnected
 	connectionsEstCh chan struct{}
 	// Amount of servers connections are established.
@@ -25,10 +26,10 @@ type stats struct {
 	mutex sync.Mutex
 }
 
-func newTailStats(connectionsTotal int) *stats {
+func newTailStats(servers int) *stats {
 	return &stats{
-		connectionsTotal: connectionsTotal,
-		connectionsEstCh: make(chan struct{}, connectionsTotal),
+		servers:          servers,
+		connectionsEstCh: make(chan struct{}, servers),
 		connected:        0,
 	}
 }
@@ -59,13 +60,14 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 			continue
 		}
 
-		stats := s.statsLine(connected, newConnections, throttle)
 		switch force {
 		case true:
+			stats := s.statsLine(connected, newConnections, throttle)
 			messages = append(messages, fmt.Sprintf("Connection stats: %s", stats))
 			s.printStatsDueInterrupt(messages)
 		default:
-			logger.Info(stats)
+			data := s.statsData(connected, newConnections, throttle)
+			logger.Mapreduce("STATS", data)
 		}
 
 		connectedLast = connected
@@ -92,16 +94,37 @@ func (s *stats) printStatsDueInterrupt(messages []string) {
 	logger.Resume()
 }
 
+func (s *stats) statsData(connected, newConnections int, throttle int) map[string]interface{} {
+	percConnected := percentOf(float64(s.servers), float64(connected))
+
+	data := make(map[string]interface{})
+	data["connected"] = connected
+	data["servers"] = s.servers
+	data["connected%"] = int(percConnected)
+	data["new"] = newConnections
+	data["throttle"] = throttle
+	data["goroutines"] = runtime.NumGoroutine()
+	data["cgocalls"] = runtime.NumCgoCall()
+	data["cpu"] = runtime.NumCPU()
+
+	return data
+}
+
 func (s *stats) statsLine(connected, newConnections int, throttle int) string {
-	percConnected := percentOf(float64(s.connectionsTotal), float64(connected))
+	sb := strings.Builder{}
 
-	var stats []string
-	stats = append(stats, fmt.Sprintf("connected=%d/%d(%d%%)", connected, s.connectionsTotal, int(percConnected)))
-	stats = append(stats, fmt.Sprintf("new=%d", newConnections))
-	stats = append(stats, fmt.Sprintf("throttle=%d", throttle))
-	stats = append(stats, fmt.Sprintf("cpus/goroutines=%d/%d", runtime.NumCPU(), runtime.NumGoroutine()))
+	i := 0
+	for k, v := range s.statsData(connected, newConnections, throttle) {
+		if i > 0 {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+		sb.WriteString(k)
+		sb.WriteByte('=')
+		sb.WriteString(fmt.Sprintf("%v", v))
+		i++
+	}
 
-	return strings.Join(stats, "|")
+	return sb.String()
 }
 
 func (s *stats) numConnected() int {
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 524829b..82fa410 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -171,20 +172,21 @@ func paintServer(sb *strings.Builder, line string) {
 
 // Colorfy a given line based on the line's content.
 func Colorfy(line string) string {
-	sb := strings.Builder{}
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
 
 	switch {
 	case strings.HasPrefix(line, "REMOTE"):
-		paintRemote(&sb, line)
+		paintRemote(sb, line)
 
 	case strings.HasPrefix(line, "CLIENT"):
-		paintClient(&sb, line)
+		paintClient(sb, line)
 
 	case strings.HasPrefix(line, "SERVER"):
-		paintServer(&sb, line)
+		paintServer(sb, line)
 
 	default:
-		color.PaintWithAttr(&sb, line,
+		color.PaintWithAttr(sb, line,
 			color.FgDefault,
 			color.BgDefault,
 			color.AttrNone)
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 3a3935d..6890201 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -14,6 +14,8 @@ import (
 
 	"github.com/mimecast/dtail/internal/color/brush"
 	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 const (
@@ -132,6 +134,24 @@ func Info(args ...interface{}) string {
 	return log(clientStr, infoStr, args)
 }
 
+// Mapreduce message logging.
+func Mapreduce(table string, data map[string]interface{}) string {
+	args := make([]interface{}, len(data)+1)
+
+	args[0] = fmt.Sprintf("MAPREDUCE:%s", strings.ToUpper(table))
+	i := 1
+	for k, v := range data {
+		args[i] = fmt.Sprintf("%s=%v", k, v)
+		i++
+	}
+
+	if Mode.Server {
+		return log(serverStr, infoStr, args)
+	}
+
+	return log(clientStr, infoStr, args)
+}
+
 // Warn message logging.
 func Warn(args ...interface{}) string {
 	if !Mode.Quiet {
@@ -230,24 +250,28 @@ func log(what string, severity string, args []interface{}) string {
 		return ""
 	}
 
-	messages := []string{}
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+
+	for i, arg := range args {
+		if i > 0 {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
 
-	for _, arg := range args {
 		switch v := arg.(type) {
 		case string:
-			messages = append(messages, v)
+			sb.WriteString(v)
 		case int:
-			messages = append(messages, fmt.Sprintf("%d", v))
+			sb.WriteString(fmt.Sprintf("%d", v))
 		case error:
-			messages = append(messages, v.Error())
+			sb.WriteString(v.Error())
 		default:
-			messages = append(messages, fmt.Sprintf("%v", v))
+			sb.WriteString(fmt.Sprintf("%v", v))
 		}
 	}
 
-	message := strings.Join(messages, "|")
+	message := sb.String()
+	pool.RecycleBuilderBuffer(sb)
 	write(what, severity, message)
-
 	return fmt.Sprintf("%s|%s", severity, message)
 }
 
diff --git a/internal/io/pool/builder.go b/internal/io/pool/builder.go
new file mode 100644
index 0000000..c9dc221
--- /dev/null
+++ b/internal/io/pool/builder.go
@@ -0,0 +1,18 @@
+package pool
+
+import (
+	"strings"
+	"sync"
+)
+
+var BuilderBuffer = sync.Pool{
+	New: func() interface{} {
+		sb := strings.Builder{}
+		return &sb
+	},
+}
+
+func RecycleBuilderBuffer(sb *strings.Builder) {
+	sb.Reset()
+	BuilderBuffer.Put(sb)
+}
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 32a34bd..2881047 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -9,8 +9,8 @@ import (
 
 // MakeFieldsDEFAULT is the default log file mapreduce parser.
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
-	fields := make(map[string]string, 20)
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
+	fields := make(map[string]string, len(splitted))
 
 	fields["*"] = "*"
 	fields["$line"] = maprLine
@@ -19,10 +19,19 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields["$timezone"] = p.timeZoneName
 	fields["$timeoffset"] = p.timeZoneOffset
 
-	for _, kv := range splitted {
+	kvStart := 0
+	// DTail mapreduce format
+	if len(splitted) > 3 && strings.HasPrefix(splitted[3], "MAPREDUCE:") {
+		fields["$severity"] = splitted[0]
+		// TODO: Parse time like we do at Mimecast
+		fields["$time"] = splitted[1]
+		kvStart = 4
+	}
+
+	for _, kv := range splitted[kvStart:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
-			return fields, errors.New("Error parsing mapr token: " + kv)
+			return fields, errors.New("Error parsing mapreduce token: " + kv)
 		}
 		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
 	}
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index d7a4da4..6284008 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -10,26 +10,33 @@ func TestDefaultLogFormat(t *testing.T) {
 		t.Errorf("Unable to create parser: %s", err.Error())
 	}
 
-	fields, err := parser.MakeFields("foo=bar|baz=bay")
-
-	if err != nil {
-		t.Errorf("Unable to parse: %s", err.Error())
+	inputs := []string{
+		"foo=bar|baz=bay",
+		"INFO|20210907-065632|SERVER|MAPREDUCE:TEST|foo=bar|baz=bay",
 	}
 
-	if bar, ok := fields["foo"]; !ok {
-		t.Errorf("Expected field 'foo', but no such field there\n")
-	} else if bar != "bar" {
-		t.Errorf("Expected 'bar' stored in field 'foo', but got '%s'\n", bar)
-	}
+	for _, input := range inputs {
+		fields, err := parser.MakeFields(input)
+
+		if err != nil {
+			t.Errorf("Parser unable to make fields: %s", err.Error())
+		}
+
+		if bar, ok := fields["foo"]; !ok {
+			t.Errorf("Expected field 'foo', but no such field there\n")
+		} else if bar != "bar" {
+			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s'\n", bar)
+		}
+
+		if bay, ok := fields["baz"]; !ok {
+			t.Errorf("Expected field 'baz', but no such field there\n")
+		} else if bay != "bay" {
+			t.Errorf("Expected 'bay' stored in field 'baz', but got '%s'\n", bay)
+		}
 
-	if bay, ok := fields["baz"]; !ok {
-		t.Errorf("Expected field 'baz', but no such field there\n")
-	} else if bay != "bay" {
-		t.Errorf("Expected 'bay' stored in field 'baz', but got '%s'\n", bay)
 	}
 
-	fields, err = parser.MakeFields("foo=bar|bazbay")
-	if err == nil {
-		t.Errorf("Expected error but didn't: %s", err.Error())
+	if _, err := parser.MakeFields("foo=bar|bazbay"); err == nil {
+		t.Errorf("Expected error but didn't")
 	}
 }
diff --git a/internal/server/stats.go b/internal/server/stats.go
index ac579ad..3e8c71d 100644
--- a/internal/server/stats.go
+++ b/internal/server/stats.go
@@ -50,10 +50,14 @@ func (s *stats) logServerStats() {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 
-	currentConnections := fmt.Sprintf("currentConnections=%d", s.currentConnections)
-	lifetimeConnections := fmt.Sprintf("lifetimeConnections=%d", s.lifetimeConnections)
-	goroutines := fmt.Sprintf("goroutines=%d", runtime.NumGoroutine())
-	logger.Info("stats", currentConnections, lifetimeConnections, goroutines)
+	data := make(map[string]interface{})
+	data["currentConnections"] = s.currentConnections
+	data["lifetimeConnections"] = s.lifetimeConnections
+	data["goroutines"] = runtime.NumGoroutine()
+	data["cgocalls"] = runtime.NumCgoCall()
+	data["cpu"] = runtime.NumCPU()
+
+	logger.Mapreduce("STATS", data)
 }
 
 func (s *stats) serverLimitExceeded() error {
diff --git a/internal/version/version.go b/internal/version/version.go
index 693192f..a54b560 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,7 +13,7 @@ const (
 	// Name of DTail.
 	Name string = "DTail"
 	// Version of DTail.
-	Version string = "3.2.0"
+	Version string = "4.0.0-RC1"
 	// Additional information for DTail
 	Additional string = "Have a lot of fun!"
 )
-- 
cgit v1.2.3


From 22e4d64496ceec906b4d2397468e3d90094b7231 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 8 Sep 2021 08:39:41 +0300
Subject: Don't fail parsing whole line when one kv could not be parsed

---
 internal/mapr/logformat/default.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'internal')

diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 2881047..da976bd 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -1,9 +1,9 @@
 package logformat
 
 import (
-	"errors"
 	"strings"
 
+	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -31,7 +31,8 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	for _, kv := range splitted[kvStart:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
-			return fields, errors.New("Error parsing mapreduce token: " + kv)
+			logger.Debug("Unable to parse key-value token, ignoring it", kv)
+			continue
 		}
 		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
 	}
-- 
cgit v1.2.3


From 4bc9e30f9742c58bdb39adfcc5aa2dc55ecb86f3 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 8 Sep 2021 19:10:50 +0300
Subject: mapreduce tables are in colors now too

---
 internal/clients/baseclient.go             |   1 +
 internal/clients/handlers/basehandler.go   |  29 ++--
 internal/clients/handlers/healthhandler.go |  11 +-
 internal/clients/handlers/maprhandler.go   |  42 ++---
 internal/clients/maprclient.go             |  31 +++-
 internal/color/paint.go                    |  13 ++
 internal/config/client.go                  |  48 +++++-
 internal/io/fs/readfile.go                 |   5 +-
 internal/mapr/aggregateset.go              |  17 +-
 internal/mapr/client/aggregate.go          |  18 +-
 internal/mapr/globalgroupset.go            |   1 +
 internal/mapr/groupset.go                  | 258 ++++++++++++++++++++---------
 internal/mapr/logformat/default.go         |  28 ++--
 internal/mapr/logformat/default_test.go    |  15 +-
 internal/mapr/logformat/generickv.go       |  31 ++++
 internal/mapr/logformat/parser.go          |   2 +
 internal/mapr/server/aggregate.go          |  17 +-
 internal/protocol/protocol.go              |  11 +-
 internal/server/handlers/controlhandler.go |   3 +-
 internal/server/handlers/serverhandler.go  | 118 +++++++------
 20 files changed, 469 insertions(+), 230 deletions(-)
 create mode 100644 internal/mapr/logformat/generickv.go

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index f20156f..de0c101 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -71,6 +71,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	go c.hostKeyCallback.PromptAddHosts(ctx)
 	// Print client stats every time something on statsCh is recieved.
 	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet)
+
 	// Keep count of active connections
 	active := make(chan struct{}, len(c.connections))
 
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 63ceaac..51f33c1 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"bytes"
 	"encoding/base64"
 	"fmt"
 	"io"
@@ -17,7 +18,7 @@ type baseHandler struct {
 	server       string
 	shellStarted bool
 	commands     chan string
-	receiveBuf   []byte
+	receiveBuf   bytes.Buffer
 	status       int
 }
 
@@ -56,14 +57,23 @@ func (h *baseHandler) SendMessage(command string) error {
 // Read data from the dtail server via Writer interface.
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		if b == protocol.MessageDelimiter || b == '\n' {
-			if len(h.receiveBuf) == 0 {
+		switch b {
+		/*
+			// TODO: Next DTail version make it so that '\n' gets ignored. For now
+			// leave it for compatibility with older DTail server + ability to display
+			// the protocol mismatch warn message.
+			case '\n' {
+				continue
+		*/
+		case '\n', protocol.MessageDelimiter:
+			message := h.receiveBuf.String()
+			if len(message) == 0 {
 				continue
 			}
-			message := string(h.receiveBuf)
 			h.handleMessageType(message)
-		} else {
-			h.receiveBuf = append(h.receiveBuf, b)
+			h.receiveBuf.Reset()
+		default:
+			h.receiveBuf.WriteByte(b)
 		}
 	}
 
@@ -78,25 +88,22 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	case <-h.Done():
 		return 0, io.EOF
 	}
-
 	return
 }
 
 // Handle various message types.
 func (h *baseHandler) handleMessageType(message string) {
-	if len(h.receiveBuf) == 0 {
+	if len(message) == 0 {
 		return
 	}
 
 	// Hidden server commands starti with a dot "."
-	if h.receiveBuf[0] == '.' {
+	if message[0] == '.' {
 		h.handleHiddenMessage(message)
-		h.receiveBuf = h.receiveBuf[:0]
 		return
 	}
 
 	logger.Raw(message)
-	h.receiveBuf = h.receiveBuf[:0]
 }
 
 // Handle messages received from server which are not meant to be displayed
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 213748c..eca0348 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"time"
@@ -13,7 +14,7 @@ import (
 type HealthHandler struct {
 	done *internal.Done
 	// Buffer of incoming data from server.
-	receiveBuf []byte
+	receiveBuf bytes.Buffer
 	// To send commands to the server.
 	commands chan string
 	// To receive messages from the server.
@@ -72,10 +73,10 @@ func (h *HealthHandler) SendMessage(command string) error {
 // Server writes byte stream to client.
 func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		h.receiveBuf = append(h.receiveBuf, b)
-		if b == protocol.MessageDelimiter { // '\n' {
-			h.receive <- string(h.receiveBuf)
-			h.receiveBuf = h.receiveBuf[:0]
+		h.receiveBuf.WriteByte(b)
+		if b == protocol.MessageDelimiter {
+			h.receive <- h.receiveBuf.String()
+			h.receiveBuf.Reset()
 		}
 	}
 
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index afad507..65b1454 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -15,7 +15,6 @@ type MaprHandler struct {
 	baseHandler
 	aggregate *client.Aggregate
 	query     *mapr.Query
-	count     uint64
 }
 
 // NewMaprHandler returns a new mapreduce client handler.
@@ -36,19 +35,20 @@ func NewMaprHandler(server string, query *mapr.Query, globalGroup *mapr.GlobalGr
 // Read data from the dtail server via Writer interface.
 func (h *MaprHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		h.baseHandler.receiveBuf = append(h.baseHandler.receiveBuf, b)
-		if b == protocol.MessageDelimiter { // '\n' {
-			if len(h.baseHandler.receiveBuf) == 0 {
-				continue
+		switch b {
+		case '\n':
+			continue
+		case protocol.MessageDelimiter:
+			message := h.baseHandler.receiveBuf.String()
+			logger.Debug(message)
+			if message[0] == 'A' {
+				h.handleAggregateMessage(message)
+			} else {
+				h.baseHandler.handleMessageType(message)
 			}
-			message := string(h.baseHandler.receiveBuf)
-
-			if h.baseHandler.receiveBuf[0] == 'A' {
-				h.handleAggregateMessage(strings.TrimSpace(message))
-				h.baseHandler.receiveBuf = h.baseHandler.receiveBuf[:0]
-				continue
-			}
-			h.baseHandler.handleMessageType(message)
+			h.baseHandler.receiveBuf.Reset()
+		default:
+			h.baseHandler.receiveBuf.WriteByte(b)
 		}
 	}
 
@@ -58,12 +58,12 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 // Handle a message received from server including mapr aggregation
 // related data.
 func (h *MaprHandler) handleAggregateMessage(message string) {
-	h.count++
-	parts := strings.Split(message, protocol.AggregateDelimiter)
-
-	// Index 0 contains 'AGGREGATE', 1 contains server host.
-	// Aggregation data begins from index 2.
-	logger.Debug("Received aggregate data", h.server, h.count, parts)
-	h.aggregate.Aggregate(parts[2:])
-	logger.Debug("Aggregated aggregate data", h.server, h.count)
+	parts := strings.SplitN(message, protocol.FieldDelimiter, 3)
+	if len(parts) != 3 {
+		logger.Error("Unable to aggregate data", h.server, message, parts, len(parts), "expected 3 parts")
+		return
+	}
+	if err := h.aggregate.Aggregate(parts[2]); err != nil {
+		logger.Error("Unable to aggregate data", h.server, message, err)
+	}
 }
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 77b674b..cab9a6c 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/omode"
@@ -37,6 +39,8 @@ type MaprClient struct {
 	query *mapr.Query
 	// Additative result or new result every interval run?
 	cumulative bool
+	// The last result string received
+	lastResult string
 }
 
 // NewMaprClient returns a new mapreduce client.
@@ -154,24 +158,37 @@ func (c *MaprClient) reportResults() {
 func (c *MaprClient) printResults() {
 	var result string
 	var err error
-	var numLines int
+	var numRows int
 
 	if c.cumulative {
-		result, numLines, err = c.globalGroup.Result(c.query)
+		result, numRows, err = c.globalGroup.Result(c.query)
 	} else {
-		result, numLines, err = c.globalGroup.SwapOut().Result(c.query)
+		result, numRows, err = c.globalGroup.SwapOut().Result(c.query)
 	}
+
 	if err != nil {
 		logger.FatalExit(err)
 	}
 
-	if numLines == 0 {
-		logger.Warn("Empty result set this time...")
+	if result == c.lastResult {
+		logger.Debug("Result hasn't changed compared to last time...")
 		return
 	}
+	c.lastResult = result
 
-	//logger.Raw(fmt.Sprintf("%s\n", c.query.RawQuery))
-	logger.Raw(c.query.RawQuery)
+	if numRows == 0 {
+		logger.Debug("Empty result set this time...")
+		return
+	}
+
+	rawQuery := c.query.RawQuery
+	if config.Client.TermColorsEnable {
+		rawQuery = color.PaintStrWithAttr(rawQuery,
+			config.Client.TermColors.MaprTable.RawQueryFg,
+			config.Client.TermColors.MaprTable.RawQueryBg,
+			config.Client.TermColors.MaprTable.RawQueryAttr)
+	}
+	logger.Raw(rawQuery)
 	logger.Raw(result)
 }
 
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 5430acd..53c9abb 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -63,6 +63,19 @@ func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, att
 	sb.WriteString(string(FgDefault))
 }
 
+// PaintWithAttrs is similar to PaintWithAttr, but it takes multiple text attributes.
+func PaintWithAttrs(sb *strings.Builder, text string, fg FgColor, bg BgColor, attrs []Attribute) {
+	sb.WriteString(string(fg))
+	sb.WriteString(string(bg))
+	for _, attr := range attrs {
+		sb.WriteString(string(attr))
+	}
+	sb.WriteString(text)
+	sb.WriteString(string(AttrReset))
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
+}
+
 // ResetWithAttr resets background, foreground and attributes.
 func ResetWithAttr(sb *strings.Builder) {
 	sb.WriteString(string(AttrReset))
diff --git a/internal/config/client.go b/internal/config/client.go
index 3c2f7de..795c4a4 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -71,11 +71,32 @@ type commonTermColors struct {
 	SeverityWarnFg    color.FgColor
 }
 
+type maprTableTermColors struct {
+	DataAttr            color.Attribute
+	DataBg              color.BgColor
+	DataFg              color.FgColor
+	DelimiterAttr       color.Attribute
+	DelimiterBg         color.BgColor
+	DelimiterFg         color.FgColor
+	HeaderAttr          color.Attribute
+	HeaderBg            color.BgColor
+	HeaderDelimiterAttr color.Attribute
+	HeaderDelimiterBg   color.BgColor
+	HeaderDelimiterFg   color.FgColor
+	HeaderFg            color.FgColor
+	HeaderGroupKeyAttr  color.Attribute
+	HeaderSortKeyAttr   color.Attribute
+	RawQueryAttr        color.Attribute
+	RawQueryBg          color.BgColor
+	RawQueryFg          color.FgColor
+}
+
 type termColors struct {
-	Remote remoteTermColors
-	Client clientTermColors
-	Server serverTermColors
-	Common commonTermColors
+	Remote    remoteTermColors
+	Client    clientTermColors
+	Server    serverTermColors
+	Common    commonTermColors
+	MaprTable maprTableTermColors
 }
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
@@ -155,6 +176,25 @@ func newDefaultClientConfig() *ClientConfig {
 				SeverityWarnBg:    color.BgBlack,
 				SeverityWarnFg:    color.FgWhite,
 			},
+			MaprTable: maprTableTermColors{
+				DataAttr:            color.AttrNone,
+				DataBg:              color.BgBlue,
+				DataFg:              color.FgWhite,
+				DelimiterAttr:       color.AttrDim,
+				DelimiterBg:         color.BgBlue,
+				DelimiterFg:         color.FgWhite,
+				HeaderAttr:          color.AttrBold,
+				HeaderBg:            color.BgBlue,
+				HeaderFg:            color.FgWhite,
+				HeaderDelimiterAttr: color.AttrDim,
+				HeaderDelimiterBg:   color.BgBlue,
+				HeaderDelimiterFg:   color.FgWhite,
+				HeaderSortKeyAttr:   color.AttrUnderline,
+				HeaderGroupKeyAttr:  color.AttrReverse,
+				RawQueryAttr:        color.AttrDim,
+				RawQueryBg:          color.BgBlack,
+				RawQueryFg:          color.FgCyan,
+			},
 		},
 	}
 }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index f2f672a..c0d44dd 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -16,7 +16,6 @@ import (
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/io/pool"
-	"github.com/mimecast/dtail/internal/protocol"
 	"github.com/mimecast/dtail/internal/regex"
 
 	"github.com/DataDog/zstd"
@@ -187,7 +186,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 				time.Sleep(time.Millisecond * 100)
 				continue
 			}
-			message.WriteByte(protocol.MessageDelimiter)
+			message.WriteString("\n")
 			select {
 			case rawLines <- message:
 				message = pool.BytesBuffer.Get().(*bytes.Buffer)
@@ -202,7 +201,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 					f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
-				message.WriteByte(protocol.MessageDelimiter)
+				message.WriteString("\n")
 				select {
 				case rawLines <- message:
 					message = pool.BytesBuffer.Get().(*bytes.Buffer)
diff --git a/internal/mapr/aggregateset.go b/internal/mapr/aggregateset.go
index 029d87b..47f4925 100644
--- a/internal/mapr/aggregateset.go
+++ b/internal/mapr/aggregateset.go
@@ -6,6 +6,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -68,22 +70,25 @@ func (s *AggregateSet) Merge(query *Query, set *AggregateSet) error {
 
 // Serialize the aggregate set so it can be sent over the wire.
 func (s *AggregateSet) Serialize(ctx context.Context, groupKey string, ch chan<- string) {
-	//logger.Trace("Serialising mapr.AggregateSet", s)
-	var sb strings.Builder
+	logger.Trace("Serialising mapr.AggregateSet", s)
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
 
 	sb.WriteString(groupKey)
 	sb.WriteString(protocol.AggregateDelimiter)
-	sb.WriteString(fmt.Sprintf("%d%s", s.Samples, protocol.AggregateDelimiter))
+	sb.WriteString(fmt.Sprintf("%d", s.Samples))
+	sb.WriteString(protocol.AggregateDelimiter)
 
 	for k, v := range s.FValues {
 		sb.WriteString(k)
-		sb.WriteString("=")
-		sb.WriteString(fmt.Sprintf("%v%s", v, protocol.AggregateDelimiter))
+		sb.WriteString(protocol.AggregateKVDelimiter)
+		sb.WriteString(fmt.Sprintf("%v", v))
+		sb.WriteString(protocol.AggregateDelimiter)
 	}
 
 	for k, v := range s.SValues {
 		sb.WriteString(k)
-		sb.WriteString("=")
+		sb.WriteString(protocol.AggregateKVDelimiter)
 		sb.WriteString(v)
 		sb.WriteString(protocol.AggregateDelimiter)
 	}
diff --git a/internal/mapr/client/aggregate.go b/internal/mapr/client/aggregate.go
index 10b34d4..5cc09a1 100644
--- a/internal/mapr/client/aggregate.go
+++ b/internal/mapr/client/aggregate.go
@@ -1,11 +1,13 @@
 package client
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
 
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // Aggregate mapreduce data on the DTail client side.
@@ -31,12 +33,18 @@ func NewAggregate(server string, query *mapr.Query, globalGroup *mapr.GlobalGrou
 }
 
 // Aggregate data from mapr log line into local (and global) group sets.
-func (a *Aggregate) Aggregate(parts []string) {
+func (a *Aggregate) Aggregate(message string) error {
+	parts := strings.Split(message, protocol.AggregateDelimiter)
+	if len(parts) < 4 {
+		return fmt.Errorf("aggregate message without any real data")
+	}
+
 	groupKey := parts[0]
 	samples, err := strconv.Atoi(parts[1])
 	if err != nil {
-		logger.FatalExit("Unable to parse sample count", parts[1], err, parts)
+		return fmt.Errorf("unable to parse sample count '%s': %v", parts[1], err)
 	}
+
 	fields := a.makeFields(parts[2:])
 	set := a.group.GetSet(groupKey)
 
@@ -63,6 +71,8 @@ func (a *Aggregate) Aggregate(parts []string) {
 		// Re-init local group (make it empty again).
 		a.group.InitSet()
 	}
+
+	return nil
 }
 
 // Create a map of key-value pairs from a part list such as ["foo=bar",  "bar=baz"].
@@ -70,8 +80,8 @@ func (a *Aggregate) makeFields(parts []string) map[string]string {
 	fields := make(map[string]string, len(parts))
 
 	for _, part := range parts {
-		kv := strings.SplitN(part, "=", 2)
-		if len(kv) < 2 {
+		kv := strings.SplitN(part, protocol.AggregateKVDelimiter, 2)
+		if len(kv) != 2 {
 			continue
 		}
 		fields[kv[0]] = kv[1]
diff --git a/internal/mapr/globalgroupset.go b/internal/mapr/globalgroupset.go
index cfab506..21bf990 100644
--- a/internal/mapr/globalgroupset.go
+++ b/internal/mapr/globalgroupset.go
@@ -48,6 +48,7 @@ func (g *GlobalGroupSet) MergeNoblock(query *Query, group *GroupSet) (bool, erro
 
 // Merge a group set into the global group set.
 func (g *GlobalGroupSet) merge(query *Query, group *GroupSet) error {
+
 	for groupKey, set := range group.sets {
 		s := g.GetSet(groupKey)
 		if err := s.Merge(query, set); err != nil {
diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index 6ee2811..d29559a 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -4,13 +4,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"sort"
 	"strconv"
 	"strings"
 
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // GroupSet represents a map of aggregate sets. The group sets
@@ -22,6 +25,14 @@ type GroupSet struct {
 	sets map[string]*AggregateSet
 }
 
+// Internal helper type
+type result struct {
+	groupKey string
+	values   []string
+	widths   []int
+	orderBy  float64
+}
+
 // NewGroupSet returns a new empty group set.
 func NewGroupSet() *GroupSet {
 	g := GroupSet{}
@@ -58,17 +69,118 @@ func (g *GroupSet) Serialize(ctx context.Context, ch chan<- string) {
 
 // Result returns a nicely formated result of the query from the group set.
 func (g *GroupSet) Result(query *Query) (string, int, error) {
-	return g.limitedResult(query, query.Limit, "\t", " ", false)
+	rows, widths, err := g.result(query, true)
+	if err != nil {
+		return "", 0, err
+	}
+
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
+
+	// Generate header now
+	lastIndex := len(query.Select) - 1
+	for i, sc := range query.Select {
+		format := fmt.Sprintf(" %%%ds ", widths[i])
+		str := fmt.Sprintf(format, sc.FieldStorage)
+		if config.Client.TermColorsEnable {
+			attrs := []color.Attribute{config.Client.TermColors.MaprTable.HeaderAttr}
+			if sc.FieldStorage == query.OrderBy {
+				attrs = append(attrs, config.Client.TermColors.MaprTable.HeaderSortKeyAttr)
+			}
+			for _, groupBy := range query.GroupBy {
+				if sc.FieldStorage == groupBy {
+					attrs = append(attrs, config.Client.TermColors.MaprTable.HeaderGroupKeyAttr)
+					break
+				}
+			}
+			color.PaintWithAttrs(sb, str,
+				config.Client.TermColors.MaprTable.HeaderFg,
+				config.Client.TermColors.MaprTable.HeaderBg,
+				attrs)
+		} else {
+			sb.WriteString(str)
+		}
+
+		if i == lastIndex {
+			continue
+		}
+		if config.Client.TermColorsEnable {
+			color.PaintWithAttr(sb, protocol.FieldDelimiter,
+				config.Client.TermColors.MaprTable.HeaderDelimiterFg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterBg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterAttr)
+		} else {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+	}
+	sb.WriteString("\n")
+
+	for i := 0; i < len(query.Select); i++ {
+		str := fmt.Sprintf("-%s-", strings.Repeat("-", widths[i]))
+		if config.Client.TermColorsEnable {
+			color.PaintWithAttr(sb, str,
+				config.Client.TermColors.MaprTable.HeaderDelimiterFg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterBg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterAttr)
+		} else {
+			sb.WriteString(str)
+		}
+		if i == lastIndex {
+			continue
+		}
+		if config.Client.TermColorsEnable {
+			color.PaintWithAttr(sb, protocol.FieldDelimiter,
+				config.Client.TermColors.MaprTable.HeaderDelimiterFg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterBg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterAttr)
+		} else {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+	}
+	sb.WriteString("\n")
+
+	// And now write the data
+	for i, r := range rows {
+		if i == query.Limit {
+			break
+		}
+		for j, value := range r.values {
+			format := fmt.Sprintf(" %%%ds ", widths[j])
+			str := fmt.Sprintf(format, value)
+			if config.Client.TermColorsEnable {
+				color.PaintWithAttr(sb, str,
+					config.Client.TermColors.MaprTable.DataFg,
+					config.Client.TermColors.MaprTable.DataBg,
+					config.Client.TermColors.MaprTable.DataAttr)
+			} else {
+				sb.WriteString(str)
+			}
+
+			if j == lastIndex {
+				continue
+			}
+			if config.Client.TermColorsEnable {
+				color.PaintWithAttr(sb, protocol.FieldDelimiter,
+					config.Client.TermColors.MaprTable.DelimiterFg,
+					config.Client.TermColors.MaprTable.DelimiterBg,
+					config.Client.TermColors.MaprTable.DelimiterAttr)
+			} else {
+				sb.WriteString(protocol.FieldDelimiter)
+			}
+		}
+		sb.WriteString("\n")
+	}
+
+	return sb.String(), len(rows), nil
 }
 
-// WriteResult writes the result to an outfile.
+// WriteResult writes the result to an CSV outfile.
 func (g *GroupSet) WriteResult(query *Query) error {
 	if !query.HasOutfile() {
 		return errors.New("No outfile specified")
 	}
 
-	// -1: Don't limit the result, include all data sets
-	result, _, err := g.limitedResult(query, -1, "", ",", true)
+	rows, _, err := g.result(query, false)
 	if err != nil {
 		return err
 	}
@@ -76,9 +188,37 @@ func (g *GroupSet) WriteResult(query *Query) error {
 	logger.Info("Writing outfile", query.Outfile)
 	tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile)
 
-	if err := ioutil.WriteFile(tmpOutfile, []byte(result), 0644); err != nil {
+	file, err := os.Create(tmpOutfile)
+	if err != nil {
 		return err
 	}
+	defer file.Close()
+
+	// Generate header now
+	lastIndex := len(query.Select) - 1
+	for i, sc := range query.Select {
+		file.WriteString(sc.FieldStorage)
+		if i == lastIndex {
+			continue
+		}
+		file.WriteString(protocol.CSVDelimiter)
+	}
+	file.WriteString("\n")
+
+	// And now write the data
+	for i, r := range rows {
+		if i == query.Limit {
+			break
+		}
+		for j, value := range r.values {
+			file.WriteString(value)
+			if j == lastIndex {
+				continue
+			}
+			file.WriteString(protocol.CSVDelimiter)
+		}
+		file.WriteString("\n")
+	}
 
 	if err := os.Rename(tmpOutfile, query.Outfile); err != nil {
 		os.Remove(tmpOutfile)
@@ -88,32 +228,22 @@ func (g *GroupSet) WriteResult(query *Query) error {
 	return nil
 }
 
-// Return a nicely formated result of the query from the group set.
-func (g *GroupSet) limitedResult(query *Query, limit int, lineStarter, fieldSeparator string, addHeader bool) (string, int, error) {
-	type result struct {
-		groupKey  string
-		resultStr string
-		orderBy   float64
-	}
+// Return a sorted result slice of the query from the group set.
+func (g *GroupSet) result(query *Query, gatherWidths bool) ([]result, []int, error) {
+	var rows []result
+	widths := make([]int, len(query.Select))
 
-	var resultSlice []result
+	var valueStr string
+	var value float64
 
 	for groupKey, set := range g.sets {
-		var sb strings.Builder
 		r := result{groupKey: groupKey}
 
-		lastIndex := len(query.Select) - 1
 		for i, sc := range query.Select {
-			storage := sc.FieldStorage
-			orderByThis := storage == query.OrderBy
-
 			switch sc.Operation {
 			case Count:
-				value := set.FValues[storage]
-				sb.WriteString(fmt.Sprintf("%d", int(value)))
-				if orderByThis {
-					r.orderBy = value
-				}
+				value = set.FValues[sc.FieldStorage]
+				valueStr = fmt.Sprintf("%d", int(value))
 			case Len:
 				fallthrough
 			case Sum:
@@ -121,74 +251,48 @@ func (g *GroupSet) limitedResult(query *Query, limit int, lineStarter, fieldSepa
 			case Min:
 				fallthrough
 			case Max:
-				value := set.FValues[storage]
-				sb.WriteString(fmt.Sprintf("%f", value))
-				if orderByThis {
-					r.orderBy = value
-				}
+				value = set.FValues[sc.FieldStorage]
+				valueStr = fmt.Sprintf("%f", value)
 			case Last:
-				value := set.SValues[storage]
-				if orderByThis {
-					f, err := strconv.ParseFloat(value, 64)
-					if err == nil {
-						r.orderBy = f
-					}
-				}
-				sb.WriteString(value)
+				valueStr = set.SValues[sc.FieldStorage]
+				value, _ = strconv.ParseFloat(valueStr, 64)
 			case Avg:
-				value := set.FValues[storage] / float64(set.Samples)
-				sb.WriteString(fmt.Sprintf("%f", value))
-				if orderByThis {
-					r.orderBy = value
-				}
+				value = set.FValues[sc.FieldStorage] / float64(set.Samples)
+				valueStr = fmt.Sprintf("%f", value)
 			default:
-				return "", 0, fmt.Errorf("Unknown aggregation method '%v'", sc.Operation)
+				return rows, widths, fmt.Errorf("Unknown aggregation method '%v'", sc.Operation)
 			}
-			if i != lastIndex {
-				sb.WriteString(fieldSeparator)
+
+			if sc.FieldStorage == query.OrderBy {
+				r.orderBy = value
+			}
+			r.values = append(r.values, valueStr)
+
+			if !gatherWidths {
+				continue
+			}
+			if widths[i] < len(sc.FieldStorage) {
+				widths[i] = len(sc.FieldStorage)
+			}
+			if widths[i] < len(valueStr) {
+				widths[i] = len(valueStr)
 			}
 		}
 
-		r.resultStr = sb.String()
-		resultSlice = append(resultSlice, r)
+		rows = append(rows, r)
 	}
 
 	if query.OrderBy != "" {
 		if query.ReverseOrder {
-			sort.SliceStable(resultSlice, func(i, j int) bool {
-				return resultSlice[i].orderBy < resultSlice[j].orderBy
+			sort.SliceStable(rows, func(i, j int) bool {
+				return rows[i].orderBy < rows[j].orderBy
 			})
 		} else {
-			sort.SliceStable(resultSlice, func(i, j int) bool {
-				return resultSlice[i].orderBy > resultSlice[j].orderBy
+			sort.SliceStable(rows, func(i, j int) bool {
+				return rows[i].orderBy > rows[j].orderBy
 			})
 		}
 	}
 
-	var sb strings.Builder
-
-	// Write header first
-	if addHeader {
-		lastIndex := len(query.Select) - 1
-		sb.WriteString(lineStarter)
-		for i, sc := range query.Select {
-			sb.WriteString(sc.FieldStorage)
-			if i != lastIndex {
-				sb.WriteString(fieldSeparator)
-			}
-		}
-		sb.WriteString("\n")
-	}
-
-	// And now write the data
-	for i, r := range resultSlice {
-		if i == limit {
-			break
-		}
-		sb.WriteString(lineStarter)
-		sb.WriteString(r.resultStr)
-		sb.WriteString("\n")
-	}
-
-	return sb.String(), len(resultSlice), nil
+	return rows, widths, nil
 }
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index da976bd..c67137e 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -1,16 +1,22 @@
 package logformat
 
 import (
+	"fmt"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// MakeFieldsDEFAULT is the default log file mapreduce parser.
+// MakeFieldsDEFAULT is the default DTail log file key-value parser.
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
-	fields := make(map[string]string, len(splitted))
+
+	if len(splitted) < 3 || !strings.HasPrefix(splitted[3], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") {
+		// Not a DTail mapreduce log line.
+		return nil, IgnoreFieldsErr
+	}
+
+	fields := make(map[string]string, len(splitted)+8)
 
 	fields["*"] = "*"
 	fields["$line"] = maprLine
@@ -19,20 +25,14 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields["$timezone"] = p.timeZoneName
 	fields["$timeoffset"] = p.timeZoneOffset
 
-	kvStart := 0
-	// DTail mapreduce format
-	if len(splitted) > 3 && strings.HasPrefix(splitted[3], "MAPREDUCE:") {
-		fields["$severity"] = splitted[0]
-		// TODO: Parse time like we do at Mimecast
-		fields["$time"] = splitted[1]
-		kvStart = 4
-	}
+	fields["$severity"] = splitted[0]
+	// TODO: Parse time like we do at Mimecast
+	fields["$time"] = splitted[1]
 
-	for _, kv := range splitted[kvStart:] {
+	for _, kv := range splitted[4:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
-			logger.Debug("Unable to parse key-value token, ignoring it", kv)
-			continue
+			return fields, fmt.Errorf("Unable to parse key-value token '%s'", kv)
 		}
 		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
 	}
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index 6284008..79911d1 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -11,8 +11,8 @@ func TestDefaultLogFormat(t *testing.T) {
 	}
 
 	inputs := []string{
-		"foo=bar|baz=bay",
 		"INFO|20210907-065632|SERVER|MAPREDUCE:TEST|foo=bar|baz=bay",
+		"INFO|20210907-065732|CLIENT|MAPREDUCE:YOMAN|baz=bay|foo=bar",
 	}
 
 	for _, input := range inputs {
@@ -23,20 +23,21 @@ func TestDefaultLogFormat(t *testing.T) {
 		}
 
 		if bar, ok := fields["foo"]; !ok {
-			t.Errorf("Expected field 'foo', but no such field there\n")
+			t.Errorf("Expected field 'foo', but no such field there in '%s'\n", input)
 		} else if bar != "bar" {
-			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s'\n", bar)
+			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n", bar, input)
 		}
 
 		if bay, ok := fields["baz"]; !ok {
-			t.Errorf("Expected field 'baz', but no such field there\n")
+			t.Errorf("Expected field 'baz', but no such field there in '%s'\n", input)
 		} else if bay != "bay" {
-			t.Errorf("Expected 'bay' stored in field 'baz', but got '%s'\n", bay)
+			t.Errorf("Expected 'bay' stored in field 'baz', but got '%s' in '%s'\n", bay, input)
 		}
 
 	}
 
-	if _, err := parser.MakeFields("foo=bar|bazbay"); err == nil {
-		t.Errorf("Expected error but didn't")
+	fields, err := parser.MakeFields("foozoo=bar|bazbay")
+	if _, ok := fields["foo"]; ok {
+		t.Errorf("Expected fiending field 'foo', but found it\n")
 	}
 }
diff --git a/internal/mapr/logformat/generickv.go b/internal/mapr/logformat/generickv.go
new file mode 100644
index 0000000..23d75cb
--- /dev/null
+++ b/internal/mapr/logformat/generickv.go
@@ -0,0 +1,31 @@
+package logformat
+
+import (
+	"strings"
+
+	"github.com/mimecast/dtail/internal/protocol"
+)
+
+// MakeFieldsGENERICKV is the generic key-value logfile parser.
+func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error) {
+	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
+	fields := make(map[string]string, len(splitted))
+
+	fields["*"] = "*"
+	fields["$line"] = maprLine
+	fields["$empty"] = ""
+	fields["$hostname"] = p.hostname
+	fields["$timezone"] = p.timeZoneName
+	fields["$timeoffset"] = p.timeZoneOffset
+
+	for _, kv := range splitted[0:] {
+		keyAndValue := strings.SplitN(kv, "=", 2)
+		if len(keyAndValue) != 2 {
+			//logger.Debug("Unable to parse key-value token, ignoring it", kv)
+			continue
+		}
+		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
+	}
+
+	return fields, nil
+}
diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go
index c53729a..6582b5f 100644
--- a/internal/mapr/logformat/parser.go
+++ b/internal/mapr/logformat/parser.go
@@ -12,6 +12,8 @@ import (
 	"github.com/mimecast/dtail/internal/mapr"
 )
 
+var IgnoreFieldsErr error = errors.New("Ignore this field set")
+
 // Parser is used to parse the mapreduce information from the server log files.
 type Parser struct {
 	hostname           string
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 9106f52..a6d6bb1 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -13,6 +13,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/logformat"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // Aggregate is for aggregating mapreduce data on the DTail server side.
@@ -89,7 +90,6 @@ func (a *Aggregate) Shutdown() {
 
 // Start an aggregation.
 func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
-
 	myCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -109,6 +109,7 @@ func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
 		fieldsCh = a.addFields(myCtx, fieldsCh)
 	}
 
+	// Periodically pre-aggregate data every a.query.Interval seconds.
 	go a.aggregateTimer(myCtx)
 	a.makeMaprLines(myCtx, fieldsCh, maprLines)
 }
@@ -139,13 +140,16 @@ func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 
 				maprLine := strings.TrimSpace(line.Content.String())
 				pool.RecycleBytesBuffer(line.Content)
-				fields, err := a.parser.MakeFields(maprLine)
-				logger.Debug(fields, err)
 
+				fields, err := a.parser.MakeFields(maprLine)
 				if err != nil {
-					logger.Error(err)
+					// Should fields be ignored anyway?
+					if err != logformat.IgnoreFieldsErr {
+						logger.Error(fields, err)
+					}
 					continue
 				}
+
 				if !a.query.WhereClause(fields) {
 					continue
 				}
@@ -170,7 +174,7 @@ func (a *Aggregate) addFields(ctx context.Context, fieldsCh <-chan map[string]st
 		defer close(ch)
 
 		for {
-			// fieldsCh will be closed via 'makeFields' if ctx is done
+			// fieldsCh will be closed via 'makeFields' when ctx is done
 			fields, ok := <-fieldsCh
 			if !ok {
 				return
@@ -219,12 +223,11 @@ func (a *Aggregate) makeMaprLines(ctx context.Context, fieldsCh <-chan map[strin
 }
 
 func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
-	//logger.Trace("Aggregating", group, fields)
 	var sb strings.Builder
 
 	for i, field := range a.query.GroupBy {
 		if i > 0 {
-			sb.WriteString(" ")
+			sb.WriteString(protocol.AggregateGroupKeyCombinator)
 		}
 		if val, ok := fields[field]; ok {
 			sb.WriteString(val)
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index d3035e4..8c9e861 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -5,8 +5,15 @@ const (
 	ProtocolCompat string = "4"
 	// MessageDelimiter delimits separate messages.
 	MessageDelimiter byte = '¬'
-	// FieldDelimiter delimits aggregation fields.
+	// FieldDelimiter delimits messagefields.
 	FieldDelimiter string = "|"
+	// CSVDelimiter delimits CSV file fields.kj:w
+	CSVDelimiter string = ","
+	// AggregateKVDelimiter delimits key-values of an aggregation message.
+	AggregateKVDelimiter string = "≔"
 	// AggregateDelimiter delimits parts of an aggregation message.
-	AggregateDelimiter string = "➔"
+	AggregateDelimiter string = "∥"
+	// AggregateDelimiter string = "⦀"
+	// AggregateGroupKeyCombinator combines the group set keys.
+	AggregateGroupKeyCombinator string = ","
 )
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
index a217b40..1e17c78 100644
--- a/internal/server/handlers/controlhandler.go
+++ b/internal/server/handlers/controlhandler.go
@@ -8,7 +8,6 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
@@ -57,7 +56,7 @@ func (h *ControlHandler) Read(p []byte) (n int, err error) {
 	for {
 		select {
 		case message := <-h.serverMessages:
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%b", h.hostname, message, protocol.MessageDelimiter))
+			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
 			n = copy(p, wholePayload)
 			return
 		case <-h.done.Done():
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 14f46a3..e74e686 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -38,7 +38,6 @@ type ServerHandler struct {
 	aggregate          *server.Aggregate
 	aggregatedMessages chan string
 	serverMessages     chan string
-	payload            []byte
 	hostname           string
 	user               *user.User
 	catLimiter         chan struct{}
@@ -47,6 +46,8 @@ type ServerHandler struct {
 	activeCommands     int32
 	activeReaders      int32
 	quiet              bool
+	readBuf            bytes.Buffer
+	writeBuf           bytes.Buffer
 }
 
 // NewServerHandler returns the server handler.
@@ -86,77 +87,74 @@ func (h *ServerHandler) Done() <-chan struct{} {
 
 // Read is to send data to the dtail client via Reader interface.
 func (h *ServerHandler) Read(p []byte) (n int, err error) {
-	for {
-		select {
-		case message := <-h.serverMessages:
-			if len(message) == 0 {
-				logger.Warn(h.user, "Empty message recieved")
-				return
-			}
-			if message[0] == '.' {
-				// Handle hidden message (don't display to the user, interpreted by dtail client)
-				payload := []byte(fmt.Sprintf("%s%s", message, string(protocol.MessageDelimiter)))
-				n = copy(p, payload)
-				return
-			}
-
-			// Handle normal server message (display to the user)
-			payload := []byte(fmt.Sprintf("SERVER|%s|%s%s", h.hostname, message, string(protocol.MessageDelimiter)))
-			n = copy(p, payload)
-			return
+	defer h.readBuf.Reset()
 
-		case message := <-h.aggregatedMessages:
-			// Send mapreduce-aggregated data as a message.
-			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
-			buf.WriteString("AGGREGATE")
-			buf.WriteString(protocol.AggregateDelimiter)
-			buf.WriteString(h.hostname)
-			buf.WriteString(protocol.AggregateDelimiter)
-			buf.WriteString(message)
-			buf.WriteByte(protocol.MessageDelimiter)
-			n = copy(p, buf.Bytes())
-			pool.RecycleBytesBuffer(buf)
+	select {
+	case message := <-h.serverMessages:
+		if message[0] == '.' {
+			// Handle hidden message (don't display to the user, interpreted by dtail client)
+			h.readBuf.WriteString(message)
+			h.readBuf.WriteByte(protocol.MessageDelimiter)
+			n = copy(p, h.readBuf.Bytes())
 			return
+		}
 
-		case line := <-h.lines:
-			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
-			buf.WriteString("REMOTE")
-			buf.WriteString(protocol.FieldDelimiter)
-			buf.WriteString(h.hostname)
-			buf.WriteString(protocol.FieldDelimiter)
-			buf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
-			buf.WriteString(protocol.FieldDelimiter)
-			buf.WriteString(fmt.Sprintf("%v", line.Count))
-			buf.WriteString(protocol.FieldDelimiter)
-			buf.WriteString(line.SourceID)
-			buf.WriteString(protocol.FieldDelimiter)
-			payload := append(buf.Bytes(), line.Content.Bytes()...)
-			n = copy(p, payload)
-			pool.RecycleBytesBuffer(buf)
-			pool.RecycleBytesBuffer(line.Content)
+		// Handle normal server message (display to the user)
+		h.readBuf.WriteString("SERVER")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(message)
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+
+	case message := <-h.aggregatedMessages:
+		// Send mapreduce-aggregated data as a message.
+		h.readBuf.WriteString("AGGREGATE")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(message)
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+
+	case line := <-h.lines:
+		h.readBuf.WriteString("REMOTE")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(line.SourceID)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(line.Content.String())
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+		pool.RecycleBytesBuffer(line.Content)
+
+	case <-time.After(time.Second):
+		// Once in a while check whether we are done.
+		select {
+		case <-h.done.Done():
+			err = io.EOF
 			return
-
-		case <-time.After(time.Second):
-			// Once in a while check whether we are done.
-			select {
-			case <-h.done.Done():
-				return 0, io.EOF
-			default:
-			}
+		default:
 		}
 	}
+	return
 }
 
 // Write is to receive data from the dtail client via Writer interface.
 func (h *ServerHandler) Write(p []byte) (n int, err error) {
-	for _, c := range p {
-		switch c {
+	for _, b := range p {
+		switch b {
 		case ';':
-			commandStr := strings.TrimSpace(string(h.payload))
-			h.handleCommand(commandStr)
-			h.payload = nil
+			h.handleCommand(string(h.writeBuf.Bytes()))
+			h.writeBuf.Reset()
 		default:
-			h.payload = append(h.payload, c)
+			h.writeBuf.WriteByte(b)
 		}
 	}
 
-- 
cgit v1.2.3


From 49a83f79e1b71c12b8efb7d0d106818b2541d77d Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 12 Sep 2021 19:01:34 +0300
Subject: limit mapreduce table output to 10 rows by default

---
 internal/clients/maprclient.go  | 17 ++++++++++++++---
 internal/mapr/globalgroupset.go |  4 ++--
 internal/mapr/groupset.go       |  8 ++++++--
 3 files changed, 22 insertions(+), 7 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index cab9a6c..2cad15d 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -158,12 +158,18 @@ func (c *MaprClient) reportResults() {
 func (c *MaprClient) printResults() {
 	var result string
 	var err error
-	var numRows int
+	var numRows, rowsLimit int
+
+	if c.query.Limit == -1 {
+		// Limit output to 10 rows when the result is printed to stdout.
+		// This can be overriden with the limit clause though.
+		rowsLimit = 10
+	}
 
 	if c.cumulative {
-		result, numRows, err = c.globalGroup.Result(c.query)
+		result, numRows, err = c.globalGroup.Result(c.query, rowsLimit)
 	} else {
-		result, numRows, err = c.globalGroup.SwapOut().Result(c.query)
+		result, numRows, err = c.globalGroup.SwapOut().Result(c.query, rowsLimit)
 	}
 
 	if err != nil {
@@ -189,6 +195,11 @@ func (c *MaprClient) printResults() {
 			config.Client.TermColors.MaprTable.RawQueryAttr)
 	}
 	logger.Raw(rawQuery)
+
+	if rowsLimit > 0 && numRows > rowsLimit {
+		logger.Warn(fmt.Sprintf("Got %d results but limited output to %d rows! Use 'limit' clause to override!",
+			numRows, rowsLimit))
+	}
 	logger.Raw(result)
 }
 
diff --git a/internal/mapr/globalgroupset.go b/internal/mapr/globalgroupset.go
index 21bf990..50bac37 100644
--- a/internal/mapr/globalgroupset.go
+++ b/internal/mapr/globalgroupset.go
@@ -93,9 +93,9 @@ func (g *GlobalGroupSet) WriteResult(query *Query) error {
 }
 
 // Result returns the result of the mapreduce aggregation as a string.
-func (g *GlobalGroupSet) Result(query *Query) (string, int, error) {
+func (g *GlobalGroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
 
-	return g.GroupSet.Result(query)
+	return g.GroupSet.Result(query, rowsLimit)
 }
diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index d29559a..9bff790 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -68,12 +68,16 @@ func (g *GroupSet) Serialize(ctx context.Context, ch chan<- string) {
 }
 
 // Result returns a nicely formated result of the query from the group set.
-func (g *GroupSet) Result(query *Query) (string, int, error) {
+func (g *GroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	rows, widths, err := g.result(query, true)
 	if err != nil {
 		return "", 0, err
 	}
 
+	if query.Limit != -1 {
+		rowsLimit = query.Limit
+	}
+
 	sb := pool.BuilderBuffer.Get().(*strings.Builder)
 	defer pool.RecycleBuilderBuffer(sb)
 
@@ -141,7 +145,7 @@ func (g *GroupSet) Result(query *Query) (string, int, error) {
 
 	// And now write the data
 	for i, r := range rows {
-		if i == query.Limit {
+		if i == rowsLimit {
 			break
 		}
 		for j, value := range r.values {
-- 
cgit v1.2.3


From 62bf0b0e9c2d5420349ad9c365d9484fc0a1c46b Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 12 Sep 2021 19:04:42 +0300
Subject: bugfix: dmap skipped the last couple of mapreduce lines

---
 internal/clients/maprclient.go            |   3 +-
 internal/mapr/server/aggregate.go         | 102 ++++++++++++------------------
 internal/server/handlers/readcommand.go   |  15 +++--
 internal/server/handlers/serverhandler.go |  86 +++++++++----------------
 4 files changed, 84 insertions(+), 122 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 2cad15d..68352ea 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -158,7 +158,8 @@ func (c *MaprClient) reportResults() {
 func (c *MaprClient) printResults() {
 	var result string
 	var err error
-	var numRows, rowsLimit int
+	var numRows int
+	rowsLimit := -1
 
 	if c.query.Limit == -1 {
 		// Limit output to 10 rows when the result is printed to stdout.
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index a6d6bb1..d11ed7d 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -19,16 +19,12 @@ import (
 // Aggregate is for aggregating mapreduce data on the DTail server side.
 type Aggregate struct {
 	done *internal.Done
-	// Log lines to process (parsing MAPREDUCE lines).
-	Lines chan line.Line
+	// NextLinesCh can be used to use a new line ch.
+	NextLinesCh chan chan line.Line
 	// Hostname of the current server (used to populate $hostname field).
 	hostname string
 	// Signals to serialize data.
 	serialize chan struct{}
-	// Signals to flush data.
-	flush chan struct{}
-	// Signals that data has been flushed
-	flushed chan struct{}
 	// The mapr query
 	query *mapr.Query
 	// The mapr log format parser
@@ -69,14 +65,12 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 	}
 
 	a := Aggregate{
-		done:      internal.NewDone(),
-		Lines:     make(chan line.Line, 100),
-		serialize: make(chan struct{}),
-		flush:     make(chan struct{}),
-		flushed:   make(chan struct{}),
-		hostname:  s[0],
-		query:     query,
-		parser:    logParser,
+		done:        internal.NewDone(),
+		NextLinesCh: make(chan chan line.Line, 10),
+		serialize:   make(chan struct{}),
+		hostname:    s[0],
+		query:       query,
+		parser:      logParser,
 	}
 
 	return &a, nil
@@ -84,12 +78,11 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 
 // Shutdown the aggregation engine.
 func (a *Aggregate) Shutdown() {
-	a.Flush()
 	a.done.Shutdown()
 }
 
 // Start an aggregation.
-func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
+func (a *Aggregate) Start(ctx context.Context, maprMessages chan<- string) {
 	myCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -102,16 +95,16 @@ func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
 		}
 	}()
 
-	fieldsCh := a.makeFields(myCtx)
+	fieldsCh := a.fieldsFromLines(myCtx)
 
 	// Add fields (e.g. via 'set' clause)
 	if len(a.query.Set) > 0 {
-		fieldsCh = a.addFields(myCtx, fieldsCh)
+		fieldsCh = a.setAdditionalFields(myCtx, fieldsCh)
 	}
 
 	// Periodically pre-aggregate data every a.query.Interval seconds.
 	go a.aggregateTimer(myCtx)
-	a.makeMaprLines(myCtx, fieldsCh, maprLines)
+	a.aggregateAndSerialize(myCtx, fieldsCh, maprMessages)
 }
 
 func (a *Aggregate) aggregateTimer(ctx context.Context) {
@@ -125,23 +118,38 @@ func (a *Aggregate) aggregateTimer(ctx context.Context) {
 	}
 }
 
-func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
-	ch := make(chan map[string]string)
+func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]string {
+	fieldsCh := make(chan map[string]string)
 
 	go func() {
-		defer close(ch)
+		defer close(fieldsCh)
+		var lines chan line.Line
+
+		// Gather first lines channel (first input file)
+		select {
+		case lines = <-a.NextLinesCh:
+		case <-ctx.Done():
+			return
+		}
 
 		for {
 			select {
-			case line, ok := <-a.Lines:
+			case line, ok := <-lines:
 				if !ok {
-					return
+					select {
+					case lines = <-a.NextLinesCh:
+						// Have a new lines channel (e.g. new input file)
+					case <-ctx.Done():
+					default:
+						// No new lines channel found.
+						return
+					}
 				}
 
 				maprLine := strings.TrimSpace(line.Content.String())
+				fields, err := a.parser.MakeFields(maprLine)
 				pool.RecycleBytesBuffer(line.Content)
 
-				fields, err := a.parser.MakeFields(maprLine)
 				if err != nil {
 					// Should fields be ignored anyway?
 					if err != logformat.IgnoreFieldsErr {
@@ -155,7 +163,7 @@ func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 				}
 
 				select {
-				case ch <- fields:
+				case fieldsCh <- fields:
 				case <-ctx.Done():
 				}
 			case <-ctx.Done():
@@ -164,17 +172,16 @@ func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 		}
 	}()
 
-	return ch
+	return fieldsCh
 }
 
-func (a *Aggregate) addFields(ctx context.Context, fieldsCh <-chan map[string]string) <-chan map[string]string {
-	ch := make(chan map[string]string)
+func (a *Aggregate) setAdditionalFields(ctx context.Context, fieldsCh <-chan map[string]string) <-chan map[string]string {
+	newFieldsCh := make(chan map[string]string)
 
 	go func() {
-		defer close(ch)
+		defer close(newFieldsCh)
 
 		for {
-			// fieldsCh will be closed via 'makeFields' when ctx is done
 			fields, ok := <-fieldsCh
 			if !ok {
 				return
@@ -184,23 +191,22 @@ func (a *Aggregate) addFields(ctx context.Context, fieldsCh <-chan map[string]st
 			}
 
 			select {
-			case ch <- fields:
+			case newFieldsCh <- fields:
 			case <-ctx.Done():
 			}
 		}
 	}()
 
-	return ch
+	return newFieldsCh
 }
 
-func (a *Aggregate) makeMaprLines(ctx context.Context, fieldsCh <-chan map[string]string, maprLines chan<- string) {
+func (a *Aggregate) aggregateAndSerialize(ctx context.Context, fieldsCh <-chan map[string]string, maprMessages chan<- string) {
 	group := mapr.NewGroupSet()
 
 	serialize := func() {
 		logger.Info("Serializing mapreduce result")
-		group.Serialize(ctx, maprLines)
+		group.Serialize(ctx, maprMessages)
 		group = mapr.NewGroupSet()
-		logger.Info("Done serializing mapreduce result")
 	}
 
 	for {
@@ -213,9 +219,6 @@ func (a *Aggregate) makeMaprLines(ctx context.Context, fieldsCh <-chan map[strin
 			a.aggregate(group, fields)
 		case <-a.serialize:
 			serialize()
-		case <-a.flush:
-			serialize()
-			a.flushed <- struct{}{}
 		case <-ctx.Done():
 			return
 		}
@@ -264,24 +267,3 @@ func (a *Aggregate) Serialize(ctx context.Context) {
 	case <-ctx.Done():
 	}
 }
-
-// Flush all data.
-func (a *Aggregate) Flush() {
-	select {
-	case a.flush <- struct{}{}:
-		logger.Info("Flushing mapreduce data")
-	case <-time.After(time.Minute):
-		logger.Warn("Starting to flush mapreduce data takes over a minute")
-		return
-	case <-a.done.Done():
-		return
-	}
-
-	select {
-	case <-a.flushed:
-		logger.Info("Done flushing")
-	case <-time.After(time.Minute):
-		logger.Warn("Waiting for data to be flushed takes over a minute")
-	case <-a.done.Done():
-	}
-}
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 5bab26f..69dd4a5 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/io/fs"
+	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
@@ -113,16 +114,20 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 	}
 
 	lines := r.server.lines
-
-	// Plug in mappreduce engine
-	if r.server.aggregate != nil {
-		lines = r.server.aggregate.Lines
-	}
+	aggregate := r.server.aggregate
 
 	for {
+		if aggregate != nil {
+			lines = make(chan line.Line, 100)
+			aggregate.NextLinesCh <- lines
+		}
 		if err := reader.Start(ctx, lines, re); err != nil {
 			logger.Error(r.server.user, path, globID, err)
 		}
+		if aggregate != nil {
+			// Also makes aggregate to Flush
+			close(lines)
+		}
 
 		select {
 		case <-ctx.Done():
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index e74e686..ed19412 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -32,36 +32,35 @@ const (
 // the Bi-directional communication between SSH client and server.
 // This handler implements the handler of the SSH server.
 type ServerHandler struct {
-	done               *internal.Done
-	lines              chan line.Line
-	regex              string
-	aggregate          *server.Aggregate
-	aggregatedMessages chan string
-	serverMessages     chan string
-	hostname           string
-	user               *user.User
-	catLimiter         chan struct{}
-	tailLimiter        chan struct{}
-	ackCloseReceived   chan struct{}
-	activeCommands     int32
-	activeReaders      int32
-	quiet              bool
-	readBuf            bytes.Buffer
-	writeBuf           bytes.Buffer
+	done             *internal.Done
+	lines            chan line.Line
+	regex            string
+	aggregate        *server.Aggregate
+	maprMessages     chan string
+	serverMessages   chan string
+	hostname         string
+	user             *user.User
+	catLimiter       chan struct{}
+	tailLimiter      chan struct{}
+	ackCloseReceived chan struct{}
+	activeCommands   int32
+	quiet            bool
+	readBuf          bytes.Buffer
+	writeBuf         bytes.Buffer
 }
 
 // NewServerHandler returns the server handler.
 func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
 	h := ServerHandler{
-		done:               internal.NewDone(),
-		lines:              make(chan line.Line, 100),
-		serverMessages:     make(chan string, 10),
-		aggregatedMessages: make(chan string, 10),
-		ackCloseReceived:   make(chan struct{}),
-		catLimiter:         catLimiter,
-		tailLimiter:        tailLimiter,
-		regex:              ".",
-		user:               user,
+		done:             internal.NewDone(),
+		lines:            make(chan line.Line, 100),
+		serverMessages:   make(chan string, 10),
+		maprMessages:     make(chan string, 10),
+		ackCloseReceived: make(chan struct{}),
+		catLimiter:       catLimiter,
+		tailLimiter:      tailLimiter,
+		regex:            ".",
+		user:             user,
 	}
 
 	fqdn, err := os.Hostname()
@@ -108,7 +107,7 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		h.readBuf.WriteByte(protocol.MessageDelimiter)
 		n = copy(p, h.readBuf.Bytes())
 
-	case message := <-h.aggregatedMessages:
+	case message := <-h.maprMessages:
 		// Send mapreduce-aggregated data as a message.
 		h.readBuf.WriteString("AGGREGATE")
 		h.readBuf.WriteString(protocol.FieldDelimiter)
@@ -260,14 +259,6 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			h.shutdown()
 		}
 	}
-	readerFinished := func() {
-		if h.decrementActiveReaders() == 0 {
-			if h.aggregate == nil {
-				return
-			}
-			h.aggregate.Shutdown()
-		}
-	}
 
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
@@ -289,18 +280,14 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 	case "grep", "cat":
 		command := newReadCommand(h, omode.CatClient)
 		go func() {
-			h.incrementActiveReaders()
 			command.Start(ctx, argc, args, 1)
-			readerFinished()
 			commandFinished()
 		}()
 
 	case "tail":
 		command := newReadCommand(h, omode.TailClient)
 		go func() {
-			h.incrementActiveReaders()
 			command.Start(ctx, argc, args, 10)
-			readerFinished()
 			commandFinished()
 		}()
 
@@ -315,7 +302,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 
 		h.aggregate = aggregate
 		go func() {
-			command.Start(ctx, h.aggregatedMessages)
+			command.Start(ctx, h.maprMessages)
 			commandFinished()
 		}()
 
@@ -361,15 +348,11 @@ func (h *ServerHandler) serverMessageC() chan<- string {
 	return h.serverMessages
 }
 
-func (h *ServerHandler) flush() {
-	logger.Debug(h.user, "flush()")
-
-	if h.aggregate != nil {
-		h.aggregate.Flush()
-	}
+func (h *ServerHandler) flushMessages() {
+	logger.Debug(h.user, "flushMessages()")
 
 	unsentMessages := func() int {
-		return len(h.lines) + len(h.serverMessages) + len(h.aggregatedMessages)
+		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
 	}
 	for i := 0; i < 3; i++ {
 		if unsentMessages() == 0 {
@@ -385,7 +368,7 @@ func (h *ServerHandler) flush() {
 
 func (h *ServerHandler) shutdown() {
 	logger.Debug(h.user, "shutdown()")
-	h.flush()
+	h.flushMessages()
 
 	go func() {
 		select {
@@ -413,15 +396,6 @@ func (h *ServerHandler) decrementActiveCommands() int32 {
 	return atomic.LoadInt32(&h.activeCommands)
 }
 
-func (h *ServerHandler) incrementActiveReaders() {
-	atomic.AddInt32(&h.activeReaders, 1)
-}
-
-func (h *ServerHandler) decrementActiveReaders() int32 {
-	atomic.AddInt32(&h.activeReaders, -1)
-	return atomic.LoadInt32(&h.activeReaders)
-}
-
 func readOptions(opts []string) (map[string]string, error) {
 	options := make(map[string]string, len(opts))
 
-- 
cgit v1.2.3


From 382943f3c475260bd5eb3bc787f480c2c0533f75 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 18 Sep 2021 11:57:09 +0300
Subject: new docker test cases - also change default FATAL bg color to magenta

---
 internal/clients/args.go                 | 46 +++++++++++++++++++++++++++++++-
 internal/clients/baseclient.go           |  3 +++
 internal/clients/handlers/basehandler.go |  7 -----
 internal/config/client.go                |  4 +--
 4 files changed, 50 insertions(+), 10 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
index 7f782f1..7ce1634 100644
--- a/internal/clients/args.go
+++ b/internal/clients/args.go
@@ -1,6 +1,13 @@
 package clients
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/omode"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -22,5 +29,42 @@ type Args struct {
 	SSHAuthMethods     []gossh.AuthMethod
 	SSHHostKeyCallback gossh.HostKeyCallback
 	PrivateKeyPathFile string
-	Quiet            bool
+	Quiet              bool
+}
+
+// When no servers are given, connect to localhost!
+func (a *Args) handleEmptyServer() {
+	if a.Discovery != "" || a.ServersStr != "" {
+		return
+	}
+
+	fqdn, err := os.Hostname()
+	if err != nil {
+		logger.FatalExit(err)
+	}
+	a.ServersStr = fmt.Sprintf("%s:%d", fqdn, config.Common.SSHPort)
+	// I am trusting my own hostname.
+	a.TrustAllHosts = true
+	logger.Debug("Will connect to local server", a.ServersStr)
+
+	cleanPath := func(dirtyPath string) string {
+		cleanPath, err := filepath.EvalSymlinks(dirtyPath)
+		if err != nil {
+			logger.FatalExit("Unable to evaluate symlinks", dirtyPath, err)
+		}
+		cleanPath, err = filepath.Abs(cleanPath)
+		if err != nil {
+			logger.FatalExit("Unable to make file path absolute", dirtyPath, cleanPath, err)
+		}
+		return cleanPath
+	}
+
+	logger.Debug("Dirty file paths", a.What)
+	var filePaths []string
+	for _, dirtyPath := range strings.Split(a.What, ",") {
+		filePaths = append(filePaths, cleanPath(dirtyPath))
+	}
+
+	a.What = strings.Join(filePaths, ",")
+	logger.Debug("Clean file paths", a.What)
 }
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index de0c101..455174e 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -41,6 +41,9 @@ type baseClient struct {
 func (c *baseClient) init() {
 	logger.Debug("Initiating base client")
 
+	// When empty server list provided, connect to localhost by default.
+	c.Args.handleEmptyServer()
+
 	flag := regex.Default
 	if c.Args.RegexInvert {
 		flag = regex.Invert
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 51f33c1..74559e9 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -112,12 +112,5 @@ func (h *baseHandler) handleHiddenMessage(message string) {
 	switch {
 	case strings.HasPrefix(message, ".syn close connection"):
 		h.SendMessage(".ack close connection")
-		select {
-		case <-time.After(time.Second * 5):
-			logger.Debug("Shutting down client after timeout and sending ack to server")
-			h.Shutdown()
-		case <-h.Done():
-			return
-		}
 	}
 }
diff --git a/internal/config/client.go b/internal/config/client.go
index 795c4a4..152ab15 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -169,8 +169,8 @@ func newDefaultClientConfig() *ClientConfig {
 				SeverityErrorAttr: color.AttrBold,
 				SeverityErrorBg:   color.BgRed,
 				SeverityErrorFg:   color.FgWhite,
-				SeverityFatalAttr: color.AttrBlink,
-				SeverityFatalBg:   color.BgRed,
+				SeverityFatalAttr: color.AttrBold,
+				SeverityFatalBg:   color.BgMagenta,
 				SeverityFatalFg:   color.FgWhite,
 				SeverityWarnAttr:  color.AttrBold,
 				SeverityWarnBg:    color.BgBlack,
-- 
cgit v1.2.3


From 03df87ee9464f6d7a1ca50254dcc5b81d1cf2625 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 18 Sep 2021 14:41:25 +0300
Subject: add spartan mode

---
 internal/clients/args.go                  | 34 +++++++++++++++++++++++++++----
 internal/clients/baseclient.go            |  3 ---
 internal/clients/catclient.go             |  6 ++++--
 internal/clients/grepclient.go            |  7 +++++--
 internal/clients/handlers/basehandler.go  | 15 +++++++-------
 internal/clients/maprclient.go            |  7 +++++--
 internal/clients/tailclient.go            |  7 +++++--
 internal/config/read.go                   |  5 ++++-
 internal/io/fs/readfile.go                | 13 +++++++-----
 internal/server/handlers/serverhandler.go | 31 +++++++++++++++++++---------
 10 files changed, 89 insertions(+), 39 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
index 7ce1634..081b911 100644
--- a/internal/clients/args.go
+++ b/internal/clients/args.go
@@ -1,6 +1,7 @@
 package clients
 
 import (
+	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -30,14 +31,35 @@ type Args struct {
 	SSHHostKeyCallback gossh.HostKeyCallback
 	PrivateKeyPathFile string
 	Quiet              bool
+	Spartan            bool
+	NoColor            bool
 }
 
-// When no servers are given, connect to localhost!
-func (a *Args) handleEmptyServer() {
-	if a.Discovery != "" || a.ServersStr != "" {
-		return
+// Transform the arguments based on certain conditions.
+func (a *Args) Transform(args []string) {
+	// Interpret additional args as file list.
+	if a.What == "" {
+		var files []string
+		for _, file := range flag.Args() {
+			files = append(files, file)
+		}
+		a.What = strings.Join(files, ",")
+	}
+
+	if a.Spartan {
+		a.Quiet = true
+		a.NoColor = true
 	}
+}
 
+// TransformAfterConfigFile same as Transform, but after the config file has been read.
+func (a *Args) TransformAfterConfigFile() {
+	if a.Discovery == "" && a.ServersStr == "" {
+		a.handleEmptyServer()
+	}
+}
+
+func (a *Args) handleEmptyServer() {
 	fqdn, err := os.Hostname()
 	if err != nil {
 		logger.FatalExit(err)
@@ -68,3 +90,7 @@ func (a *Args) handleEmptyServer() {
 	a.What = strings.Join(filePaths, ",")
 	logger.Debug("Clean file paths", a.What)
 }
+
+func (a *Args) SerializeOptions() string {
+	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
+}
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 455174e..de0c101 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -41,9 +41,6 @@ type baseClient struct {
 func (c *baseClient) init() {
 	logger.Debug("Initiating base client")
 
-	// When empty server list provided, connect to localhost by default.
-	c.Args.handleEmptyServer()
-
 	flag := regex.Default
 	if c.Args.RegexInvert {
 		flag = regex.Invert
diff --git a/internal/clients/catclient.go b/internal/clients/catclient.go
index b7b6131..d14cdcc 100644
--- a/internal/clients/catclient.go
+++ b/internal/clients/catclient.go
@@ -42,9 +42,11 @@ func (c CatClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c CatClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
+			c.Mode.String(),
+			c.Args.SerializeOptions(),
+			file, c.Regex.Serialize()))
 	}
 	return
 }
diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index 652c31b..ea5022b 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -41,9 +41,12 @@ func (c GrepClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c GrepClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
+			c.Mode.String(),
+			c.Args.SerializeOptions(),
+			file,
+			c.Regex.Serialize()))
 	}
 
 	return
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 74559e9..0f2d1b5 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -67,9 +67,12 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 		*/
 		case '\n', protocol.MessageDelimiter:
 			message := h.receiveBuf.String()
-			if len(message) == 0 {
-				continue
-			}
+			/*
+				// dcat/grep should actually display empty lines.
+					if len(message) == 0 {
+						continue
+					}
+			*/
 			h.handleMessageType(message)
 			h.receiveBuf.Reset()
 		default:
@@ -93,12 +96,8 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 
 // Handle various message types.
 func (h *baseHandler) handleMessageType(message string) {
-	if len(message) == 0 {
-		return
-	}
-
 	// Hidden server commands starti with a dot "."
-	if message[0] == '.' {
+	if len(message) > 0 && message[0] == '.' {
 		h.handleHiddenMessage(message)
 		return
 	}
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 68352ea..e6ab96b 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -116,7 +116,6 @@ func (c MaprClient) makeHandler(server string) handlers.Handler {
 
 func (c MaprClient) makeCommands() (commands []string) {
 	commands = append(commands, fmt.Sprintf("map %s", c.query.RawQuery))
-	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 
 	modeStr := "cat"
 	if c.Mode == omode.TailClient {
@@ -128,7 +127,11 @@ func (c MaprClient) makeCommands() (commands []string) {
 			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout, modeStr, file, c.Regex.Serialize()))
 			continue
 		}
-		commands = append(commands, fmt.Sprintf("%s:%s %s %s", modeStr, options, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
+			modeStr,
+			c.Args.SerializeOptions(),
+			file,
+			c.Regex.Serialize()))
 	}
 
 	return
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index cefbaa7..360354b 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -38,9 +38,12 @@ func (c TailClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c TailClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
+			c.Mode.String(),
+			c.Args.SerializeOptions(),
+			file,
+			c.Regex.Serialize()))
 	}
 	logger.Debug(commands)
 
diff --git a/internal/config/read.go b/internal/config/read.go
index a4e605b..ea358f8 100644
--- a/internal/config/read.go
+++ b/internal/config/read.go
@@ -5,7 +5,7 @@ import (
 )
 
 // Read the DTail configuration.
-func Read(configFile string, sshPort int) {
+func Read(configFile string, sshPort int, noColor bool) {
 	initializer := configInitializer{
 		Common: newDefaultCommonConfig(),
 		Server: newDefaultServerConfig(),
@@ -34,4 +34,7 @@ func Read(configFile string, sshPort int) {
 	if sshPort != 2222 {
 		Common.SSHPort = sshPort
 	}
+	if noColor {
+		Client.TermColorsEnable = false
+	}
 }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index c0d44dd..ec33c60 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -182,11 +182,14 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 
 		switch b {
 		case '\n':
-			if message.Len() == 0 {
-				time.Sleep(time.Millisecond * 100)
-				continue
-			}
-			message.WriteString("\n")
+			/*
+				// dcat/dgrep should actually transfer empty lines
+					if message.Len() == 0 {
+						time.Sleep(time.Millisecond * 100)
+						continue
+					}
+			*/
+			//message.WriteString("\n")
 			select {
 			case rawLines <- message:
 				message = pool.BytesBuffer.Get().(*bytes.Buffer)
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index ed19412..2f3b73b 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -45,6 +45,7 @@ type ServerHandler struct {
 	ackCloseReceived chan struct{}
 	activeCommands   int32
 	quiet            bool
+	spartan          bool
 	readBuf          bytes.Buffer
 	writeBuf         bytes.Buffer
 }
@@ -118,16 +119,18 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		n = copy(p, h.readBuf.Bytes())
 
 	case line := <-h.lines:
-		h.readBuf.WriteString("REMOTE")
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(h.hostname)
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(line.SourceID)
-		h.readBuf.WriteString(protocol.FieldDelimiter)
+		if !h.spartan {
+			h.readBuf.WriteString("REMOTE")
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(h.hostname)
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(line.SourceID)
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+		}
 		h.readBuf.WriteString(line.Content.String())
 		h.readBuf.WriteByte(protocol.MessageDelimiter)
 		n = copy(p, h.readBuf.Bytes())
@@ -275,6 +278,12 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			h.quiet = true
 		}
 	}
+	if spartan, ok := options["spartan"]; ok {
+		if spartan == "true" {
+			logger.Debug(h.user, "Enabling spartan mode")
+			h.spartan = true
+		}
+	}
 
 	switch commandName {
 	case "grep", "cat":
@@ -397,6 +406,7 @@ func (h *ServerHandler) decrementActiveCommands() int32 {
 }
 
 func readOptions(opts []string) (map[string]string, error) {
+	logger.Debug("Parsing options", opts)
 	options := make(map[string]string, len(opts))
 
 	for _, o := range opts {
@@ -416,6 +426,7 @@ func readOptions(opts []string) (map[string]string, error) {
 			val = string(decoded)
 		}
 
+		logger.Debug("Setting option", key, val)
 		options[key] = val
 	}
 
-- 
cgit v1.2.3


From f796df4c2f4bc8b61152c7d3e363152fb7bbc6f9 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 18 Sep 2021 18:43:19 +0300
Subject: remote connector is now an interface

---
 internal/clients/baseclient.go                  |  24 +--
 internal/clients/connectors/connector.go        |  17 ++
 internal/clients/connectors/serverconnection.go | 225 ++++++++++++++++++++++++
 internal/clients/healthclient.go                |  12 +-
 internal/clients/remote/connection.go           | 212 ----------------------
 5 files changed, 259 insertions(+), 231 deletions(-)
 create mode 100644 internal/clients/connectors/connector.go
 create mode 100644 internal/clients/connectors/serverconnection.go
 delete mode 100644 internal/clients/remote/connection.go

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index de0c101..d0631fc 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mimecast/dtail/internal/clients/remote"
+	"github.com/mimecast/dtail/internal/clients/connectors"
 	"github.com/mimecast/dtail/internal/discovery"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/omode"
@@ -23,7 +23,7 @@ type baseClient struct {
 	// List of remote servers to connect to.
 	servers []string
 	// We have one connection per remote server.
-	connections []*remote.Connection
+	connections []connectors.Connector
 	// SSH auth methods to use to connect to the remote servers.
 	sshAuthMethods []gossh.AuthMethod
 	// To deal with SSH host keys
@@ -77,7 +77,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 
 	var mutex sync.Mutex
 	for i, conn := range c.connections {
-		go func(i int, conn *remote.Connection) {
+		go func(i int, conn connectors.Connector) {
 			connStatus := c.start(ctx, active, i, conn)
 
 			// Update global status.
@@ -93,7 +93,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	return
 }
 
-func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, conn *remote.Connection) (status int) {
+func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, conn connectors.Connector) (status int) {
 	// Increment connection count
 	active <- struct{}{}
 	// Derement connection count
@@ -105,26 +105,20 @@ func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, con
 
 		conn.Start(connCtx, cancel, c.throttleCh, c.stats.connectionsEstCh)
 		// Retrieve status code from handler (dtail client will exit with that status)
-		status = conn.Handler.Status()
+		status = conn.Handler().Status()
 
 		if !c.retry {
 			return
 		}
 
 		time.Sleep(time.Second * 2)
-		logger.Debug(conn.Server, "Reconnecting")
-
-		conn = c.makeConnection(conn.Server, c.sshAuthMethods, c.hostKeyCallback)
-		c.connections[i] = conn
+		logger.Debug(conn.Server(), "Reconnecting")
+		c.connections[i] = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback)
 	}
 }
 
-func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod, hostKeyCallback client.HostKeyCallback) *remote.Connection {
-	conn := remote.NewConnection(server, c.UserName, sshAuthMethods, hostKeyCallback)
-	conn.Handler = c.maker.makeHandler(server)
-	conn.Commands = c.maker.makeCommands()
-
-	return conn
+func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod, hostKeyCallback client.HostKeyCallback) connectors.Connector {
+	return connectors.NewServerConnection(server, c.UserName, sshAuthMethods, hostKeyCallback, c.maker.makeHandler(server), c.maker.makeCommands())
 }
 
 func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
diff --git a/internal/clients/connectors/connector.go b/internal/clients/connectors/connector.go
new file mode 100644
index 0000000..3ab6a08
--- /dev/null
+++ b/internal/clients/connectors/connector.go
@@ -0,0 +1,17 @@
+package connectors
+
+import (
+	"context"
+
+	"github.com/mimecast/dtail/internal/clients/handlers"
+)
+
+// Connector interface.
+type Connector interface {
+	// Start the connection.
+	Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{})
+	// Server hostname.
+	Server() string
+	// Handler for the connection.
+	Handler() handlers.Handler
+}
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
new file mode 100644
index 0000000..fab2f87
--- /dev/null
+++ b/internal/clients/connectors/serverconnection.go
@@ -0,0 +1,225 @@
+package connectors
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/ssh/client"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// ServerConnection represents a client connection connection to a single server.
+type ServerConnection struct {
+	// The remote server's hostname connected to.
+	server string
+	// The remote server's port connected to.
+	port int
+	// The SSH client configuration used.
+	config *ssh.ClientConfig
+	// The SSH client handler to use.
+	handler handlers.Handler
+	// DTail commands sent from client to server. When client loses
+	// connection to the server it re-connects automatically and sends the
+	// same commands again.
+	commands []string
+	// Is it a persistent connection or a one-off?
+	isOneOff bool
+	// To deal with SSH server host keys
+	hostKeyCallback client.HostKeyCallback
+	// To determine if connection throttling has finished or not
+	throttlingDone bool
+}
+
+// NewServerConnection returns a new connection.
+func NewServerConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback, handler handlers.Handler, commands []string) *ServerConnection {
+	logger.Debug(server, "Creating new connection")
+
+	c := ServerConnection{
+		hostKeyCallback: hostKeyCallback,
+		server:          server,
+		handler:         handler,
+		commands:        commands,
+		config: &ssh.ClientConfig{
+			User:            userName,
+			Auth:            authMethods,
+			HostKeyCallback: hostKeyCallback.Wrap(),
+			Timeout:         time.Second * 2,
+		},
+	}
+
+	c.initServerPort()
+	return &c
+}
+
+// NewOneOffServerConnection creates new one-off connection (only for sending a series of commands and then quit).
+func NewOneOffServerConnection(server string, userName string, authMethods []ssh.AuthMethod, handler handlers.Handler, commands []string) *ServerConnection {
+	c := ServerConnection{
+		server:   server,
+		handler:  handler,
+		commands: commands,
+		config: &ssh.ClientConfig{
+			User:            userName,
+			Auth:            authMethods,
+			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		},
+		isOneOff: true,
+	}
+
+	c.initServerPort()
+	return &c
+}
+
+// Server hostname
+func (c *ServerConnection) Server() string {
+	return c.server
+}
+
+// Handler for the connection
+func (c *ServerConnection) Handler() handlers.Handler {
+	return c.handler
+}
+
+// Attempt to parse the server port address from the provided server FQDN.
+func (c *ServerConnection) initServerPort() {
+	c.port = config.Common.SSHPort
+	parts := strings.Split(c.server, ":")
+
+	if len(parts) == 2 {
+		logger.Debug("Parsing port from hostname", parts)
+		port, err := strconv.Atoi(parts[1])
+		if err != nil {
+			logger.FatalExit("Unable to parse client port", c.server, parts, err)
+		}
+		c.server = parts[0]
+		c.port = port
+	}
+}
+
+// Start the server connection. Build up SSH session and send some DTail commands.
+func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+	// Throttle how many connections can be established concurrently (based on ch length)
+	logger.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
+
+	select {
+	case throttleCh <- struct{}{}:
+	case <-ctx.Done():
+		logger.Debug(c.server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
+		return
+	}
+
+	logger.Debug(c.server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
+
+	go func() {
+		defer func() {
+			if !c.throttlingDone {
+				logger.Debug(c.server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
+				c.throttlingDone = true
+				<-throttleCh
+			}
+			cancel()
+		}()
+
+		if err := c.dial(ctx, cancel, throttleCh, statsCh); err != nil {
+			logger.Warn(c.server, c.port, err)
+			if c.hostKeyCallback.Untrusted(fmt.Sprintf("%s:%d", c.server, c.port)) {
+				logger.Debug(c.server, "Not trusting host")
+			}
+		}
+	}()
+
+	<-ctx.Done()
+}
+
+// Dail into a new SSH connection. Close connection in case of an error.
+func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) error {
+	logger.Debug(c.server, "Incrementing connection stats")
+	statsCh <- struct{}{}
+	defer func() {
+		logger.Debug(c.server, "Decrementing connection stats")
+		<-statsCh
+	}()
+
+	logger.Debug(c.server, "Dialing into the connection")
+	address := fmt.Sprintf("%s:%d", c.server, c.port)
+
+	client, err := ssh.Dial("tcp", address, c.config)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	return c.session(ctx, cancel, client, throttleCh)
+}
+
+// Create the SSH session. Close the session in case of an error.
+func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
+	logger.Debug(c.server, "session")
+
+	session, err := client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	return c.handle(ctx, cancel, session, throttleCh)
+}
+
+func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
+	logger.Debug(c.server, "handle")
+
+	stdinPipe, err := session.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	stdoutPipe, err := session.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	if err := session.Shell(); err != nil {
+		return err
+	}
+
+	go func() {
+		io.Copy(stdinPipe, c.handler)
+		cancel()
+	}()
+
+	go func() {
+		io.Copy(c.handler, stdoutPipe)
+		cancel()
+	}()
+
+	go func() {
+		select {
+		case <-c.handler.Done():
+		case <-ctx.Done():
+		}
+		cancel()
+	}()
+
+	// Send all commands to client.
+	for _, command := range c.commands {
+		logger.Debug(command)
+		c.handler.SendMessage(command)
+	}
+
+	if !c.throttlingDone {
+		logger.Debug(c.server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
+		c.throttlingDone = true
+		<-throttleCh
+	}
+
+	<-ctx.Done()
+	c.handler.Shutdown()
+	return nil
+}
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index 692464c..47007b6 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -7,8 +7,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mimecast/dtail/internal/clients/connectors"
 	"github.com/mimecast/dtail/internal/clients/handlers"
-	"github.com/mimecast/dtail/internal/clients/remote"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/protocol"
@@ -47,9 +47,13 @@ func (c *HealthClient) Start(ctx context.Context) (status int) {
 	throttleCh := make(chan struct{}, runtime.NumCPU())
 	statsCh := make(chan struct{}, 1)
 
-	conn := remote.NewOneOffConnection(c.server, c.userName, c.sshAuthMethods)
-	conn.Handler = handlers.NewHealthHandler(c.server, receive)
-	conn.Commands = []string{c.mode.String()}
+	conn := connectors.NewOneOffServerConnection(
+		c.server,
+		c.userName,
+		c.sshAuthMethods,
+		handlers.NewHealthHandler(c.server, receive),
+		[]string{c.mode.String()},
+	)
 
 	connCtx, cancel := context.WithCancel(ctx)
 	go conn.Start(connCtx, cancel, throttleCh, statsCh)
diff --git a/internal/clients/remote/connection.go b/internal/clients/remote/connection.go
deleted file mode 100644
index b29ffed..0000000
--- a/internal/clients/remote/connection.go
+++ /dev/null
@@ -1,212 +0,0 @@
-package remote
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/mimecast/dtail/internal/clients/handlers"
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/ssh/client"
-
-	"golang.org/x/crypto/ssh"
-)
-
-// Connection represents a client connection connection to a single server.
-type Connection struct {
-	// The remote server's hostname connected to.
-	Server string
-	// The remote server's port connected to.
-	port int
-	// The SSH client configuration used.
-	config *ssh.ClientConfig
-	// The SSH client handler to use.
-	Handler handlers.Handler
-	// DTail commands sent from client to server. When client loses
-	// connection to the server it re-connects automatically and sends the
-	// same commands again.
-	Commands []string
-	// Is it a persistent connection or a one-off?
-	isOneOff bool
-	// To deal with SSH server host keys
-	hostKeyCallback client.HostKeyCallback
-	// To determine if connection throttling has finished or not
-	throttlingDone bool
-}
-
-// NewConnection returns a new connection.
-func NewConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback) *Connection {
-	logger.Debug(server, "Creating new connection")
-
-	c := Connection{
-		hostKeyCallback: hostKeyCallback,
-		config: &ssh.ClientConfig{
-			User:            userName,
-			Auth:            authMethods,
-			HostKeyCallback: hostKeyCallback.Wrap(),
-			Timeout:         time.Second * 3,
-		},
-	}
-
-	c.initServerPort(server)
-
-	return &c
-}
-
-// NewOneOffConnection creates new one-off connection (only for sending a series of commands and then quit).
-func NewOneOffConnection(server string, userName string, authMethods []ssh.AuthMethod) *Connection {
-	c := Connection{
-		config: &ssh.ClientConfig{
-			User:            userName,
-			Auth:            authMethods,
-			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		},
-		isOneOff: true,
-	}
-
-	c.initServerPort(server)
-
-	return &c
-}
-
-// Attempt to parse the server port address from the provided server FQDN.
-func (c *Connection) initServerPort(server string) {
-	c.Server = server
-	c.port = config.Common.SSHPort
-	parts := strings.Split(server, ":")
-
-	if len(parts) == 2 {
-		logger.Debug("Parsing port from hostname", parts)
-		port, err := strconv.Atoi(parts[1])
-		if err != nil {
-			logger.FatalExit("Unable to parse client port", server, parts, err)
-		}
-		c.Server = parts[0]
-		c.port = port
-	}
-}
-
-// Start the server connection. Build up SSH session and send some DTail commands.
-func (c *Connection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
-	// Throttle how many connections can be established concurrently (based on ch length)
-	logger.Debug(c.Server, "Throttling connection", len(throttleCh), cap(throttleCh))
-
-	select {
-	case throttleCh <- struct{}{}:
-	case <-ctx.Done():
-		logger.Debug(c.Server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
-		return
-	}
-
-	logger.Debug(c.Server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
-
-	go func() {
-		defer func() {
-			if !c.throttlingDone {
-				logger.Debug(c.Server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
-				c.throttlingDone = true
-				<-throttleCh
-			}
-			cancel()
-		}()
-
-		if err := c.dial(ctx, cancel, throttleCh, statsCh); err != nil {
-			logger.Warn(c.Server, c.port, err)
-			if c.hostKeyCallback.Untrusted(fmt.Sprintf("%s:%d", c.Server, c.port)) {
-				logger.Debug(c.Server, "Not trusting host")
-			}
-		}
-	}()
-
-	<-ctx.Done()
-}
-
-// Dail into a new SSH connection. Close connection in case of an error.
-func (c *Connection) dial(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) error {
-	logger.Debug(c.Server, "Incrementing connection stats")
-	statsCh <- struct{}{}
-	defer func() {
-		logger.Debug(c.Server, "Decrementing connection stats")
-		<-statsCh
-	}()
-
-	logger.Debug(c.Server, "Dialing into the connection")
-	address := fmt.Sprintf("%s:%d", c.Server, c.port)
-
-	client, err := ssh.Dial("tcp", address, c.config)
-	if err != nil {
-		return err
-	}
-	defer client.Close()
-
-	return c.session(ctx, cancel, client, throttleCh)
-}
-
-// Create the SSH session. Close the session in case of an error.
-func (c *Connection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
-	logger.Debug(c.Server, "session")
-
-	session, err := client.NewSession()
-	if err != nil {
-		return err
-	}
-	defer session.Close()
-
-	return c.handle(ctx, cancel, session, throttleCh)
-}
-
-func (c *Connection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
-	logger.Debug(c.Server, "handle")
-
-	stdinPipe, err := session.StdinPipe()
-	if err != nil {
-		return err
-	}
-
-	stdoutPipe, err := session.StdoutPipe()
-	if err != nil {
-		return err
-	}
-
-	if err := session.Shell(); err != nil {
-		return err
-	}
-
-	go func() {
-		io.Copy(stdinPipe, c.Handler)
-		cancel()
-	}()
-
-	go func() {
-		io.Copy(c.Handler, stdoutPipe)
-		cancel()
-	}()
-
-	go func() {
-		select {
-		case <-c.Handler.Done():
-		case <-ctx.Done():
-		}
-		cancel()
-	}()
-
-	// Send all commands to client.
-	for _, command := range c.Commands {
-		logger.Debug(command)
-		c.Handler.SendMessage(command)
-	}
-
-	if !c.throttlingDone {
-		logger.Debug(c.Server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
-		c.throttlingDone = true
-		<-throttleCh
-	}
-
-	<-ctx.Done()
-	c.Handler.Shutdown()
-	return nil
-}
-- 
cgit v1.2.3


From 8ba431b0953c3b1458baefaeebf4ad92916315d9 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 18 Sep 2021 19:27:50 +0300
Subject: fix auto reconnect

---
 internal/clients/args.go                        | 43 ++----------
 internal/clients/baseclient.go                  |  9 ++-
 internal/clients/connectors/serverconnection.go | 38 ++++-------
 internal/clients/connectors/serverless.go       | 90 +++++++++++++++++++++++++
 internal/clients/handlers/basehandler.go        | 10 +++
 internal/done.go                                |  9 +++
 internal/server/handlers/serverhandler.go       |  6 +-
 7 files changed, 139 insertions(+), 66 deletions(-)
 create mode 100644 internal/clients/connectors/serverless.go

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
index 081b911..dc71e83 100644
--- a/internal/clients/args.go
+++ b/internal/clients/args.go
@@ -3,12 +3,8 @@ package clients
 import (
 	"flag"
 	"fmt"
-	"os"
-	"path/filepath"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/omode"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -33,6 +29,7 @@ type Args struct {
 	Quiet              bool
 	Spartan            bool
 	NoColor            bool
+	Serverless         bool
 }
 
 // Transform the arguments based on certain conditions.
@@ -50,45 +47,15 @@ func (a *Args) Transform(args []string) {
 		a.Quiet = true
 		a.NoColor = true
 	}
-}
 
-// TransformAfterConfigFile same as Transform, but after the config file has been read.
-func (a *Args) TransformAfterConfigFile() {
 	if a.Discovery == "" && a.ServersStr == "" {
-		a.handleEmptyServer()
+		a.Serverless = true
 	}
 }
 
-func (a *Args) handleEmptyServer() {
-	fqdn, err := os.Hostname()
-	if err != nil {
-		logger.FatalExit(err)
-	}
-	a.ServersStr = fmt.Sprintf("%s:%d", fqdn, config.Common.SSHPort)
-	// I am trusting my own hostname.
-	a.TrustAllHosts = true
-	logger.Debug("Will connect to local server", a.ServersStr)
-
-	cleanPath := func(dirtyPath string) string {
-		cleanPath, err := filepath.EvalSymlinks(dirtyPath)
-		if err != nil {
-			logger.FatalExit("Unable to evaluate symlinks", dirtyPath, err)
-		}
-		cleanPath, err = filepath.Abs(cleanPath)
-		if err != nil {
-			logger.FatalExit("Unable to make file path absolute", dirtyPath, cleanPath, err)
-		}
-		return cleanPath
-	}
-
-	logger.Debug("Dirty file paths", a.What)
-	var filePaths []string
-	for _, dirtyPath := range strings.Split(a.What, ",") {
-		filePaths = append(filePaths, cleanPath(dirtyPath))
-	}
-
-	a.What = strings.Join(filePaths, ",")
-	logger.Debug("Clean file paths", a.What)
+// TransformAfterConfigFile same as Transform, but after the config file has been read.
+func (a *Args) TransformAfterConfigFile() {
+	// TODO: Remove this method. It's not used.
 }
 
 func (a *Args) SerializeOptions() string {
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index d0631fc..5523052 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -113,12 +113,17 @@ func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, con
 
 		time.Sleep(time.Second * 2)
 		logger.Debug(conn.Server(), "Reconnecting")
-		c.connections[i] = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback)
+		conn = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback)
+		c.connections[i] = conn
 	}
 }
 
 func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod, hostKeyCallback client.HostKeyCallback) connectors.Connector {
-	return connectors.NewServerConnection(server, c.UserName, sshAuthMethods, hostKeyCallback, c.maker.makeHandler(server), c.maker.makeCommands())
+	if c.Args.Serverless {
+		return connectors.NewServerless(c.UserName, c.maker.makeHandler(server), c.maker.makeCommands())
+	}
+	return connectors.NewServerConnection(server, c.UserName, sshAuthMethods,
+		hostKeyCallback, c.maker.makeHandler(server), c.maker.makeCommands())
 }
 
 func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index fab2f87..0904ba1 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -16,31 +16,21 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-// ServerConnection represents a client connection connection to a single server.
+// ServerConnection represents a connection to a single remote dtail server via SSH protocol.
 type ServerConnection struct {
-	// The remote server's hostname connected to.
-	server string
-	// The remote server's port connected to.
-	port int
-	// The SSH client configuration used.
-	config *ssh.ClientConfig
-	// The SSH client handler to use.
-	handler handlers.Handler
-	// DTail commands sent from client to server. When client loses
-	// connection to the server it re-connects automatically and sends the
-	// same commands again.
-	commands []string
-	// Is it a persistent connection or a one-off?
-	isOneOff bool
-	// To deal with SSH server host keys
+	server          string
+	port            int
+	config          *ssh.ClientConfig
+	handler         handlers.Handler
+	commands        []string
+	isOneOff        bool
 	hostKeyCallback client.HostKeyCallback
-	// To determine if connection throttling has finished or not
-	throttlingDone bool
+	throttlingDone  bool
 }
 
-// NewServerConnection returns a new connection.
+// NewServerConnection returns a new DTail SSH server connection.
 func NewServerConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback, handler handlers.Handler, commands []string) *ServerConnection {
-	logger.Debug(server, "Creating new connection")
+	logger.Debug(server, "Creating new connection", server, handler, commands)
 
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
@@ -77,12 +67,10 @@ func NewOneOffServerConnection(server string, userName string, authMethods []ssh
 	return &c
 }
 
-// Server hostname
 func (c *ServerConnection) Server() string {
 	return c.server
 }
 
-// Handler for the connection
 func (c *ServerConnection) Handler() handlers.Handler {
 	return c.handler
 }
@@ -103,7 +91,6 @@ func (c *ServerConnection) initServerPort() {
 	}
 }
 
-// Start the server connection. Build up SSH session and send some DTail commands.
 func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
 	// Throttle how many connections can be established concurrently (based on ch length)
 	logger.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
@@ -161,7 +148,7 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 
 // Create the SSH session. Close the session in case of an error.
 func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
-	logger.Debug(c.server, "session")
+	logger.Debug(c.server, "Creating SSH session")
 
 	session, err := client.NewSession()
 	if err != nil {
@@ -173,7 +160,7 @@ func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFun
 }
 
 func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
-	logger.Debug(c.server, "handle")
+	logger.Debug(c.server, "Creating handler for SSH session")
 
 	stdinPipe, err := session.StdinPipe()
 	if err != nil {
@@ -221,5 +208,6 @@ func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc
 
 	<-ctx.Done()
 	c.handler.Shutdown()
+
 	return nil
 }
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
new file mode 100644
index 0000000..0500645
--- /dev/null
+++ b/internal/clients/connectors/serverless.go
@@ -0,0 +1,90 @@
+package connectors
+
+import (
+	"context"
+	"io"
+
+	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/logger"
+	serverHandlers "github.com/mimecast/dtail/internal/server/handlers"
+	user "github.com/mimecast/dtail/internal/user/server"
+)
+
+// Serverless creates a server object directly without TCP.
+type Serverless struct {
+	handler  handlers.Handler
+	commands []string
+	userName string
+}
+
+// NewServerConnection returns a new connection.
+func NewServerless(userName string, handler handlers.Handler, commands []string) *Serverless {
+	s := Serverless{
+		userName: userName,
+		handler:  handler,
+		commands: commands,
+	}
+
+	logger.Debug("Creating new serverless connector", handler, commands)
+	return &s
+}
+
+func (s *Serverless) Server() string {
+	return "local(serverless)"
+}
+
+func (s *Serverless) Handler() handlers.Handler {
+	return s.handler
+}
+
+func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+	go func() {
+		defer cancel()
+
+		if err := s.handle(ctx, cancel); err != nil {
+			logger.Warn(err)
+		}
+	}()
+
+	<-ctx.Done()
+}
+
+func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) error {
+	logger.Debug("Creating server handler for a serverless session")
+
+	serverHandler := serverHandlers.NewServerHandler(
+		user.New(s.userName, s.Server()),
+		make(chan struct{}, config.Server.MaxConcurrentCats),
+		make(chan struct{}, config.Server.MaxConcurrentTails),
+	)
+
+	go func() {
+		io.Copy(serverHandler, s.handler)
+		cancel()
+	}()
+
+	go func() {
+		io.Copy(s.handler, serverHandler)
+		cancel()
+	}()
+
+	go func() {
+		select {
+		case <-s.handler.Done():
+		case <-ctx.Done():
+		}
+		cancel()
+	}()
+
+	// Send all commands to client.
+	for _, command := range s.commands {
+		logger.Debug(command)
+		s.handler.SendMessage(command)
+	}
+
+	<-ctx.Done()
+	s.handler.Shutdown()
+
+	return nil
+}
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 0f2d1b5..af1ad62 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -22,6 +22,16 @@ type baseHandler struct {
 	status       int
 }
 
+func (h *baseHandler) String() string {
+	return fmt.Sprintf("baseHandler(%s,server:%s,shellStarted:%v,status:%d)@%p",
+		h.done,
+		h.server,
+		h.shellStarted,
+		h.status,
+		h,
+	)
+}
+
 func (h *baseHandler) Server() string {
 	return h.server
 }
diff --git a/internal/done.go b/internal/done.go
index 54e5e8e..5ea22a0 100644
--- a/internal/done.go
+++ b/internal/done.go
@@ -17,6 +17,15 @@ func NewDone() *Done {
 	}
 }
 
+func (d *Done) String() string {
+	select {
+	case <-d.Done():
+		return "Done(yes)"
+	default:
+		return "Done(no)"
+	}
+}
+
 // Done returns the done channel (closed when done)
 func (d *Done) Done() <-chan struct{} {
 	return d.ch
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 2f3b73b..4820476 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -331,7 +331,11 @@ func (h *ServerHandler) handleAckCommand(argc int, args []string) {
 		return
 	}
 	if args[1] == "close" && args[2] == "connection" {
-		close(h.ackCloseReceived)
+		select {
+		case <-h.ackCloseReceived:
+		default:
+			close(h.ackCloseReceived)
+		}
 	}
 }
 
-- 
cgit v1.2.3


From 3223be4cf95d0b6828196ac7a84277c18f3f5655 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 19 Sep 2021 13:22:59 +0300
Subject: move args to config package

logger package rewrite as dlog
---
 internal/clients/args.go                        |  63 --------
 internal/clients/baseclient.go                  |  28 ++--
 internal/clients/catclient.go                   |  12 +-
 internal/clients/connectors/serverconnection.go |  34 ++--
 internal/clients/connectors/serverless.go       |  21 ++-
 internal/clients/grepclient.go                  |  13 +-
 internal/clients/handlers/basehandler.go        |   6 +-
 internal/clients/handlers/clienthandler.go      |   4 +-
 internal/clients/handlers/maprhandler.go        |   8 +-
 internal/clients/maprclient.go                  |  40 ++---
 internal/clients/stats.go                       |   8 +-
 internal/clients/tailclient.go                  |  16 +-
 internal/color/table.go                         |   8 +-
 internal/config/args.go                         | 105 ++++++++++++
 internal/config/common.go                       |   8 +-
 internal/config/config.go                       |   3 +
 internal/config/read.go                         |  40 -----
 internal/config/server.go                       |   1 +
 internal/config/setup.go                        |  35 ++++
 internal/discovery/comma.go                     |   4 +-
 internal/discovery/discovery.go                 |  18 +--
 internal/discovery/file.go                      |   8 +-
 internal/io/dlog/dlog.go                        | 206 ++++++++++++++++++++++++
 internal/io/dlog/level.go                       |  89 ++++++++++
 internal/io/dlog/loggers/factory.go             |  60 +++++++
 internal/io/dlog/loggers/file.go                | 156 ++++++++++++++++++
 internal/io/dlog/loggers/fout.go                |  46 ++++++
 internal/io/dlog/loggers/logger.go              |  18 +++
 internal/io/dlog/loggers/none.go                |  21 +++
 internal/io/dlog/loggers/stdout.go              |  73 +++++++++
 internal/io/dlog/rotation.go                    |  27 ++++
 internal/io/dlog/source.go                      |  19 +++
 internal/io/dlog/strategy.go                    |  22 +++
 internal/io/fs/permissions/permission.go        |   4 +-
 internal/io/fs/readfile.go                      |  16 +-
 internal/io/logger/logger.go                    |  23 ++-
 internal/io/logger/modes.go                     |   9 +-
 internal/io/prompt/prompt.go                    |   6 +-
 internal/mapr/aggregateset.go                   |   6 +-
 internal/mapr/client/aggregate.go               |   4 +-
 internal/mapr/funcs/function.go                 |   4 +-
 internal/mapr/groupset.go                       |   4 +-
 internal/mapr/logformat/default.go              |   1 +
 internal/mapr/logformat/generickv.go            |   2 +-
 internal/mapr/logformat/parser.go               |   3 -
 internal/mapr/query.go                          |   4 -
 internal/mapr/server/aggregate.go               |  22 +--
 internal/mapr/token.go                          |  10 +-
 internal/mapr/whereclause.go                    |   6 +-
 internal/mapr/wherecondition.go                 |   6 +-
 internal/regex/regex.go                         |  11 +-
 internal/regex/regex_test.go                    |  16 +-
 internal/server/continuous.go                   |  21 ++-
 internal/server/handlers/controlhandler.go      |  12 +-
 internal/server/handlers/readcommand.go         |  26 +--
 internal/server/handlers/serverhandler.go       |  46 +++---
 internal/server/scheduler.go                    |  22 +--
 internal/server/server.go                       |  52 +++---
 internal/server/stats.go                        |   6 +-
 internal/ssh/client/authmethods.go              |  28 ++--
 internal/ssh/client/knownhostscallback.go       |  14 +-
 internal/ssh/server/hostkey.go                  |  17 +-
 internal/ssh/server/publickeycallback.go        |  12 +-
 internal/ssh/ssh.go                             |   6 +-
 internal/user/server/user.go                    |  22 +--
 65 files changed, 1236 insertions(+), 425 deletions(-)
 delete mode 100644 internal/clients/args.go
 create mode 100644 internal/config/args.go
 delete mode 100644 internal/config/read.go
 create mode 100644 internal/config/setup.go
 create mode 100644 internal/io/dlog/dlog.go
 create mode 100644 internal/io/dlog/level.go
 create mode 100644 internal/io/dlog/loggers/factory.go
 create mode 100644 internal/io/dlog/loggers/file.go
 create mode 100644 internal/io/dlog/loggers/fout.go
 create mode 100644 internal/io/dlog/loggers/logger.go
 create mode 100644 internal/io/dlog/loggers/none.go
 create mode 100644 internal/io/dlog/loggers/stdout.go
 create mode 100644 internal/io/dlog/rotation.go
 create mode 100644 internal/io/dlog/source.go
 create mode 100644 internal/io/dlog/strategy.go

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
deleted file mode 100644
index dc71e83..0000000
--- a/internal/clients/args.go
+++ /dev/null
@@ -1,63 +0,0 @@
-package clients
-
-import (
-	"flag"
-	"fmt"
-	"strings"
-
-	"github.com/mimecast/dtail/internal/omode"
-
-	gossh "golang.org/x/crypto/ssh"
-)
-
-// Args is a helper struct to summarize common client arguments.
-type Args struct {
-	Mode               omode.Mode
-	ServersStr         string
-	UserName           string
-	What               string
-	Arguments          []string
-	RegexStr           string
-	RegexInvert        bool
-	TrustAllHosts      bool
-	Discovery          string
-	ConnectionsPerCPU  int
-	Timeout            int
-	SSHAuthMethods     []gossh.AuthMethod
-	SSHHostKeyCallback gossh.HostKeyCallback
-	PrivateKeyPathFile string
-	Quiet              bool
-	Spartan            bool
-	NoColor            bool
-	Serverless         bool
-}
-
-// Transform the arguments based on certain conditions.
-func (a *Args) Transform(args []string) {
-	// Interpret additional args as file list.
-	if a.What == "" {
-		var files []string
-		for _, file := range flag.Args() {
-			files = append(files, file)
-		}
-		a.What = strings.Join(files, ",")
-	}
-
-	if a.Spartan {
-		a.Quiet = true
-		a.NoColor = true
-	}
-
-	if a.Discovery == "" && a.ServersStr == "" {
-		a.Serverless = true
-	}
-}
-
-// TransformAfterConfigFile same as Transform, but after the config file has been read.
-func (a *Args) TransformAfterConfigFile() {
-	// TODO: Remove this method. It's not used.
-}
-
-func (a *Args) SerializeOptions() string {
-	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
-}
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 5523052..fc01955 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -6,8 +6,9 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/clients/connectors"
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/discovery"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 	"github.com/mimecast/dtail/internal/ssh/client"
@@ -17,7 +18,7 @@ import (
 
 // This is the main client data structure.
 type baseClient struct {
-	Args
+	config.Args
 	// To display client side stats
 	stats *stats
 	// List of remote servers to connect to.
@@ -39,7 +40,8 @@ type baseClient struct {
 }
 
 func (c *baseClient) init() {
-	logger.Debug("Initiating base client")
+	dlog.Client.Debug("Initiating base client")
+	dlog.Client.Debug(c.Args.String())
 
 	flag := regex.Default
 	if c.Args.RegexInvert {
@@ -47,11 +49,14 @@ func (c *baseClient) init() {
 	}
 	regex, err := regex.New(c.Args.RegexStr, flag)
 	if err != nil {
-		logger.FatalExit(c.Regex, "invalid regex!", err, regex)
+		dlog.Client.FatalPanic(c.Regex, "invalid regex!", err, regex)
 	}
 	c.Regex = regex
-	logger.Debug("Regex", c.Regex)
+	dlog.Client.Debug("Regex", c.Regex)
 
+	if c.Args.Serverless {
+		return
+	}
 	c.sshAuthMethods, c.hostKeyCallback = client.InitSSHAuthMethods(c.Args.SSHAuthMethods, c.Args.SSHHostKeyCallback, c.Args.TrustAllHosts, c.throttleCh, c.Args.PrivateKeyPathFile)
 }
 
@@ -67,8 +72,11 @@ func (c *baseClient) makeConnections(maker maker) {
 }
 
 func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status int) {
-	// Periodically check for unknown hosts, and ask the user whether to trust them or not.
-	go c.hostKeyCallback.PromptAddHosts(ctx)
+	// Can be nil when serverless.
+	if c.hostKeyCallback != nil {
+		// Periodically check for unknown hosts, and ask the user whether to trust them or not.
+		go c.hostKeyCallback.PromptAddHosts(ctx)
+	}
 	// Print client stats every time something on statsCh is recieved.
 	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet)
 
@@ -112,7 +120,7 @@ func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, con
 		}
 
 		time.Sleep(time.Second * 2)
-		logger.Debug(conn.Server(), "Reconnecting")
+		dlog.Client.Debug(conn.Server(), "Reconnecting")
 		conn = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback)
 		c.connections[i] = conn
 	}
@@ -127,7 +135,7 @@ func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMe
 }
 
 func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
-	defer logger.Debug("Terminated connection")
+	defer dlog.Client.Debug("Terminated connection")
 
 	// We want to have at least one active connection
 	<-active
@@ -143,7 +151,7 @@ func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
 		if numActive == 0 {
 			return
 		}
-		logger.Debug("Active connections", numActive)
+		dlog.Client.Debug("Active connections", numActive)
 		time.Sleep(time.Second)
 	}
 }
diff --git a/internal/clients/catclient.go b/internal/clients/catclient.go
index d14cdcc..2726e7e 100644
--- a/internal/clients/catclient.go
+++ b/internal/clients/catclient.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 )
 
@@ -16,7 +18,7 @@ type CatClient struct {
 }
 
 // NewCatClient returns a new cat client.
-func NewCatClient(args Args) (*CatClient, error) {
+func NewCatClient(args config.Args) (*CatClient, error) {
 	if args.RegexStr != "" {
 		return nil, errors.New("Can't use regex with 'cat' operating mode")
 	}
@@ -42,11 +44,13 @@ func (c CatClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c CatClient) makeCommands() (commands []string) {
+	regex, err := c.Regex.Serialize()
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
-			c.Mode.String(),
-			c.Args.SerializeOptions(),
-			file, c.Regex.Serialize()))
+			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
 	return
 }
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 0904ba1..5bc63ee 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/ssh/client"
 
 	"golang.org/x/crypto/ssh"
@@ -30,7 +30,7 @@ type ServerConnection struct {
 
 // NewServerConnection returns a new DTail SSH server connection.
 func NewServerConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback, handler handlers.Handler, commands []string) *ServerConnection {
-	logger.Debug(server, "Creating new connection", server, handler, commands)
+	dlog.Client.Debug(server, "Creating new connection", server, handler, commands)
 
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
@@ -81,10 +81,10 @@ func (c *ServerConnection) initServerPort() {
 	parts := strings.Split(c.server, ":")
 
 	if len(parts) == 2 {
-		logger.Debug("Parsing port from hostname", parts)
+		dlog.Client.Debug("Parsing port from hostname", parts)
 		port, err := strconv.Atoi(parts[1])
 		if err != nil {
-			logger.FatalExit("Unable to parse client port", c.server, parts, err)
+			dlog.Client.FatalPanic("Unable to parse client port", c.server, parts, err)
 		}
 		c.server = parts[0]
 		c.port = port
@@ -93,21 +93,21 @@ func (c *ServerConnection) initServerPort() {
 
 func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
 	// Throttle how many connections can be established concurrently (based on ch length)
-	logger.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
+	dlog.Client.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
 
 	select {
 	case throttleCh <- struct{}{}:
 	case <-ctx.Done():
-		logger.Debug(c.server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
+		dlog.Client.Debug(c.server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
 		return
 	}
 
-	logger.Debug(c.server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
+	dlog.Client.Debug(c.server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
 
 	go func() {
 		defer func() {
 			if !c.throttlingDone {
-				logger.Debug(c.server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
+				dlog.Client.Debug(c.server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
 				c.throttlingDone = true
 				<-throttleCh
 			}
@@ -115,9 +115,9 @@ func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
 		}()
 
 		if err := c.dial(ctx, cancel, throttleCh, statsCh); err != nil {
-			logger.Warn(c.server, c.port, err)
+			dlog.Client.Warn(c.server, c.port, err)
 			if c.hostKeyCallback.Untrusted(fmt.Sprintf("%s:%d", c.server, c.port)) {
-				logger.Debug(c.server, "Not trusting host")
+				dlog.Client.Debug(c.server, "Not trusting host")
 			}
 		}
 	}()
@@ -127,14 +127,14 @@ func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
 
 // Dail into a new SSH connection. Close connection in case of an error.
 func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) error {
-	logger.Debug(c.server, "Incrementing connection stats")
+	dlog.Client.Debug(c.server, "Incrementing connection stats")
 	statsCh <- struct{}{}
 	defer func() {
-		logger.Debug(c.server, "Decrementing connection stats")
+		dlog.Client.Debug(c.server, "Decrementing connection stats")
 		<-statsCh
 	}()
 
-	logger.Debug(c.server, "Dialing into the connection")
+	dlog.Client.Debug(c.server, "Dialing into the connection")
 	address := fmt.Sprintf("%s:%d", c.server, c.port)
 
 	client, err := ssh.Dial("tcp", address, c.config)
@@ -148,7 +148,7 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 
 // Create the SSH session. Close the session in case of an error.
 func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
-	logger.Debug(c.server, "Creating SSH session")
+	dlog.Client.Debug(c.server, "Creating SSH session")
 
 	session, err := client.NewSession()
 	if err != nil {
@@ -160,7 +160,7 @@ func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFun
 }
 
 func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
-	logger.Debug(c.server, "Creating handler for SSH session")
+	dlog.Client.Debug(c.server, "Creating handler for SSH session")
 
 	stdinPipe, err := session.StdinPipe()
 	if err != nil {
@@ -196,12 +196,12 @@ func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc
 
 	// Send all commands to client.
 	for _, command := range c.commands {
-		logger.Debug(command)
+		dlog.Client.Debug(command)
 		c.handler.SendMessage(command)
 	}
 
 	if !c.throttlingDone {
-		logger.Debug(c.server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
+		dlog.Client.Debug(c.server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
 		c.throttlingDone = true
 		<-throttleCh
 	}
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index 0500645..c7b5f62 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -6,7 +6,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	serverHandlers "github.com/mimecast/dtail/internal/server/handlers"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
@@ -26,7 +26,7 @@ func NewServerless(userName string, handler handlers.Handler, commands []string)
 		commands: commands,
 	}
 
-	logger.Debug("Creating new serverless connector", handler, commands)
+	dlog.Client.Debug("Creating new serverless connector", handler, commands)
 	return &s
 }
 
@@ -43,7 +43,7 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 		defer cancel()
 
 		if err := s.handle(ctx, cancel); err != nil {
-			logger.Warn(err)
+			dlog.Client.Warn(err)
 		}
 	}()
 
@@ -51,7 +51,7 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 }
 
 func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) error {
-	logger.Debug("Creating server handler for a serverless session")
+	dlog.Client.Debug("Creating server handler for a serverless session")
 
 	serverHandler := serverHandlers.NewServerHandler(
 		user.New(s.userName, s.Server()),
@@ -59,14 +59,19 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 		make(chan struct{}, config.Server.MaxConcurrentTails),
 	)
 
+	terminate := func() {
+		serverHandler.Shutdown()
+		cancel()
+	}
+
 	go func() {
 		io.Copy(serverHandler, s.handler)
-		cancel()
+		terminate()
 	}()
 
 	go func() {
 		io.Copy(s.handler, serverHandler)
-		cancel()
+		terminate()
 	}()
 
 	go func() {
@@ -74,12 +79,12 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 		case <-s.handler.Done():
 		case <-ctx.Done():
 		}
-		cancel()
+		terminate()
 	}()
 
 	// Send all commands to client.
 	for _, command := range s.commands {
-		logger.Debug(command)
+		dlog.Client.Debug(command)
 		s.handler.SendMessage(command)
 	}
 
diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index ea5022b..ae21ff2 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 )
 
@@ -16,7 +18,7 @@ type GrepClient struct {
 }
 
 // NewGrepClient creates a new grep client.
-func NewGrepClient(args Args) (*GrepClient, error) {
+func NewGrepClient(args config.Args) (*GrepClient, error) {
 	if args.RegexStr == "" {
 		return nil, errors.New("No regex specified, use '-regex' flag")
 	}
@@ -41,12 +43,13 @@ func (c GrepClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c GrepClient) makeCommands() (commands []string) {
+	regex, err := c.Regex.Serialize()
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
-			c.Mode.String(),
-			c.Args.SerializeOptions(),
-			file,
-			c.Regex.Serialize()))
+			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
 
 	return
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index af1ad62..3291b43 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -51,7 +51,7 @@ func (h *baseHandler) Shutdown() {
 // SendMessage to the server.
 func (h *baseHandler) SendMessage(command string) error {
 	encoded := base64.StdEncoding.EncodeToString([]byte(command))
-	logger.Debug("Sending command", h.server, command, encoded)
+	dlog.Client.Debug("Sending command", h.server, command, encoded)
 
 	select {
 	case h.commands <- fmt.Sprintf("protocol %s base64 %v;", protocol.ProtocolCompat, encoded):
@@ -112,7 +112,7 @@ func (h *baseHandler) handleMessageType(message string) {
 		return
 	}
 
-	logger.Raw(message)
+	dlog.Client.Raw(message)
 }
 
 // Handle messages received from server which are not meant to be displayed
diff --git a/internal/clients/handlers/clienthandler.go b/internal/clients/handlers/clienthandler.go
index 2bcb038..27ac85e 100644
--- a/internal/clients/handlers/clienthandler.go
+++ b/internal/clients/handlers/clienthandler.go
@@ -2,7 +2,7 @@ package handlers
 
 import (
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ClientHandler is the basic client handler interface.
@@ -12,7 +12,7 @@ type ClientHandler struct {
 
 // NewClientHandler creates a new client handler.
 func NewClientHandler(server string) *ClientHandler {
-	logger.Debug(server, "Creating new client handler")
+	dlog.Client.Debug(server, "Creating new client handler")
 
 	return &ClientHandler{
 		baseHandler{
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index 65b1454..848e7f0 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -4,7 +4,7 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/client"
 	"github.com/mimecast/dtail/internal/protocol"
@@ -40,7 +40,7 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 			continue
 		case protocol.MessageDelimiter:
 			message := h.baseHandler.receiveBuf.String()
-			logger.Debug(message)
+			dlog.Client.Debug(message)
 			if message[0] == 'A' {
 				h.handleAggregateMessage(message)
 			} else {
@@ -60,10 +60,10 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 func (h *MaprHandler) handleAggregateMessage(message string) {
 	parts := strings.SplitN(message, protocol.FieldDelimiter, 3)
 	if len(parts) != 3 {
-		logger.Error("Unable to aggregate data", h.server, message, parts, len(parts), "expected 3 parts")
+		dlog.Client.Error("Unable to aggregate data", h.server, message, parts, len(parts), "expected 3 parts")
 		return
 	}
 	if err := h.aggregate.Aggregate(parts[2]); err != nil {
-		logger.Error("Unable to aggregate data", h.server, message, err)
+		dlog.Client.Error("Unable to aggregate data", h.server, message, err)
 	}
 }
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index e6ab96b..f23aa08 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -11,7 +11,7 @@ import (
 	"github.com/mimecast/dtail/internal/clients/handlers"
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/omode"
 )
@@ -44,14 +44,14 @@ type MaprClient struct {
 }
 
 // NewMaprClient returns a new mapreduce client.
-func NewMaprClient(args Args, queryStr string, maprClientMode MaprClientMode) (*MaprClient, error) {
+func NewMaprClient(args config.Args, queryStr string, maprClientMode MaprClientMode) (*MaprClient, error) {
 	if queryStr == "" {
 		return nil, errors.New("No mapreduce query specified, use '-query' flag")
 	}
 
 	query, err := mapr.NewQuery(queryStr)
 	if err != nil {
-		logger.FatalExit(queryStr, "Can't parse mapr query", err)
+		dlog.Client.FatalPanic(queryStr, "Can't parse mapr query", err)
 	}
 
 	// Don't retry connection if in tail mode and no outfile specified.
@@ -68,7 +68,7 @@ func NewMaprClient(args Args, queryStr string, maprClientMode MaprClientMode) (*
 		cumulative = args.Mode == omode.MapClient || query.HasOutfile()
 	}
 
-	logger.Debug("Cumulative mapreduce mode?", cumulative)
+	dlog.Client.Debug("Cumulative mapreduce mode?", cumulative)
 
 	c := MaprClient{
 		baseClient: baseClient{
@@ -103,7 +103,7 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 
 	status = c.baseClient.Start(ctx, statsCh)
 	if c.cumulative {
-		logger.Debug("Received final mapreduce result")
+		dlog.Client.Debug("Received final mapreduce result")
 		c.reportResults()
 	}
 
@@ -123,15 +123,17 @@ func (c MaprClient) makeCommands() (commands []string) {
 	}
 
 	for _, file := range strings.Split(c.What, ",") {
+		regex, err := c.Regex.Serialize()
+		if err != nil {
+			dlog.Client.FatalPanic(err)
+		}
 		if c.Timeout > 0 {
-			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout, modeStr, file, c.Regex.Serialize()))
+			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout,
+				modeStr, file, regex))
 			continue
 		}
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
-			modeStr,
-			c.Args.SerializeOptions(),
-			file,
-			c.Regex.Serialize()))
+			modeStr, c.Args.SerializeOptions(), file, regex))
 	}
 
 	return
@@ -141,7 +143,7 @@ func (c *MaprClient) periodicReportResults(ctx context.Context) {
 	for {
 		select {
 		case <-time.After(c.query.Interval):
-			logger.Debug("Gathering interim mapreduce result")
+			dlog.Client.Debug("Gathering interim mapreduce result")
 			c.reportResults()
 		case <-ctx.Done():
 			return
@@ -177,17 +179,17 @@ func (c *MaprClient) printResults() {
 	}
 
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Client.FatalPanic(err)
 	}
 
 	if result == c.lastResult {
-		logger.Debug("Result hasn't changed compared to last time...")
+		dlog.Client.Debug("Result hasn't changed compared to last time...")
 		return
 	}
 	c.lastResult = result
 
 	if numRows == 0 {
-		logger.Debug("Empty result set this time...")
+		dlog.Client.Debug("Empty result set this time...")
 		return
 	}
 
@@ -198,24 +200,24 @@ func (c *MaprClient) printResults() {
 			config.Client.TermColors.MaprTable.RawQueryBg,
 			config.Client.TermColors.MaprTable.RawQueryAttr)
 	}
-	logger.Raw(rawQuery)
+	dlog.Client.Raw(rawQuery)
 
 	if rowsLimit > 0 && numRows > rowsLimit {
-		logger.Warn(fmt.Sprintf("Got %d results but limited output to %d rows! Use 'limit' clause to override!",
+		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited output to %d rows! Use 'limit' clause to override!",
 			numRows, rowsLimit))
 	}
-	logger.Raw(result)
+	dlog.Client.Raw(result)
 }
 
 func (c *MaprClient) writeResultsToOutfile() {
 	if c.cumulative {
 		if err := c.globalGroup.WriteResult(c.query); err != nil {
-			logger.FatalExit(err)
+			dlog.Client.FatalPanic(err)
 		}
 		return
 	}
 
 	if err := c.globalGroup.SwapOut().WriteResult(c.query); err != nil {
-		logger.FatalExit(err)
+		dlog.Client.FatalPanic(err)
 	}
 }
diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index 6da443c..fbef572 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -67,7 +67,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 			s.printStatsDueInterrupt(messages)
 		default:
 			data := s.statsData(connected, newConnections, throttle)
-			logger.Mapreduce("STATS", data)
+			dlog.Client.Mapreduce("STATS", data)
 		}
 
 		connectedLast = connected
@@ -78,7 +78,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 }
 
 func (s *stats) printStatsDueInterrupt(messages []string) {
-	logger.Pause()
+	dlog.Client.Pause()
 	for i, message := range messages {
 		if i > 0 && config.Client.TermColorsEnable {
 			fmt.Println(color.PaintStrWithAttr(message,
@@ -91,7 +91,7 @@ func (s *stats) printStatsDueInterrupt(messages []string) {
 		fmt.Println(fmt.Sprintf(" %s", message))
 	}
 	time.Sleep(time.Second * time.Duration(config.InterruptTimeoutS))
-	logger.Resume()
+	dlog.Client.Resume()
 }
 
 func (s *stats) statsData(connected, newConnections int, throttle int) map[string]interface{} {
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index 360354b..d42a0e4 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -6,7 +6,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 )
 
@@ -16,7 +17,7 @@ type TailClient struct {
 }
 
 // NewTailClient returns a new TailClient.
-func NewTailClient(args Args) (*TailClient, error) {
+func NewTailClient(args config.Args) (*TailClient, error) {
 	args.Mode = omode.TailClient
 
 	c := TailClient{
@@ -38,14 +39,15 @@ func (c TailClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c TailClient) makeCommands() (commands []string) {
+	regex, err := c.Regex.Serialize()
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
-			c.Mode.String(),
-			c.Args.SerializeOptions(),
-			file,
-			c.Regex.Serialize()))
+			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
-	logger.Debug(commands)
+	dlog.Client.Debug(commands)
 
 	return
 }
diff --git a/internal/color/table.go b/internal/color/table.go
index 7ecfbca..2115edf 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -7,17 +7,17 @@ import (
 
 const sampleParagraph string = "Mimecast is Making Email Safer for Business. We believe that securely operating a business in the cloud requires new levels of IT preparedness, centered around cyber resilience. This is why we unify the delivery and management of security, continuity and data protection for email via one, simple-to-use cloud platform. Thousands of organizations trust us to increase their cyber resilience preparedness, streamline compliance, reduce IT complexity and keep their business running. We give employees fast and secure access to sensitive business information, and ensure email keeps flowing in the event of an outage. Mimecast will remain committed to protecting your IT assets through constant innovation and focus on your success."
 
-func TablePrintAndExit(useSampleParagraph bool) {
+func TablePrintAndExit(displaySampleParagraph bool) {
 	for _, attr := range AttributeNames {
 		if attr == "Hidden" || attr == "SlowBlink" {
 			continue
 		}
-		printColorTable(attr, useSampleParagraph)
+		printColorTable(attr, displaySampleParagraph)
 	}
 	os.Exit(0)
 }
 
-func printColorTable(attr string, useSampleParagraph bool) {
+func printColorTable(attr string, displaySampleParagraph bool) {
 	for _, fg := range ColorNames {
 		fgColor, _ := ToFgColor(fg)
 		for _, bg := range ColorNames {
@@ -29,7 +29,7 @@ func printColorTable(attr string, useSampleParagraph bool) {
 
 			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
 			fmt.Print(PaintStrWithAttr(text, fgColor, bgColor, attribute))
-			if useSampleParagraph {
+			if displaySampleParagraph {
 				fmt.Print("\n")
 				fmt.Print(PaintStrWithAttr(sampleParagraph, fgColor, bgColor, attribute))
 				fmt.Print("\n")
diff --git a/internal/config/args.go b/internal/config/args.go
new file mode 100644
index 0000000..89e4bc9
--- /dev/null
+++ b/internal/config/args.go
@@ -0,0 +1,105 @@
+package config
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+
+	"github.com/mimecast/dtail/internal/omode"
+
+	gossh "golang.org/x/crypto/ssh"
+)
+
+// Args is a helper struct to summarize common client arguments.
+type Args struct {
+	Arguments          []string
+	ConfigFile         string
+	ConnectionsPerCPU  int
+	Discovery          string
+	LogLevel           string
+	Mode               omode.Mode
+	NoColor            bool
+	PrivateKeyPathFile string
+	Quiet              bool
+	RegexInvert        bool
+	RegexStr           string
+	Serverless         bool
+	ServersStr         string
+	Spartan            bool
+	SSHAuthMethods     []gossh.AuthMethod
+	SSHHostKeyCallback gossh.HostKeyCallback
+	SSHPort            int
+	Timeout            int
+	TrustAllHosts      bool
+	UserName           string
+	What               string
+}
+
+func (a *Args) String() string {
+	var sb strings.Builder
+
+	sb.WriteString("Args(")
+	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "ConnectionsPerCPU", a.ConnectionsPerCPU))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Discovery", a.Discovery))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Spartan", a.Spartan))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHAuthMethods", a.SSHAuthMethods))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHHostKeyCallback", a.SSHHostKeyCallback))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHPort", a.SSHPort))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Timeout", a.Timeout))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "TrustAllHosts", a.TrustAllHosts))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "UserName", a.UserName))
+	sb.WriteString(fmt.Sprintf("%s:%v", "What", a.What))
+	sb.WriteString(")")
+
+	return sb.String()
+}
+
+// Based on the argument list, transform/manipulate some of the arguments.
+func (a *Args) transform(args []string) {
+	if a.LogLevel != "" {
+		Common.LogLevel = a.LogLevel
+	}
+
+	if a.SSHPort != 2222 {
+		Common.SSHPort = a.SSHPort
+	}
+	if a.NoColor {
+		Client.TermColorsEnable = false
+	}
+
+	if a.Spartan {
+		a.Quiet = true
+		a.NoColor = true
+	}
+
+	if a.Discovery == "" && a.ServersStr == "" {
+		a.Serverless = true
+	}
+
+	// Interpret additional args as file list.
+	if a.What == "" {
+		var files []string
+		for _, file := range flag.Args() {
+			files = append(files, file)
+		}
+		a.What = strings.Join(files, ",")
+	}
+}
+
+// SerializeOptions returns a string ready to be sent over the wire to the server.
+func (a *Args) SerializeOptions() string {
+	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
+}
+
+// TODO: Put the DeseializeOptions function here (move it away from the internal/server package)
diff --git a/internal/config/common.go b/internal/config/common.go
index c3e203e..7d45261 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -6,10 +6,8 @@ type CommonConfig struct {
 	SSHPort int
 	// Enable experimental features (mainly for dev purposes)
 	ExperimentalFeaturesEnable bool `json:",omitempty"`
-	// Enable debug logging. Don't enable in production.
-	DebugEnable bool `json:",omitempty"`
-	// Enable trace logging. Don't enable in production.
-	TraceEnable bool `json:",omitempty"`
+	// LogLevel defines how much is logged. TODO: Adjust JSONschema
+	LogLevel string `json:",omitempty"`
 	// The log strategy to use, one of
 	//   stdout: only log to stdout (useful when used with systemd)
 	//   daily: create a log file for every day
@@ -26,8 +24,6 @@ type CommonConfig struct {
 func newDefaultCommonConfig() *CommonConfig {
 	return &CommonConfig{
 		SSHPort:                    2222,
-		DebugEnable:                false,
-		TraceEnable:                false,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
 		CacheDir:                   "cache",
diff --git a/internal/config/config.go b/internal/config/config.go
index 276ddcf..2d77041 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -15,6 +15,9 @@ const ScheduleUser string = "DTAIL-SCHEDULE"
 // ContinuousUser is used for non-interactive continuous mapreduce queries.
 const ContinuousUser string = "DTAIL-CONTINUOUS"
 
+// TestUser is used for unit tests and potentially also for integration tests.
+const TestUser string = "DTAIL-TEST"
+
 // InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
 const InterruptTimeoutS int = 3
 
diff --git a/internal/config/read.go b/internal/config/read.go
deleted file mode 100644
index ea358f8..0000000
--- a/internal/config/read.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package config
-
-import (
-	"os"
-)
-
-// Read the DTail configuration.
-func Read(configFile string, sshPort int, noColor bool) {
-	initializer := configInitializer{
-		Common: newDefaultCommonConfig(),
-		Server: newDefaultServerConfig(),
-		Client: newDefaultClientConfig(),
-	}
-
-	if configFile == "" {
-		configFile = "./cfg/dtail.json"
-	}
-
-	if _, err := os.Stat(configFile); !os.IsNotExist(err) {
-		initializer.parseConfig(configFile)
-	}
-
-	// Assign pointers to global variables, so that we can access the
-	// configuration from any place of the program.
-	Common = initializer.Common
-	Server = initializer.Server
-	Client = initializer.Client
-
-	if Server.MapreduceLogFormat == "" {
-		Server.MapreduceLogFormat = "default"
-	}
-
-	// If non-standard port specified, overwrite config
-	if sshPort != 2222 {
-		Common.SSHPort = sshPort
-	}
-	if noColor {
-		Client.TermColorsEnable = false
-	}
-}
diff --git a/internal/config/server.go b/internal/config/server.go
index dc0d587..8bbb394 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -76,6 +76,7 @@ func newDefaultServerConfig() *ServerConfig {
 		MaxConcurrentTails: 50,
 		HostKeyFile:        "./cache/ssh_host_key",
 		HostKeyBits:        4096,
+		MapreduceLogFormat: "default",
 		Permissions: Permissions{
 			Default: defaultPermissions,
 		},
diff --git a/internal/config/setup.go b/internal/config/setup.go
new file mode 100644
index 0000000..3c4bcc4
--- /dev/null
+++ b/internal/config/setup.go
@@ -0,0 +1,35 @@
+package config
+
+import (
+	"os"
+)
+
+const NoConfigFile string = "Don't read a config file - use defaults only"
+
+// Setup the DTail configuration.
+func Setup(args *Args, additionalArgs []string) {
+	initializer := configInitializer{
+		Common: newDefaultCommonConfig(),
+		Server: newDefaultServerConfig(),
+		Client: newDefaultClientConfig(),
+	}
+
+	if args.ConfigFile == "" {
+		// TODO: Search more paths for config file (e.g. in /etc and in ~/.config/...
+		args.ConfigFile = "./cfg/dtail.json"
+	}
+
+	if args.ConfigFile != NoConfigFile {
+		if _, err := os.Stat(args.ConfigFile); !os.IsNotExist(err) {
+			initializer.parseConfig(args.ConfigFile)
+		}
+	}
+
+	// Assign pointers to global variables, so that we can access the
+	// configuration from any place of the program.
+	Common = initializer.Common
+	Server = initializer.Server
+	Client = initializer.Client
+
+	args.transform(additionalArgs)
+}
diff --git a/internal/discovery/comma.go b/internal/discovery/comma.go
index 4344240..9bea89c 100644
--- a/internal/discovery/comma.go
+++ b/internal/discovery/comma.go
@@ -3,11 +3,11 @@ package discovery
 import (
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ServerListFromCOMMA retrieves a list of servers from comma separated input list.
 func (d *Discovery) ServerListFromCOMMA() []string {
-	logger.Debug("Retrieving server list from comma separated list", d.server)
+	dlog.Common.Debug("Retrieving server list from comma separated list", d.server)
 	return strings.Split(d.server, ",")
 }
diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go
index 3608ce7..83ee95e 100644
--- a/internal/discovery/discovery.go
+++ b/internal/discovery/discovery.go
@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ServerOrder to specify how to sort the server list.
@@ -42,7 +42,7 @@ func New(method, server string, order ServerOrder) *Discovery {
 	if strings.Contains(module, ":") {
 		s := strings.Split(module, ":")
 		if len(s) != 2 {
-			logger.FatalExit("Unable to parse discovery module", module)
+			dlog.Common.FatalPanic("Unable to parse discovery module", module)
 		}
 		module = s[0]
 		options = s[1]
@@ -72,11 +72,11 @@ func (d *Discovery) initRegex() {
 	}
 
 	regexStr := string(runes)
-	logger.Debug("Using filter regex", regexStr)
+	dlog.Common.Debug("Using filter regex", regexStr)
 
 	regex, err := regexp.Compile(regexStr)
 	if err != nil {
-		logger.FatalExit("Could not compile regex", regexStr, err)
+		dlog.Common.FatalPanic("Could not compile regex", regexStr, err)
 	}
 
 	d.regex = regex
@@ -97,7 +97,7 @@ func (d *Discovery) ServerList() []string {
 		servers = d.shuffleList(servers)
 	}
 
-	logger.Debug("Discovered servers", len(servers), servers)
+	dlog.Common.Debug("Discovered servers", len(servers), servers)
 	return servers
 }
 
@@ -124,7 +124,7 @@ func (d *Discovery) serverListFromReflectedModule() []string {
 	rt := reflect.TypeOf(d)
 	reflectedMethod, ok := rt.MethodByName(methodName)
 	if !ok {
-		logger.FatalExit("No such server discovery module", d.module, methodName)
+		dlog.Common.FatalPanic("No such server discovery module", d.module, methodName)
 	}
 
 	inputValues := make([]reflect.Value, 1)
@@ -138,7 +138,7 @@ func (d *Discovery) serverListFromReflectedModule() []string {
 
 // Filter server list based on a regexp.
 func (d *Discovery) filterList(servers []string) (filtered []string) {
-	logger.Debug("Filtering server list")
+	dlog.Common.Debug("Filtering server list")
 
 	for _, server := range servers {
 		if d.regex.MatchString(server) {
@@ -160,13 +160,13 @@ func (d *Discovery) dedupList(servers []string) (deduped []string) {
 		}
 	}
 
-	logger.Debug("Deduped server list", len(servers), len(deduped))
+	dlog.Common.Debug("Deduped server list", len(servers), len(deduped))
 	return
 }
 
 // Randomly shuffle the server list.
 func (d *Discovery) shuffleList(servers []string) []string {
-	logger.Debug("Shuffling server list")
+	dlog.Common.Debug("Shuffling server list")
 
 	r := rand.New(rand.NewSource(time.Now().Unix()))
 	shuffled := make([]string, len(servers))
diff --git a/internal/discovery/file.go b/internal/discovery/file.go
index 1250755..fb46eeb 100644
--- a/internal/discovery/file.go
+++ b/internal/discovery/file.go
@@ -4,16 +4,16 @@ import (
 	"bufio"
 	"os"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ServerListFromFILE retrieves a list of servers from a file.
 func (d *Discovery) ServerListFromFILE() (servers []string) {
-	logger.Debug("Retrieving server list from file", d.server)
+	dlog.Common.Debug("Retrieving server list from file", d.server)
 
 	file, err := os.Open(d.server)
 	if err != nil {
-		logger.FatalExit(d.server, err)
+		dlog.Common.FatalPanic(d.server, err)
 	}
 	defer file.Close()
 
@@ -22,7 +22,7 @@ func (d *Discovery) ServerListFromFILE() (servers []string) {
 		servers = append(servers, scanner.Text())
 	}
 	if err := scanner.Err(); err != nil {
-		logger.FatalExit(d.server, err)
+		dlog.Common.FatalPanic(d.server, err)
 	}
 
 	return
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
new file mode 100644
index 0000000..7282741
--- /dev/null
+++ b/internal/io/dlog/dlog.go
@@ -0,0 +1,206 @@
+package dlog
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mimecast/dtail/internal/color/brush"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog/loggers"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/protocol"
+)
+
+// Client is the log handler for the client packages.
+var Client *DLog
+
+// Server is the log handler for the server packages.
+var Server *DLog
+
+// Common is the log handler for all other packages.
+// TODO: Rename Common to Common
+var Common *DLog
+
+var mutex sync.Mutex
+var started bool
+
+// Start logger(s).
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLevel string) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	if started {
+		Common.FatalPanic("Logger already started")
+	}
+
+	level := newLevel(logLevel)
+	switch sourceProcess {
+	case CLIENT:
+		// This is a DTail client process running.
+		impl := loggers.FOUT
+		Client = New(CLIENT, CLIENT, impl, level)
+		Server = New(CLIENT, SERVER, impl, level)
+		Common = Client
+	case SERVER:
+		// This is a DTail server process running.
+		impl := loggers.FILE
+		Client = New(SERVER, CLIENT, impl, level)
+		Server = New(SERVER, SERVER, impl, level)
+		Common = Server
+	}
+
+	var wg2 sync.WaitGroup
+	wg2.Add(2)
+	Client.start(ctx, &wg2)
+	Server.start(ctx, &wg2)
+	started = true
+
+	go rotation(ctx)
+	go func() {
+		wg2.Wait()
+		wg.Done()
+	}()
+}
+
+// DLog is the DTail logger.
+type DLog struct {
+	logger loggers.Logger
+	// Is this a DTail server or client process logging?
+	sourceProcess source
+	// Is this a DTail server or client package logging? In serverless mode
+	// the client can also execute code from the server package.
+	sourcePackage source
+	// Max log level to log.
+	maxLevel level
+}
+
+// New creates a new DTail logger.
+func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+	return &DLog{
+		logger:        loggers.Factory(sourceProcess.String(), impl),
+		sourceProcess: sourceProcess,
+		sourcePackage: sourcePackage,
+		maxLevel:      maxLevel,
+	}
+}
+
+func (d *DLog) start(ctx context.Context, wg *sync.WaitGroup) {
+	go func() {
+		defer wg.Done()
+		var wg2 sync.WaitGroup
+		wg2.Add(1)
+		d.logger.Start(ctx, &wg2)
+		<-ctx.Done()
+		wg2.Wait()
+	}()
+}
+
+func (d *DLog) log(level level, args []interface{}) string {
+	if d.maxLevel < level {
+		return ""
+	}
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
+	now := time.Now()
+
+	sb.WriteString(d.sourcePackage.String())
+	sb.WriteString(protocol.FieldDelimiter)
+	sb.WriteString(now.Format("20060102-150405"))
+	sb.WriteString(protocol.FieldDelimiter)
+	sb.WriteString(level.String())
+	sb.WriteString(protocol.FieldDelimiter)
+	d.writeArgStrings(sb, args)
+
+	message := sb.String()
+	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
+		d.logger.Log(now, message)
+		return message
+	}
+
+	d.logger.LogWithColors(now, message, brush.Colorfy(message))
+	return message
+}
+
+func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
+	for i, arg := range args {
+		if i > 0 {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+		switch v := arg.(type) {
+		case string:
+			sb.WriteString(v)
+		case error:
+			sb.WriteString(v.Error())
+		default:
+			sb.WriteString(fmt.Sprintf("%v", v))
+		}
+	}
+}
+
+func (d *DLog) FatalPanic(args ...interface{}) {
+	d.log(FATAL, args)
+	d.logger.Flush()
+	panic("Not recovering from this fatal error...")
+}
+
+func (d *DLog) Fatal(args ...interface{}) string {
+	return d.log(FATAL, args)
+}
+
+func (d *DLog) Error(args ...interface{}) string {
+	return d.log(ERROR, args)
+}
+
+func (d *DLog) Warn(args ...interface{}) string {
+	return d.log(WARN, args)
+}
+
+func (d *DLog) Info(args ...interface{}) string {
+	return d.log(INFO, args)
+}
+
+func (d *DLog) Verbose(args ...interface{}) string {
+	return d.log(VERBOSE, args)
+}
+
+func (d *DLog) Debug(args ...interface{}) string {
+	return d.log(DEBUG, args)
+}
+
+func (d *DLog) Trace(args ...interface{}) string {
+	return d.log(TRACE, args)
+}
+
+func (d *DLog) Devel(args ...interface{}) string {
+	return d.log(DEVEL, args)
+}
+
+func (d *DLog) Raw(message string) string {
+	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
+		d.logger.Log(time.Now(), message)
+		return message
+	}
+
+	d.logger.Log(time.Now(), brush.Colorfy(message))
+	return message
+}
+
+func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
+	args := make([]interface{}, len(data)+1)
+	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
+
+	i := 1
+	for k, v := range data {
+		args[i] = fmt.Sprintf("%s=%v", k, v)
+		i++
+	}
+
+	return d.log(INFO, args)
+}
+
+func (d *DLog) Flush()  { d.logger.Flush() }
+func (d *DLog) Pause()  { d.logger.Pause() }
+func (d *DLog) Resume() { d.logger.Resume() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
new file mode 100644
index 0000000..84550f0
--- /dev/null
+++ b/internal/io/dlog/level.go
@@ -0,0 +1,89 @@
+package dlog
+
+import (
+	"fmt"
+	"strings"
+)
+
+type level int
+
+const (
+	FATAL   level = iota
+	ERROR   level = iota
+	WARN    level = iota
+	INFO    level = iota
+	DEFAULT level = iota
+	VERBOSE level = iota
+	DEBUG   level = iota
+	DEVEL   level = iota
+	TRACE   level = iota
+	ALL     level = iota
+)
+
+var allLevels = []level{
+	FATAL,
+	ERROR,
+	WARN,
+	INFO,
+	DEFAULT,
+	VERBOSE,
+	DEBUG,
+	DEVEL,
+	TRACE,
+	ALL,
+}
+
+func newLevel(l string) level {
+	switch strings.ToUpper(l) {
+	case "FATAL":
+		return FATAL
+	case "ERROR":
+		return ERROR
+	case "WARN":
+		return WARN
+	case "INFO":
+		return INFO
+	case "":
+		fallthrough
+	case "DEFAULT":
+		return DEFAULT
+	case "VERBOSE":
+		return VERBOSE
+	case "DEBUG":
+		return DEBUG
+	case "DEVEL":
+		return DEVEL
+	case "TRACE":
+		return TRACE
+	case "ALL":
+		return ALL
+	}
+	panic(fmt.Sprintf("Unknown log level %s, must be one of: %v", l, allLevels))
+}
+
+func (l level) String() string {
+	switch l {
+	case FATAL:
+		return "FATAL"
+	case ERROR:
+		return "ERROR"
+	case WARN:
+		return "WARN"
+	case INFO:
+		return "INFO"
+	case DEFAULT:
+		return "DEFAULT"
+	case VERBOSE:
+		return "VERBOSE"
+	case DEBUG:
+		return "DEBUG"
+	case DEVEL:
+		return "DEVEL"
+	case TRACE:
+		return "TRACE"
+	case ALL:
+		return "ALL"
+	}
+
+	panic("Unknown log level " + fmt.Sprintf("%d", l))
+}
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
new file mode 100644
index 0000000..3eb29c5
--- /dev/null
+++ b/internal/io/dlog/loggers/factory.go
@@ -0,0 +1,60 @@
+package loggers
+
+import (
+	"fmt"
+	"sync"
+)
+
+type Impl int
+
+const (
+	NONE   Impl = iota
+	STDOUT Impl = iota
+	FILE   Impl = iota
+	FOUT   Impl = iota
+)
+
+var factoryMap map[string]Logger
+var factoryMutex sync.Mutex
+
+func Factory(name string, impl Impl) Logger {
+	factoryMutex.Lock()
+	defer factoryMutex.Unlock()
+
+	id := fmt.Sprintf("name:%s,impl:%v", name, impl)
+
+	if factoryMap == nil {
+		factoryMap = make(map[string]Logger)
+	}
+
+	singleton, ok := factoryMap[id]
+	if !ok {
+		switch impl {
+		case NONE:
+			singleton = none{}
+		case STDOUT:
+			singleton = newStdout()
+			factoryMap[id] = singleton
+		case FILE:
+			singleton = newFile()
+			factoryMap[id] = singleton
+		case FOUT:
+			singleton = newFout()
+			factoryMap[id] = singleton
+		}
+	}
+
+	return singleton
+}
+
+func FactoryRotate() {
+	factoryMutex.Lock()
+	defer factoryMutex.Unlock()
+	if factoryMap == nil {
+		return
+	}
+
+	for _, impl := range factoryMap {
+		impl.Rotate()
+	}
+}
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
new file mode 100644
index 0000000..1c525c9
--- /dev/null
+++ b/internal/io/dlog/loggers/file.go
@@ -0,0 +1,156 @@
+package loggers
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/mimecast/dtail/internal/config"
+)
+
+type fileMessageBuf struct {
+	now     time.Time
+	message string
+}
+
+type file struct {
+	bufferCh    chan *fileMessageBuf
+	pauseCh     chan struct{}
+	resumeCh    chan struct{}
+	rotateCh    chan struct{}
+	flushCh     chan struct{}
+	lastDateStr string
+	fd          *os.File
+	writer      *bufio.Writer
+	mutex       sync.Mutex
+	started     bool
+}
+
+func newFile() *file {
+	f := file{
+		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
+		pauseCh:  make(chan struct{}),
+		resumeCh: make(chan struct{}),
+		rotateCh: make(chan struct{}),
+		flushCh:  make(chan struct{}),
+	}
+	f.getWriter(time.Now().Format("20060102"))
+	return &f
+}
+
+func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	// Logger already started from another Goroutine.
+	if s.started {
+		wg.Done()
+		return
+	}
+
+	pause := func(ctx context.Context) {
+		select {
+		case <-s.resumeCh:
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	go func() {
+		defer wg.Done()
+
+		for {
+			select {
+			case m := <-s.bufferCh:
+				s.write(m)
+			case <-s.pauseCh:
+				pause(ctx)
+			case <-s.flushCh:
+				s.flush()
+			case <-ctx.Done():
+				s.flush()
+				s.fd.Close()
+				return
+			}
+		}
+	}()
+
+	s.started = true
+}
+
+func (s *file) Log(now time.Time, message string) {
+	s.bufferCh <- &fileMessageBuf{now, message}
+}
+
+func (s *file) LogWithColors(now time.Time, message, coloredMessage string) {
+	panic("Colors not supported in file logger")
+}
+
+func (s *file) Pause()  { s.pauseCh <- struct{}{} }
+func (s *file) Resume() { s.resumeCh <- struct{}{} }
+func (s *file) Flush()  { s.flushCh <- struct{}{} }
+
+// TODO: Test that Rotate() actually works.
+func (s *file) Rotate()           { s.rotateCh <- struct{}{} }
+func (file) SupportsColors() bool { return false }
+
+func (s *file) write(m *fileMessageBuf) {
+	select {
+	case <-s.rotateCh:
+		// Force re-opening the outfile.
+		s.lastDateStr = ""
+	default:
+	}
+
+	writer := s.getWriter(m.now.Format("20060102"))
+	writer.WriteString(m.message)
+	writer.WriteByte('\n')
+}
+
+func (s *file) getWriter(dateStr string) *bufio.Writer {
+	if s.lastDateStr == dateStr {
+		return s.writer
+	}
+
+	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
+		if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil {
+			panic(err)
+		}
+	}
+
+	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr)
+	newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
+	if err != nil {
+		panic(err)
+	}
+
+	// Close old writer.
+	if s.fd != nil {
+		s.writer.Flush()
+		s.fd.Close()
+	}
+
+	s.fd = newFd
+	s.writer = bufio.NewWriterSize(s.fd, 1)
+	s.lastDateStr = dateStr
+
+	return s.writer
+}
+
+func (s *file) flush() {
+	defer s.writer.Flush()
+
+	for {
+		select {
+		case m := <-s.bufferCh:
+			s.write(m)
+		default:
+			return
+		}
+	}
+}
diff --git a/internal/io/dlog/loggers/fout.go b/internal/io/dlog/loggers/fout.go
new file mode 100644
index 0000000..603dbe9
--- /dev/null
+++ b/internal/io/dlog/loggers/fout.go
@@ -0,0 +1,46 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+type fout struct {
+	file   *file
+	stdout *stdout
+}
+
+// Logs to both, a file and stdout
+func newFout() *fout {
+	return &fout{file: newFile(), stdout: newStdout()}
+}
+
+func (f *fout) Start(ctx context.Context, wg *sync.WaitGroup) {
+	go func() {
+		defer wg.Done()
+
+		var wg2 sync.WaitGroup
+		wg2.Add(2)
+		f.file.Start(ctx, &wg2)
+		f.stdout.Start(ctx, &wg2)
+		wg2.Wait()
+	}()
+}
+
+func (f *fout) Log(now time.Time, message string) {
+	f.stdout.Log(now, message)
+	f.file.Log(now, message)
+}
+
+func (f *fout) LogWithColors(now time.Time, message, coloredMessage string) {
+	f.stdout.LogWithColors(now, "", coloredMessage)
+	f.file.Log(now, message)
+}
+
+func (f *fout) Flush()  { f.stdout.Flush(); f.file.Flush() }
+func (f *fout) Pause()  { f.stdout.Pause(); f.file.Pause() }
+func (f *fout) Resume() { f.stdout.Resume(); f.file.Resume() }
+func (f *fout) Rotate() { f.file.Rotate() }
+
+func (fout) SupportsColors() bool { return true }
diff --git a/internal/io/dlog/loggers/logger.go b/internal/io/dlog/loggers/logger.go
new file mode 100644
index 0000000..c88900d
--- /dev/null
+++ b/internal/io/dlog/loggers/logger.go
@@ -0,0 +1,18 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+type Logger interface {
+	Log(now time.Time, message string)
+	LogWithColors(now time.Time, message, messageWithColors string)
+	Start(ctx context.Context, wg *sync.WaitGroup)
+	Flush()
+	Pause()
+	Resume()
+	Rotate()
+	SupportsColors() bool
+}
diff --git a/internal/io/dlog/loggers/none.go b/internal/io/dlog/loggers/none.go
new file mode 100644
index 0000000..270027f
--- /dev/null
+++ b/internal/io/dlog/loggers/none.go
@@ -0,0 +1,21 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+// don't log anything
+type none struct{}
+
+func (none) Start(ctx context.Context, wg *sync.WaitGroup) { wg.Done() }
+func (none) Log(now time.Time, message string)             {}
+
+func (none) LogWithColors(now time.Time, message, coloredMessage string) {}
+
+func (none) Flush()               {}
+func (none) Pause()               {}
+func (none) Resume()              {}
+func (none) Rotate()              {}
+func (none) SupportsColors() bool { return false }
diff --git a/internal/io/dlog/loggers/stdout.go b/internal/io/dlog/loggers/stdout.go
new file mode 100644
index 0000000..9738323
--- /dev/null
+++ b/internal/io/dlog/loggers/stdout.go
@@ -0,0 +1,73 @@
+package loggers
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type stdout struct {
+	bufferCh chan string
+	pauseCh  chan struct{}
+	resumeCh chan struct{}
+}
+
+func newStdout() *stdout {
+	return &stdout{
+		bufferCh: make(chan string, 100),
+		pauseCh:  make(chan struct{}),
+		resumeCh: make(chan struct{}),
+	}
+}
+
+func (s *stdout) Start(ctx context.Context, wg *sync.WaitGroup) {
+	pause := func(ctx context.Context) {
+		select {
+		case <-s.resumeCh:
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	go func() {
+		defer wg.Done()
+
+		for {
+			select {
+			case message := <-s.bufferCh:
+				fmt.Println(message)
+			case <-s.pauseCh:
+				pause(ctx)
+			case <-ctx.Done():
+				s.Flush()
+				return
+			}
+		}
+	}()
+}
+
+func (s *stdout) Log(now time.Time, message string) {
+	s.bufferCh <- message
+}
+
+func (s *stdout) LogWithColors(now time.Time, message, coloredMessage string) {
+	s.bufferCh <- coloredMessage
+}
+
+func (s *stdout) Flush() {
+	for {
+		select {
+		case message := <-s.bufferCh:
+			fmt.Println(message)
+		default:
+			return
+		}
+	}
+}
+
+func (s *stdout) Pause()            { s.pauseCh <- struct{}{} }
+func (s *stdout) Resume()           { s.resumeCh <- struct{}{} }
+func (s *stdout) Rotate()           {}
+func (stdout) SupportsColors() bool { return true }
diff --git a/internal/io/dlog/rotation.go b/internal/io/dlog/rotation.go
new file mode 100644
index 0000000..15ce1fd
--- /dev/null
+++ b/internal/io/dlog/rotation.go
@@ -0,0 +1,27 @@
+package dlog
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/mimecast/dtail/internal/io/dlog/loggers"
+)
+
+func rotation(ctx context.Context) {
+	rotateCh := make(chan os.Signal, 1)
+	signal.Notify(rotateCh, syscall.SIGHUP)
+	go func() {
+		for {
+			select {
+			case <-rotateCh:
+				Common.Debug("Invoking log rotation")
+				loggers.FactoryRotate()
+				return
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+}
diff --git a/internal/io/dlog/source.go b/internal/io/dlog/source.go
new file mode 100644
index 0000000..265885e
--- /dev/null
+++ b/internal/io/dlog/source.go
@@ -0,0 +1,19 @@
+package dlog
+
+type source int
+
+const (
+	CLIENT source = iota
+	SERVER source = iota
+)
+
+func (s source) String() string {
+	switch s {
+	case CLIENT:
+		return "CLIENT"
+	case SERVER:
+		return "SERVER"
+	}
+
+	panic("Unknown log source type")
+}
diff --git a/internal/io/dlog/strategy.go b/internal/io/dlog/strategy.go
new file mode 100644
index 0000000..32d8298
--- /dev/null
+++ b/internal/io/dlog/strategy.go
@@ -0,0 +1,22 @@
+package dlog
+
+import "github.com/mimecast/dtail/internal/config"
+
+// Strategy allows to specify a log rotation strategy.
+type Strategy int
+
+// Possible log strategies.
+const (
+	NormalStrategy Strategy = iota
+	DailyStrategy  Strategy = iota
+	StdoutStrategy Strategy = iota
+)
+
+func logStrategy() Strategy {
+	switch config.Common.LogStrategy {
+	case "daily":
+		return DailyStrategy
+	default:
+	}
+	return StdoutStrategy
+}
diff --git a/internal/io/fs/permissions/permission.go b/internal/io/fs/permissions/permission.go
index cc5dd9b..bbcb74e 100644
--- a/internal/io/fs/permissions/permission.go
+++ b/internal/io/fs/permissions/permission.go
@@ -3,12 +3,12 @@
 package permissions
 
 import (
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ToRead is to check whether user has read permissions to a given file.
 func ToRead(user, filePath string) (bool, error) {
 	// Only implemented for Linux, always expect true
-	logger.Warn(user, filePath, "Not performing ACL check, not supported on this platform")
+	dlog.Common.Warn(user, filePath, "Not performing ACL check, not supported on this platform")
 	return true, nil
 }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index ec33c60..07486a1 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/regex"
 
@@ -62,7 +62,7 @@ func (f readFile) Retry() bool {
 
 // Start tailing a log file.
 func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Regex) error {
-	logger.Debug("readFile", f)
+	dlog.Common.Debug("readFile", f)
 	defer func() {
 		select {
 		case <-f.limiter:
@@ -74,7 +74,7 @@ func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Re
 	case f.limiter <- struct{}{}:
 	default:
 		select {
-		case f.serverMessages <- logger.Warn(f.filePath, f.globID, "Server limit reached. Queuing file..."):
+		case f.serverMessages <- dlog.Common.Warn(f.filePath, f.globID, "Server limit reached. Queuing file..."):
 		case <-ctx.Done():
 			return nil
 		}
@@ -126,7 +126,7 @@ func (f readFile) makeReader(fd *os.File) (reader *bufio.Reader, err error) {
 	case strings.HasSuffix(f.FilePath(), ".gz"):
 		fallthrough
 	case strings.HasSuffix(f.FilePath(), ".gzip"):
-		logger.Info(f.FilePath(), "Detected gzip compression format")
+		dlog.Common.Info(f.FilePath(), "Detected gzip compression format")
 		var gzipReader *gzip.Reader
 		gzipReader, err = gzip.NewReader(fd)
 		if err != nil {
@@ -134,7 +134,7 @@ func (f readFile) makeReader(fd *os.File) (reader *bufio.Reader, err error) {
 		}
 		reader = bufio.NewReader(gzipReader)
 	case strings.HasSuffix(f.FilePath(), ".zst"):
-		logger.Info(f.FilePath(), "Detected zstd compression format")
+		dlog.Common.Info(f.FilePath(), "Detected zstd compression format")
 		reader = bufio.NewReader(zstd.NewReader(fd))
 	default:
 		reader = bufio.NewReader(fd)
@@ -172,7 +172,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 			default:
 			}
 			if !f.seekEOF {
-				logger.Info(f.FilePath(), "End of file reached")
+				dlog.Common.Info(f.FilePath(), "End of file reached")
 				return nil
 			}
 			time.Sleep(time.Millisecond * 100)
@@ -201,7 +201,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 		default:
 			if message.Len() >= lineLengthThreshold {
 				if !warnedAboutLongLine {
-					f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
+					f.serverMessages <- dlog.Common.Warn(f.filePath, "Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
 				message.WriteString("\n")
@@ -268,7 +268,7 @@ func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int, r
 
 // Check wether log file is truncated. Returns nil if not.
 func (f readFile) truncated(fd *os.File) (bool, error) {
-	logger.Debug(f.filePath, "File truncation check")
+	dlog.Common.Debug(f.filePath, "File truncation check")
 
 	// Can not seek currently open FD.
 	curPos, err := fd.Seek(0, os.SEEK_CUR)
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 6890201..6a6b5ec 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -1,5 +1,7 @@
 package logger
 
+// TODO: Rewrite this logger
+
 import (
 	"bufio"
 	"context"
@@ -64,12 +66,25 @@ var resumeCh chan struct{}
 // Tell the logger about logrotation
 var rotateCh chan os.Signal
 
+// Override the logger with a custom callack (e.g. for the t.Log for unit tests)
+type unitTestCallback func(message string)
+
+var unitTestOkCb unitTestCallback
+var unitTestErrorCb unitTestCallback
+
 // Helper type to make logging non-blocking.
 type buf struct {
 	time    time.Time
 	message string
 }
 
+// StartUnitTests enables to log all messages to the unit tests.
+func StartUnitTests(ctx context.Context, okCb, errCb unitTestCallback) {
+	unitTestOkCb = okCb
+	unitTestErrorCb = errCb
+	Start(ctx, Modes{UnitTest: true})
+}
+
 // Start logging.
 func Start(ctx context.Context, mode Modes) {
 	Mode = mode
@@ -91,12 +106,12 @@ func Start(ctx context.Context, mode Modes) {
 	switch strategy {
 	case DailyStrategy:
 		_, err := os.Stat(config.Common.LogDir)
-		Mode.logToFile = !os.IsNotExist(err)
+		Mode.logToFile = !os.IsNotExist(err) && !Mode.UnitTest
 		Mode.logToStdout = !Mode.Server || Mode.Debug || Mode.Trace || Mode.Quiet
 	case StdoutStrategy:
 		fallthrough
 	default:
-		Mode.logToFile = !Mode.Server
+		Mode.logToFile = !Mode.Server && !Mode.UnitTest
 		Mode.logToStdout = true
 	}
 
@@ -182,8 +197,8 @@ func Fatal(args ...interface{}) string {
 	return log(clientStr, fatalStr, args)
 }
 
-// FatalExit logs an error and exists the process.
-func FatalExit(args ...interface{}) {
+// FatalPanic logs an error and exists the process.
+func FatalPanic(args ...interface{}) {
 	what := clientStr
 	if Mode.Server {
 		what = serverStr
diff --git a/internal/io/logger/modes.go b/internal/io/logger/modes.go
index 8864179..85f90a5 100644
--- a/internal/io/logger/modes.go
+++ b/internal/io/logger/modes.go
@@ -2,11 +2,12 @@ package logger
 
 // Modes specifies the logging mode.
 type Modes struct {
-	Server      bool
-	Trace       bool
 	Debug       bool
+	logToFile   bool
+	logToStdout bool
 	Nothing     bool
 	Quiet       bool
-	logToStdout bool
-	logToFile   bool
+	Server      bool
+	Trace       bool
+	UnitTest    bool
 }
diff --git a/internal/io/prompt/prompt.go b/internal/io/prompt/prompt.go
index 36ebdb5..7c3cdb5 100644
--- a/internal/io/prompt/prompt.go
+++ b/internal/io/prompt/prompt.go
@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // Answer is a user input of a prompt question.
@@ -58,7 +58,7 @@ func (p *Prompt) Add(answer Answer) {
 // Ask a question.
 func (p *Prompt) Ask() {
 	reader := bufio.NewReader(os.Stdin)
-	logger.Pause()
+	dlog.Common.Pause()
 
 	for {
 		fmt.Print(p.askString())
@@ -70,7 +70,7 @@ func (p *Prompt) Ask() {
 			}
 
 			if !a.AskAgain {
-				logger.Resume()
+				dlog.Common.Resume()
 				if a.EndCallback != nil {
 					a.EndCallback()
 				}
diff --git a/internal/mapr/aggregateset.go b/internal/mapr/aggregateset.go
index 47f4925..14e6943 100644
--- a/internal/mapr/aggregateset.go
+++ b/internal/mapr/aggregateset.go
@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 )
@@ -37,7 +37,7 @@ func (s *AggregateSet) String() string {
 // Merge one aggregate set into this one.
 func (s *AggregateSet) Merge(query *Query, set *AggregateSet) error {
 	s.Samples += set.Samples
-	//logger.Trace("Merge", set)
+	//dlog.Common.Trace("Merge", set)
 
 	for _, sc := range query.Select {
 		storage := sc.FieldStorage
@@ -70,7 +70,7 @@ func (s *AggregateSet) Merge(query *Query, set *AggregateSet) error {
 
 // Serialize the aggregate set so it can be sent over the wire.
 func (s *AggregateSet) Serialize(ctx context.Context, groupKey string, ch chan<- string) {
-	logger.Trace("Serialising mapr.AggregateSet", s)
+	dlog.Common.Trace("Serialising mapr.AggregateSet", s)
 	sb := pool.BuilderBuffer.Get().(*strings.Builder)
 	defer pool.RecycleBuilderBuffer(sb)
 
diff --git a/internal/mapr/client/aggregate.go b/internal/mapr/client/aggregate.go
index 5cc09a1..d0c1d70 100644
--- a/internal/mapr/client/aggregate.go
+++ b/internal/mapr/client/aggregate.go
@@ -5,7 +5,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/protocol"
 )
@@ -52,7 +52,7 @@ func (a *Aggregate) Aggregate(message string) error {
 	for _, sc := range a.query.Select {
 		if val, ok := fields[sc.FieldStorage]; ok {
 			if err := set.Aggregate(sc.FieldStorage, sc.Operation, val, true); err != nil {
-				logger.Error(err)
+				dlog.Common.Error(err)
 				continue
 			}
 			addedSamples = true
diff --git a/internal/mapr/funcs/function.go b/internal/mapr/funcs/function.go
index 1a89c3a..0433b9a 100644
--- a/internal/mapr/funcs/function.go
+++ b/internal/mapr/funcs/function.go
@@ -58,9 +58,9 @@ func NewFunctionStack(in string) (FunctionStack, string, error) {
 // Call the function stack.
 func (fs FunctionStack) Call(str string) string {
 	for i := len(fs) - 1; i >= 0; i-- {
-		//logger.Debug("Call", fs[i].Name, str)
+		//dlog.Common.Debug("Call", fs[i].Name, str)
 		str = fs[i].call(str)
-		//logger.Debug("Call.result", fs[i].Name, str)
+		//dlog.Common.Debug("Call.result", fs[i].Name, str)
 	}
 
 	return str
diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index 9bff790..df8c603 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -11,7 +11,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 )
@@ -189,7 +189,7 @@ func (g *GroupSet) WriteResult(query *Query) error {
 		return err
 	}
 
-	logger.Info("Writing outfile", query.Outfile)
+	dlog.Common.Info("Writing outfile", query.Outfile)
 	tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile)
 
 	file, err := os.Create(tmpOutfile)
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index c67137e..e0bbc30 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -26,6 +26,7 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields["$timeoffset"] = p.timeZoneOffset
 
 	fields["$severity"] = splitted[0]
+	fields["$loglevel"] = splitted[0]
 	// TODO: Parse time like we do at Mimecast
 	fields["$time"] = splitted[1]
 
diff --git a/internal/mapr/logformat/generickv.go b/internal/mapr/logformat/generickv.go
index 23d75cb..3769c22 100644
--- a/internal/mapr/logformat/generickv.go
+++ b/internal/mapr/logformat/generickv.go
@@ -21,7 +21,7 @@ func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error)
 	for _, kv := range splitted[0:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
-			//logger.Debug("Unable to parse key-value token, ignoring it", kv)
+			//dlog.Common.Debug("Unable to parse key-value token, ignoring it", kv)
 			continue
 		}
 		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go
index 6582b5f..a352580 100644
--- a/internal/mapr/logformat/parser.go
+++ b/internal/mapr/logformat/parser.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
 )
 
@@ -76,12 +75,10 @@ func (p *Parser) MakeFields(maprLine string) (fields map[string]string, err erro
 
 	if errInterface == nil {
 		fields, err = returnValues[0].Interface().(map[string]string), nil
-		logger.Trace("parser.MakeFields", fields, err)
 		return
 	}
 
 	fields, err = returnValues[0].Interface().(map[string]string), errInterface.(error)
-	logger.Trace("parser.MakeFields", fields, err)
 
 	return
 }
diff --git a/internal/mapr/query.go b/internal/mapr/query.go
index 01852da..6c1d849 100644
--- a/internal/mapr/query.go
+++ b/internal/mapr/query.go
@@ -6,8 +6,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/mimecast/dtail/internal/io/logger"
 )
 
 const (
@@ -67,8 +65,6 @@ func NewQuery(queryStr string) (*Query, error) {
 	}
 
 	err := q.parse(tokens)
-
-	logger.Debug(q)
 	return &q, err
 }
 
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index d11ed7d..767aada 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -9,7 +9,7 @@ import (
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/logformat"
@@ -40,7 +40,7 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 
 	fqdn, err := os.Hostname()
 	if err != nil {
-		logger.Error(err)
+		dlog.Common.Error(err)
 	}
 	s := strings.Split(fqdn, ".")
 
@@ -55,12 +55,12 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 		parserName = query.LogFormat
 	}
 
-	logger.Info("Creating log format parser", parserName)
+	dlog.Common.Info("Creating log format parser", parserName)
 	logParser, err := logformat.NewParser(parserName, query)
 	if err != nil {
-		logger.Error("Could not create log format parser. Falling back to 'generic'", err)
+		dlog.Common.Error("Could not create log format parser. Falling back to 'generic'", err)
 		if logParser, err = logformat.NewParser("generic", query); err != nil {
-			logger.FatalExit("Could not create log format parser", err)
+			dlog.Common.FatalPanic("Could not create log format parser", err)
 		}
 	}
 
@@ -153,7 +153,7 @@ func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]strin
 				if err != nil {
 					// Should fields be ignored anyway?
 					if err != logformat.IgnoreFieldsErr {
-						logger.Error(fields, err)
+						dlog.Common.Error(fields, err)
 					}
 					continue
 				}
@@ -187,7 +187,7 @@ func (a *Aggregate) setAdditionalFields(ctx context.Context, fieldsCh <-chan map
 				return
 			}
 			if err := a.query.SetClause(fields); err != nil {
-				logger.Error(err)
+				dlog.Common.Error(err)
 			}
 
 			select {
@@ -204,7 +204,7 @@ func (a *Aggregate) aggregateAndSerialize(ctx context.Context, fieldsCh <-chan m
 	group := mapr.NewGroupSet()
 
 	serialize := func() {
-		logger.Info("Serializing mapreduce result")
+		dlog.Common.Info("Serializing mapreduce result")
 		group.Serialize(ctx, maprMessages)
 		group = mapr.NewGroupSet()
 	}
@@ -243,7 +243,7 @@ func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 	for _, sc := range a.query.Select {
 		if val, ok := fields[sc.Field]; ok {
 			if err := set.Aggregate(sc.FieldStorage, sc.Operation, val, false); err != nil {
-				logger.Error(err)
+				dlog.Common.Error(err)
 				continue
 			}
 			addedSample = true
@@ -255,7 +255,7 @@ func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 		return
 	}
 
-	logger.Trace("Aggregated data locally without adding new samples")
+	dlog.Common.Trace("Aggregated data locally without adding new samples")
 }
 
 // Serialize all the aggregated data.
@@ -263,7 +263,7 @@ func (a *Aggregate) Serialize(ctx context.Context) {
 	select {
 	case a.serialize <- struct{}{}:
 	case <-time.After(time.Minute):
-		logger.Warn("Starting to serialize mapredice data takes over a minute")
+		dlog.Common.Warn("Starting to serialize mapredice data takes over a minute")
 	case <-ctx.Done():
 	}
 }
diff --git a/internal/mapr/token.go b/internal/mapr/token.go
index 8972188..7c6578b 100644
--- a/internal/mapr/token.go
+++ b/internal/mapr/token.go
@@ -58,12 +58,12 @@ func tokenize(queryStr string) []token {
 }
 
 func tokensConsume(tokens []token) ([]token, []token) {
-	//logger.Trace("=====================")
+	//dlog.Common.Trace("=====================")
 	var consumed []token
 
 	for i, t := range tokens {
 		if t.isKeyword() {
-			//logger.Trace("keyword", t)
+			//dlog.Common.Trace("keyword", t)
 			return tokens[i:], consumed
 		}
 		// strip escapes, such as ` from `foo`, this allows to use keywords as field names
@@ -73,7 +73,7 @@ func tokensConsume(tokens []token) ([]token, []token) {
 		}
 		if t.str[0] == '`' && t.str[length-1] == '`' {
 			stripped := t.str[1 : length-1]
-			//logger.Trace("stripped", stripped)
+			//dlog.Common.Trace("stripped", stripped)
 			t := token{
 				str:        stripped,
 				isBareword: t.isBareword,
@@ -81,11 +81,11 @@ func tokensConsume(tokens []token) ([]token, []token) {
 			consumed = append(consumed, t)
 			continue
 		}
-		//logger.Trace("bare", token)
+		//dlog.Common.Trace("bare", token)
 		consumed = append(consumed, t)
 	}
 
-	//logger.Trace("result", consumed)
+	//dlog.Common.Trace("result", consumed)
 	return nil, consumed
 }
 
diff --git a/internal/mapr/whereclause.go b/internal/mapr/whereclause.go
index cc1c164..6356d94 100644
--- a/internal/mapr/whereclause.go
+++ b/internal/mapr/whereclause.go
@@ -3,7 +3,7 @@ package mapr
 import (
 	"strconv"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // WhereClause interprets the where clause of the mapreduce query.
@@ -55,7 +55,7 @@ func whereClauseFloatValue(fields map[string]string, str string, float float64,
 		}
 		return f, true
 	default:
-		logger.Error("Unexpected argument in 'where' clause", str, float, t)
+		dlog.Common.Error("Unexpected argument in 'where' clause", str, float, t)
 		return 0, false
 	}
 }
@@ -71,7 +71,7 @@ func whereClauseStringValue(fields map[string]string, str string, t fieldType) (
 	case String:
 		return str, true
 	default:
-		logger.Error("Unexpected argument in 'where' clause", str, t)
+		dlog.Common.Error("Unexpected argument in 'where' clause", str, t)
 		return str, false
 	}
 }
diff --git a/internal/mapr/wherecondition.go b/internal/mapr/wherecondition.go
index 7a60dba..c60c0a5 100644
--- a/internal/mapr/wherecondition.go
+++ b/internal/mapr/wherecondition.go
@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // QueryOperation determines the mapreduce operation.
@@ -168,7 +168,7 @@ func (wc *whereCondition) floatClause(lValue float64, rValue float64) bool {
 	case FloatGe:
 		return lValue >= rValue
 	default:
-		logger.Error("Unknown float operation", lValue, wc.Operation, rValue)
+		dlog.Common.Error("Unknown float operation", lValue, wc.Operation, rValue)
 	}
 
 	return false
@@ -193,7 +193,7 @@ func (wc *whereCondition) stringClause(lValue string, rValue string) bool {
 	case StringNotHasSuffix:
 		return !strings.HasSuffix(lValue, rValue)
 	default:
-		logger.Error("Unknown string operation", lValue, wc.Operation, rValue)
+		dlog.Common.Error("Unknown string operation", lValue, wc.Operation, rValue)
 	}
 
 	return false
diff --git a/internal/regex/regex.go b/internal/regex/regex.go
index 2561659..352ffd6 100644
--- a/internal/regex/regex.go
+++ b/internal/regex/regex.go
@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
-
-	"github.com/mimecast/dtail/internal/io/logger"
 )
 
 // Regex for filtering lines.
@@ -91,17 +89,17 @@ func (r Regex) MatchString(str string) bool {
 }
 
 // Serialize the regex.
-func (r Regex) Serialize() string {
+func (r Regex) Serialize() (string, error) {
 	var flags []string
 	for _, flag := range r.flags {
 		flags = append(flags, flag.String())
 	}
 
 	if !r.initialized {
-		logger.FatalExit("Unable to serialize regex as not initialized properly", r)
+		return "", fmt.Errorf("Unable to serialize regex as not initialized properly: %v", r)
 	}
 
-	return fmt.Sprintf("regex:%s %s", strings.Join(flags, ","), r.regexStr)
+	return fmt.Sprintf("regex:%s %s", strings.Join(flags, ","), r.regexStr), nil
 }
 
 // Deserialize the regex.
@@ -109,7 +107,6 @@ func Deserialize(str string) (Regex, error) {
 	// Get regex string
 	s := strings.SplitN(str, " ", 2)
 	if len(s) < 2 {
-		logger.Debug("Using noop regex", str)
 		return NewNoop(), nil
 	}
 
@@ -127,10 +124,8 @@ func Deserialize(str string) (Regex, error) {
 		for _, flagStr := range strings.Split(s[1], ",") {
 			flag, err := NewFlag(flagStr)
 			if err != nil {
-				logger.Error("ignoring flag", err)
 				continue
 			}
-			logger.Debug("Adding regex flag", flag)
 			flags = append(flags, flag)
 		}
 	}
diff --git a/internal/regex/regex_test.go b/internal/regex/regex_test.go
index a5e7faf..2ce49ac 100644
--- a/internal/regex/regex_test.go
+++ b/internal/regex/regex_test.go
@@ -20,9 +20,13 @@ func TestRegex(t *testing.T) {
 		t.Errorf("expected to match string '%s' with regex '%v' but didn't\n", input, r)
 	}
 
-	r2, err := Deserialize(r.Serialize())
+	serialized, err := r.Serialize()
 	if err != nil {
-		t.Errorf("unable to serialize deserialized regex: %v: %v\n", r.Serialize(), err)
+		t.Errorf("unable to serialize regex: %v: %v\n", serialized, err)
+	}
+	r2, err := Deserialize(serialized)
+	if err != nil {
+		t.Errorf("unable to serialize deserialized regex: %v: %v\n", serialized, err)
 	}
 	if r.String() != r2.String() {
 		t.Errorf("regex should be the same after deserialize(serialize(..)), got '%s' but expected '%s'.\n",
@@ -37,9 +41,13 @@ func TestRegex(t *testing.T) {
 		t.Errorf("expected to not match string '%s' with regex '%v' but matched\n", input, r)
 	}
 
-	r2, err = Deserialize(r.Serialize())
+	serialized, err = r.Serialize()
+	if err != nil {
+		t.Errorf("unable to serialize regex: %v: %v\n", serialized, err)
+	}
+	r2, err = Deserialize(serialized)
 	if err != nil {
-		t.Errorf("unable to serialize deserialized regex: %v: %v\n", r.Serialize(), err)
+		t.Errorf("unable to serialize deserialized regex: %v: %v\n", serialized, err)
 	}
 	if r.String() != r2.String() {
 		t.Errorf("regex should be the same after deserialize(serialize(..)), got '%s' but expected '%s'.\n",
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index f75c732..5f4c454 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -8,9 +8,8 @@ import (
 
 	"github.com/mimecast/dtail/internal/clients"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
-
 	gossh "golang.org/x/crypto/ssh"
 )
 
@@ -22,7 +21,7 @@ func newContinuous() *continuous {
 }
 
 func (c *continuous) start(ctx context.Context) {
-	logger.Info("Starting continuous job runner after 10s")
+	dlog.Server.Info("Starting continuous job runner after 10s")
 	time.Sleep(time.Second * 10)
 
 	c.runJobs(ctx)
@@ -31,7 +30,7 @@ func (c *continuous) start(ctx context.Context) {
 func (c *continuous) runJobs(ctx context.Context) {
 	for _, job := range config.Server.Continuous {
 		if !job.Enable {
-			logger.Debug(job.Name, "Not running job as not enabled")
+			dlog.Server.Debug(job.Name, "Not running job as not enabled")
 			continue
 		}
 
@@ -51,7 +50,7 @@ func (c *continuous) runJobs(ctx context.Context) {
 }
 
 func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
-	logger.Debug(job.Name, "Processing job")
+	dlog.Server.Debug(job.Name, "Processing job")
 
 	files := fillDates(job.Files)
 	outfile := fillDates(job.Outfile)
@@ -61,7 +60,7 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 		servers = config.Server.SSHBindAddress
 	}
 
-	args := clients.Args{
+	args := config.Args{
 		ConnectionsPerCPU: 10,
 		Discovery:         job.Discovery,
 		ServersStr:        servers,
@@ -75,7 +74,7 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	query := fmt.Sprintf("%s outfile %s", job.Query, outfile)
 	client, err := clients.NewMaprClient(args, query, clients.NonCumulativeMode)
 	if err != nil {
-		logger.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
+		dlog.Server.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
 		return
 	}
 
@@ -85,21 +84,21 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	if job.RestartOnDayChange {
 		go func() {
 			if c.waitForDayChange(ctx) {
-				logger.Info(fmt.Sprintf("Canceling job %s due to day change", job.Name))
+				dlog.Server.Info(fmt.Sprintf("Canceling job %s due to day change", job.Name))
 				cancel()
 			}
 		}()
 	}
 
-	logger.Info(fmt.Sprintf("Starting job %s", job.Name))
+	dlog.Server.Info(fmt.Sprintf("Starting job %s", job.Name))
 	status := client.Start(jobCtx, make(chan string))
 	logMessage := fmt.Sprintf("Job exited with status %d", status)
 
 	if status != 0 {
-		logger.Warn(logMessage)
+		dlog.Server.Warn(logMessage)
 		return
 	}
-	logger.Info(logMessage)
+	dlog.Server.Info(logMessage)
 }
 
 func (c *continuous) waitForDayChange(ctx context.Context) bool {
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
index 1e17c78..ae70675 100644
--- a/internal/server/handlers/controlhandler.go
+++ b/internal/server/handlers/controlhandler.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
@@ -22,7 +22,7 @@ type ControlHandler struct {
 
 // NewControlHandler returns a new control handler.
 func NewControlHandler(user *user.User) *ControlHandler {
-	logger.Debug(user, "Creating control handler")
+	dlog.Server.Debug(user, "Creating control handler")
 
 	h := ControlHandler{
 		done:           internal.NewDone(),
@@ -32,7 +32,7 @@ func NewControlHandler(user *user.User) *ControlHandler {
 
 	fqdn, err := os.Hostname()
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Server.FatalPanic(err)
 	}
 
 	s := strings.Split(fqdn, ".")
@@ -84,15 +84,15 @@ func (h *ControlHandler) Write(p []byte) (n int, err error) {
 }
 
 func (h *ControlHandler) handleCommand(command string) {
-	logger.Info(h.user, command)
+	dlog.Server.Info(h.user, command)
 	s := strings.Split(command, " ")
-	logger.Debug(h.user, "Receiving command", command, s)
+	dlog.Server.Debug(h.user, "Receiving command", command, s)
 
 	switch s[0] {
 	case "health":
 		h.serverMessages <- "OK: DTail SSH Server seems fine"
 		h.serverMessages <- "done;"
 	default:
-		h.serverMessages <- logger.Error(h.user, "Received unknown control command", command, s)
+		h.serverMessages <- dlog.Server.Error(h.user, "Received unknown control command", command, s)
 	}
 }
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 69dd4a5..60ad2a0 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/io/fs"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 )
@@ -32,13 +32,13 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string, retrie
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			r.server.sendServerMessage(logger.Error(r.server.user, commandParseWarning, err))
+			r.server.sendServerMessage(dlog.Server.Error(r.server.user, commandParseWarning, err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.sendServerWarnMessage(logger.Warn(r.server.user, commandParseWarning, args, argc))
+		r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, commandParseWarning, args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re, retries)
@@ -51,14 +51,14 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 	for retryCount := 0; retryCount < retries; retryCount++ {
 		paths, err := filepath.Glob(glob)
 		if err != nil {
-			logger.Warn(r.server.user, glob, err)
+			dlog.Server.Warn(r.server.user, glob, err)
 			time.Sleep(retryInterval)
 			continue
 		}
 
 		if numPaths := len(paths); numPaths == 0 {
-			logger.Error(r.server.user, "No such file(s) to read", glob)
-			r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Unable to read file(s), check server logs"))
+			dlog.Server.Error(r.server.user, "No such file(s) to read", glob)
+			r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
 			select {
 			case <-ctx.Done():
 				return
@@ -72,7 +72,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 		return
 	}
 
-	r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Giving up to read file(s)"))
+	r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Giving up to read file(s)"))
 	return
 }
 
@@ -92,8 +92,8 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGr
 	globID := r.makeGlobID(path, glob)
 
 	if !r.server.user.HasFilePermission(path, "readfiles") {
-		logger.Error(r.server.user, "No permission to read file", path, globID)
-		r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Unable to read file(s), check server logs"))
+		dlog.Server.Error(r.server.user, "No permission to read file", path, globID)
+		r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
 		return
 	}
 
@@ -101,7 +101,7 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGr
 }
 
 func (r *readCommand) readFile(ctx context.Context, path, globID string, re regex.Regex) {
-	logger.Info(r.server.user, "Start reading file", path, globID)
+	dlog.Server.Info(r.server.user, "Start reading file", path, globID)
 
 	var reader fs.FileReader
 	switch r.mode {
@@ -122,7 +122,7 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 			aggregate.NextLinesCh <- lines
 		}
 		if err := reader.Start(ctx, lines, re); err != nil {
-			logger.Error(r.server.user, path, globID, err)
+			dlog.Server.Error(r.server.user, path, globID, err)
 		}
 		if aggregate != nil {
 			// Also makes aggregate to Flush
@@ -139,7 +139,7 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 		}
 
 		time.Sleep(time.Second * 2)
-		logger.Info(path, globID, "Reading file again")
+		dlog.Server.Info(path, globID, "Reading file again")
 	}
 }
 
@@ -161,6 +161,6 @@ func (r *readCommand) makeGlobID(path, glob string) string {
 		return pathParts[len(pathParts)-1]
 	}
 
-	r.server.sendServerWarnMessage(logger.Warn("Empty file path given?", path, glob))
+	r.server.sendServerWarnMessage(dlog.Server.Warn("Empty file path given?", path, glob))
 	return ""
 }
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 4820476..b664566 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -16,7 +16,7 @@ import (
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
@@ -66,7 +66,7 @@ func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *S
 
 	fqdn, err := os.Hostname()
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Server.FatalPanic(err)
 	}
 
 	s := strings.Split(fqdn, ".")
@@ -165,18 +165,18 @@ func (h *ServerHandler) Write(p []byte) (n int, err error) {
 }
 
 func (h *ServerHandler) handleCommand(commandStr string) {
-	logger.Debug(h.user, commandStr)
+	dlog.Server.Debug(h.user, commandStr)
 	ctx := context.Background()
 
 	args, argc, add, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
 	if err != nil {
-		h.send(h.serverMessages, logger.Error(h.user, err)+add)
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err)+add)
 		return
 	}
 
 	args, argc, err = h.handleBase64(args, argc)
 	if err != nil {
-		h.send(h.serverMessages, logger.Error(h.user, err))
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
 	}
 
@@ -239,7 +239,7 @@ func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, er
 
 	args = strings.Split(decodedStr, " ")
 	argc = len(decodedStr)
-	logger.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
+	dlog.Server.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
 
 	return args, argc, nil
 }
@@ -247,14 +247,14 @@ func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, er
 func (h *ServerHandler) handleControlCommand(argc int, args []string) {
 	switch args[0] {
 	case "debug":
-		h.send(h.serverMessages, logger.Debug(h.user, "Receiving debug command", argc, args))
+		h.send(h.serverMessages, dlog.Server.Debug(h.user, "Receiving debug command", argc, args))
 	default:
-		logger.Warn(h.user, "Received unknown control command", argc, args)
+		dlog.Server.Warn(h.user, "Received unknown control command", argc, args)
 	}
 }
 
 func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string) {
-	logger.Debug(h.user, "handleUserCommand", argc, args)
+	dlog.Server.Debug(h.user, "handleUserCommand", argc, args)
 
 	h.incrementActiveCommands()
 	commandFinished := func() {
@@ -268,19 +268,19 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 
 	options, err := readOptions(splitted[1:])
 	if err != nil {
-		h.sendServerMessage(logger.Error(h.user, err))
+		h.sendServerMessage(dlog.Server.Error(h.user, err))
 		commandFinished()
 		return
 	}
 	if quiet, ok := options["quiet"]; ok {
 		if quiet == "true" {
-			logger.Debug(h.user, "Enabling quiet mode")
+			dlog.Server.Debug(h.user, "Enabling quiet mode")
 			h.quiet = true
 		}
 	}
 	if spartan, ok := options["spartan"]; ok {
 		if spartan == "true" {
-			logger.Debug(h.user, "Enabling spartan mode")
+			dlog.Server.Debug(h.user, "Enabling spartan mode")
 			h.spartan = true
 		}
 	}
@@ -304,7 +304,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		command, aggregate, err := newMapCommand(h, argc, args)
 		if err != nil {
 			h.sendServerMessage(err.Error())
-			logger.Error(h.user, err)
+			dlog.Server.Error(h.user, err)
 			commandFinished()
 			return
 		}
@@ -320,14 +320,14 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		commandFinished()
 
 	default:
-		h.sendServerMessage(logger.Error(h.user, "Received unknown user command", commandName, argc, args, options))
+		h.sendServerMessage(dlog.Server.Error(h.user, "Received unknown user command", commandName, argc, args, options))
 		commandFinished()
 	}
 }
 
 func (h *ServerHandler) handleAckCommand(argc int, args []string) {
 	if argc < 3 {
-		h.sendServerWarnMessage(logger.Warn(h.user, commandParseWarning, args, argc))
+		h.sendServerWarnMessage(dlog.Server.Warn(h.user, commandParseWarning, args, argc))
 		return
 	}
 	if args[1] == "close" && args[2] == "connection" {
@@ -362,25 +362,25 @@ func (h *ServerHandler) serverMessageC() chan<- string {
 }
 
 func (h *ServerHandler) flushMessages() {
-	logger.Debug(h.user, "flushMessages()")
+	dlog.Server.Debug(h.user, "flushMessages()")
 
 	unsentMessages := func() int {
 		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
 	}
 	for i := 0; i < 3; i++ {
 		if unsentMessages() == 0 {
-			logger.Debug(h.user, "All lines sent")
+			dlog.Server.Debug(h.user, "All lines sent")
 			return
 		}
-		logger.Debug(h.user, "Still lines to be sent")
+		dlog.Server.Debug(h.user, "Still lines to be sent")
 		time.Sleep(time.Second)
 	}
 
-	logger.Warn(h.user, "Some lines remain unsent", unsentMessages())
+	dlog.Server.Warn(h.user, "Some lines remain unsent", unsentMessages())
 }
 
 func (h *ServerHandler) shutdown() {
-	logger.Debug(h.user, "shutdown()")
+	dlog.Server.Debug(h.user, "shutdown()")
 	h.flushMessages()
 
 	go func() {
@@ -393,7 +393,7 @@ func (h *ServerHandler) shutdown() {
 	select {
 	case <-h.ackCloseReceived:
 	case <-time.After(time.Second * 5):
-		logger.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
+		dlog.Server.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
 	case <-h.done.Done():
 	}
 
@@ -410,7 +410,7 @@ func (h *ServerHandler) decrementActiveCommands() int32 {
 }
 
 func readOptions(opts []string) (map[string]string, error) {
-	logger.Debug("Parsing options", opts)
+	dlog.Server.Debug("Parsing options", opts)
 	options := make(map[string]string, len(opts))
 
 	for _, o := range opts {
@@ -430,7 +430,7 @@ func readOptions(opts []string) (map[string]string, error) {
 			val = string(decoded)
 		}
 
-		logger.Debug("Setting option", key, val)
+		dlog.Server.Debug("Setting option", key, val)
 		options[key] = val
 	}
 
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index a1e9e36..f474cc8 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/clients"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -24,7 +24,7 @@ func newScheduler() *scheduler {
 }
 
 func (s *scheduler) start(ctx context.Context) {
-	logger.Info("Starting scheduled job runner after 10s")
+	dlog.Server.Info("Starting scheduled job runner after 10s")
 	// First run after just 10s!
 	time.Sleep(time.Second * 10)
 	s.runJobs(ctx)
@@ -42,18 +42,18 @@ func (s *scheduler) start(ctx context.Context) {
 func (s *scheduler) runJobs(ctx context.Context) {
 	for _, job := range config.Server.Schedule {
 		if !job.Enable {
-			logger.Debug(job.Name, "Not running job as not enabled")
+			dlog.Server.Debug(job.Name, "Not running job as not enabled")
 			continue
 		}
 
 		hour, err := strconv.Atoi(time.Now().Format("15"))
 		if err != nil {
-			logger.Error(job.Name, "Unable to create job", err)
+			dlog.Server.Error(job.Name, "Unable to create job", err)
 			continue
 		}
 
 		if hour < job.TimeRange[0] || hour >= job.TimeRange[1] {
-			logger.Debug(job.Name, "Not running job out of time range")
+			dlog.Server.Debug(job.Name, "Not running job out of time range")
 			continue
 		}
 
@@ -62,7 +62,7 @@ func (s *scheduler) runJobs(ctx context.Context) {
 
 		_, err = os.Stat(outfile)
 		if !os.IsNotExist(err) {
-			logger.Debug(job.Name, "Not running job as outfile already exists", outfile)
+			dlog.Server.Debug(job.Name, "Not running job as outfile already exists", outfile)
 			continue
 		}
 
@@ -71,7 +71,7 @@ func (s *scheduler) runJobs(ctx context.Context) {
 			servers = config.Server.SSHBindAddress
 		}
 
-		args := clients.Args{
+		args := config.Args{
 			ConnectionsPerCPU: 10,
 			Discovery:         job.Discovery,
 			ServersStr:        servers,
@@ -85,21 +85,21 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		query := fmt.Sprintf("%s outfile %s", job.Query, outfile)
 		client, err := clients.NewMaprClient(args, query, clients.CumulativeMode)
 		if err != nil {
-			logger.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
+			dlog.Server.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
 			continue
 		}
 
 		jobCtx, cancel := context.WithCancel(ctx)
 		defer cancel()
 
-		logger.Info(fmt.Sprintf("Starting job %s", job.Name))
+		dlog.Server.Info(fmt.Sprintf("Starting job %s", job.Name))
 		status := client.Start(jobCtx, make(chan string))
 		logMessage := fmt.Sprintf("Job exited with status %d", status)
 
 		if status != 0 {
-			logger.Warn(logMessage)
+			dlog.Server.Warn(logMessage)
 			continue
 		}
-		logger.Info(logMessage)
+		dlog.Server.Info(logMessage)
 	}
 }
diff --git a/internal/server/server.go b/internal/server/server.go
index a20737e..a8f541b 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/server/handlers"
 	"github.com/mimecast/dtail/internal/ssh/server"
 	user "github.com/mimecast/dtail/internal/user/server"
@@ -36,7 +36,7 @@ type Server struct {
 
 // New returns a new server.
 func New() *Server {
-	logger.Info("Creating server", version.String())
+	dlog.Server.Info("Creating server", version.String())
 
 	s := Server{
 		sshServerConfig: &gossh.ServerConfig{},
@@ -51,7 +51,7 @@ func New() *Server {
 
 	private, err := gossh.ParsePrivateKey(server.PrivateHostKey())
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Server.FatalPanic(err)
 	}
 	s.sshServerConfig.AddHostKey(private)
 
@@ -60,14 +60,14 @@ func New() *Server {
 
 // Start the server.
 func (s *Server) Start(ctx context.Context) int {
-	logger.Info("Starting server")
+	dlog.Server.Info("Starting server")
 
 	bindAt := fmt.Sprintf("%s:%d", config.Server.SSHBindAddress, config.Common.SSHPort)
-	logger.Info("Binding server", bindAt)
+	dlog.Server.Info("Binding server", bindAt)
 
 	listener, err := net.Listen("tcp", bindAt)
 	if err != nil {
-		logger.FatalExit("Failed to open listening TCP socket", err)
+		dlog.Server.FatalPanic("Failed to open listening TCP socket", err)
 	}
 
 	go s.stats.start(ctx)
@@ -82,7 +82,7 @@ func (s *Server) Start(ctx context.Context) int {
 }
 
 func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
-	logger.Debug("Starting listener loop")
+	dlog.Server.Debug("Starting listener loop")
 
 	for {
 		conn, err := listener.Accept() // Blocking
@@ -92,12 +92,12 @@ func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 				return
 			default:
 			}
-			logger.Error("Failed to accept incoming connection", err)
+			dlog.Server.Error("Failed to accept incoming connection", err)
 			continue
 		}
 
 		if err := s.stats.serverLimitExceeded(); err != nil {
-			logger.Error(err)
+			dlog.Server.Error(err)
 			conn.Close()
 			continue
 		}
@@ -107,11 +107,11 @@ func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 }
 
 func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
-	logger.Info("Handling connection")
+	dlog.Server.Info("Handling connection")
 
 	sshConn, chans, reqs, err := gossh.NewServerConn(conn, s.sshServerConfig)
 	if err != nil {
-		logger.Error("Something just happened", err)
+		dlog.Server.Error("Something just happened", err)
 		return
 	}
 
@@ -125,29 +125,29 @@ func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
 
 func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChannel gossh.NewChannel) {
 	user := user.New(sshConn.User(), sshConn.RemoteAddr().String())
-	logger.Info(user, "Invoking channel handler")
+	dlog.Server.Info(user, "Invoking channel handler")
 
 	if newChannel.ChannelType() != "session" {
 		err := errors.New("Don'w allow other channel types than session")
-		logger.Error(user, err)
+		dlog.Server.Error(user, err)
 		newChannel.Reject(gossh.Prohibited, err.Error())
 		return
 	}
 
 	channel, requests, err := newChannel.Accept()
 	if err != nil {
-		logger.Error(user, "Could not accept channel", err)
+		dlog.Server.Error(user, "Could not accept channel", err)
 		return
 	}
 
 	if err := s.handleRequests(ctx, sshConn, requests, channel, user); err != nil {
-		logger.Error(user, err)
+		dlog.Server.Error(user, err)
 		sshConn.Close()
 	}
 }
 
 func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-chan *gossh.Request, channel gossh.Channel, user *user.User) error {
-	logger.Info(user, "Invoking request handler")
+	dlog.Server.Info(user, "Invoking request handler")
 
 	for req := range in {
 		var payload = struct{ Value string }{}
@@ -190,10 +190,10 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 
 			go func() {
 				if err := sshConn.Wait(); err != nil && err != io.EOF {
-					logger.Error(user, err)
+					dlog.Server.Error(user, err)
 				}
 				s.stats.decrementConnections()
-				logger.Info(user, "Good bye Mister!")
+				dlog.Server.Info(user, "Good bye Mister!")
 				terminate()
 			}()
 
@@ -216,7 +216,7 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	user := user.New(c.User(), c.RemoteAddr().String())
 
 	if config.ServerRelaxedAuthEnable {
-		logger.Fatal(user, "Granting permissions via relaxed-auth")
+		dlog.Server.Fatal(user, "Granting permissions via relaxed-auth")
 		return nil, nil
 	}
 
@@ -228,20 +228,20 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	switch user.Name {
 	case config.ControlUser:
 		if authInfo == config.ControlUser {
-			logger.Debug(user, "Granting permissions to control user")
+			dlog.Server.Debug(user, "Granting permissions to control user")
 			return nil, nil
 		}
 	case config.ScheduleUser:
 		for _, job := range config.Server.Schedule {
 			if s.backgroundCanSSH(user, authInfo, remoteIP, job.Name, job.AllowFrom) {
-				logger.Debug(user, "Granting SSH connection")
+				dlog.Server.Debug(user, "Granting SSH connection")
 				return nil, nil
 			}
 		}
 	case config.ContinuousUser:
 		for _, job := range config.Server.Continuous {
 			if s.backgroundCanSSH(user, authInfo, remoteIP, job.Name, job.AllowFrom) {
-				logger.Debug(user, "Granting SSH connection")
+				dlog.Server.Debug(user, "Granting SSH connection")
 				return nil, nil
 			}
 		}
@@ -252,22 +252,22 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 }
 
 func (s *Server) backgroundCanSSH(user *user.User, jobName, remoteIP, allowedJobName string, allowFrom []string) bool {
-	logger.Debug("backgroundCanSSH", user, jobName, remoteIP, allowedJobName, allowFrom)
+	dlog.Server.Debug("backgroundCanSSH", user, jobName, remoteIP, allowedJobName, allowFrom)
 
 	if jobName != allowedJobName {
-		logger.Debug(user, jobName, "backgroundCanSSH", "Job name does not match, skipping to next one...", allowedJobName)
+		dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Job name does not match, skipping to next one...", allowedJobName)
 		return false
 	}
 
 	for _, myAddr := range allowFrom {
 		ips, err := net.LookupIP(myAddr)
 		if err != nil {
-			logger.Debug(user, jobName, "backgroundCanSSH", "Unable to lookup IP address for allowed hosts lookup, skipping to next one...", myAddr, err)
+			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Unable to lookup IP address for allowed hosts lookup, skipping to next one...", myAddr, err)
 			continue
 		}
 
 		for _, ip := range ips {
-			logger.Debug(user, jobName, "backgroundCanSSH", "Comparing IP addresses", remoteIP, ip.String())
+			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Comparing IP addresses", remoteIP, ip.String())
 			if remoteIP == ip.String() {
 				return true
 			}
diff --git a/internal/server/stats.go b/internal/server/stats.go
index 3e8c71d..8583318 100644
--- a/internal/server/stats.go
+++ b/internal/server/stats.go
@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // Used to collect and display various server stats.
@@ -41,7 +41,7 @@ func (s *stats) hasConnections() bool {
 	s.mutex.Unlock()
 
 	has := currentConnections > 0
-	logger.Info("stats", "Server with open connections?", has, currentConnections)
+	dlog.Server.Info("stats", "Server with open connections?", has, currentConnections)
 
 	return has
 }
@@ -57,7 +57,7 @@ func (s *stats) logServerStats() {
 	data["cgocalls"] = runtime.NumCgoCall()
 	data["cpu"] = runtime.NumCPU()
 
-	logger.Mapreduce("STATS", data)
+	dlog.Server.Mapreduce("STATS", data)
 }
 
 func (s *stats) serverLimitExceeded() error {
diff --git a/internal/ssh/client/authmethods.go b/internal/ssh/client/authmethods.go
index 2ff80b2..4508319 100644
--- a/internal/ssh/client/authmethods.go
+++ b/internal/ssh/client/authmethods.go
@@ -4,7 +4,7 @@ import (
 	"os"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/ssh"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -15,7 +15,7 @@ func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod, hostKeyCallback gossh
 	if len(sshAuthMethods) > 0 {
 		simpleCallback, err := NewSimpleCallback()
 		if err != nil {
-			logger.FatalExit(err)
+			dlog.Common.FatalPanic(err)
 		}
 		return sshAuthMethods, simpleCallback
 	}
@@ -29,13 +29,13 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, pri
 	knownHostsPath := os.Getenv("HOME") + "/.ssh/known_hosts"
 	knownHostsCallback, err := NewKnownHostsCallback(knownHostsPath, trustAllHosts, throttleCh)
 	if err != nil {
-		logger.FatalExit(knownHostsPath, err)
+		dlog.Common.FatalPanic(knownHostsPath, err)
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
 
 	if config.Common.ExperimentalFeaturesEnable {
 		sshAuthMethods = append(sshAuthMethods, gossh.Password("experimental feature test"))
-		logger.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
+		dlog.Common.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
 	}
 
 	// First try to read custom private key path.
@@ -43,41 +43,41 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, pri
 		authMethod, err := ssh.PrivateKey(privateKeyPath)
 		if err == nil {
 			sshAuthMethods = append(sshAuthMethods, authMethod)
-			logger.Debug("initKnownHostsAuthMethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+			dlog.Common.Debug("initKnownHostsAuthMethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
 			return sshAuthMethods, knownHostsCallback
 		}
-		logger.FatalExit("Unable to use private SSH key", privateKeyPath, err)
+		dlog.Common.FatalPanic("Unable to use private SSH key", privateKeyPath, err)
 	}
 
 	// Second, try SSH Agent
 	authMethod, err := ssh.Agent()
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK) to list of auth methods, not adding further methods")
+		dlog.Common.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK) to list of auth methods, not adding further methods")
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err)
 
 	// Third, try Linux/UNIX default key paths
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_rsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Common.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
 
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_dsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Common.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
 
-	logger.FatalExit("Unable to find private SSH key information")
+	dlog.Common.FatalPanic("Unable to find private SSH key information")
 
 	// Never reach this point.
 	return sshAuthMethods, knownHostsCallback
diff --git a/internal/ssh/client/knownhostscallback.go b/internal/ssh/client/knownhostscallback.go
index 1ccf6c6..a73d612 100644
--- a/internal/ssh/client/knownhostscallback.go
+++ b/internal/ssh/client/knownhostscallback.go
@@ -10,7 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/prompt"
 
 	"golang.org/x/crypto/ssh"
@@ -97,7 +97,7 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 			responseCh: make(chan response),
 		}
 
-		logger.Warn("Encountered unknown host", unknown)
+		dlog.Common.Warn("Encountered unknown host", unknown)
 		// Notify user that there is an unknown host
 		c.unknownCh <- unknown
 
@@ -139,7 +139,7 @@ func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 				hosts = []unknownHost{}
 			}
 		case <-ctx.Done():
-			logger.Debug("Stopping goroutine prompting new hosts...")
+			dlog.Common.Debug("Stopping goroutine prompting new hosts...")
 			return
 		}
 	}
@@ -154,7 +154,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 
 	select {
 	case <-c.trustAllHostsCh:
-		logger.Warn("Trusting host keys of servers", servers)
+		dlog.Common.Warn("Trusting host keys of servers", servers)
 		c.trustHosts(hosts)
 		return
 	default:
@@ -175,7 +175,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.trustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Added hosts to known hosts file", c.knownHostsPath)
+			dlog.Common.Info("Added hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
@@ -188,7 +188,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.trustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Added hosts to known hosts file", c.knownHostsPath)
+			dlog.Common.Info("Added hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
@@ -200,7 +200,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.dontTrustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Didn't add hosts to known hosts file", c.knownHostsPath)
+			dlog.Common.Info("Didn't add hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
diff --git a/internal/ssh/server/hostkey.go b/internal/ssh/server/hostkey.go
index 07790ad..20de1f0 100644
--- a/internal/ssh/server/hostkey.go
+++ b/internal/ssh/server/hostkey.go
@@ -1,11 +1,12 @@
 package server
 
 import (
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/ssh"
 	"io/ioutil"
 	"os"
+
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/ssh"
 )
 
 // PrivateHostKey retrieves the private server RSA host key.
@@ -14,24 +15,24 @@ func PrivateHostKey() []byte {
 	_, err := os.Stat(hostKeyFile)
 
 	if os.IsNotExist(err) {
-		logger.Info("Generating private server RSA host key")
+		dlog.Common.Info("Generating private server RSA host key")
 		privateKey, err := ssh.GeneratePrivateRSAKey(config.Server.HostKeyBits)
 
 		if err != nil {
-			logger.FatalExit("Failed to generate private server RSA host key", err)
+			dlog.Common.FatalPanic("Failed to generate private server RSA host key", err)
 		}
 
 		pem := ssh.EncodePrivateKeyToPEM(privateKey)
 		if err := ioutil.WriteFile(hostKeyFile, pem, 0600); err != nil {
-			logger.Error("Unable to write private server RSA host key to file", hostKeyFile, err)
+			dlog.Common.Error("Unable to write private server RSA host key to file", hostKeyFile, err)
 		}
 		return pem
 	}
 
-	logger.Info("Reading private server RSA host key from file", hostKeyFile)
+	dlog.Common.Info("Reading private server RSA host key from file", hostKeyFile)
 	pem, err := ioutil.ReadFile(hostKeyFile)
 	if err != nil {
-		logger.FatalExit("Failed to load private server RSA host key", err)
+		dlog.Common.FatalPanic("Failed to load private server RSA host key", err)
 	}
 	return pem
 }
diff --git a/internal/ssh/server/publickeycallback.go b/internal/ssh/server/publickeycallback.go
index e81f019..65ecdd1 100644
--- a/internal/ssh/server/publickeycallback.go
+++ b/internal/ssh/server/publickeycallback.go
@@ -7,7 +7,7 @@ import (
 	osUser "os/user"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	user "github.com/mimecast/dtail/internal/user/server"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -16,7 +16,7 @@ import (
 // PublicKeyCallback is for the server to check whether a public SSH key is authorized ot not.
 func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
 	user := user.New(c.User(), c.RemoteAddr().String())
-	logger.Info(user, "Incoming authorization")
+	dlog.Common.Info(user, "Incoming authorization")
 
 	cwd, err := os.Getwd()
 	if err != nil {
@@ -24,7 +24,7 @@ func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*go
 	}
 
 	if config.ServerRelaxedAuthEnable {
-		logger.Fatal(user, "Granting permissions via relaxed-auth")
+		dlog.Common.Fatal(user, "Granting permissions via relaxed-auth")
 		return nil, nil
 	}
 
@@ -38,7 +38,7 @@ func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*go
 		authorizedKeysFile = user.HomeDir + "/.ssh/authorized_keys"
 	}
 
-	logger.Info(user, "Reading", authorizedKeysFile)
+	dlog.Common.Info(user, "Reading", authorizedKeysFile)
 	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysFile)
 	if err != nil {
 		return nil, fmt.Errorf("Unable to read authorized keys file|%s|%s|%s", authorizedKeysFile, user, err.Error())
@@ -53,10 +53,10 @@ func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*go
 		authorizedKeysMap[string(authorizedPubKey.Marshal())] = true
 		authorizedKeysBytes = restBytes
 
-		logger.Debug(user, "Authorized public key fingerprint", gossh.FingerprintSHA256(authorizedPubKey))
+		dlog.Common.Debug(user, "Authorized public key fingerprint", gossh.FingerprintSHA256(authorizedPubKey))
 	}
 
-	logger.Debug(user, "Offered public key fingerprint", gossh.FingerprintSHA256(offeredPubKey))
+	dlog.Common.Debug(user, "Offered public key fingerprint", gossh.FingerprintSHA256(offeredPubKey))
 
 	if authorizedKeysMap[string(offeredPubKey.Marshal())] {
 		return &gossh.Permissions{
diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go
index 78bf99e..56494a7 100644
--- a/internal/ssh/ssh.go
+++ b/internal/ssh/ssh.go
@@ -11,7 +11,7 @@ import (
 	"os"
 	"syscall"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 
 	gossh "golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
@@ -58,7 +58,7 @@ func Agent() (gossh.AuthMethod, error) {
 		return nil, err
 	}
 	for i, key := range keys {
-		logger.Debug("Public key", i, key)
+		dlog.Common.Debug("Public key", i, key)
 	}
 	return gossh.PublicKeysCallback(agentClient.Signers), nil
 }
@@ -106,7 +106,7 @@ func KeyFile(keyFile string) (gossh.AuthMethod, error) {
 func PrivateKey(keyFile string) (gossh.AuthMethod, error) {
 	signer, err := KeyFile(keyFile)
 	if err != nil {
-		logger.Debug(keyFile, err)
+		dlog.Common.Debug(keyFile, err)
 		return nil, err
 	}
 	return gossh.AuthMethod(signer), nil
diff --git a/internal/user/server/user.go b/internal/user/server/user.go
index 637945c..99cd211 100644
--- a/internal/user/server/user.go
+++ b/internal/user/server/user.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/fs/permissions"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 const maxLinkDepth int = 100
@@ -39,9 +39,9 @@ func (u *User) String() string {
 
 // HasFilePermission is used to determine whether user is alowed to read a file.
 func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission bool) {
-	logger.Debug(u, filePath, permissionType, "Checking config permissions")
+	dlog.Common.Debug(u, filePath, permissionType, "Checking config permissions")
 	if config.ServerRelaxedAuthEnable {
-		logger.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
+		dlog.Common.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
 		return true
 	}
 
@@ -52,25 +52,25 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 
 	cleanPath, err := filepath.EvalSymlinks(filePath)
 	if err != nil {
-		logger.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
+		dlog.Common.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
 		hasPermission = false
 		return
 	}
 
 	cleanPath, err = filepath.Abs(cleanPath)
 	if err != nil {
-		logger.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
+		dlog.Common.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
 		hasPermission = false
 		return
 	}
 
 	if cleanPath != filePath {
-		logger.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
+		dlog.Common.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
 	}
 
 	hasPermission, err = u.hasFilePermission(cleanPath, permissionType)
 	if err != nil {
-		logger.Warn(u, cleanPath, err)
+		dlog.Common.Warn(u, cleanPath, err)
 	}
 
 	return
@@ -81,7 +81,7 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 	if _, err := permissions.ToRead(u.Name, cleanPath); err != nil {
 		return false, fmt.Errorf("User without OS file system permissions to read path: '%v'", err)
 	}
-	logger.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
+	dlog.Common.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
 
 	// Only allow to follow regular files or symlinks.
 	info, err := os.Lstat(cleanPath)
@@ -123,7 +123,7 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 			permission = strings.Join(splitted[1:], ":")
 		}
 
-		logger.Debug(u, cleanPath, typeStr, permission)
+		dlog.Common.Debug(u, cleanPath, typeStr, permission)
 
 		if typeStr != permissionType {
 			continue
@@ -141,12 +141,12 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 		}
 
 		if negate && re.MatchString(cleanPath) {
-			logger.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
+			dlog.Common.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
 			hasPermission = false
 		}
 
 		if !negate && re.MatchString(cleanPath) {
-			logger.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
+			dlog.Common.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
 			hasPermission = true
 		}
 	}
-- 
cgit v1.2.3


From a4d25626414ee36f937badd13e164aaf271c65d5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 26 Sep 2021 16:42:47 +0300
Subject: refactor config reader - also looks in additional search paths for
 config file unless NONE is specified

---
 internal/clients/connectors/serverless.go |  7 +++-
 internal/clients/handlers/basehandler.go  |  2 +-
 internal/config/args.go                   | 25 ++++++++++---
 internal/config/common.go                 |  2 +-
 internal/config/config.go                 | 61 +++++++++++++++++++++----------
 internal/config/setup.go                  | 32 ++++------------
 internal/io/dlog/dlog.go                  | 37 ++++++++++++++-----
 internal/io/logger/logger.go              |  2 -
 internal/mapr/logformat/default.go        |  2 +-
 internal/server/continuous.go             |  2 +-
 internal/server/handlers/readcommand.go   |  2 +-
 internal/server/scheduler.go              |  2 +-
 internal/server/server.go                 | 12 +++++-
 internal/ssh/server/publickeycallback.go  |  5 ++-
 internal/user/server/user.go              | 37 +++++++++----------
 15 files changed, 140 insertions(+), 90 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index c7b5f62..7740aab 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -53,8 +53,13 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) error {
 	dlog.Client.Debug("Creating server handler for a serverless session")
 
+	user, err := user.New(s.userName, s.Server())
+	if err != nil {
+		return err
+	}
+
 	serverHandler := serverHandlers.NewServerHandler(
-		user.New(s.userName, s.Server()),
+		user,
 		make(chan struct{}, config.Server.MaxConcurrentCats),
 		make(chan struct{}, config.Server.MaxConcurrentTails),
 	)
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 3291b43..8acb45f 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -69,7 +69,7 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		switch b {
 		/*
-			// TODO: Next DTail version make it so that '\n' gets ignored. For now
+			// NEXT: Next DTail version make it so that '\n' gets ignored. For now
 			// leave it for compatibility with older DTail server + ability to display
 			// the protocol mismatch warn message.
 			case '\n' {
diff --git a/internal/config/args.go b/internal/config/args.go
index 89e4bc9..767cc65 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -16,6 +16,7 @@ type Args struct {
 	ConfigFile         string
 	ConnectionsPerCPU  int
 	Discovery          string
+	LogDir             string
 	LogLevel           string
 	Mode               omode.Mode
 	NoColor            bool
@@ -39,6 +40,8 @@ func (a *Args) String() string {
 	var sb strings.Builder
 
 	sb.WriteString("Args(")
+	// TODO: All commands should make use of this
+	sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir))
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile))
@@ -66,16 +69,24 @@ func (a *Args) String() string {
 }
 
 // Based on the argument list, transform/manipulate some of the arguments.
-func (a *Args) transform(args []string) {
+func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
+	if a.LogDir != "" {
+		common.LogDir = a.LogDir
+		if common.LogStrategy == "" {
+			// TODO: Implement the other (not-daily) log strategy for the server.
+			common.LogStrategy = "daily"
+		}
+	}
+
 	if a.LogLevel != "" {
-		Common.LogLevel = a.LogLevel
+		common.LogLevel = a.LogLevel
 	}
 
-	if a.SSHPort != 2222 {
-		Common.SSHPort = a.SSHPort
+	if a.SSHPort != DefaultSSHPort {
+		common.SSHPort = a.SSHPort
 	}
 	if a.NoColor {
-		Client.TermColorsEnable = false
+		client.TermColorsEnable = false
 	}
 
 	if a.Spartan {
@@ -95,6 +106,8 @@ func (a *Args) transform(args []string) {
 		}
 		a.What = strings.Join(files, ",")
 	}
+
+	return client, server, common
 }
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
@@ -102,4 +115,4 @@ func (a *Args) SerializeOptions() string {
 	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
 }
 
-// TODO: Put the DeseializeOptions function here (move it away from the internal/server package)
+// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package)
diff --git a/internal/config/common.go b/internal/config/common.go
index 7d45261..acc8e6a 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -23,7 +23,7 @@ type CommonConfig struct {
 // Create a new default configuration.
 func newDefaultCommonConfig() *CommonConfig {
 	return &CommonConfig{
-		SSHPort:                    2222,
+		SSHPort:                    DefaultSSHPort,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
 		CacheDir:                   "cache",
diff --git a/internal/config/config.go b/internal/config/config.go
index 2d77041..c9f411c 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -2,24 +2,26 @@ package config
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 )
 
-// ControlUser is used for various DTail specific operations.
-const ControlUser string = "DTAIL-CONTROL"
-
-// ScheduleUser is used for non-interactive scheduled mapreduce queries.
-const ScheduleUser string = "DTAIL-SCHEDULE"
-
-// ContinuousUser is used for non-interactive continuous mapreduce queries.
-const ContinuousUser string = "DTAIL-CONTINUOUS"
-
-// TestUser is used for unit tests and potentially also for integration tests.
-const TestUser string = "DTAIL-TEST"
-
-// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
-const InterruptTimeoutS int = 3
+const (
+	// ControlUser is used for various DTail specific operations.
+	ControlUser string = "DTAIL-CONTROL"
+	// ScheduleUser is used for non-interactive scheduled mapreduce queries.
+	ScheduleUser string = "DTAIL-SCHEDULE"
+	// ContinuousUser is used for non-interactive continuous mapreduce queries.
+	ContinuousUser string = "DTAIL-CONTINUOUS"
+	// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
+	InterruptTimeoutS int = 3
+	// ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start)
+	DefaultConnectionsPerCPU int = 10
+	// DTailSSHServerDefaultPort is the default DServer port.
+	DefaultSSHPort int = 2222
+)
 
 // Client holds a DTail client configuration.
 var Client *ClientConfig
@@ -37,21 +39,42 @@ type configInitializer struct {
 	Client *ClientConfig
 }
 
-// Parse and read a given config file in JSON format.
-func (c *configInitializer) parseConfig(configFile string) {
+func (c *configInitializer) parseConfig(args *Args) {
+	if strings.ToUpper(args.ConfigFile) == "NONE" {
+		return
+	}
+
+	if args.ConfigFile != "" {
+		c.parseSpecificConfig(args.ConfigFile)
+		return
+	}
+
+	if homeDir, err := os.UserHomeDir(); err != nil {
+		var paths []string
+		paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir))
+		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
+		for _, configPath := range paths {
+			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
+				c.parseSpecificConfig(configPath)
+			}
+		}
+	}
+}
+
+func (c *configInitializer) parseSpecificConfig(configFile string) {
 	fd, err := os.Open(configFile)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to read config file: %v", err))
 	}
 	defer fd.Close()
 
 	cfgBytes, err := ioutil.ReadAll(fd)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
 	}
 
 	err = json.Unmarshal([]byte(cfgBytes), c)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
 	}
 }
diff --git a/internal/config/setup.go b/internal/config/setup.go
index 3c4bcc4..be8e867 100644
--- a/internal/config/setup.go
+++ b/internal/config/setup.go
@@ -1,11 +1,5 @@
 package config
 
-import (
-	"os"
-)
-
-const NoConfigFile string = "Don't read a config file - use defaults only"
-
 // Setup the DTail configuration.
 func Setup(args *Args, additionalArgs []string) {
 	initializer := configInitializer{
@@ -13,23 +7,11 @@ func Setup(args *Args, additionalArgs []string) {
 		Server: newDefaultServerConfig(),
 		Client: newDefaultClientConfig(),
 	}
-
-	if args.ConfigFile == "" {
-		// TODO: Search more paths for config file (e.g. in /etc and in ~/.config/...
-		args.ConfigFile = "./cfg/dtail.json"
-	}
-
-	if args.ConfigFile != NoConfigFile {
-		if _, err := os.Stat(args.ConfigFile); !os.IsNotExist(err) {
-			initializer.parseConfig(args.ConfigFile)
-		}
-	}
-
-	// Assign pointers to global variables, so that we can access the
-	// configuration from any place of the program.
-	Common = initializer.Common
-	Server = initializer.Server
-	Client = initializer.Client
-
-	args.transform(additionalArgs)
+	initializer.parseConfig(args)
+	Client, Server, Common = args.transformConfig(
+		additionalArgs,
+		initializer.Client,
+		initializer.Server,
+		initializer.Common,
+	)
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 7282741..49b405d 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -3,6 +3,7 @@ package dlog
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -21,7 +22,6 @@ var Client *DLog
 var Server *DLog
 
 // Common is the log handler for all other packages.
-// TODO: Rename Common to Common
 var Common *DLog
 
 var mutex sync.Mutex
@@ -75,15 +75,22 @@ type DLog struct {
 	sourcePackage source
 	// Max log level to log.
 	maxLevel level
+	// Current hostname.
+	hostname string
 }
 
 // New creates a new DTail logger.
 func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+	hostname, err := os.Hostname()
+	if err != nil {
+		panic(err)
+	}
 	return &DLog{
 		logger:        loggers.Factory(sourceProcess.String(), impl),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      maxLevel,
+		hostname:      hostname,
 	}
 }
 
@@ -106,11 +113,18 @@ func (d *DLog) log(level level, args []interface{}) string {
 	defer pool.RecycleBuilderBuffer(sb)
 	now := time.Now()
 
-	sb.WriteString(d.sourcePackage.String())
-	sb.WriteString(protocol.FieldDelimiter)
-	sb.WriteString(now.Format("20060102-150405"))
-	sb.WriteString(protocol.FieldDelimiter)
-	sb.WriteString(level.String())
+	switch d.sourceProcess {
+	case CLIENT:
+		sb.WriteString(d.sourcePackage.String())
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(d.hostname)
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(level.String())
+	default:
+		sb.WriteString(level.String())
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(now.Format("20060102-150405"))
+	}
 	sb.WriteString(protocol.FieldDelimiter)
 	d.writeArgStrings(sb, args)
 
@@ -159,6 +173,11 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
+	if d.sourcePackage == SERVER && d.sourceProcess != CLIENT {
+		// This can be dtail client in serverless mode. In this case log all
+		// info server messages as verbose.
+		return d.log(VERBOSE, args)
+	}
 	return d.log(INFO, args)
 }
 
@@ -183,13 +202,14 @@ func (d *DLog) Raw(message string) string {
 		d.logger.Log(time.Now(), message)
 		return message
 	}
-
-	d.logger.Log(time.Now(), brush.Colorfy(message))
+	d.logger.LogWithColors(time.Now(), message, brush.Colorfy(message))
 	return message
 }
 
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
+
+	// TODO: mC compatible SERVER mapreduce fields, no MAPREDUCE keyword in CLIENT mode
 	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
 
 	i := 1
@@ -197,7 +217,6 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 		args[i] = fmt.Sprintf("%s=%v", k, v)
 		i++
 	}
-
 	return d.log(INFO, args)
 }
 
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 6a6b5ec..905d1cf 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -1,7 +1,5 @@
 package logger
 
-// TODO: Rewrite this logger
-
 import (
 	"bufio"
 	"context"
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index e0bbc30..7fb1700 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -27,7 +27,7 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 
 	fields["$severity"] = splitted[0]
 	fields["$loglevel"] = splitted[0]
-	// TODO: Parse time like we do at Mimecast
+	// NEXT: Parse time like we do at Mimecast
 	fields["$time"] = splitted[1]
 
 	for _, kv := range splitted[4:] {
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index 5f4c454..87c8889 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -61,7 +61,7 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	}
 
 	args := config.Args{
-		ConnectionsPerCPU: 10,
+		ConnectionsPerCPU: config.DefaultConnectionsPerCPU,
 		Discovery:         job.Discovery,
 		ServersStr:        servers,
 		What:              files,
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 60ad2a0..c76ae2a 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -7,9 +7,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/fs"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 )
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index f474cc8..64e6573 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -72,7 +72,7 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		}
 
 		args := config.Args{
-			ConnectionsPerCPU: 10,
+			ConnectionsPerCPU: config.DefaultConnectionsPerCPU,
 			Discovery:         job.Discovery,
 			ServersStr:        servers,
 			What:              files,
diff --git a/internal/server/server.go b/internal/server/server.go
index a8f541b..d1cd57d 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -124,7 +124,12 @@ func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
 }
 
 func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChannel gossh.NewChannel) {
-	user := user.New(sshConn.User(), sshConn.RemoteAddr().String())
+	user, err := user.New(sshConn.User(), sshConn.RemoteAddr().String())
+	if err != nil {
+		dlog.Server.Error(user, err)
+		newChannel.Reject(gossh.Prohibited, err.Error())
+		return
+	}
 	dlog.Server.Info(user, "Invoking channel handler")
 
 	if newChannel.ChannelType() != "session" {
@@ -213,7 +218,10 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 
 // Callback for SSH authentication.
 func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Permissions, error) {
-	user := user.New(c.User(), c.RemoteAddr().String())
+	user, err := user.New(c.User(), c.RemoteAddr().String())
+	if err != nil {
+		return nil, err
+	}
 
 	if config.ServerRelaxedAuthEnable {
 		dlog.Server.Fatal(user, "Granting permissions via relaxed-auth")
diff --git a/internal/ssh/server/publickeycallback.go b/internal/ssh/server/publickeycallback.go
index 65ecdd1..59d1f31 100644
--- a/internal/ssh/server/publickeycallback.go
+++ b/internal/ssh/server/publickeycallback.go
@@ -15,7 +15,10 @@ import (
 
 // PublicKeyCallback is for the server to check whether a public SSH key is authorized ot not.
 func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
-	user := user.New(c.User(), c.RemoteAddr().String())
+	user, err := user.New(c.User(), c.RemoteAddr().String())
+	if err != nil {
+		return nil, err
+	}
 	dlog.Common.Info(user, "Incoming authorization")
 
 	cwd, err := os.Getwd()
diff --git a/internal/user/server/user.go b/internal/user/server/user.go
index 99cd211..70ead1c 100644
--- a/internal/user/server/user.go
+++ b/internal/user/server/user.go
@@ -8,8 +8,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/fs/permissions"
 	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/fs/permissions"
 )
 
 const maxLinkDepth int = 100
@@ -25,11 +25,16 @@ type User struct {
 }
 
 // New returns a new user.
-func New(name, remoteAddress string) *User {
+func New(name, remoteAddress string) (*User, error) {
+	permissions, err := config.ServerUserPermissions(name)
+	if err != nil {
+		return nil, err
+	}
 	return &User{
 		Name:          name,
 		remoteAddress: remoteAddress,
-	}
+		permissions:   permissions,
+	}, nil
 }
 
 // String representation of the user.
@@ -39,9 +44,9 @@ func (u *User) String() string {
 
 // HasFilePermission is used to determine whether user is alowed to read a file.
 func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission bool) {
-	dlog.Common.Debug(u, filePath, permissionType, "Checking config permissions")
+	dlog.Server.Debug(u, filePath, permissionType, "Checking config permissions")
 	if config.ServerRelaxedAuthEnable {
-		dlog.Common.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
+		dlog.Server.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
 		return true
 	}
 
@@ -52,25 +57,25 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 
 	cleanPath, err := filepath.EvalSymlinks(filePath)
 	if err != nil {
-		dlog.Common.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
+		dlog.Server.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
 		hasPermission = false
 		return
 	}
 
 	cleanPath, err = filepath.Abs(cleanPath)
 	if err != nil {
-		dlog.Common.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
+		dlog.Server.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
 		hasPermission = false
 		return
 	}
 
 	if cleanPath != filePath {
-		dlog.Common.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
+		dlog.Server.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
 	}
 
 	hasPermission, err = u.hasFilePermission(cleanPath, permissionType)
 	if err != nil {
-		dlog.Common.Warn(u, cleanPath, err)
+		dlog.Server.Warn(u, cleanPath, err)
 	}
 
 	return
@@ -81,7 +86,7 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 	if _, err := permissions.ToRead(u.Name, cleanPath); err != nil {
 		return false, fmt.Errorf("User without OS file system permissions to read path: '%v'", err)
 	}
-	dlog.Common.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
+	dlog.Server.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
 
 	// Only allow to follow regular files or symlinks.
 	info, err := os.Lstat(cleanPath)
@@ -93,12 +98,6 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 		return false, fmt.Errorf("Can only open regular files or follow symlinks")
 	}
 
-	permissions, err := config.ServerUserPermissions(u.Name)
-	if err != nil {
-		return false, err
-	}
-	u.permissions = permissions
-
 	hasPermission, err := u.iteratePaths(cleanPath, permissionType)
 	if err != nil {
 		return false, err
@@ -123,7 +122,7 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 			permission = strings.Join(splitted[1:], ":")
 		}
 
-		dlog.Common.Debug(u, cleanPath, typeStr, permission)
+		dlog.Server.Debug(u, cleanPath, typeStr, permission)
 
 		if typeStr != permissionType {
 			continue
@@ -141,12 +140,12 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 		}
 
 		if negate && re.MatchString(cleanPath) {
-			dlog.Common.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
 			hasPermission = false
 		}
 
 		if !negate && re.MatchString(cleanPath) {
-			dlog.Common.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
 			hasPermission = true
 		}
 	}
-- 
cgit v1.2.3


From 81b829e4e9f5ca0ac1df71c17f709af213b837ed Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 28 Sep 2021 21:11:50 +0300
Subject: can have daily and normal file log rotation

---
 internal/config/args.go             |  43 ----
 internal/config/config.go           |  43 ++++
 internal/config/setup.go            |   4 +-
 internal/io/dlog/dlog.go            |  14 +-
 internal/io/dlog/loggers/factory.go |   8 +-
 internal/io/dlog/loggers/file.go    | 117 +++++-----
 internal/io/dlog/loggers/fout.go    |   4 +-
 internal/io/dlog/strategy.go        |  22 --
 internal/io/logger/logger.go        | 443 ------------------------------------
 internal/io/logger/modes.go         |  13 --
 internal/io/logger/strategy.go      |  22 --
 11 files changed, 123 insertions(+), 610 deletions(-)
 delete mode 100644 internal/io/dlog/strategy.go
 delete mode 100644 internal/io/logger/logger.go
 delete mode 100644 internal/io/logger/modes.go
 delete mode 100644 internal/io/logger/strategy.go

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index 767cc65..7f24348 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -1,7 +1,6 @@
 package config
 
 import (
-	"flag"
 	"fmt"
 	"strings"
 
@@ -68,48 +67,6 @@ func (a *Args) String() string {
 	return sb.String()
 }
 
-// Based on the argument list, transform/manipulate some of the arguments.
-func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
-	if a.LogDir != "" {
-		common.LogDir = a.LogDir
-		if common.LogStrategy == "" {
-			// TODO: Implement the other (not-daily) log strategy for the server.
-			common.LogStrategy = "daily"
-		}
-	}
-
-	if a.LogLevel != "" {
-		common.LogLevel = a.LogLevel
-	}
-
-	if a.SSHPort != DefaultSSHPort {
-		common.SSHPort = a.SSHPort
-	}
-	if a.NoColor {
-		client.TermColorsEnable = false
-	}
-
-	if a.Spartan {
-		a.Quiet = true
-		a.NoColor = true
-	}
-
-	if a.Discovery == "" && a.ServersStr == "" {
-		a.Serverless = true
-	}
-
-	// Interpret additional args as file list.
-	if a.What == "" {
-		var files []string
-		for _, file := range flag.Args() {
-			files = append(files, file)
-		}
-		a.What = strings.Join(files, ",")
-	}
-
-	return client, server, common
-}
-
 // SerializeOptions returns a string ready to be sent over the wire to the server.
 func (a *Args) SerializeOptions() string {
 	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
diff --git a/internal/config/config.go b/internal/config/config.go
index c9f411c..76dcc65 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -2,6 +2,7 @@ package config
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -78,3 +79,45 @@ func (c *configInitializer) parseSpecificConfig(configFile string) {
 		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
 	}
 }
+
+func (c *configInitializer) transformConfig(args *Args, additionalArgs []string,
+	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
+	if args.LogDir != "" {
+		common.LogDir = args.LogDir
+		if common.LogStrategy == "" {
+			// TODO: Implement the other (not-daily) log strategy for the server.
+			common.LogStrategy = "daily"
+		}
+	}
+
+	if args.LogLevel != "" {
+		common.LogLevel = args.LogLevel
+	}
+
+	if args.SSHPort != DefaultSSHPort {
+		common.SSHPort = args.SSHPort
+	}
+	if args.NoColor {
+		client.TermColorsEnable = false
+	}
+
+	if args.Spartan {
+		args.Quiet = true
+		args.NoColor = true
+	}
+
+	if args.Discovery == "" && args.ServersStr == "" {
+		args.Serverless = true
+	}
+
+	// Interpret additional args as file list.
+	if args.What == "" {
+		var files []string
+		for _, file := range flag.Args() {
+			files = append(files, file)
+		}
+		args.What = strings.Join(files, ",")
+	}
+
+	return client, server, common
+}
diff --git a/internal/config/setup.go b/internal/config/setup.go
index be8e867..7800914 100644
--- a/internal/config/setup.go
+++ b/internal/config/setup.go
@@ -8,8 +8,8 @@ func Setup(args *Args, additionalArgs []string) {
 		Client: newDefaultClientConfig(),
 	}
 	initializer.parseConfig(args)
-	Client, Server, Common = args.transformConfig(
-		additionalArgs,
+	Client, Server, Common = initializer.transformConfig(
+		args, additionalArgs,
 		initializer.Client,
 		initializer.Server,
 		initializer.Common,
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 49b405d..49533a5 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -36,19 +36,21 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 		Common.FatalPanic("Logger already started")
 	}
 
+	strategy := loggers.GetStrategy(config.Common.LogStrategy)
 	level := newLevel(logLevel)
+
 	switch sourceProcess {
 	case CLIENT:
 		// This is a DTail client process running.
 		impl := loggers.FOUT
-		Client = New(CLIENT, CLIENT, impl, level)
-		Server = New(CLIENT, SERVER, impl, level)
+		Client = New(CLIENT, CLIENT, level, impl, strategy)
+		Server = New(CLIENT, SERVER, level, impl, strategy)
 		Common = Client
 	case SERVER:
 		// This is a DTail server process running.
 		impl := loggers.FILE
-		Client = New(SERVER, CLIENT, impl, level)
-		Server = New(SERVER, SERVER, impl, level)
+		Client = New(SERVER, CLIENT, level, impl, strategy)
+		Server = New(SERVER, SERVER, level, impl, strategy)
 		Common = Server
 	}
 
@@ -80,13 +82,13 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+func New(sourceProcess, sourcePackage source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
 	}
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), impl),
+		logger:        loggers.Factory(sourceProcess.String(), impl, strategy),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      maxLevel,
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 3eb29c5..8697dc4 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -17,11 +17,11 @@ const (
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
-func Factory(name string, impl Impl) Logger {
+func Factory(name string, impl Impl, strategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("name:%s,impl:%v", name, impl)
+	id := fmt.Sprintf("name:%s,fileBase:%s,impl:%v", name, strategy.FileBase, impl)
 
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
@@ -36,10 +36,10 @@ func Factory(name string, impl Impl) Logger {
 			singleton = newStdout()
 			factoryMap[id] = singleton
 		case FILE:
-			singleton = newFile()
+			singleton = newFile(strategy)
 			factoryMap[id] = singleton
 		case FOUT:
-			singleton = newFout()
+			singleton = newFout(strategy)
 			factoryMap[id] = singleton
 		}
 	}
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 1c525c9..dcdd7d0 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -12,49 +12,54 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 )
 
+type fileWriter struct {
+}
+
 type fileMessageBuf struct {
 	now     time.Time
 	message string
 }
 
 type file struct {
-	bufferCh    chan *fileMessageBuf
-	pauseCh     chan struct{}
-	resumeCh    chan struct{}
-	rotateCh    chan struct{}
-	flushCh     chan struct{}
-	lastDateStr string
-	fd          *os.File
-	writer      *bufio.Writer
-	mutex       sync.Mutex
-	started     bool
+	bufferCh     chan *fileMessageBuf
+	pauseCh      chan struct{}
+	resumeCh     chan struct{}
+	rotateCh     chan struct{}
+	flushCh      chan struct{}
+	fd           *os.File
+	writer       *bufio.Writer
+	mutex        sync.Mutex
+	started      bool
+	lastFileName string
+	strategy     Strategy
 }
 
-func newFile() *file {
+func newFile(strategy Strategy) *file {
 	f := file{
 		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
 		rotateCh: make(chan struct{}),
 		flushCh:  make(chan struct{}),
+		strategy: strategy,
 	}
-	f.getWriter(time.Now().Format("20060102"))
+
 	return &f
 }
 
-func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
-	s.mutex.Lock()
-	defer s.mutex.Unlock()
+func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
 
 	// Logger already started from another Goroutine.
-	if s.started {
+	if f.started {
 		wg.Done()
 		return
 	}
 
 	pause := func(ctx context.Context) {
 		select {
-		case <-s.resumeCh:
+		case <-f.resumeCh:
 			return
 		case <-ctx.Done():
 			return
@@ -66,55 +71,61 @@ func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 
 		for {
 			select {
-			case m := <-s.bufferCh:
-				s.write(m)
-			case <-s.pauseCh:
+			case m := <-f.bufferCh:
+				f.write(m)
+			case <-f.pauseCh:
 				pause(ctx)
-			case <-s.flushCh:
-				s.flush()
+			case <-f.flushCh:
+				f.flush()
 			case <-ctx.Done():
-				s.flush()
-				s.fd.Close()
+				f.flush()
+				f.fd.Close()
 				return
 			}
 		}
 	}()
 
-	s.started = true
+	f.started = true
 }
 
-func (s *file) Log(now time.Time, message string) {
-	s.bufferCh <- &fileMessageBuf{now, message}
+func (f *file) Log(now time.Time, message string) {
+	f.bufferCh <- &fileMessageBuf{now, message}
 }
 
-func (s *file) LogWithColors(now time.Time, message, coloredMessage string) {
+func (f *file) LogWithColors(now time.Time, message, coloredMessage string) {
 	panic("Colors not supported in file logger")
 }
 
-func (s *file) Pause()  { s.pauseCh <- struct{}{} }
-func (s *file) Resume() { s.resumeCh <- struct{}{} }
-func (s *file) Flush()  { s.flushCh <- struct{}{} }
+func (f *file) Pause()  { f.pauseCh <- struct{}{} }
+func (f *file) Resume() { f.resumeCh <- struct{}{} }
+func (f *file) Flush()  { f.flushCh <- struct{}{} }
 
 // TODO: Test that Rotate() actually works.
-func (s *file) Rotate()           { s.rotateCh <- struct{}{} }
-func (file) SupportsColors() bool { return false }
+func (f *file) Rotate()            { f.rotateCh <- struct{}{} }
+func (*file) SupportsColors() bool { return false }
 
-func (s *file) write(m *fileMessageBuf) {
+func (f *file) write(m *fileMessageBuf) {
 	select {
-	case <-s.rotateCh:
-		// Force re-opening the outfile.
-		s.lastDateStr = ""
+	case <-f.rotateCh:
+		// Force re-opening the outfile next time in getWriter.
+		f.lastFileName = ""
 	default:
 	}
 
-	writer := s.getWriter(m.now.Format("20060102"))
+	var writer *bufio.Writer
+	if f.strategy.Rotation == DailyRotation {
+		writer = f.getWriter(m.now.Format("20060102"))
+	} else {
+		writer = f.getWriter(f.strategy.FileBase)
+	}
+
 	writer.WriteString(m.message)
 	writer.WriteByte('\n')
 }
 
-func (s *file) getWriter(dateStr string) *bufio.Writer {
-	if s.lastDateStr == dateStr {
-		return s.writer
+func (f *file) getWriter(name string) *bufio.Writer {
+	if f.lastFileName == name {
+		return f.writer
 	}
 
 	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
@@ -123,32 +134,32 @@ func (s *file) getWriter(dateStr string) *bufio.Writer {
 		}
 	}
 
-	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr)
+	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, name)
 	newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
 	if err != nil {
 		panic(err)
 	}
 
 	// Close old writer.
-	if s.fd != nil {
-		s.writer.Flush()
-		s.fd.Close()
+	if f.fd != nil {
+		f.writer.Flush()
+		f.fd.Close()
 	}
 
-	s.fd = newFd
-	s.writer = bufio.NewWriterSize(s.fd, 1)
-	s.lastDateStr = dateStr
+	f.fd = newFd
+	f.writer = bufio.NewWriterSize(f.fd, 1)
+	f.lastFileName = name
 
-	return s.writer
+	return f.writer
 }
 
-func (s *file) flush() {
-	defer s.writer.Flush()
+func (f *file) flush() {
+	defer f.writer.Flush()
 
 	for {
 		select {
-		case m := <-s.bufferCh:
-			s.write(m)
+		case m := <-f.bufferCh:
+			f.write(m)
 		default:
 			return
 		}
diff --git a/internal/io/dlog/loggers/fout.go b/internal/io/dlog/loggers/fout.go
index 603dbe9..60c318d 100644
--- a/internal/io/dlog/loggers/fout.go
+++ b/internal/io/dlog/loggers/fout.go
@@ -12,8 +12,8 @@ type fout struct {
 }
 
 // Logs to both, a file and stdout
-func newFout() *fout {
-	return &fout{file: newFile(), stdout: newStdout()}
+func newFout(strategy Strategy) *fout {
+	return &fout{file: newFile(strategy), stdout: newStdout()}
 }
 
 func (f *fout) Start(ctx context.Context, wg *sync.WaitGroup) {
diff --git a/internal/io/dlog/strategy.go b/internal/io/dlog/strategy.go
deleted file mode 100644
index 32d8298..0000000
--- a/internal/io/dlog/strategy.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package dlog
-
-import "github.com/mimecast/dtail/internal/config"
-
-// Strategy allows to specify a log rotation strategy.
-type Strategy int
-
-// Possible log strategies.
-const (
-	NormalStrategy Strategy = iota
-	DailyStrategy  Strategy = iota
-	StdoutStrategy Strategy = iota
-)
-
-func logStrategy() Strategy {
-	switch config.Common.LogStrategy {
-	case "daily":
-		return DailyStrategy
-	default:
-	}
-	return StdoutStrategy
-}
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
deleted file mode 100644
index 905d1cf..0000000
--- a/internal/io/logger/logger.go
+++ /dev/null
@@ -1,443 +0,0 @@
-package logger
-
-import (
-	"bufio"
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"runtime"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/mimecast/dtail/internal/color/brush"
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/pool"
-	"github.com/mimecast/dtail/internal/protocol"
-)
-
-const (
-	clientStr string = "CLIENT"
-	serverStr string = "SERVER"
-	infoStr   string = "INFO"
-	warnStr   string = "WARN"
-	errorStr  string = "ERROR"
-	fatalStr  string = "FATAL"
-	debugStr  string = "DEBUG"
-	traceStr  string = "TRACE"
-)
-
-// Mode specifies the configured logging mode(s)
-var Mode Modes
-
-// Strategy is the current log strattegy used.
-var strategy Strategy
-
-// Synchronise access to logging.
-var mutex sync.Mutex
-
-// File descriptor of log file when Mode.logToFile enabled.
-var fd *os.File
-
-// File write buffer of log file when Mode.logToFile enabled.
-var writer *bufio.Writer
-
-// File write buffer of stdout when Mode.logToStdout enabled.
-var stdoutWriter *bufio.Writer
-
-// Current hostname.
-var hostname string
-
-// Used to detect change of day (create one log file per day0
-var lastDateStr string
-
-// Used to make logging non-blocking.
-var fileLogBufCh chan buf
-var stdoutBufCh chan string
-
-// Stdout channel, required to pause output
-var pauseCh chan struct{}
-var resumeCh chan struct{}
-
-// Tell the logger about logrotation
-var rotateCh chan os.Signal
-
-// Override the logger with a custom callack (e.g. for the t.Log for unit tests)
-type unitTestCallback func(message string)
-
-var unitTestOkCb unitTestCallback
-var unitTestErrorCb unitTestCallback
-
-// Helper type to make logging non-blocking.
-type buf struct {
-	time    time.Time
-	message string
-}
-
-// StartUnitTests enables to log all messages to the unit tests.
-func StartUnitTests(ctx context.Context, okCb, errCb unitTestCallback) {
-	unitTestOkCb = okCb
-	unitTestErrorCb = errCb
-	Start(ctx, Modes{UnitTest: true})
-}
-
-// Start logging.
-func Start(ctx context.Context, mode Modes) {
-	Mode = mode
-
-	switch {
-	case Mode.Nothing:
-		return
-	case Mode.Quiet:
-		Mode.Trace = false
-		Mode.Debug = false
-	case Mode.Trace:
-		Mode.Debug = true
-	default:
-	}
-
-	strategy := logStrategy()
-	stdoutWriter = bufio.NewWriter(os.Stdout)
-
-	switch strategy {
-	case DailyStrategy:
-		_, err := os.Stat(config.Common.LogDir)
-		Mode.logToFile = !os.IsNotExist(err) && !Mode.UnitTest
-		Mode.logToStdout = !Mode.Server || Mode.Debug || Mode.Trace || Mode.Quiet
-	case StdoutStrategy:
-		fallthrough
-	default:
-		Mode.logToFile = !Mode.Server && !Mode.UnitTest
-		Mode.logToStdout = true
-	}
-
-	fqdn, err := os.Hostname()
-	if err != nil {
-		panic(err)
-	}
-	s := strings.Split(fqdn, ".")
-	hostname = s[0]
-
-	pauseCh = make(chan struct{})
-	resumeCh = make(chan struct{})
-
-	// Setup logrotation
-	rotateCh = make(chan os.Signal, 1)
-	signal.Notify(rotateCh, syscall.SIGHUP)
-
-	if Mode.logToStdout {
-		stdoutBufCh = make(chan string, runtime.NumCPU()*100)
-		go writeToStdout(ctx)
-	}
-
-	if Mode.logToFile {
-		fileLogBufCh = make(chan buf, runtime.NumCPU()*100)
-		go writeToFile(ctx)
-	}
-}
-
-// Info message logging.
-func Info(args ...interface{}) string {
-	if Mode.Server {
-		return log(serverStr, infoStr, args)
-	}
-
-	return log(clientStr, infoStr, args)
-}
-
-// Mapreduce message logging.
-func Mapreduce(table string, data map[string]interface{}) string {
-	args := make([]interface{}, len(data)+1)
-
-	args[0] = fmt.Sprintf("MAPREDUCE:%s", strings.ToUpper(table))
-	i := 1
-	for k, v := range data {
-		args[i] = fmt.Sprintf("%s=%v", k, v)
-		i++
-	}
-
-	if Mode.Server {
-		return log(serverStr, infoStr, args)
-	}
-
-	return log(clientStr, infoStr, args)
-}
-
-// Warn message logging.
-func Warn(args ...interface{}) string {
-	if !Mode.Quiet {
-		if Mode.Server {
-			return log(serverStr, warnStr, args)
-		}
-		return log(clientStr, warnStr, args)
-	}
-
-	return ""
-}
-
-// Error message logging.
-func Error(args ...interface{}) string {
-	if Mode.Server {
-		return log(serverStr, errorStr, args)
-	}
-
-	return log(clientStr, errorStr, args)
-}
-
-// Fatal message logging.
-func Fatal(args ...interface{}) string {
-	if Mode.Server {
-		return log(serverStr, fatalStr, args)
-	}
-
-	return log(clientStr, fatalStr, args)
-}
-
-// FatalPanic logs an error and exists the process.
-func FatalPanic(args ...interface{}) {
-	what := clientStr
-	if Mode.Server {
-		what = serverStr
-	}
-	log(what, fatalStr, args)
-
-	time.Sleep(time.Second)
-	mutex.Lock()
-	defer mutex.Unlock()
-
-	closeWriter()
-	os.Exit(3)
-}
-
-// Debug message logging.
-func Debug(args ...interface{}) string {
-	if Mode.Debug {
-		if Mode.Server {
-			return log(serverStr, debugStr, args)
-		}
-		return log(clientStr, debugStr, args)
-	}
-
-	return ""
-}
-
-// Trace message logging.
-func Trace(args ...interface{}) string {
-	if Mode.Trace {
-		if Mode.Server {
-			return log(serverStr, traceStr, args)
-		}
-		return log(clientStr, traceStr, args)
-	}
-
-	return ""
-}
-
-// Write log line to buffer and/or log file.
-func write(what, severity, message string) {
-	if Mode.logToStdout {
-		line := fmt.Sprintf("%s|%s|%s|%s", what, hostname, severity, message)
-
-		if config.Client.TermColorsEnable {
-			line = brush.Colorfy(line)
-		}
-
-		stdoutBufCh <- line
-	}
-
-	if Mode.logToFile {
-		t := time.Now()
-		timeStr := t.Format("20060102-150405")
-		fileLogBufCh <- buf{
-			time:    t,
-			message: fmt.Sprintf("%s|%s|%s|%s", severity, timeStr, what, message),
-		}
-	}
-}
-
-// Generig log message.
-func log(what string, severity string, args []interface{}) string {
-	if Mode.Nothing {
-		return ""
-	}
-
-	sb := pool.BuilderBuffer.Get().(*strings.Builder)
-
-	for i, arg := range args {
-		if i > 0 {
-			sb.WriteString(protocol.FieldDelimiter)
-		}
-
-		switch v := arg.(type) {
-		case string:
-			sb.WriteString(v)
-		case int:
-			sb.WriteString(fmt.Sprintf("%d", v))
-		case error:
-			sb.WriteString(v.Error())
-		default:
-			sb.WriteString(fmt.Sprintf("%v", v))
-		}
-	}
-
-	message := sb.String()
-	pool.RecycleBuilderBuffer(sb)
-	write(what, severity, message)
-	return fmt.Sprintf("%s|%s", severity, message)
-}
-
-// Raw message logging.
-func Raw(message string) {
-	if Mode.Nothing {
-		return
-	}
-
-	if Mode.logToFile {
-		fileLogBufCh <- buf{time.Now(), message}
-	}
-
-	if Mode.logToStdout {
-		if config.Client.TermColorsEnable {
-			message = brush.Colorfy(message)
-		}
-		stdoutBufCh <- message
-	}
-}
-
-// Close log writer (e.g. on change of day).
-func closeWriter() {
-	if writer != nil {
-		writer.Flush()
-		fd.Close()
-	}
-}
-
-// Return the correct log file writer
-func fileWriter(dateStr string) *bufio.Writer {
-	if dateStr != lastDateStr {
-		return updateFileWriter(dateStr)
-	}
-
-	// Check for log rotation signal
-	select {
-	case <-rotateCh:
-		stdoutWriter.WriteString("Received signal for logrotation\n")
-		return updateFileWriter(dateStr)
-	default:
-	}
-
-	return writer
-}
-
-// Update log file writer
-func updateFileWriter(dateStr string) *bufio.Writer {
-	// Detected change of day. Close current writer and create a new one.
-	mutex.Lock()
-	defer mutex.Unlock()
-	closeWriter()
-
-	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
-		if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil {
-			panic(err)
-		}
-	}
-
-	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr)
-	newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
-	if err != nil {
-		panic(err)
-	}
-
-	fd = newFd
-	writer = bufio.NewWriterSize(fd, 1)
-	lastDateStr = dateStr
-
-	return writer
-}
-
-// Flush all outstanding lines.
-func Flush() {
-	for {
-		select {
-		case message := <-stdoutBufCh:
-			stdoutWriter.WriteString(message)
-			stdoutWriter.WriteString("\n")
-		default:
-			stdoutWriter.Flush()
-			return
-		}
-	}
-}
-
-func writeToStdout(ctx context.Context) {
-	for {
-		select {
-		case message := <-stdoutBufCh:
-			stdoutWriter.WriteString(message)
-			stdoutWriter.WriteString("\n")
-		case <-time.After(time.Millisecond * 100):
-			stdoutWriter.Flush()
-		case <-pauseCh:
-		PAUSE:
-			for {
-				select {
-				case <-stdoutBufCh:
-				case <-resumeCh:
-					break PAUSE
-				case <-ctx.Done():
-					return
-				}
-			}
-		case <-ctx.Done():
-			Flush()
-			return
-		}
-	}
-}
-
-func writeToFile(ctx context.Context) {
-	for {
-		select {
-		case buf := <-fileLogBufCh:
-			dateStr := buf.time.Format("20060102")
-			w := fileWriter(dateStr)
-			w.WriteString(buf.message)
-			w.WriteString("\n")
-		case <-pauseCh:
-		PAUSE:
-			for {
-				select {
-				case <-stdoutBufCh:
-				case <-resumeCh:
-					break PAUSE
-				case <-ctx.Done():
-					return
-				}
-			}
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-// Pause logging.
-func Pause() {
-	if Mode.logToStdout {
-		pauseCh <- struct{}{}
-	}
-	if Mode.logToFile {
-		pauseCh <- struct{}{}
-	}
-}
-
-// Resume logging (after pausing).
-func Resume() {
-	if Mode.logToStdout {
-		resumeCh <- struct{}{}
-	}
-	if Mode.logToFile {
-		resumeCh <- struct{}{}
-	}
-}
diff --git a/internal/io/logger/modes.go b/internal/io/logger/modes.go
deleted file mode 100644
index 85f90a5..0000000
--- a/internal/io/logger/modes.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package logger
-
-// Modes specifies the logging mode.
-type Modes struct {
-	Debug       bool
-	logToFile   bool
-	logToStdout bool
-	Nothing     bool
-	Quiet       bool
-	Server      bool
-	Trace       bool
-	UnitTest    bool
-}
diff --git a/internal/io/logger/strategy.go b/internal/io/logger/strategy.go
deleted file mode 100644
index 44bf393..0000000
--- a/internal/io/logger/strategy.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package logger
-
-import "github.com/mimecast/dtail/internal/config"
-
-// Strategy allows to specify a log rotation strategy.
-type Strategy int
-
-// Possible log strategies.
-const (
-	NormalStrategy Strategy = iota
-	DailyStrategy  Strategy = iota
-	StdoutStrategy Strategy = iota
-)
-
-func logStrategy() Strategy {
-	switch config.Common.LogStrategy {
-	case "daily":
-		return DailyStrategy
-	default:
-	}
-	return StdoutStrategy
-}
-- 
cgit v1.2.3


From 5484754cca4653b050f9b5075883db1d62dc0d18 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 10:46:47 +0300
Subject: add more default fields to MAPREDUCE

---
 internal/config/args.go                 |  2 +-
 internal/config/config.go               |  1 -
 internal/io/dlog/dlog.go                | 38 ++++++++++++++++++--
 internal/io/dlog/loggers/file.go        |  1 -
 internal/mapr/logformat/default.go      | 23 +++++++++---
 internal/mapr/logformat/default_test.go | 62 +++++++++++++++++++++++++++------
 internal/server/stats.go                |  4 ---
 7 files changed, 108 insertions(+), 23 deletions(-)

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index 7f24348..484aa8b 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -39,7 +39,7 @@ func (a *Args) String() string {
 	var sb strings.Builder
 
 	sb.WriteString("Args(")
-	// TODO: All commands should make use of this
+
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir))
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
diff --git a/internal/config/config.go b/internal/config/config.go
index 76dcc65..3d05a11 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -85,7 +85,6 @@ func (c *configInitializer) transformConfig(args *Args, additionalArgs []string,
 	if args.LogDir != "" {
 		common.LogDir = args.LogDir
 		if common.LogStrategy == "" {
-			// TODO: Implement the other (not-daily) log strategy for the server.
 			common.LogStrategy = "daily"
 		}
 	}
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 49533a5..bc9b2f8 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -3,7 +3,11 @@ package dlog
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -211,8 +215,38 @@ func (d *DLog) Raw(message string) string {
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
-	// TODO: mC compatible SERVER mapreduce fields, no MAPREDUCE keyword in CLIENT mode
-	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
+	if d.sourceProcess == SERVER {
+		// level|date-time|process|caller|cpus|goroutines|cgocalls|loadavg|uptime|MAPREDUCE:TABLE|key=value|...
+
+		var loadAvg string
+		if loadAvgBytes, err := ioutil.ReadFile("/proc/loadavg"); err == nil {
+			tmp := string(loadAvgBytes)
+			s := strings.SplitN(tmp, " ", 2)
+			loadAvg = s[0]
+		}
+
+		var uptime string
+		if uptimeBytes, err := ioutil.ReadFile("/proc/uptime"); err == nil {
+			tmp := string(uptimeBytes)
+			s := strings.SplitN(tmp, ".", 2)
+			i, _ := strconv.ParseInt(s[0], 10, 64)
+			t := time.Duration(i) * time.Second
+			uptime = fmt.Sprintf("%v", t)
+		}
+
+		_, file, line, _ := runtime.Caller(1)
+		args[0] = fmt.Sprintf("%d|%s:%d|%d|%d|%d|%s|%s|MAPREDUCE:%s",
+			os.Getpid(),
+			filepath.Base(file), line,
+			runtime.NumCPU(),
+			runtime.NumGoroutine(),
+			runtime.NumCgoCall(),
+			loadAvg,
+			uptime,
+			strings.ToUpper(table))
+	} else {
+		args[0] = fmt.Sprintf("STATS:%s", strings.ToUpper(table))
+	}
 
 	i := 1
 	for k, v := range data {
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index dcdd7d0..6e692a3 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -100,7 +100,6 @@ func (f *file) Pause()  { f.pauseCh <- struct{}{} }
 func (f *file) Resume() { f.resumeCh <- struct{}{} }
 func (f *file) Flush()  { f.flushCh <- struct{}{} }
 
-// TODO: Test that Rotate() actually works.
 func (f *file) Rotate()            { f.rotateCh <- struct{}{} }
 func (*file) SupportsColors() bool { return false }
 
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 7fb1700..8016667 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -11,7 +11,7 @@ import (
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
 
-	if len(splitted) < 3 || !strings.HasPrefix(splitted[3], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") {
+	if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") {
 		// Not a DTail mapreduce log line.
 		return nil, IgnoreFieldsErr
 	}
@@ -27,10 +27,25 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 
 	fields["$severity"] = splitted[0]
 	fields["$loglevel"] = splitted[0]
-	// NEXT: Parse time like we do at Mimecast
-	fields["$time"] = splitted[1]
 
-	for _, kv := range splitted[4:] {
+	time := splitted[1]
+	fields["$time"] = time
+	if len(time) == 15 {
+		// Example: 20211002-071209
+		fields["$date"] = time[0:8]
+		fields["$hour"] = time[9:11]
+		fields["$minute"] = time[11:13]
+		fields["$second"] = time[13:]
+	}
+	fields["$pid"] = splitted[2]
+	fields["$caller"] = splitted[3]
+	fields["$cpus"] = splitted[4]
+	fields["$goroutines"] = splitted[5]
+	fields["$cgocalls"] = splitted[6]
+	fields["$loadavg"] = splitted[7]
+	fields["$uptime"] = splitted[8]
+
+	for _, kv := range splitted[10:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
 			return fields, fmt.Errorf("Unable to parse key-value token '%s'", kv)
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index 79911d1..02c03a3 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -1,6 +1,7 @@
 package logformat
 
 import (
+	"fmt"
 	"testing"
 )
 
@@ -10,9 +11,15 @@ func TestDefaultLogFormat(t *testing.T) {
 		t.Errorf("Unable to create parser: %s", err.Error())
 	}
 
+	date := "20211002"
+	hour := "07"
+	minute := "23"
+	second := "42"
+	time := fmt.Sprintf("%s-%s%s%s", date, hour, minute, second)
+
 	inputs := []string{
-		"INFO|20210907-065632|SERVER|MAPREDUCE:TEST|foo=bar|baz=bay",
-		"INFO|20210907-065732|CLIENT|MAPREDUCE:YOMAN|baz=bay|foo=bar",
+		fmt.Sprintf("INFO|%s|1|default_test.go:0|8|14|7|0.21|471h0m21s|MAPREDUCE:STATS|foo=bar|bar=foo", time),
+		fmt.Sprintf("INFO|%s|1|default_test.go:0|8|14|7|0.21|471h0m21s|MAPREDUCE:STATS|bar=foo|foo=bar", time),
 	}
 
 	for _, input := range inputs {
@@ -22,18 +29,53 @@ func TestDefaultLogFormat(t *testing.T) {
 			t.Errorf("Parser unable to make fields: %s", err.Error())
 		}
 
-		if bar, ok := fields["foo"]; !ok {
-			t.Errorf("Expected field 'foo', but no such field there in '%s'\n", input)
-		} else if bar != "bar" {
-			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n", bar, input)
+		if val, ok := fields["$severity"]; !ok {
+			t.Errorf("Expected field '$severity', but no such field there in '%s'\n", input)
+		} else if val != "INFO" {
+			t.Errorf("Expected 'INFO' stored in field '$severity', but got '%s' in '%s'\n", val, input)
+		}
+
+		if val, ok := fields["$time"]; !ok {
+			t.Errorf("Expected field '$time', but no such field there in '%s'\n", input)
+		} else if val != time {
+			t.Errorf("Expected '%s' stored in field '$time', but got '%s' in '%s'\n", time, val, input)
+		}
+
+		if val, ok := fields["$date"]; !ok {
+			t.Errorf("Expected field '$date', but no such field there in '%s'\n", input)
+		} else if val != date {
+			t.Errorf("Expected '%s' stored in field '$date', but got '%s' in '%s'\n", date, val, input)
+		}
+
+		if val, ok := fields["$hour"]; !ok {
+			t.Errorf("Expected field '$hour', but no such field there in '%s'\n", input)
+		} else if val != hour {
+			t.Errorf("Expected '%s' stored in field '$hour', but got '%s' in '%s'\n", hour, val, input)
+		}
+
+		if val, ok := fields["$minute"]; !ok {
+			t.Errorf("Expected field '$minute', but no such field there in '%s'\n", input)
+		} else if val != minute {
+			t.Errorf("Expected '%s' stored in field '$minute', but got '%s' in '%s'\n", minute, val, input)
 		}
 
-		if bay, ok := fields["baz"]; !ok {
-			t.Errorf("Expected field 'baz', but no such field there in '%s'\n", input)
-		} else if bay != "bay" {
-			t.Errorf("Expected 'bay' stored in field 'baz', but got '%s' in '%s'\n", bay, input)
+		if val, ok := fields["$second"]; !ok {
+			t.Errorf("Expected field '$second', but no such field there in '%s'\n", input)
+		} else if val != second {
+			t.Errorf("Expected '%s' stored in field '$second', but got '%s' in '%s'\n", second, val, input)
 		}
 
+		if val, ok := fields["foo"]; !ok {
+			t.Errorf("Expected field 'foo', but no such field there in '%s'\n", input)
+		} else if val != "bar" {
+			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n", val, input)
+		}
+
+		if val, ok := fields["bar"]; !ok {
+			t.Errorf("Expected field 'bar', but no such field there in '%s'\n", input)
+		} else if val != "foo" {
+			t.Errorf("Expected 'foo' stored in field 'bar', but got '%s' in '%s'\n", val, input)
+		}
 	}
 
 	fields, err := parser.MakeFields("foozoo=bar|bazbay")
diff --git a/internal/server/stats.go b/internal/server/stats.go
index 8583318..c07634d 100644
--- a/internal/server/stats.go
+++ b/internal/server/stats.go
@@ -3,7 +3,6 @@ package server
 import (
 	"context"
 	"fmt"
-	"runtime"
 	"sync"
 	"time"
 
@@ -53,9 +52,6 @@ func (s *stats) logServerStats() {
 	data := make(map[string]interface{})
 	data["currentConnections"] = s.currentConnections
 	data["lifetimeConnections"] = s.lifetimeConnections
-	data["goroutines"] = runtime.NumGoroutine()
-	data["cgocalls"] = runtime.NumCgoCall()
-	data["cpu"] = runtime.NumCPU()
 
 	dlog.Server.Mapreduce("STATS", data)
 }
-- 
cgit v1.2.3


From e922e0f7a955fd40753930b818f580f714578ca3 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 11:23:08 +0300
Subject: Client default log dir is ~/log

---
 internal/config/config.go            | 11 +++++++++--
 internal/io/dlog/dlog.go             | 27 ++++++++++++++-------------
 internal/io/dlog/loggers/strategy.go | 27 +++++++++++++++++++++++++++
 internal/io/dlog/source.go           | 19 -------------------
 internal/source/source.go            | 19 +++++++++++++++++++
 5 files changed, 69 insertions(+), 34 deletions(-)
 create mode 100644 internal/io/dlog/loggers/strategy.go
 delete mode 100644 internal/io/dlog/source.go
 create mode 100644 internal/source/source.go

(limited to 'internal')

diff --git a/internal/config/config.go b/internal/config/config.go
index 3d05a11..6d4730a 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -84,9 +84,16 @@ func (c *configInitializer) transformConfig(args *Args, additionalArgs []string,
 	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
 	if args.LogDir != "" {
 		common.LogDir = args.LogDir
-		if common.LogStrategy == "" {
-			common.LogStrategy = "daily"
+	}
+	if strings.Contains(common.LogDir, "~/") {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			panic(err)
 		}
+		common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
+	}
+	if common.LogStrategy == "" {
+		common.LogStrategy = "daily"
 	}
 
 	if args.LogLevel != "" {
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index bc9b2f8..3de3120 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -17,6 +17,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog/loggers"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
+	"github.com/mimecast/dtail/internal/source"
 )
 
 // Client is the log handler for the client packages.
@@ -32,7 +33,7 @@ var mutex sync.Mutex
 var started bool
 
 // Start logger(s).
-func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLevel string) {
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source, logLevel string) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
@@ -44,17 +45,17 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 	level := newLevel(logLevel)
 
 	switch sourceProcess {
-	case CLIENT:
+	case source.Client:
 		// This is a DTail client process running.
 		impl := loggers.FOUT
-		Client = New(CLIENT, CLIENT, level, impl, strategy)
-		Server = New(CLIENT, SERVER, level, impl, strategy)
+		Client = New(source.Client, source.Client, level, impl, strategy)
+		Server = New(source.Client, source.Server, level, impl, strategy)
 		Common = Client
-	case SERVER:
+	case source.Server:
 		// This is a DTail server process running.
 		impl := loggers.FILE
-		Client = New(SERVER, CLIENT, level, impl, strategy)
-		Server = New(SERVER, SERVER, level, impl, strategy)
+		Client = New(source.Server, source.Client, level, impl, strategy)
+		Server = New(source.Server, source.Server, level, impl, strategy)
 		Common = Server
 	}
 
@@ -75,10 +76,10 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 type DLog struct {
 	logger loggers.Logger
 	// Is this a DTail server or client process logging?
-	sourceProcess source
+	sourceProcess source.Source
 	// Is this a DTail server or client package logging? In serverless mode
 	// the client can also execute code from the server package.
-	sourcePackage source
+	sourcePackage source.Source
 	// Max log level to log.
 	maxLevel level
 	// Current hostname.
@@ -86,7 +87,7 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
+func New(sourceProcess, sourcePackage source.Source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
@@ -120,7 +121,7 @@ func (d *DLog) log(level level, args []interface{}) string {
 	now := time.Now()
 
 	switch d.sourceProcess {
-	case CLIENT:
+	case source.Client:
 		sb.WriteString(d.sourcePackage.String())
 		sb.WriteString(protocol.FieldDelimiter)
 		sb.WriteString(d.hostname)
@@ -179,7 +180,7 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	if d.sourcePackage == SERVER && d.sourceProcess != CLIENT {
+	if d.sourcePackage == source.Server && d.sourceProcess != source.Client {
 		// This can be dtail client in serverless mode. In this case log all
 		// info server messages as verbose.
 		return d.log(VERBOSE, args)
@@ -215,7 +216,7 @@ func (d *DLog) Raw(message string) string {
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
-	if d.sourceProcess == SERVER {
+	if d.sourceProcess == source.Server {
 		// level|date-time|process|caller|cpus|goroutines|cgocalls|loadavg|uptime|MAPREDUCE:TABLE|key=value|...
 
 		var loadAvg string
diff --git a/internal/io/dlog/loggers/strategy.go b/internal/io/dlog/loggers/strategy.go
new file mode 100644
index 0000000..a1b9355
--- /dev/null
+++ b/internal/io/dlog/loggers/strategy.go
@@ -0,0 +1,27 @@
+package loggers
+
+import (
+	"os"
+	"path/filepath"
+)
+
+type Rotation int
+
+const (
+	DailyRotation  Rotation = iota
+	SignalRotation Rotation = iota
+)
+
+type Strategy struct {
+	Rotation Rotation
+	FileBase string
+}
+
+func GetStrategy(name string) Strategy {
+	switch name {
+	case "daily":
+		return Strategy{DailyRotation, ""}
+	default:
+		return Strategy{SignalRotation, filepath.Base(os.Args[0])}
+	}
+}
diff --git a/internal/io/dlog/source.go b/internal/io/dlog/source.go
deleted file mode 100644
index 265885e..0000000
--- a/internal/io/dlog/source.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package dlog
-
-type source int
-
-const (
-	CLIENT source = iota
-	SERVER source = iota
-)
-
-func (s source) String() string {
-	switch s {
-	case CLIENT:
-		return "CLIENT"
-	case SERVER:
-		return "SERVER"
-	}
-
-	panic("Unknown log source type")
-}
diff --git a/internal/source/source.go b/internal/source/source.go
new file mode 100644
index 0000000..bf1c880
--- /dev/null
+++ b/internal/source/source.go
@@ -0,0 +1,19 @@
+package source
+
+type Source int
+
+const (
+	Client Source = iota
+	Server Source = iota
+)
+
+func (s Source) String() string {
+	switch s {
+	case Client:
+		return "Client"
+	case Server:
+		return "Server"
+	}
+
+	panic("Unknown log source type")
+}
-- 
cgit v1.2.3


From dde2a1702ea1283dc5b2754f751f9381e8d0f005 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 11:54:07 +0300
Subject: reduce logging in serverless mode

---
 internal/config/args.go                   |  30 +++++++-
 internal/config/config.go                 | 116 ++++--------------------------
 internal/config/initializer.go            | 109 ++++++++++++++++++++++++++++
 internal/config/setup.go                  |  17 -----
 internal/io/dlog/dlog.go                  |   5 --
 internal/server/handlers/readcommand.go   |  12 ++--
 internal/server/handlers/serverhandler.go |  93 ++++++++----------------
 internal/source/source.go                 |   4 +-
 internal/version/version.go               |   2 +-
 9 files changed, 190 insertions(+), 198 deletions(-)
 create mode 100644 internal/config/initializer.go
 delete mode 100644 internal/config/setup.go

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index 484aa8b..3e2eb1f 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -1,6 +1,7 @@
 package config
 
 import (
+	"encoding/base64"
 	"fmt"
 	"strings"
 
@@ -69,7 +70,32 @@ func (a *Args) String() string {
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
 func (a *Args) SerializeOptions() string {
-	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
+	return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, a.Serverless)
 }
 
-// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package)
+// DeserializeOptions deserializes the options, but into a map.
+func DeserializeOptions(opts []string) (map[string]string, error) {
+	options := make(map[string]string, len(opts))
+
+	for _, o := range opts {
+		kv := strings.SplitN(o, "=", 2)
+		if len(kv) != 2 {
+			return options, fmt.Errorf("Unable to parse options: %v", kv)
+		}
+		key := kv[0]
+		val := kv[1]
+
+		if strings.HasPrefix(val, "base64%") {
+			s := strings.SplitN(val, "%", 2)
+			decoded, err := base64.StdEncoding.DecodeString(s[1])
+			if err != nil {
+				return options, err
+			}
+			val = string(decoded)
+		}
+
+		options[key] = val
+	}
+
+	return options, nil
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index 6d4730a..09ae994 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1,14 +1,5 @@
 package config
 
-import (
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-)
-
 const (
 	// ControlUser is used for various DTail specific operations.
 	ControlUser string = "DTAIL-CONTROL"
@@ -33,97 +24,18 @@ var Server *ServerConfig
 // Common holds common configs of both both, client and server.
 var Common *CommonConfig
 
-// Used to initialize the configuration.
-type configInitializer struct {
-	Common *CommonConfig
-	Server *ServerConfig
-	Client *ClientConfig
-}
-
-func (c *configInitializer) parseConfig(args *Args) {
-	if strings.ToUpper(args.ConfigFile) == "NONE" {
-		return
-	}
-
-	if args.ConfigFile != "" {
-		c.parseSpecificConfig(args.ConfigFile)
-		return
-	}
-
-	if homeDir, err := os.UserHomeDir(); err != nil {
-		var paths []string
-		paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir))
-		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
-		for _, configPath := range paths {
-			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
-				c.parseSpecificConfig(configPath)
-			}
-		}
-	}
-}
-
-func (c *configInitializer) parseSpecificConfig(configFile string) {
-	fd, err := os.Open(configFile)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to read config file: %v", err))
-	}
-	defer fd.Close()
-
-	cfgBytes, err := ioutil.ReadAll(fd)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
-	}
-
-	err = json.Unmarshal([]byte(cfgBytes), c)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
-	}
-}
-
-func (c *configInitializer) transformConfig(args *Args, additionalArgs []string,
-	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
-	if args.LogDir != "" {
-		common.LogDir = args.LogDir
-	}
-	if strings.Contains(common.LogDir, "~/") {
-		homeDir, err := os.UserHomeDir()
-		if err != nil {
-			panic(err)
-		}
-		common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
-	}
-	if common.LogStrategy == "" {
-		common.LogStrategy = "daily"
-	}
-
-	if args.LogLevel != "" {
-		common.LogLevel = args.LogLevel
-	}
-
-	if args.SSHPort != DefaultSSHPort {
-		common.SSHPort = args.SSHPort
-	}
-	if args.NoColor {
-		client.TermColorsEnable = false
-	}
-
-	if args.Spartan {
-		args.Quiet = true
-		args.NoColor = true
-	}
-
-	if args.Discovery == "" && args.ServersStr == "" {
-		args.Serverless = true
-	}
-
-	// Interpret additional args as file list.
-	if args.What == "" {
-		var files []string
-		for _, file := range flag.Args() {
-			files = append(files, file)
-		}
-		args.What = strings.Join(files, ",")
-	}
-
-	return client, server, common
+// Setup the DTail configuration.
+func Setup(args *Args, additionalArgs []string) {
+	initializer := initializer{
+		Common: newDefaultCommonConfig(),
+		Server: newDefaultServerConfig(),
+		Client: newDefaultClientConfig(),
+	}
+	initializer.parseConfig(args)
+	Client, Server, Common = initializer.transformConfig(
+		args, additionalArgs,
+		initializer.Client,
+		initializer.Server,
+		initializer.Common,
+	)
 }
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
new file mode 100644
index 0000000..f1a9ec4
--- /dev/null
+++ b/internal/config/initializer.go
@@ -0,0 +1,109 @@
+package config
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+// Used to initialize the configuration.
+type initializer struct {
+	Common *CommonConfig
+	Server *ServerConfig
+	Client *ClientConfig
+}
+
+func (c *initializer) parseConfig(args *Args) {
+	if strings.ToUpper(args.ConfigFile) == "NONE" {
+		return
+	}
+
+	if args.ConfigFile != "" {
+		c.parseSpecificConfig(args.ConfigFile)
+		return
+	}
+
+	if homeDir, err := os.UserHomeDir(); err != nil {
+		var paths []string
+		paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir))
+		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
+		for _, configPath := range paths {
+			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
+				c.parseSpecificConfig(configPath)
+			}
+		}
+	}
+}
+
+func (c *initializer) parseSpecificConfig(configFile string) {
+	fd, err := os.Open(configFile)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to read config file: %v", err))
+	}
+	defer fd.Close()
+
+	cfgBytes, err := ioutil.ReadAll(fd)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
+	}
+
+	err = json.Unmarshal([]byte(cfgBytes), c)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
+	}
+}
+
+func (c *initializer) transformConfig(args *Args, additionalArgs []string,
+	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
+	if args.LogDir != "" {
+		common.LogDir = args.LogDir
+	}
+	if strings.Contains(common.LogDir, "~/") {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			panic(err)
+		}
+		common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
+	}
+	if common.LogStrategy == "" {
+		common.LogStrategy = "daily"
+	}
+
+	if args.LogLevel != "" {
+		common.LogLevel = args.LogLevel
+	} else if args.ServersStr == "" && args.Discovery == "" {
+		// We are in serverless mode. Default log level is WARN.
+		common.LogLevel = "WARN"
+	}
+
+	if args.SSHPort != DefaultSSHPort {
+		common.SSHPort = args.SSHPort
+	}
+	if args.NoColor {
+		client.TermColorsEnable = false
+	}
+
+	if args.Spartan {
+		args.Quiet = true
+		args.NoColor = true
+	}
+
+	if args.Discovery == "" && args.ServersStr == "" {
+		// We are not connecting to any servers.
+		args.Serverless = true
+	}
+
+	// Interpret additional args as file list.
+	if args.What == "" {
+		var files []string
+		for _, file := range flag.Args() {
+			files = append(files, file)
+		}
+		args.What = strings.Join(files, ",")
+	}
+
+	return client, server, common
+}
diff --git a/internal/config/setup.go b/internal/config/setup.go
deleted file mode 100644
index 7800914..0000000
--- a/internal/config/setup.go
+++ /dev/null
@@ -1,17 +0,0 @@
-package config
-
-// Setup the DTail configuration.
-func Setup(args *Args, additionalArgs []string) {
-	initializer := configInitializer{
-		Common: newDefaultCommonConfig(),
-		Server: newDefaultServerConfig(),
-		Client: newDefaultClientConfig(),
-	}
-	initializer.parseConfig(args)
-	Client, Server, Common = initializer.transformConfig(
-		args, additionalArgs,
-		initializer.Client,
-		initializer.Server,
-		initializer.Common,
-	)
-}
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 3de3120..6cacfe2 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -180,11 +180,6 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	if d.sourcePackage == source.Server && d.sourceProcess != source.Client {
-		// This can be dtail client in serverless mode. In this case log all
-		// info server messages as verbose.
-		return d.log(VERBOSE, args)
-	}
 	return d.log(INFO, args)
 }
 
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index c76ae2a..6579018 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -32,13 +32,13 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string, retrie
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			r.server.sendServerMessage(dlog.Server.Error(r.server.user, commandParseWarning, err))
+			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user, commandParseWarning, err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, commandParseWarning, args, argc))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, commandParseWarning, args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re, retries)
@@ -58,7 +58,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 
 		if numPaths := len(paths); numPaths == 0 {
 			dlog.Server.Error(r.server.user, "No such file(s) to read", glob)
-			r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
+			r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
 			select {
 			case <-ctx.Done():
 				return
@@ -72,7 +72,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 		return
 	}
 
-	r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Giving up to read file(s)"))
+	r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Giving up to read file(s)"))
 	return
 }
 
@@ -93,7 +93,7 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGr
 
 	if !r.server.user.HasFilePermission(path, "readfiles") {
 		dlog.Server.Error(r.server.user, "No permission to read file", path, globID)
-		r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
 		return
 	}
 
@@ -161,6 +161,6 @@ func (r *readCommand) makeGlobID(path, glob string) string {
 		return pathParts[len(pathParts)-1]
 	}
 
-	r.server.sendServerWarnMessage(dlog.Server.Warn("Empty file path given?", path, glob))
+	r.server.send(r.server.serverMessages, dlog.Server.Warn("Empty file path given?", path, glob))
 	return ""
 }
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index b664566..ace2626 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -15,8 +15,8 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
@@ -46,6 +46,7 @@ type ServerHandler struct {
 	activeCommands   int32
 	quiet            bool
 	spartan          bool
+	serverless       bool
 	readBuf          bytes.Buffer
 	writeBuf         bytes.Buffer
 }
@@ -99,6 +100,12 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			return
 		}
 
+		if h.serverless {
+			// In serverless mode we have logged the server message already via the
+			// dlog logger, no need to send the message again to the client part.
+			return
+		}
+
 		// Handle normal server message (display to the user)
 		h.readBuf.WriteString("SERVER")
 		h.readBuf.WriteString(protocol.FieldDelimiter)
@@ -266,23 +273,24 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
 
-	options, err := readOptions(splitted[1:])
+	options, err := config.DeserializeOptions(splitted[1:])
 	if err != nil {
-		h.sendServerMessage(dlog.Server.Error(h.user, err))
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		commandFinished()
 		return
 	}
-	if quiet, ok := options["quiet"]; ok {
-		if quiet == "true" {
-			dlog.Server.Debug(h.user, "Enabling quiet mode")
-			h.quiet = true
-		}
+
+	if quiet, _ := options["quiet"]; quiet == "true" {
+		dlog.Server.Debug(h.user, "Enabling quiet mode")
+		h.quiet = true
 	}
-	if spartan, ok := options["spartan"]; ok {
-		if spartan == "true" {
-			dlog.Server.Debug(h.user, "Enabling spartan mode")
-			h.spartan = true
-		}
+	if spartan, _ := options["spartan"]; spartan == "true" {
+		dlog.Server.Debug(h.user, "Enabling spartan mode")
+		h.spartan = true
+	}
+	if serverless, _ := options["serverless"]; serverless == "true" {
+		dlog.Server.Debug(h.user, "Enabling serverless mode")
+		h.serverless = true
 	}
 
 	switch commandName {
@@ -303,7 +311,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 	case "map":
 		command, aggregate, err := newMapCommand(h, argc, args)
 		if err != nil {
-			h.sendServerMessage(err.Error())
+			h.send(h.serverMessages, err.Error())
 			dlog.Server.Error(h.user, err)
 			commandFinished()
 			return
@@ -320,14 +328,16 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		commandFinished()
 
 	default:
-		h.sendServerMessage(dlog.Server.Error(h.user, "Received unknown user command", commandName, argc, args, options))
+		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown user command", commandName, argc, args, options))
 		commandFinished()
 	}
 }
 
 func (h *ServerHandler) handleAckCommand(argc int, args []string) {
 	if argc < 3 {
-		h.sendServerWarnMessage(dlog.Server.Warn(h.user, commandParseWarning, args, argc))
+		if !h.quiet {
+			h.send(h.serverMessages, dlog.Server.Warn(h.user, commandParseWarning, args, argc))
+		}
 		return
 	}
 	if args[1] == "close" && args[2] == "connection" {
@@ -346,23 +356,8 @@ func (h *ServerHandler) send(ch chan<- string, message string) {
 	}
 }
 
-func (h *ServerHandler) sendServerMessage(message string) {
-	h.send(h.serverMessageC(), message)
-}
-
-func (h *ServerHandler) sendServerWarnMessage(message string) {
-	if h.quiet {
-		return
-	}
-	h.send(h.serverMessageC(), message)
-}
-
-func (h *ServerHandler) serverMessageC() chan<- string {
-	return h.serverMessages
-}
-
-func (h *ServerHandler) flushMessages() {
-	dlog.Server.Debug(h.user, "flushMessages()")
+func (h *ServerHandler) flush() {
+	dlog.Server.Debug(h.user, "flush()")
 
 	unsentMessages := func() int {
 		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
@@ -381,11 +376,11 @@ func (h *ServerHandler) flushMessages() {
 
 func (h *ServerHandler) shutdown() {
 	dlog.Server.Debug(h.user, "shutdown()")
-	h.flushMessages()
+	h.flush()
 
 	go func() {
 		select {
-		case h.serverMessageC() <- ".syn close connection":
+		case h.serverMessages <- ".syn close connection":
 		case <-h.done.Done():
 		}
 	}()
@@ -408,31 +403,3 @@ func (h *ServerHandler) decrementActiveCommands() int32 {
 	atomic.AddInt32(&h.activeCommands, -1)
 	return atomic.LoadInt32(&h.activeCommands)
 }
-
-func readOptions(opts []string) (map[string]string, error) {
-	dlog.Server.Debug("Parsing options", opts)
-	options := make(map[string]string, len(opts))
-
-	for _, o := range opts {
-		kv := strings.SplitN(o, "=", 2)
-		if len(kv) != 2 {
-			return options, fmt.Errorf("Unable to parse options: %v", kv)
-		}
-		key := kv[0]
-		val := kv[1]
-
-		if strings.HasPrefix(val, "base64%") {
-			s := strings.SplitN(val, "%", 2)
-			decoded, err := base64.StdEncoding.DecodeString(s[1])
-			if err != nil {
-				return options, err
-			}
-			val = string(decoded)
-		}
-
-		dlog.Server.Debug("Setting option", key, val)
-		options[key] = val
-	}
-
-	return options, nil
-}
diff --git a/internal/source/source.go b/internal/source/source.go
index bf1c880..73dccb2 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -10,9 +10,9 @@ const (
 func (s Source) String() string {
 	switch s {
 	case Client:
-		return "Client"
+		return "CLIENT"
 	case Server:
-		return "Server"
+		return "SERVER"
 	}
 
 	panic("Unknown log source type")
diff --git a/internal/version/version.go b/internal/version/version.go
index a54b560..4ff6eae 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -39,7 +39,7 @@ func PaintedString() string {
 		color.FgBlack, color.BgGreen)
 
 	additional := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Additional),
-		color.FgWhite, color.BgMagenta, color.AttrBlink)
+		color.FgWhite, color.BgMagenta, color.AttrUnderline)
 
 	return fmt.Sprintf("%s%v%s%s", name, version, protocol, additional)
 }
-- 
cgit v1.2.3


From 168087fc3372a0000a19692948bec3555b9cabd0 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 13:44:27 +0300
Subject: add dcat test

---
 internal/config/common.go          |  2 +-
 internal/config/initializer.go     | 19 ++++++------
 internal/io/dlog/loggers/stdout.go | 59 +++++++++++++-------------------------
 internal/io/fs/readfile.go         | 10 +------
 4 files changed, 33 insertions(+), 57 deletions(-)

(limited to 'internal')

diff --git a/internal/config/common.go b/internal/config/common.go
index acc8e6a..255bd28 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -6,7 +6,7 @@ type CommonConfig struct {
 	SSHPort int
 	// Enable experimental features (mainly for dev purposes)
 	ExperimentalFeaturesEnable bool `json:",omitempty"`
-	// LogLevel defines how much is logged. TODO: Adjust JSONschema
+	// LogLevel defines how much is logged.
 	LogLevel string `json:",omitempty"`
 	// The log strategy to use, one of
 	//   stdout: only log to stdout (useful when used with systemd)
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index f1a9ec4..0e725a6 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -72,6 +72,17 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string,
 		common.LogStrategy = "daily"
 	}
 
+	if args.Spartan {
+		args.Quiet = true
+		args.NoColor = true
+		if args.LogLevel == "" {
+			args.LogLevel = "ERROR"
+		}
+	}
+	if args.NoColor {
+		client.TermColorsEnable = false
+	}
+
 	if args.LogLevel != "" {
 		common.LogLevel = args.LogLevel
 	} else if args.ServersStr == "" && args.Discovery == "" {
@@ -82,14 +93,6 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string,
 	if args.SSHPort != DefaultSSHPort {
 		common.SSHPort = args.SSHPort
 	}
-	if args.NoColor {
-		client.TermColorsEnable = false
-	}
-
-	if args.Spartan {
-		args.Quiet = true
-		args.NoColor = true
-	}
 
 	if args.Discovery == "" && args.ServersStr == "" {
 		// We are not connecting to any servers.
diff --git a/internal/io/dlog/loggers/stdout.go b/internal/io/dlog/loggers/stdout.go
index 9738323..05485c6 100644
--- a/internal/io/dlog/loggers/stdout.go
+++ b/internal/io/dlog/loggers/stdout.go
@@ -8,66 +8,47 @@ import (
 )
 
 type stdout struct {
-	bufferCh chan string
 	pauseCh  chan struct{}
 	resumeCh chan struct{}
+	mutex    sync.Mutex
 }
 
 func newStdout() *stdout {
 	return &stdout{
-		bufferCh: make(chan string, 100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
 	}
 }
 
 func (s *stdout) Start(ctx context.Context, wg *sync.WaitGroup) {
-	pause := func(ctx context.Context) {
-		select {
-		case <-s.resumeCh:
-			return
-		case <-ctx.Done():
-			return
-		}
-	}
-
-	go func() {
-		defer wg.Done()
-
-		for {
-			select {
-			case message := <-s.bufferCh:
-				fmt.Println(message)
-			case <-s.pauseCh:
-				pause(ctx)
-			case <-ctx.Done():
-				s.Flush()
-				return
-			}
-		}
-	}()
+	wg.Done()
 }
 
 func (s *stdout) Log(now time.Time, message string) {
-	s.bufferCh <- message
+	s.log(message)
 }
 
 func (s *stdout) LogWithColors(now time.Time, message, coloredMessage string) {
-	s.bufferCh <- coloredMessage
+	s.log(coloredMessage)
 }
 
-func (s *stdout) Flush() {
-	for {
-		select {
-		case message := <-s.bufferCh:
-			fmt.Println(message)
-		default:
-			return
-		}
+func (s *stdout) log(message string) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	select {
+	case <-s.pauseCh:
+		// Pause until resumed.
+		<-s.resumeCh
+	default:
 	}
+
+	fmt.Println(message)
 }
 
-func (s *stdout) Pause()            { s.pauseCh <- struct{}{} }
-func (s *stdout) Resume()           { s.resumeCh <- struct{}{} }
-func (s *stdout) Rotate()           {}
+func (s *stdout) Pause()  { s.pauseCh <- struct{}{} }
+func (s *stdout) Resume() { s.resumeCh <- struct{}{} }
+func (s *stdout) Flush()  {}
+func (s *stdout) Rotate() {}
+
 func (stdout) SupportsColors() bool { return true }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 07486a1..f128c07 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -13,8 +13,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/regex"
 
@@ -182,14 +182,6 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 
 		switch b {
 		case '\n':
-			/*
-				// dcat/dgrep should actually transfer empty lines
-					if message.Len() == 0 {
-						time.Sleep(time.Millisecond * 100)
-						continue
-					}
-			*/
-			//message.WriteString("\n")
 			select {
 			case rawLines <- message:
 				message = pool.BytesBuffer.Get().(*bytes.Buffer)
-- 
cgit v1.2.3


From b0973dd7d17269ad9d4ebfd4842072e739c59eca Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 3 Oct 2021 13:09:32 +0300
Subject: add dmap tests

---
 internal/clients/maprclient.go | 2 +-
 internal/config/common.go      | 1 +
 internal/config/config.go      | 5 ++++-
 internal/config/initializer.go | 6 ++++--
 internal/io/dlog/dlog.go       | 7 +++++--
 5 files changed, 15 insertions(+), 6 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index f23aa08..19fe119 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -203,7 +203,7 @@ func (c *MaprClient) printResults() {
 	dlog.Client.Raw(rawQuery)
 
 	if rowsLimit > 0 && numRows > rowsLimit {
-		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited output to %d rows! Use 'limit' clause to override!",
+		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited terminal output to %d rows! Use 'limit' clause to override!",
 			numRows, rowsLimit))
 	}
 	dlog.Client.Raw(result)
diff --git a/internal/config/common.go b/internal/config/common.go
index 255bd28..5e81bc9 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -26,6 +26,7 @@ func newDefaultCommonConfig() *CommonConfig {
 		SSHPort:                    DefaultSSHPort,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
+		LogLevel:                   "INFO",
 		CacheDir:                   "cache",
 		TmpDir:                     "/tmp",
 	}
diff --git a/internal/config/config.go b/internal/config/config.go
index 09ae994..d58162f 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1,5 +1,7 @@
 package config
 
+import "github.com/mimecast/dtail/internal/source"
+
 const (
 	// ControlUser is used for various DTail specific operations.
 	ControlUser string = "DTAIL-CONTROL"
@@ -25,7 +27,7 @@ var Server *ServerConfig
 var Common *CommonConfig
 
 // Setup the DTail configuration.
-func Setup(args *Args, additionalArgs []string) {
+func Setup(sourceProcess source.Source, args *Args, additionalArgs []string) {
 	initializer := initializer{
 		Common: newDefaultCommonConfig(),
 		Server: newDefaultServerConfig(),
@@ -33,6 +35,7 @@ func Setup(args *Args, additionalArgs []string) {
 	}
 	initializer.parseConfig(args)
 	Client, Server, Common = initializer.transformConfig(
+		sourceProcess,
 		args, additionalArgs,
 		initializer.Client,
 		initializer.Server,
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 0e725a6..a58f82a 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -7,6 +7,8 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/source"
 )
 
 // Used to initialize the configuration.
@@ -56,7 +58,7 @@ func (c *initializer) parseSpecificConfig(configFile string) {
 	}
 }
 
-func (c *initializer) transformConfig(args *Args, additionalArgs []string,
+func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string,
 	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
 	if args.LogDir != "" {
 		common.LogDir = args.LogDir
@@ -85,7 +87,7 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string,
 
 	if args.LogLevel != "" {
 		common.LogLevel = args.LogLevel
-	} else if args.ServersStr == "" && args.Discovery == "" {
+	} else if sourceProcess == source.Client && args.ServersStr == "" && args.Discovery == "" {
 		// We are in serverless mode. Default log level is WARN.
 		common.LogLevel = "WARN"
 	}
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 6cacfe2..2beda75 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -163,8 +163,11 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 
 func (d *DLog) FatalPanic(args ...interface{}) {
 	d.log(FATAL, args)
-	d.logger.Flush()
-	panic("Not recovering from this fatal error...")
+	d.Flush()
+
+	var sb strings.Builder
+	d.writeArgStrings(&sb, args)
+	panic(sb.String())
 }
 
 func (d *DLog) Fatal(args ...interface{}) string {
-- 
cgit v1.2.3


From 117f062a01e9d456e19f738a0b7047d9bb01285f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 3 Oct 2021 13:32:20 +0300
Subject: when a mapreduce outfile is specified also always write a
 outfile.query file

---
 internal/mapr/groupset.go | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

(limited to 'internal')

diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index df8c603..ce7630d 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -178,11 +178,31 @@ func (g *GroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	return sb.String(), len(rows), nil
 }
 
+func (*GroupSet) writeQueryFile(query *Query) error {
+	queryFile := fmt.Sprintf("%s.query", query.Outfile)
+	tmpQueryFile := fmt.Sprintf("%s.tmp", queryFile)
+	dlog.Common.Debug("Writing query file", queryFile)
+
+	fd, err := os.Create(tmpQueryFile)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	fd.WriteString(query.RawQuery)
+	os.Rename(tmpQueryFile, queryFile)
+
+	return nil
+}
+
 // WriteResult writes the result to an CSV outfile.
 func (g *GroupSet) WriteResult(query *Query) error {
 	if !query.HasOutfile() {
 		return errors.New("No outfile specified")
 	}
+	if err := g.writeQueryFile(query); err != nil {
+		return err
+	}
 
 	rows, _, err := g.result(query, false)
 	if err != nil {
@@ -192,22 +212,22 @@ func (g *GroupSet) WriteResult(query *Query) error {
 	dlog.Common.Info("Writing outfile", query.Outfile)
 	tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile)
 
-	file, err := os.Create(tmpOutfile)
+	fd, err := os.Create(tmpOutfile)
 	if err != nil {
 		return err
 	}
-	defer file.Close()
+	defer fd.Close()
 
 	// Generate header now
 	lastIndex := len(query.Select) - 1
 	for i, sc := range query.Select {
-		file.WriteString(sc.FieldStorage)
+		fd.WriteString(sc.FieldStorage)
 		if i == lastIndex {
 			continue
 		}
-		file.WriteString(protocol.CSVDelimiter)
+		fd.WriteString(protocol.CSVDelimiter)
 	}
-	file.WriteString("\n")
+	fd.WriteString("\n")
 
 	// And now write the data
 	for i, r := range rows {
@@ -215,13 +235,13 @@ func (g *GroupSet) WriteResult(query *Query) error {
 			break
 		}
 		for j, value := range r.values {
-			file.WriteString(value)
+			fd.WriteString(value)
 			if j == lastIndex {
 				continue
 			}
-			file.WriteString(protocol.CSVDelimiter)
+			fd.WriteString(protocol.CSVDelimiter)
 		}
-		file.WriteString("\n")
+		fd.WriteString("\n")
 	}
 
 	if err := os.Rename(tmpOutfile, query.Outfile); err != nil {
-- 
cgit v1.2.3


From 2ff78a71087d4ba95121996338d0c418b907acfe Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 5 Oct 2021 10:00:38 +0300
Subject: more on this

---
 internal/clients/baseclient.go                  |   8 +-
 internal/clients/connectors/serverconnection.go |  19 --
 internal/clients/connectors/serverless.go       |  22 +-
 internal/clients/handlers/healthhandler.go      | 114 ++++----
 internal/clients/healthclient.go                |  97 ++-----
 internal/io/dlog/dlog.go                        |   6 +
 internal/server/handlers/basehandler.go         | 283 ++++++++++++++++++++
 internal/server/handlers/readcommand.go         |   4 +-
 internal/server/handlers/serverhandler.go       | 332 +++---------------------
 internal/source/source.go                       |   9 +-
 10 files changed, 417 insertions(+), 477 deletions(-)
 create mode 100644 internal/server/handlers/basehandler.go

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index fc01955..5ac298f 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -86,7 +86,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	var mutex sync.Mutex
 	for i, conn := range c.connections {
 		go func(i int, conn connectors.Connector) {
-			connStatus := c.start(ctx, active, i, conn)
+			connStatus := c.startConnection(ctx, active, i, conn)
 
 			// Update global status.
 			mutex.Lock()
@@ -97,11 +97,12 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 		}(i, conn)
 	}
 
+	time.Sleep(time.Second * 2)
 	c.waitUntilDone(ctx, active)
 	return
 }
 
-func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, conn connectors.Connector) (status int) {
+func (c *baseClient) startConnection(ctx context.Context, active chan struct{}, i int, conn connectors.Connector) (status int) {
 	// Increment connection count
 	active <- struct{}{}
 	// Derement connection count
@@ -146,12 +147,13 @@ func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
 		<-ctx.Done()
 	}
 
+	// TODO: Rewrite this to use a wait group.
 	for {
 		numActive := len(active)
 		if numActive == 0 {
 			return
 		}
 		dlog.Client.Debug("Active connections", numActive)
-		time.Sleep(time.Second)
+		time.Sleep(time.Second * time.Millisecond * 100)
 	}
 }
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 5bc63ee..1666a79 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -23,7 +23,6 @@ type ServerConnection struct {
 	config          *ssh.ClientConfig
 	handler         handlers.Handler
 	commands        []string
-	isOneOff        bool
 	hostKeyCallback client.HostKeyCallback
 	throttlingDone  bool
 }
@@ -49,24 +48,6 @@ func NewServerConnection(server string, userName string, authMethods []ssh.AuthM
 	return &c
 }
 
-// NewOneOffServerConnection creates new one-off connection (only for sending a series of commands and then quit).
-func NewOneOffServerConnection(server string, userName string, authMethods []ssh.AuthMethod, handler handlers.Handler, commands []string) *ServerConnection {
-	c := ServerConnection{
-		server:   server,
-		handler:  handler,
-		commands: commands,
-		config: &ssh.ClientConfig{
-			User:            userName,
-			Auth:            authMethods,
-			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		},
-		isOneOff: true,
-	}
-
-	c.initServerPort()
-	return &c
-}
-
 func (c *ServerConnection) Server() string {
 	return c.server
 }
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index 7740aab..ae72c9b 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -20,14 +20,12 @@ type Serverless struct {
 
 // NewServerConnection returns a new connection.
 func NewServerless(userName string, handler handlers.Handler, commands []string) *Serverless {
-	s := Serverless{
+	dlog.Client.Debug("Creating new serverless connector", handler, commands)
+	return &Serverless{
 		userName: userName,
 		handler:  handler,
 		commands: commands,
 	}
-
-	dlog.Client.Debug("Creating new serverless connector", handler, commands)
-	return &s
 }
 
 func (s *Serverless) Server() string {
@@ -58,11 +56,17 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 		return err
 	}
 
-	serverHandler := serverHandlers.NewServerHandler(
-		user,
-		make(chan struct{}, config.Server.MaxConcurrentCats),
-		make(chan struct{}, config.Server.MaxConcurrentTails),
-	)
+	var serverHandler serverHandlers.Handler
+	switch s.userName {
+	case config.ControlUser:
+		serverHandler = serverHandlers.NewControlHandler(user)
+	default:
+		serverHandler = serverHandlers.NewServerHandler(
+			user,
+			make(chan struct{}, config.Server.MaxConcurrentCats),
+			make(chan struct{}, config.Server.MaxConcurrentTails),
+		)
+	}
 
 	terminate := func() {
 		serverHandler.Shutdown()
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index eca0348..4949985 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -1,90 +1,72 @@
 package handlers
 
 import (
-	"bytes"
-	"errors"
 	"fmt"
-	"time"
+	"strings"
 
 	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// HealthHandler implements the handler required for health checks.
+// HealthHandler is the handler used on the client side for running mapreduce aggregations.
 type HealthHandler struct {
-	done *internal.Done
-	// Buffer of incoming data from server.
-	receiveBuf bytes.Buffer
-	// To send commands to the server.
-	commands chan string
-	// To receive messages from the server.
-	receive chan<- string
-	// The remote server address
-	server string
-	// The return status.
-	status int
+	baseHandler
+	HealthStatusCh chan<- int
 }
 
-// NewHealthHandler returns a new health check handler.
-func NewHealthHandler(server string, receive chan<- string) *HealthHandler {
-	h := HealthHandler{
-		server:   server,
-		receive:  receive,
-		commands: make(chan string),
-		status:   -1,
-		done:     internal.NewDone(),
+// NewHealthHandler returns a new health client handler.
+func NewHealthHandler(server string) *HealthHandler {
+	dlog.Client.Debug(server, "Creating new health handler")
+	return &HealthHandler{
+		baseHandler: baseHandler{
+			server:       server,
+			shellStarted: false,
+			commands:     make(chan string),
+			status:       -1,
+			done:         internal.NewDone(),
+		},
+		HealthStatusCh: make(chan int),
 	}
-
-	return &h
-}
-
-// Server returns the remote server name.
-func (h *HealthHandler) Server() string {
-	return h.server
-}
-
-// Status of the handler.
-func (h *HealthHandler) Status() int {
-	return h.status
-}
-
-// Done returns done channel of the handler.
-func (h *HealthHandler) Done() <-chan struct{} {
-	return h.done.Done()
 }
 
-// Shutdown the handler.
-func (h *HealthHandler) Shutdown() {
-	h.done.Shutdown()
-}
-
-// SendMessage sends a DTail command to the server.
-func (h *HealthHandler) SendMessage(command string) error {
-	select {
-	case h.commands <- fmt.Sprintf("%s;", command):
-	case <-time.NewTimer(time.Second * 10).C:
-		return errors.New("Timed out sending command " + command)
-	case <-h.Done():
-	}
-
-	return nil
-}
-
-// Server writes byte stream to client.
+// Read data from the dtail server via Writer interface.
 func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		h.receiveBuf.WriteByte(b)
-		if b == protocol.MessageDelimiter {
-			h.receive <- h.receiveBuf.String()
-			h.receiveBuf.Reset()
+		switch b {
+		case '\n':
+			continue
+		case protocol.MessageDelimiter:
+			message := h.baseHandler.receiveBuf.String()
+			dlog.Client.Debug(message)
+			h.handleHealthMessage(message)
+			h.baseHandler.receiveBuf.Reset()
+		default:
+			h.baseHandler.receiveBuf.WriteByte(b)
 		}
 	}
 
 	return len(p), nil
 }
 
-// Server reads byte stream from client.
-func (h *HealthHandler) Read(p []byte) (n int, err error) {
-	n = copy(p, []byte(<-h.commands))
-	return
+func (h *HealthHandler) handleHealthMessage(message string) {
+	s := strings.Split(message, protocol.FieldDelimiter)
+	message = s[len(s)-1]
+	status := strings.Split(message, ":")
+	fmt.Println(status)
+	/*
+		switch status {
+		case "OK":
+			h.HealthStatusCh <- 0
+		case "WARNING":
+			h.HealthStatusCh <- 1
+		case "CRITICAL":
+			h.HealthStatusCh <- 2
+		case "UNKNOWN":
+			h.HealthStatusCh <- 3
+		default:
+			fmt.Println("CRITICAL: Unexpected server response: '%s'")
+			h.HealthStatusCh <- 2
+		}
+	*/
 }
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index 47007b6..df919ae 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -1,101 +1,44 @@
 package clients
 
 import (
-	"context"
-	"fmt"
 	"runtime"
-	"strings"
-	"time"
 
-	"github.com/mimecast/dtail/internal/clients/connectors"
 	"github.com/mimecast/dtail/internal/clients/handlers"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/omode"
-	"github.com/mimecast/dtail/internal/protocol"
 
 	gossh "golang.org/x/crypto/ssh"
 )
 
-// HealthClient is used for health checking (e.g. via Nagios)
+// HealthClient is used to perform a basic server health check.
 type HealthClient struct {
-	// Client operating mode
-	mode omode.Mode
-	// The remote server address
-	server string
-	// SSH user name
-	userName string
-	// SSH auth methods to use to connect to the remote servers.
-	sshAuthMethods []gossh.AuthMethod
+	baseClient
 }
 
-// NewHealthClient returns a new healh client.
-func NewHealthClient(mode omode.Mode) (*HealthClient, error) {
+// NewHealthClient returns a new health client.
+func NewHealthClient(args config.Args) (*HealthClient, error) {
+	args.Mode = omode.HealthClient
+	args.UserName = config.ControlUser
 	c := HealthClient{
-		mode:     mode,
-		server:   fmt.Sprintf("%s:%d", config.Server.SSHBindAddress, config.Common.SSHPort),
-		userName: config.ControlUser,
+		baseClient: baseClient{
+			Args:       args,
+			throttleCh: make(chan struct{}, args.ConnectionsPerCPU*runtime.NumCPU()),
+			retry:      false,
+		},
 	}
-	c.initSSHAuthMethods()
+
+	c.init()
+	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.ControlUser))
+	c.makeConnections(c)
 
 	return &c, nil
 }
 
-// Start the health client.
-func (c *HealthClient) Start(ctx context.Context) (status int) {
-	receive := make(chan string)
-
-	throttleCh := make(chan struct{}, runtime.NumCPU())
-	statsCh := make(chan struct{}, 1)
-
-	conn := connectors.NewOneOffServerConnection(
-		c.server,
-		c.userName,
-		c.sshAuthMethods,
-		handlers.NewHealthHandler(c.server, receive),
-		[]string{c.mode.String()},
-	)
-
-	connCtx, cancel := context.WithCancel(ctx)
-	go conn.Start(connCtx, cancel, throttleCh, statsCh)
-
-	for {
-		select {
-		case data := <-receive:
-			// Parse recieved data.
-			s := strings.Split(data, protocol.FieldDelimiter)
-			message := s[len(s)-1]
-			if strings.HasPrefix(message, "done;") {
-				return
-			}
-
-			// Set severity.
-			s = strings.Split(message, ":")
-			switch s[0] {
-			case "OK":
-			case "WARNING":
-				if status < 1 {
-					status = 1
-				}
-			case "CRITICAL":
-				status = 2
-			case "UNKNOWN":
-				status = 3
-			default:
-				fmt.Printf("CRITICAL: Unexpected server response: '%s'\n", message)
-				status = 2
-				return
-			}
-			fmt.Print(message)
-
-		case <-time.After(time.Second * 2):
-			status = 2
-			fmt.Println("CRITICAL: Could not communicate with DTail server")
-			return
-		}
-	}
+func (c HealthClient) makeHandler(server string) handlers.Handler {
+	return handlers.NewHealthHandler(server)
 }
 
-// Initialize SSH auth methods.
-func (c *HealthClient) initSSHAuthMethods() {
-	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.ControlUser))
+func (c HealthClient) makeCommands() (commands []string) {
+	commands = append(commands, "health")
+	return
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 2beda75..db99307 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -57,6 +57,12 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Client = New(source.Server, source.Client, level, impl, strategy)
 		Server = New(source.Server, source.Server, level, impl, strategy)
 		Common = Server
+	case source.HealthCheck:
+		// Health check isn't logging anything.
+		impl := loggers.STDOUT
+		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
+		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
+		Common = Client
 	}
 
 	var wg2 sync.WaitGroup
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
new file mode 100644
index 0000000..12fb2b3
--- /dev/null
+++ b/internal/server/handlers/basehandler.go
@@ -0,0 +1,283 @@
+package handlers
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/mapr/server"
+	"github.com/mimecast/dtail/internal/protocol"
+	user "github.com/mimecast/dtail/internal/user/server"
+)
+
+type handleCommandCb func(context.Context, int, []string)
+
+type baseHandler struct {
+	done             *internal.Done
+	handleCommandCb  handleCommandCb
+	lines            chan line.Line
+	aggregate        *server.Aggregate
+	maprMessages     chan string
+	serverMessages   chan string
+	hostname         string
+	user             *user.User
+	ackCloseReceived chan struct{}
+	activeCommands   int32
+	quiet            bool
+	spartan          bool
+	serverless       bool
+	readBuf          bytes.Buffer
+	writeBuf         bytes.Buffer
+}
+
+// Shutdown the handler.
+func (h *baseHandler) Shutdown() {
+	h.done.Shutdown()
+}
+
+// Done channel of the handler.
+func (h *baseHandler) Done() <-chan struct{} {
+	return h.done.Done()
+}
+
+// Read is to send data to the dtail client via Reader interface.
+func (h *baseHandler) Read(p []byte) (n int, err error) {
+	defer h.readBuf.Reset()
+
+	select {
+	case message := <-h.serverMessages:
+		if message[0] == '.' {
+			// Handle hidden message (don't display to the user, interpreted by dtail client)
+			h.readBuf.WriteString(message)
+			h.readBuf.WriteByte(protocol.MessageDelimiter)
+			n = copy(p, h.readBuf.Bytes())
+			return
+		}
+
+		if h.serverless {
+			// In serverless mode we have logged the server message already via the
+			// dlog logger, no need to send the message again to the client part.
+			return
+		}
+
+		// Handle normal server message (display to the user)
+		h.readBuf.WriteString("SERVER")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(message)
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+
+	case message := <-h.maprMessages:
+		// Send mapreduce-aggregated data as a message.
+		h.readBuf.WriteString("AGGREGATE")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(message)
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+
+	case line := <-h.lines:
+		if !h.spartan {
+			h.readBuf.WriteString("REMOTE")
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(h.hostname)
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(line.SourceID)
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+		}
+		h.readBuf.WriteString(line.Content.String())
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+		pool.RecycleBytesBuffer(line.Content)
+
+	case <-time.After(time.Second):
+		// Once in a while check whether we are done.
+		select {
+		case <-h.done.Done():
+			err = io.EOF
+			return
+		default:
+		}
+	}
+	return
+}
+
+// Write is to receive data from the dtail client via Writer interface.
+func (h *baseHandler) Write(p []byte) (n int, err error) {
+	for _, b := range p {
+		switch b {
+		case ';':
+			h.handleCommand(string(h.writeBuf.Bytes()))
+			h.writeBuf.Reset()
+		default:
+			h.writeBuf.WriteByte(b)
+		}
+	}
+
+	n = len(p)
+	return
+}
+
+func (h *baseHandler) handleCommand(commandStr string) {
+	dlog.Server.Debug(h.user, commandStr)
+
+	args, argc, add, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
+	if err != nil {
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err)+add)
+		return
+	}
+
+	args, argc, err = h.handleBase64(args, argc)
+	if err != nil {
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		<-h.done.Done()
+		cancel()
+	}()
+
+	h.handleCommandCb(ctx, argc, args)
+}
+
+func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
+	argc := len(args)
+	var add string
+
+	if argc <= 2 || args[0] != "protocol" {
+		return args, argc, add, errors.New("unable to determine protocol version")
+	}
+
+	if args[1] != protocol.ProtocolCompat {
+		clientCompat, _ := strconv.Atoi(args[1])
+		serverCompat, _ := strconv.Atoi(protocol.ProtocolCompat)
+		if clientCompat <= 3 {
+			// Protocol version 3 or lower expect a newline as message separator
+			// One day (after 2 major versions) this exception may be removed!
+			add = "\n"
+		}
+
+		toUpdate := "client"
+		if clientCompat > serverCompat {
+			toUpdate = "server"
+		}
+
+		err := fmt.Errorf("DTail server protocol version '%s' does not match client protocol version '%s', please update DTail %s!",
+			protocol.ProtocolCompat, args[1], toUpdate)
+		return args, argc, add, err
+	}
+
+	return args[2:], argc - 2, add, nil
+}
+
+func (h *baseHandler) handleBase64(args []string, argc int) ([]string, int, error) {
+	err := errors.New("Unable to decode client message, DTail server and client versions may not be compatible")
+
+	if argc != 2 || args[0] != "base64" {
+		return args, argc, err
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(args[1])
+	if err != nil {
+		return args, argc, err
+	}
+	decodedStr := string(decoded)
+
+	args = strings.Split(decodedStr, " ")
+	argc = len(decodedStr)
+	dlog.Server.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
+
+	return args, argc, nil
+}
+
+func (h *baseHandler) handleAckCommand(argc int, args []string) {
+	if argc < 3 {
+		if !h.quiet {
+			h.send(h.serverMessages, dlog.Server.Warn(h.user, "Unable to parse command", args, argc))
+		}
+		return
+	}
+	if args[1] == "close" && args[2] == "connection" {
+		select {
+		case <-h.ackCloseReceived:
+		default:
+			close(h.ackCloseReceived)
+		}
+	}
+}
+
+func (h *baseHandler) send(ch chan<- string, message string) {
+	select {
+	case ch <- message:
+	case <-h.done.Done():
+	}
+}
+
+func (h *baseHandler) flush() {
+	dlog.Server.Debug(h.user, "flush()")
+
+	numUnsentMessages := func() int {
+		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
+	}
+
+	for i := 0; i < 3; i++ {
+		if numUnsentMessages() == 0 {
+			dlog.Server.Debug(h.user, "All lines sent")
+			return
+		}
+		dlog.Server.Debug(h.user, "Still lines to be sent")
+		time.Sleep(time.Second)
+	}
+
+	dlog.Server.Warn(h.user, "Some lines remain unsent", numUnsentMessages())
+}
+
+func (h *baseHandler) shutdown() {
+	dlog.Server.Debug(h.user, "shutdown()")
+	h.flush()
+
+	go func() {
+		select {
+		case h.serverMessages <- ".syn close connection":
+		case <-h.done.Done():
+		}
+	}()
+
+	select {
+	case <-h.ackCloseReceived:
+	case <-time.After(time.Second * 5):
+		dlog.Server.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
+	case <-h.done.Done():
+	}
+
+	h.done.Shutdown()
+}
+
+func (h *baseHandler) incrementActiveCommands() {
+	atomic.AddInt32(&h.activeCommands, 1)
+}
+
+func (h *baseHandler) decrementActiveCommands() int32 {
+	atomic.AddInt32(&h.activeCommands, -1)
+	return atomic.LoadInt32(&h.activeCommands)
+}
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 6579018..abc44c7 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -32,13 +32,13 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string, retrie
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user, commandParseWarning, err))
+			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user, "Unable to parse command", err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, commandParseWarning, args, argc))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to parse command", args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re, retries)
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index ace2626..2ec4fbf 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -1,69 +1,60 @@
 package handlers
 
 import (
-	"bytes"
 	"context"
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"io"
 	"os"
-	"strconv"
 	"strings"
-	"sync/atomic"
-	"time"
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/pool"
-	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
-	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
-const (
-	commandParseWarning string = "Unable to parse command"
-)
-
 // ServerHandler implements the Reader and Writer interfaces to handle
 // the Bi-directional communication between SSH client and server.
 // This handler implements the handler of the SSH server.
 type ServerHandler struct {
-	done             *internal.Done
-	lines            chan line.Line
-	regex            string
-	aggregate        *server.Aggregate
-	maprMessages     chan string
-	serverMessages   chan string
-	hostname         string
-	user             *user.User
-	catLimiter       chan struct{}
-	tailLimiter      chan struct{}
-	ackCloseReceived chan struct{}
-	activeCommands   int32
-	quiet            bool
-	spartan          bool
-	serverless       bool
-	readBuf          bytes.Buffer
-	writeBuf         bytes.Buffer
+	baseHandler
+	catLimiter  chan struct{}
+	tailLimiter chan struct{}
+	regex       string
+	/*
+		done             *internal.Done
+		lines            chan line.Line
+		aggregate        *server.Aggregate
+		maprMessages     chan string
+		serverMessages   chan string
+		hostname         string
+		user             *user.User
+		ackCloseReceived chan struct{}
+		activeCommands   int32
+		quiet            bool
+		spartan          bool
+		serverless       bool
+		readBuf          bytes.Buffer
+		writeBuf         bytes.Buffer
+	*/
 }
 
 // NewServerHandler returns the server handler.
 func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
 	h := ServerHandler{
-		done:             internal.NewDone(),
-		lines:            make(chan line.Line, 100),
-		serverMessages:   make(chan string, 10),
-		maprMessages:     make(chan string, 10),
-		ackCloseReceived: make(chan struct{}),
-		catLimiter:       catLimiter,
-		tailLimiter:      tailLimiter,
-		regex:            ".",
-		user:             user,
-	}
+		baseHandler: baseHandler{
+			done:             internal.NewDone(),
+			lines:            make(chan line.Line, 100),
+			serverMessages:   make(chan string, 10),
+			maprMessages:     make(chan string, 10),
+			ackCloseReceived: make(chan struct{}),
+			user:             user,
+		},
+		catLimiter:  catLimiter,
+		tailLimiter: tailLimiter,
+		regex:       ".",
+	}
+	h.handleCommandCb = h.handleUserCommand
 
 	fqdn, err := os.Hostname()
 	if err != nil {
@@ -76,192 +67,8 @@ func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *S
 	return &h
 }
 
-// Shutdown the handler.
-func (h *ServerHandler) Shutdown() {
-	h.done.Shutdown()
-}
-
-// Done channel of the handler.
-func (h *ServerHandler) Done() <-chan struct{} {
-	return h.done.Done()
-}
-
-// Read is to send data to the dtail client via Reader interface.
-func (h *ServerHandler) Read(p []byte) (n int, err error) {
-	defer h.readBuf.Reset()
-
-	select {
-	case message := <-h.serverMessages:
-		if message[0] == '.' {
-			// Handle hidden message (don't display to the user, interpreted by dtail client)
-			h.readBuf.WriteString(message)
-			h.readBuf.WriteByte(protocol.MessageDelimiter)
-			n = copy(p, h.readBuf.Bytes())
-			return
-		}
-
-		if h.serverless {
-			// In serverless mode we have logged the server message already via the
-			// dlog logger, no need to send the message again to the client part.
-			return
-		}
-
-		// Handle normal server message (display to the user)
-		h.readBuf.WriteString("SERVER")
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(h.hostname)
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(message)
-		h.readBuf.WriteByte(protocol.MessageDelimiter)
-		n = copy(p, h.readBuf.Bytes())
-
-	case message := <-h.maprMessages:
-		// Send mapreduce-aggregated data as a message.
-		h.readBuf.WriteString("AGGREGATE")
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(h.hostname)
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(message)
-		h.readBuf.WriteByte(protocol.MessageDelimiter)
-		n = copy(p, h.readBuf.Bytes())
-
-	case line := <-h.lines:
-		if !h.spartan {
-			h.readBuf.WriteString("REMOTE")
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-			h.readBuf.WriteString(h.hostname)
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-			h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-			h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-			h.readBuf.WriteString(line.SourceID)
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-		}
-		h.readBuf.WriteString(line.Content.String())
-		h.readBuf.WriteByte(protocol.MessageDelimiter)
-		n = copy(p, h.readBuf.Bytes())
-		pool.RecycleBytesBuffer(line.Content)
-
-	case <-time.After(time.Second):
-		// Once in a while check whether we are done.
-		select {
-		case <-h.done.Done():
-			err = io.EOF
-			return
-		default:
-		}
-	}
-	return
-}
-
-// Write is to receive data from the dtail client via Writer interface.
-func (h *ServerHandler) Write(p []byte) (n int, err error) {
-	for _, b := range p {
-		switch b {
-		case ';':
-			h.handleCommand(string(h.writeBuf.Bytes()))
-			h.writeBuf.Reset()
-		default:
-			h.writeBuf.WriteByte(b)
-		}
-	}
-
-	n = len(p)
-	return
-}
-
-func (h *ServerHandler) handleCommand(commandStr string) {
-	dlog.Server.Debug(h.user, commandStr)
-	ctx := context.Background()
-
-	args, argc, add, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
-	if err != nil {
-		h.send(h.serverMessages, dlog.Server.Error(h.user, err)+add)
-		return
-	}
-
-	args, argc, err = h.handleBase64(args, argc)
-	if err != nil {
-		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
-		return
-	}
-
-	if h.user.Name == config.ControlUser {
-		h.handleControlCommand(argc, args)
-		return
-	}
-
-	ctx, cancel := context.WithCancel(ctx)
-	go func() {
-		<-h.done.Done()
-		cancel()
-	}()
-
-	h.handleUserCommand(ctx, argc, args)
-}
-
-func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
-	argc := len(args)
-	var add string
-
-	if argc <= 2 || args[0] != "protocol" {
-		return args, argc, add, errors.New("unable to determine protocol version")
-	}
-
-	if args[1] != protocol.ProtocolCompat {
-		clientCompat, _ := strconv.Atoi(args[1])
-		serverCompat, _ := strconv.Atoi(protocol.ProtocolCompat)
-		if clientCompat <= 3 {
-			// Protocol version 3 or lower expect a newline as message separator
-			// One day (after 2 major versions) this exception may be removed!
-			add = "\n"
-		}
-
-		toUpdate := "client"
-		if clientCompat > serverCompat {
-			toUpdate = "server"
-		}
-
-		err := fmt.Errorf("DTail server protocol version '%s' does not match client protocol version '%s', please update DTail %s!",
-			protocol.ProtocolCompat, args[1], toUpdate)
-		return args, argc, add, err
-	}
-
-	return args[2:], argc - 2, add, nil
-}
-
-func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, error) {
-	err := errors.New("Unable to decode client message, DTail server and client versions may not be compatible")
-
-	if argc != 2 || args[0] != "base64" {
-		return args, argc, err
-	}
-
-	decoded, err := base64.StdEncoding.DecodeString(args[1])
-	if err != nil {
-		return args, argc, err
-	}
-	decodedStr := string(decoded)
-
-	args = strings.Split(decodedStr, " ")
-	argc = len(decodedStr)
-	dlog.Server.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
-
-	return args, argc, nil
-}
-
-func (h *ServerHandler) handleControlCommand(argc int, args []string) {
-	switch args[0] {
-	case "debug":
-		h.send(h.serverMessages, dlog.Server.Debug(h.user, "Receiving debug command", argc, args))
-	default:
-		dlog.Server.Warn(h.user, "Received unknown control command", argc, args)
-	}
-}
-
 func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string) {
-	dlog.Server.Debug(h.user, "handleUserCommand", argc, args)
+	dlog.Server.Debug(h.user, "Handling user command", argc, args)
 
 	h.incrementActiveCommands()
 	commandFinished := func() {
@@ -332,74 +139,3 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		commandFinished()
 	}
 }
-
-func (h *ServerHandler) handleAckCommand(argc int, args []string) {
-	if argc < 3 {
-		if !h.quiet {
-			h.send(h.serverMessages, dlog.Server.Warn(h.user, commandParseWarning, args, argc))
-		}
-		return
-	}
-	if args[1] == "close" && args[2] == "connection" {
-		select {
-		case <-h.ackCloseReceived:
-		default:
-			close(h.ackCloseReceived)
-		}
-	}
-}
-
-func (h *ServerHandler) send(ch chan<- string, message string) {
-	select {
-	case ch <- message:
-	case <-h.done.Done():
-	}
-}
-
-func (h *ServerHandler) flush() {
-	dlog.Server.Debug(h.user, "flush()")
-
-	unsentMessages := func() int {
-		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
-	}
-	for i := 0; i < 3; i++ {
-		if unsentMessages() == 0 {
-			dlog.Server.Debug(h.user, "All lines sent")
-			return
-		}
-		dlog.Server.Debug(h.user, "Still lines to be sent")
-		time.Sleep(time.Second)
-	}
-
-	dlog.Server.Warn(h.user, "Some lines remain unsent", unsentMessages())
-}
-
-func (h *ServerHandler) shutdown() {
-	dlog.Server.Debug(h.user, "shutdown()")
-	h.flush()
-
-	go func() {
-		select {
-		case h.serverMessages <- ".syn close connection":
-		case <-h.done.Done():
-		}
-	}()
-
-	select {
-	case <-h.ackCloseReceived:
-	case <-time.After(time.Second * 5):
-		dlog.Server.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
-	case <-h.done.Done():
-	}
-
-	h.done.Shutdown()
-}
-
-func (h *ServerHandler) incrementActiveCommands() {
-	atomic.AddInt32(&h.activeCommands, 1)
-}
-
-func (h *ServerHandler) decrementActiveCommands() int32 {
-	atomic.AddInt32(&h.activeCommands, -1)
-	return atomic.LoadInt32(&h.activeCommands)
-}
diff --git a/internal/source/source.go b/internal/source/source.go
index 73dccb2..be7aecd 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -3,8 +3,9 @@ package source
 type Source int
 
 const (
-	Client Source = iota
-	Server Source = iota
+	Client      Source = iota
+	Server      Source = iota
+	HealthCheck Source = iota
 )
 
 func (s Source) String() string {
@@ -13,7 +14,9 @@ func (s Source) String() string {
 		return "CLIENT"
 	case Server:
 		return "SERVER"
+	case HealthCheck:
+		return "HEALTHCHECK"
 	}
 
-	panic("Unknown log source type")
+	panic("Unknown source type")
 }
-- 
cgit v1.2.3


From 56b613e56a2598f8acd83b25005ed27a82a69067 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 5 Oct 2021 22:39:58 +0300
Subject: more on this

---
 internal/clients/baseclient.go            | 47 +++++++------------------------
 internal/clients/connectors/serverless.go |  1 +
 internal/clients/handlers/basehandler.go  | 23 ++++++---------
 internal/clients/maprclient.go            |  2 ++
 internal/config/client.go                 |  4 +--
 internal/server/handlers/basehandler.go   |  2 +-
 internal/server/handlers/serverhandler.go | 16 -----------
 7 files changed, 25 insertions(+), 70 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 5ac298f..9574e13 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -9,7 +9,6 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/discovery"
 	"github.com/mimecast/dtail/internal/io/dlog"
-	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 	"github.com/mimecast/dtail/internal/ssh/client"
 
@@ -80,13 +79,14 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	// Print client stats every time something on statsCh is recieved.
 	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet)
 
-	// Keep count of active connections
-	active := make(chan struct{}, len(c.connections))
+	var wg sync.WaitGroup
+	wg.Add(len(c.connections))
 
 	var mutex sync.Mutex
 	for i, conn := range c.connections {
 		go func(i int, conn connectors.Connector) {
-			connStatus := c.startConnection(ctx, active, i, conn)
+			defer wg.Done()
+			connStatus := c.startConnection(ctx, i, conn)
 
 			// Update global status.
 			mutex.Lock()
@@ -97,17 +97,11 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 		}(i, conn)
 	}
 
-	time.Sleep(time.Second * 2)
-	c.waitUntilDone(ctx, active)
+	wg.Wait()
 	return
 }
 
-func (c *baseClient) startConnection(ctx context.Context, active chan struct{}, i int, conn connectors.Connector) (status int) {
-	// Increment connection count
-	active <- struct{}{}
-	// Derement connection count
-	defer func() { <-active }()
-
+func (c *baseClient) startConnection(ctx context.Context, i int, conn connectors.Connector) (status int) {
 	for {
 		connCtx, cancel := context.WithCancel(ctx)
 		defer cancel()
@@ -127,33 +121,12 @@ func (c *baseClient) startConnection(ctx context.Context, active chan struct{},
 	}
 }
 
-func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod, hostKeyCallback client.HostKeyCallback) connectors.Connector {
+func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod,
+	hostKeyCallback client.HostKeyCallback) connectors.Connector {
 	if c.Args.Serverless {
-		return connectors.NewServerless(c.UserName, c.maker.makeHandler(server), c.maker.makeCommands())
+		return connectors.NewServerless(c.UserName, c.maker.makeHandler(server),
+			c.maker.makeCommands())
 	}
 	return connectors.NewServerConnection(server, c.UserName, sshAuthMethods,
 		hostKeyCallback, c.maker.makeHandler(server), c.maker.makeCommands())
 }
-
-func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
-	defer dlog.Client.Debug("Terminated connection")
-
-	// We want to have at least one active connection
-	<-active
-	// Put it back on the channel
-	active <- struct{}{}
-
-	if c.Mode == omode.TailClient && c.retry {
-		<-ctx.Done()
-	}
-
-	// TODO: Rewrite this to use a wait group.
-	for {
-		numActive := len(active)
-		if numActive == 0 {
-			return
-		}
-		dlog.Client.Debug("Active connections", numActive)
-		time.Sleep(time.Second * time.Millisecond * 100)
-	}
-}
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index ae72c9b..2a1cec4 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -69,6 +69,7 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 	}
 
 	terminate := func() {
+		dlog.Client.Debug("Terminating serverless connection")
 		serverHandler.Shutdown()
 		cancel()
 	}
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 8acb45f..04124e7 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -40,14 +40,6 @@ func (h *baseHandler) Status() int {
 	return h.status
 }
 
-func (h *baseHandler) Done() <-chan struct{} {
-	return h.done.Done()
-}
-
-func (h *baseHandler) Shutdown() {
-	h.done.Shutdown()
-}
-
 // SendMessage to the server.
 func (h *baseHandler) SendMessage(command string) error {
 	encoded := base64.StdEncoding.EncodeToString([]byte(command))
@@ -77,12 +69,6 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 		*/
 		case '\n', protocol.MessageDelimiter:
 			message := h.receiveBuf.String()
-			/*
-				// dcat/grep should actually display empty lines.
-					if len(message) == 0 {
-						continue
-					}
-			*/
 			h.handleMessageType(message)
 			h.receiveBuf.Reset()
 		default:
@@ -121,5 +107,14 @@ func (h *baseHandler) handleHiddenMessage(message string) {
 	switch {
 	case strings.HasPrefix(message, ".syn close connection"):
 		h.SendMessage(".ack close connection")
+		h.Shutdown()
 	}
 }
+
+func (h *baseHandler) Done() <-chan struct{} {
+	return h.done.Done()
+}
+
+func (h *baseHandler) Shutdown() {
+	h.done.Shutdown()
+}
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 19fe119..92bbe39 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -110,6 +110,8 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	return
 }
 
+// NEXT: Make this a callback function rather trying to use polymorphism to call this.
+// This applies to all clients.
 func (c MaprClient) makeHandler(server string) handlers.Handler {
 	return handlers.NewMaprHandler(server, c.query, c.globalGroup)
 }
diff --git a/internal/config/client.go b/internal/config/client.go
index 152ab15..ecd05c5 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -162,8 +162,8 @@ func newDefaultClientConfig() *ClientConfig {
 				HostnameBg:    color.BgCyan,
 				HostnameFg:    color.FgBlack,
 				TextAttr:      color.AttrNone,
-				TextBg:        color.BgCyan,
-				TextFg:        color.FgBlack,
+				TextBg:        color.BgBlack,
+				TextFg:        color.FgWhite,
 			},
 			Common: commonTermColors{
 				SeverityErrorAttr: color.AttrBold,
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 12fb2b3..b683578 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -242,7 +242,7 @@ func (h *baseHandler) flush() {
 
 	for i := 0; i < 3; i++ {
 		if numUnsentMessages() == 0 {
-			dlog.Server.Debug(h.user, "All lines sent")
+			dlog.Server.Debug(h.user, "All lines sent", fmt.Sprintf("%p", h))
 			return
 		}
 		dlog.Server.Debug(h.user, "Still lines to be sent")
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 2ec4fbf..25cb8ba 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -21,22 +21,6 @@ type ServerHandler struct {
 	catLimiter  chan struct{}
 	tailLimiter chan struct{}
 	regex       string
-	/*
-		done             *internal.Done
-		lines            chan line.Line
-		aggregate        *server.Aggregate
-		maprMessages     chan string
-		serverMessages   chan string
-		hostname         string
-		user             *user.User
-		ackCloseReceived chan struct{}
-		activeCommands   int32
-		quiet            bool
-		spartan          bool
-		serverless       bool
-		readBuf          bytes.Buffer
-		writeBuf         bytes.Buffer
-	*/
 }
 
 // NewServerHandler returns the server handler.
-- 
cgit v1.2.3


From a1f13af6b943a0e8f2c2304bbee86231694304ac Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 6 Oct 2021 09:50:41 +0300
Subject: enable faster shutdown - useful for dgrep/dmap and dcat commands

---
 internal/clients/baseclient.go             | 14 ++---
 internal/clients/connectors/serverless.go  | 15 +++--
 internal/clients/handlers/basehandler.go   |  8 +--
 internal/clients/handlers/healthhandler.go | 14 ++---
 internal/clients/handlers/maprhandler.go   |  2 +-
 internal/clients/healthclient.go           |  4 +-
 internal/config/config.go                  |  4 +-
 internal/io/dlog/dlog.go                   |  2 +
 internal/io/fs/permissions/permission.go   |  2 +-
 internal/server/handlers/basehandler.go    | 20 ++++--
 internal/server/handlers/controlhandler.go | 98 ------------------------------
 internal/server/handlers/healthhandler.go  | 58 ++++++++++++++++++
 internal/server/handlers/serverhandler.go  | 16 ++---
 internal/server/server.go                  | 10 +--
 14 files changed, 118 insertions(+), 149 deletions(-)
 delete mode 100644 internal/server/handlers/controlhandler.go
 create mode 100644 internal/server/handlers/healthhandler.go

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 9574e13..b474208 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -39,8 +39,7 @@ type baseClient struct {
 }
 
 func (c *baseClient) init() {
-	dlog.Client.Debug("Initiating base client")
-	dlog.Client.Debug(c.Args.String())
+	dlog.Client.Debug("Initiating base client", c.Args.String())
 
 	flag := regex.Default
 	if c.Args.RegexInvert {
@@ -48,15 +47,16 @@ func (c *baseClient) init() {
 	}
 	regex, err := regex.New(c.Args.RegexStr, flag)
 	if err != nil {
-		dlog.Client.FatalPanic(c.Regex, "invalid regex!", err, regex)
+		dlog.Client.FatalPanic(c.Regex, "Invalid regex!", err, regex)
 	}
 	c.Regex = regex
-	dlog.Client.Debug("Regex", c.Regex)
 
 	if c.Args.Serverless {
 		return
 	}
-	c.sshAuthMethods, c.hostKeyCallback = client.InitSSHAuthMethods(c.Args.SSHAuthMethods, c.Args.SSHHostKeyCallback, c.Args.TrustAllHosts, c.throttleCh, c.Args.PrivateKeyPathFile)
+	c.sshAuthMethods, c.hostKeyCallback = client.InitSSHAuthMethods(
+		c.Args.SSHAuthMethods, c.Args.SSHHostKeyCallback, c.Args.TrustAllHosts,
+		c.throttleCh, c.Args.PrivateKeyPathFile)
 }
 
 func (c *baseClient) makeConnections(maker maker) {
@@ -71,6 +71,7 @@ func (c *baseClient) makeConnections(maker maker) {
 }
 
 func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status int) {
+	dlog.Client.Trace("Starting base client")
 	// Can be nil when serverless.
 	if c.hostKeyCallback != nil {
 		// Periodically check for unknown hosts, and ask the user whether to trust them or not.
@@ -81,13 +82,12 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 
 	var wg sync.WaitGroup
 	wg.Add(len(c.connections))
-
 	var mutex sync.Mutex
+
 	for i, conn := range c.connections {
 		go func(i int, conn connectors.Connector) {
 			defer wg.Done()
 			connStatus := c.startConnection(ctx, i, conn)
-
 			// Update global status.
 			mutex.Lock()
 			defer mutex.Unlock()
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index 2a1cec4..768a5ce 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -37,6 +37,7 @@ func (s *Serverless) Handler() handlers.Handler {
 }
 
 func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+	dlog.Client.Debug("Starting serverless connector")
 	go func() {
 		defer cancel()
 
@@ -44,7 +45,6 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 			dlog.Client.Warn(err)
 		}
 	}()
-
 	<-ctx.Done()
 }
 
@@ -58,9 +58,11 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 
 	var serverHandler serverHandlers.Handler
 	switch s.userName {
-	case config.ControlUser:
-		serverHandler = serverHandlers.NewControlHandler(user)
+	case config.HealthUser:
+		dlog.Client.Debug("Creating serverless health handler")
+		serverHandler = serverHandlers.NewHealthHandler(user)
 	default:
+		dlog.Client.Debug("Creating serverless server handler")
 		serverHandler = serverHandlers.NewServerHandler(
 			user,
 			make(chan struct{}, config.Server.MaxConcurrentCats),
@@ -76,29 +78,34 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 
 	go func() {
 		io.Copy(serverHandler, s.handler)
+		dlog.Client.Trace("io.Copy(serverHandler, s.handler) => done")
 		terminate()
 	}()
 
 	go func() {
 		io.Copy(s.handler, serverHandler)
+		dlog.Client.Trace("io.Copy(s.handler, serverHandler) => done")
 		terminate()
 	}()
 
 	go func() {
 		select {
 		case <-s.handler.Done():
+			dlog.Client.Trace("<-s.handler.Done()")
 		case <-ctx.Done():
+			dlog.Client.Trace("<-ctx.Done()")
 		}
 		terminate()
 	}()
 
 	// Send all commands to client.
 	for _, command := range s.commands {
-		dlog.Client.Debug(command)
+		dlog.Client.Debug("Sending command to serverless server", command)
 		s.handler.SendMessage(command)
 	}
 
 	<-ctx.Done()
+	dlog.Client.Trace("s.handler.Shutdown()")
 	s.handler.Shutdown()
 
 	return nil
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 04124e7..b520c25 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -69,7 +69,7 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 		*/
 		case '\n', protocol.MessageDelimiter:
 			message := h.receiveBuf.String()
-			h.handleMessageType(message)
+			h.handleMessage(message)
 			h.receiveBuf.Reset()
 		default:
 			h.receiveBuf.WriteByte(b)
@@ -90,9 +90,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	return
 }
 
-// Handle various message types.
-func (h *baseHandler) handleMessageType(message string) {
-	// Hidden server commands starti with a dot "."
+func (h *baseHandler) handleMessage(message string) {
 	if len(message) > 0 && message[0] == '.' {
 		h.handleHiddenMessage(message)
 		return
@@ -106,7 +104,7 @@ func (h *baseHandler) handleMessageType(message string) {
 func (h *baseHandler) handleHiddenMessage(message string) {
 	switch {
 	case strings.HasPrefix(message, ".syn close connection"):
-		h.SendMessage(".ack close connection")
+		go h.SendMessage(".ack close connection")
 		h.Shutdown()
 	}
 }
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 4949985..4b16ce4 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -12,7 +12,6 @@ import (
 // HealthHandler is the handler used on the client side for running mapreduce aggregations.
 type HealthHandler struct {
 	baseHandler
-	HealthStatusCh chan<- int
 }
 
 // NewHealthHandler returns a new health client handler.
@@ -26,7 +25,6 @@ func NewHealthHandler(server string) *HealthHandler {
 			status:       -1,
 			done:         internal.NewDone(),
 		},
-		HealthStatusCh: make(chan int),
 	}
 }
 
@@ -34,12 +32,10 @@ func NewHealthHandler(server string) *HealthHandler {
 func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		switch b {
-		case '\n':
-			continue
-		case protocol.MessageDelimiter:
+		case '\n', protocol.MessageDelimiter:
 			message := h.baseHandler.receiveBuf.String()
 			dlog.Client.Debug(message)
-			h.handleHealthMessage(message)
+			h.handleMessage(message)
 			h.baseHandler.receiveBuf.Reset()
 		default:
 			h.baseHandler.receiveBuf.WriteByte(b)
@@ -49,7 +45,11 @@ func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-func (h *HealthHandler) handleHealthMessage(message string) {
+func (h *HealthHandler) handleMessage(message string) {
+	if len(message) > 0 && message[0] == '.' {
+		h.baseHandler.handleHiddenMessage(message)
+		return
+	}
 	s := strings.Split(message, protocol.FieldDelimiter)
 	message = s[len(s)-1]
 	status := strings.Split(message, ":")
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index 848e7f0..d1acfbd 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -44,7 +44,7 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 			if message[0] == 'A' {
 				h.handleAggregateMessage(message)
 			} else {
-				h.baseHandler.handleMessageType(message)
+				h.baseHandler.handleMessage(message)
 			}
 			h.baseHandler.receiveBuf.Reset()
 		default:
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index df919ae..e2c8ccb 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -18,7 +18,7 @@ type HealthClient struct {
 // NewHealthClient returns a new health client.
 func NewHealthClient(args config.Args) (*HealthClient, error) {
 	args.Mode = omode.HealthClient
-	args.UserName = config.ControlUser
+	args.UserName = config.HealthUser
 	c := HealthClient{
 		baseClient: baseClient{
 			Args:       args,
@@ -28,7 +28,7 @@ func NewHealthClient(args config.Args) (*HealthClient, error) {
 	}
 
 	c.init()
-	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.ControlUser))
+	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.HealthUser))
 	c.makeConnections(c)
 
 	return &c, nil
diff --git a/internal/config/config.go b/internal/config/config.go
index d58162f..f216688 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -3,8 +3,8 @@ package config
 import "github.com/mimecast/dtail/internal/source"
 
 const (
-	// ControlUser is used for various DTail specific operations.
-	ControlUser string = "DTAIL-CONTROL"
+	// HealthUser is used for the health check
+	HealthUser string = "DTAIL-HEALTH"
 	// ScheduleUser is used for non-interactive scheduled mapreduce queries.
 	ScheduleUser string = "DTAIL-SCHEDULE"
 	// ContinuousUser is used for non-interactive continuous mapreduce queries.
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index db99307..a17d6e9 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -201,6 +201,8 @@ func (d *DLog) Debug(args ...interface{}) string {
 }
 
 func (d *DLog) Trace(args ...interface{}) string {
+	_, file, line, _ := runtime.Caller(1)
+	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(TRACE, args)
 }
 
diff --git a/internal/io/fs/permissions/permission.go b/internal/io/fs/permissions/permission.go
index bbcb74e..e80dbb2 100644
--- a/internal/io/fs/permissions/permission.go
+++ b/internal/io/fs/permissions/permission.go
@@ -9,6 +9,6 @@ import (
 // ToRead is to check whether user has read permissions to a given file.
 func ToRead(user, filePath string) (bool, error) {
 	// Only implemented for Linux, always expect true
-	dlog.Common.Warn(user, filePath, "Not performing ACL check, not supported on this platform")
+	dlog.Common.Warn(user, filePath, "Not performing ACL check as not compiled in")
 	return true, nil
 }
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index b683578..4fa8f00 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
@@ -21,7 +22,7 @@ import (
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
-type handleCommandCb func(context.Context, int, []string)
+type handleCommandCb func(context.Context, int, []string, string, map[string]string)
 
 type baseHandler struct {
 	done             *internal.Done
@@ -157,7 +158,16 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		cancel()
 	}()
 
-	h.handleCommandCb(ctx, argc, args)
+	splitted := strings.Split(args[0], ":")
+	commandName := splitted[0]
+
+	options, err := config.DeserializeOptions(splitted[1:])
+	if err != nil {
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
+		return
+	}
+
+	h.handleCommandCb(ctx, argc, args, commandName, options)
 }
 
 func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
@@ -234,19 +244,19 @@ func (h *baseHandler) send(ch chan<- string, message string) {
 }
 
 func (h *baseHandler) flush() {
-	dlog.Server.Debug(h.user, "flush()")
+	dlog.Server.Trace(h.user, "flush()")
 
 	numUnsentMessages := func() int {
 		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
 	}
 
-	for i := 0; i < 3; i++ {
+	for i := 0; i < 10; i++ {
 		if numUnsentMessages() == 0 {
 			dlog.Server.Debug(h.user, "All lines sent", fmt.Sprintf("%p", h))
 			return
 		}
 		dlog.Server.Debug(h.user, "Still lines to be sent")
-		time.Sleep(time.Second)
+		time.Sleep(time.Millisecond * 10)
 	}
 
 	dlog.Server.Warn(h.user, "Some lines remain unsent", numUnsentMessages())
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
deleted file mode 100644
index ae70675..0000000
--- a/internal/server/handlers/controlhandler.go
+++ /dev/null
@@ -1,98 +0,0 @@
-package handlers
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"strings"
-
-	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/dlog"
-	user "github.com/mimecast/dtail/internal/user/server"
-)
-
-// ControlHandler is used for control functions and health monitoring.
-type ControlHandler struct {
-	done           *internal.Done
-	hostname       string
-	payload        []byte
-	serverMessages chan string
-	user           *user.User
-}
-
-// NewControlHandler returns a new control handler.
-func NewControlHandler(user *user.User) *ControlHandler {
-	dlog.Server.Debug(user, "Creating control handler")
-
-	h := ControlHandler{
-		done:           internal.NewDone(),
-		serverMessages: make(chan string, 10),
-		user:           user,
-	}
-
-	fqdn, err := os.Hostname()
-	if err != nil {
-		dlog.Server.FatalPanic(err)
-	}
-
-	s := strings.Split(fqdn, ".")
-	h.hostname = s[0]
-
-	return &h
-}
-
-// Shutdown the handler.
-func (h *ControlHandler) Shutdown() {
-	h.done.Shutdown()
-}
-
-// Done channel of the handler.
-func (h *ControlHandler) Done() <-chan struct{} {
-	return h.done.Done()
-}
-
-// Read is to send data to the client via the Reader interface.
-func (h *ControlHandler) Read(p []byte) (n int, err error) {
-	for {
-		select {
-		case message := <-h.serverMessages:
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
-			n = copy(p, wholePayload)
-			return
-		case <-h.done.Done():
-			return 0, io.EOF
-		}
-	}
-}
-
-// Write is to read data to the client via the Writer interface.
-func (h *ControlHandler) Write(p []byte) (n int, err error) {
-	for _, c := range p {
-		switch c {
-		case ';':
-			wholePayload := strings.TrimSpace(string(h.payload))
-			h.handleCommand(wholePayload)
-			h.payload = nil
-
-		default:
-			h.payload = append(h.payload, c)
-		}
-	}
-
-	n = len(p)
-	return
-}
-
-func (h *ControlHandler) handleCommand(command string) {
-	dlog.Server.Info(h.user, command)
-	s := strings.Split(command, " ")
-	dlog.Server.Debug(h.user, "Receiving command", command, s)
-
-	switch s[0] {
-	case "health":
-		h.serverMessages <- "OK: DTail SSH Server seems fine"
-		h.serverMessages <- "done;"
-	default:
-		h.serverMessages <- dlog.Server.Error(h.user, "Received unknown control command", command, s)
-	}
-}
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
new file mode 100644
index 0000000..3f3b932
--- /dev/null
+++ b/internal/server/handlers/healthhandler.go
@@ -0,0 +1,58 @@
+package handlers
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/line"
+	user "github.com/mimecast/dtail/internal/user/server"
+)
+
+// HealthHandler is for the remote health check.
+type HealthHandler struct {
+	baseHandler
+}
+
+// NewHealthHandler returns the server handler.
+func NewHealthHandler(user *user.User) *HealthHandler {
+	dlog.Server.Debug(user, "Creating new server health handler")
+	h := HealthHandler{
+		baseHandler: baseHandler{
+			done:             internal.NewDone(),
+			lines:            make(chan line.Line, 100),
+			serverMessages:   make(chan string, 10),
+			maprMessages:     make(chan string, 10),
+			ackCloseReceived: make(chan struct{}),
+			user:             user,
+		},
+	}
+	h.handleCommandCb = h.handleHealthCommand
+
+	fqdn, err := os.Hostname()
+	if err != nil {
+		dlog.Server.FatalPanic(err)
+	}
+
+	s := strings.Split(fqdn, ".")
+	h.hostname = s[0]
+
+	return &h
+}
+
+func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int, args []string,
+	commandName string, options map[string]string) {
+	dlog.Server.Debug(h.user, "Handling health command", argc, args)
+
+	switch commandName {
+	case "health":
+		h.send(h.serverMessages, "OK: DTail SSH Server seems fine")
+	case "ack", ".ack":
+		h.handleAckCommand(argc, args)
+	default:
+		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown health command", commandName, argc, args))
+	}
+	h.shutdown()
+}
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 25cb8ba..aaffe14 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/omode"
@@ -25,6 +24,7 @@ type ServerHandler struct {
 
 // NewServerHandler returns the server handler.
 func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
+	dlog.Server.Debug(user, "Creating new server handler")
 	h := ServerHandler{
 		baseHandler: baseHandler{
 			done:             internal.NewDone(),
@@ -51,7 +51,9 @@ func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *S
 	return &h
 }
 
-func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string) {
+func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string,
+	commandName string, options map[string]string) {
+
 	dlog.Server.Debug(h.user, "Handling user command", argc, args)
 
 	h.incrementActiveCommands()
@@ -61,16 +63,6 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		}
 	}
 
-	splitted := strings.Split(args[0], ":")
-	commandName := splitted[0]
-
-	options, err := config.DeserializeOptions(splitted[1:])
-	if err != nil {
-		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
-		commandFinished()
-		return
-	}
-
 	if quiet, _ := options["quiet"]; quiet == "true" {
 		dlog.Server.Debug(h.user, "Enabling quiet mode")
 		h.quiet = true
diff --git a/internal/server/server.go b/internal/server/server.go
index d1cd57d..b3d4bff 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -162,8 +162,8 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 		case "shell":
 			var handler handlers.Handler
 			switch user.Name {
-			case config.ControlUser:
-				handler = handlers.NewControlHandler(user)
+			case config.HealthUser:
+				handler = handlers.NewHealthHandler(user)
 			default:
 				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter)
 			}
@@ -234,9 +234,9 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	remoteIP := splitted[0]
 
 	switch user.Name {
-	case config.ControlUser:
-		if authInfo == config.ControlUser {
-			dlog.Server.Debug(user, "Granting permissions to control user")
+	case config.HealthUser:
+		if authInfo == config.HealthUser {
+			dlog.Server.Debug(user, "Granting permissions to health user")
 			return nil, nil
 		}
 	case config.ScheduleUser:
-- 
cgit v1.2.3


From 47365ee1effe18b2f08ea0a5472b18be60959998 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 6 Oct 2021 10:55:50 +0300
Subject: move health check to separate client binary

---
 internal/clients/baseclient.go             |  1 -
 internal/clients/handlers/healthhandler.go | 25 ++++---------------------
 internal/clients/healthclient.go           | 29 +++++++++++++++++++++++++++++
 internal/config/initializer.go             | 10 +++++++++-
 internal/io/dlog/dlog.go                   |  2 +-
 internal/io/signal/signal.go               |  5 +++++
 internal/server/handlers/healthhandler.go  |  4 ++--
 internal/server/handlers/serverhandler.go  |  2 +-
 8 files changed, 51 insertions(+), 27 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index b474208..d5d7c2c 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -88,7 +88,6 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 		go func(i int, conn connectors.Connector) {
 			defer wg.Done()
 			connStatus := c.startConnection(ctx, i, conn)
-			// Update global status.
 			mutex.Lock()
 			defer mutex.Unlock()
 			if connStatus > status {
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 4b16ce4..10ba1f7 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -1,7 +1,6 @@
 package handlers
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/mimecast/dtail/internal"
@@ -22,7 +21,7 @@ func NewHealthHandler(server string) *HealthHandler {
 			server:       server,
 			shellStarted: false,
 			commands:     make(chan string),
-			status:       -1,
+			status:       2, // Assume CRITICAL status by default.
 			done:         internal.NewDone(),
 		},
 	}
@@ -34,14 +33,12 @@ func (h *HealthHandler) Write(p []byte) (n int, err error) {
 		switch b {
 		case '\n', protocol.MessageDelimiter:
 			message := h.baseHandler.receiveBuf.String()
-			dlog.Client.Debug(message)
 			h.handleMessage(message)
 			h.baseHandler.receiveBuf.Reset()
 		default:
 			h.baseHandler.receiveBuf.WriteByte(b)
 		}
 	}
-
 	return len(p), nil
 }
 
@@ -52,21 +49,7 @@ func (h *HealthHandler) handleMessage(message string) {
 	}
 	s := strings.Split(message, protocol.FieldDelimiter)
 	message = s[len(s)-1]
-	status := strings.Split(message, ":")
-	fmt.Println(status)
-	/*
-		switch status {
-		case "OK":
-			h.HealthStatusCh <- 0
-		case "WARNING":
-			h.HealthStatusCh <- 1
-		case "CRITICAL":
-			h.HealthStatusCh <- 2
-		case "UNKNOWN":
-			h.HealthStatusCh <- 3
-		default:
-			fmt.Println("CRITICAL: Unexpected server response: '%s'")
-			h.HealthStatusCh <- 2
-		}
-	*/
+	if message == "OK" {
+		h.baseHandler.status = 0
+	}
 }
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index e2c8ccb..ac1dc20 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -1,6 +1,8 @@
 package clients
 
 import (
+	"context"
+	"fmt"
 	"runtime"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
@@ -42,3 +44,30 @@ func (c HealthClient) makeCommands() (commands []string) {
 	commands = append(commands, "health")
 	return
 }
+
+func (c *HealthClient) Start(ctx context.Context, statsCh <-chan string) int {
+	status := c.baseClient.Start(ctx, statsCh)
+
+	switch status {
+	case 0:
+		if c.Serverless {
+			fmt.Printf("WARNING: All seems fine but the check only run in serverless mode, please specify a remote server via --server hostname:port\n")
+			return 1
+		}
+		fmt.Printf("OK: All fine at %s :-)\n", c.ServersStr)
+	case 2:
+		if c.Serverless {
+			fmt.Printf("CRITICAL: DTail server not operating properly (using serverless connction)!\n")
+			return 2
+		}
+		fmt.Printf("CRITICAL: DTail server not operating properly at %s!\n", c.ServersStr)
+	default:
+		if c.Serverless {
+			fmt.Printf("UNKNOWN: Received unknown status code %d (using serverless connection)\n", status)
+			return status
+		}
+		fmt.Printf("UNKNOWN: Received unknown status code %d from %s!\n", status, c.ServersStr)
+	}
+
+	return status
+}
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index a58f82a..ec758c8 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -96,11 +96,19 @@ func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, a
 		common.SSHPort = args.SSHPort
 	}
 
-	if args.Discovery == "" && args.ServersStr == "" {
+	if args.Discovery == "" && (args.ServersStr == "" ||
+		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
 	}
 
+	if sourceProcess == source.HealthCheck {
+		args.TrustAllHosts = true
+		if !args.Serverless && strings.ToLower(args.ServersStr) == "" {
+			args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort)
+		}
+	}
+
 	// Interpret additional args as file list.
 	if args.What == "" {
 		var files []string
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index a17d6e9..2ae3b04 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -59,7 +59,7 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Common = Server
 	case source.HealthCheck:
 		// Health check isn't logging anything.
-		impl := loggers.STDOUT
+		impl := loggers.NONE
 		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
 		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
 		Common = Client
diff --git a/internal/io/signal/signal.go b/internal/io/signal/signal.go
index 500c530..14056c4 100644
--- a/internal/io/signal/signal.go
+++ b/internal/io/signal/signal.go
@@ -44,3 +44,8 @@ func InterruptCh(ctx context.Context) <-chan string {
 
 	return statsCh
 }
+
+// NoCh doesn't listen on a signal.
+func NoCh(ctx context.Context) <-chan string {
+	return make(chan string)
+}
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
index 3f3b932..347ff66 100644
--- a/internal/server/handlers/healthhandler.go
+++ b/internal/server/handlers/healthhandler.go
@@ -48,8 +48,8 @@ func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int, args
 
 	switch commandName {
 	case "health":
-		h.send(h.serverMessages, "OK: DTail SSH Server seems fine")
-	case "ack", ".ack":
+		h.send(h.serverMessages, "OK")
+	case ".ack":
 		h.handleAckCommand(argc, args)
 	default:
 		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown health command", commandName, argc, args))
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index aaffe14..aed8956 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -106,7 +106,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			commandFinished()
 		}()
 
-	case "ack", ".ack":
+	case ".ack":
 		h.handleAckCommand(argc, args)
 		commandFinished()
 
-- 
cgit v1.2.3


From 148ce987cde531978b4d5cdd30df9a67cc384ec5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Fri, 8 Oct 2021 11:43:43 +0300
Subject: refactor

---
 internal/clients/maprclient.go          |  11 +--
 internal/config/args.go                 |   4 +
 internal/config/common.go               |  14 ++--
 internal/config/config.go               |  27 +++++--
 internal/config/initializer.go          | 131 ++++++++++++++++++++------------
 internal/io/dlog/dlog.go                |  53 ++++++-------
 internal/io/dlog/level.go               |  95 ++++++++++-------------
 internal/io/dlog/loggers/factory.go     |  36 ++++-----
 internal/mapr/logformat/default_test.go |   2 +-
 internal/server/continuous.go           |   4 +-
 internal/server/handlers/basehandler.go |   2 +-
 internal/server/scheduler.go            |   4 +-
 12 files changed, 204 insertions(+), 179 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 92bbe39..412a219 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -31,8 +31,6 @@ const (
 // MaprClient is used for running mapreduce aggregations on remote files.
 type MaprClient struct {
 	baseClient
-	// Query string for mapr aggregations
-	queryStr string
 	// Global group set for merged mapr aggregation results
 	globalGroup *mapr.GlobalGroupSet
 	// The query object (constructed from queryStr)
@@ -44,14 +42,14 @@ type MaprClient struct {
 }
 
 // NewMaprClient returns a new mapreduce client.
-func NewMaprClient(args config.Args, queryStr string, maprClientMode MaprClientMode) (*MaprClient, error) {
-	if queryStr == "" {
+func NewMaprClient(args config.Args, maprClientMode MaprClientMode) (*MaprClient, error) {
+	if args.QueryStr == "" {
 		return nil, errors.New("No mapreduce query specified, use '-query' flag")
 	}
 
-	query, err := mapr.NewQuery(queryStr)
+	query, err := mapr.NewQuery(args.QueryStr)
 	if err != nil {
-		dlog.Client.FatalPanic(queryStr, "Can't parse mapr query", err)
+		dlog.Client.FatalPanic(args.QueryStr, "Can't parse mapr query", err)
 	}
 
 	// Don't retry connection if in tail mode and no outfile specified.
@@ -77,7 +75,6 @@ func NewMaprClient(args config.Args, queryStr string, maprClientMode MaprClientM
 			retry:      retry,
 		},
 		query:      query,
-		queryStr:   queryStr,
 		cumulative: cumulative,
 	}
 
diff --git a/internal/config/args.go b/internal/config/args.go
index 3e2eb1f..a671ae3 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -17,10 +17,12 @@ type Args struct {
 	ConnectionsPerCPU  int
 	Discovery          string
 	LogDir             string
+	Logger             string
 	LogLevel           string
 	Mode               omode.Mode
 	NoColor            bool
 	PrivateKeyPathFile string
+	QueryStr           string
 	Quiet              bool
 	RegexInvert        bool
 	RegexStr           string
@@ -42,6 +44,7 @@ func (a *Args) String() string {
 	sb.WriteString("Args(")
 
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir))
+	sb.WriteString(fmt.Sprintf("%s:%s,", "Logger", a.Logger))
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile))
@@ -50,6 +53,7 @@ func (a *Args) String() string {
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "QueryStr", a.QueryStr))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr))
diff --git a/internal/config/common.go b/internal/config/common.go
index 5e81bc9..9d22c95 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -6,14 +6,14 @@ type CommonConfig struct {
 	SSHPort int
 	// Enable experimental features (mainly for dev purposes)
 	ExperimentalFeaturesEnable bool `json:",omitempty"`
+	// LogDir defines the log directory.
+	LogDir string
+	// Logger defines the name of the logger implementation.
+	Logger string
 	// LogLevel defines how much is logged.
 	LogLevel string `json:",omitempty"`
-	// The log strategy to use, one of
-	//   stdout: only log to stdout (useful when used with systemd)
-	//   daily: create a log file for every day
+	// LogStrategy defines the log rotation strategy.
 	LogStrategy string
-	// The log directory
-	LogDir string
 	// The cache directory
 	CacheDir string
 	// The temp directory
@@ -26,7 +26,9 @@ func newDefaultCommonConfig() *CommonConfig {
 		SSHPort:                    DefaultSSHPort,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
-		LogLevel:                   "INFO",
+		Logger:                     "stdout",
+		LogLevel:                   DefaultLogLevel,
+		LogStrategy:                "daily",
 		CacheDir:                   "cache",
 		TmpDir:                     "/tmp",
 	}
diff --git a/internal/config/config.go b/internal/config/config.go
index f216688..077a658 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -15,6 +15,14 @@ const (
 	DefaultConnectionsPerCPU int = 10
 	// DTailSSHServerDefaultPort is the default DServer port.
 	DefaultSSHPort int = 2222
+	// DefaultLogLevel specifies the default log level (obviously)
+	DefaultLogLevel string = "INFO"
+	// DefaultClientLogger specifies the default logger for the client commands.
+	DefaultClientLogger string = "fout"
+	// DefaultServerLogger specifies the default logger for dtail server.
+	DefaultServerLogger string = "file"
+	// DefaultHealthCheckLogger specifies the default logger used for health checks.
+	DefaultHealthCheckLogger string = "none"
 )
 
 // Client holds a DTail client configuration.
@@ -33,12 +41,15 @@ func Setup(sourceProcess source.Source, args *Args, additionalArgs []string) {
 		Server: newDefaultServerConfig(),
 		Client: newDefaultClientConfig(),
 	}
-	initializer.parseConfig(args)
-	Client, Server, Common = initializer.transformConfig(
-		sourceProcess,
-		args, additionalArgs,
-		initializer.Client,
-		initializer.Server,
-		initializer.Common,
-	)
+	if err := initializer.parseConfig(args); err != nil {
+		panic(err)
+	}
+	if err := initializer.transformConfig(sourceProcess, args, additionalArgs); err != nil {
+		panic(err)
+	}
+
+	// Make config accessible globally
+	Server = initializer.Server
+	Client = initializer.Client
+	Common = initializer.Common
 }
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index ec758c8..5247699 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -18,14 +18,15 @@ type initializer struct {
 	Client *ClientConfig
 }
 
-func (c *initializer) parseConfig(args *Args) {
+type transformCb func(*initializer, *Args, []string) error
+
+func (c *initializer) parseConfig(args *Args) error {
 	if strings.ToUpper(args.ConfigFile) == "NONE" {
-		return
+		return nil
 	}
 
 	if args.ConfigFile != "" {
-		c.parseSpecificConfig(args.ConfigFile)
-		return
+		return c.parseSpecificConfig(args.ConfigFile)
 	}
 
 	if homeDir, err := os.UserHomeDir(); err != nil {
@@ -38,85 +39,119 @@ func (c *initializer) parseConfig(args *Args) {
 			}
 		}
 	}
+
+	return nil
 }
 
-func (c *initializer) parseSpecificConfig(configFile string) {
+func (c *initializer) parseSpecificConfig(configFile string) error {
 	fd, err := os.Open(configFile)
 	if err != nil {
-		panic(fmt.Sprintf("Unable to read config file: %v", err))
+		return fmt.Errorf("Unable to read config file: %v", err)
 	}
 	defer fd.Close()
 
 	cfgBytes, err := ioutil.ReadAll(fd)
 	if err != nil {
-		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
+		return fmt.Errorf("Unable to read config file %s: %v", configFile, err)
 	}
 
-	err = json.Unmarshal([]byte(cfgBytes), c)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
+	if err := json.Unmarshal([]byte(cfgBytes), c); err != nil {
+		return fmt.Errorf("Unable to parse config file %s: %v", configFile, err)
 	}
+
+	return nil
 }
 
-func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string,
-	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
-	if args.LogDir != "" {
-		common.LogDir = args.LogDir
-	}
-	if strings.Contains(common.LogDir, "~/") {
-		homeDir, err := os.UserHomeDir()
-		if err != nil {
-			panic(err)
-		}
-		common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
-	}
-	if common.LogStrategy == "" {
-		common.LogStrategy = "daily"
+func (i *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string) error {
+
+	switch sourceProcess {
+	case source.Server:
+		return i.optimusPrime(transformServer, args, additionalArgs)
+	case source.Client:
+		return i.optimusPrime(transformClient, args, additionalArgs)
+	case source.HealthCheck:
+		return i.optimusPrime(transformHealthCheck, args, additionalArgs)
+	default:
+		return fmt.Errorf("Unable to transform config, unknown source '%s'", sourceProcess)
 	}
+}
 
-	if args.Spartan {
-		args.Quiet = true
-		args.NoColor = true
-		if args.LogLevel == "" {
-			args.LogLevel = "ERROR"
-		}
+func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalArgs []string) error {
+	// Copy args to config objects.
+	if args.SSHPort != DefaultSSHPort {
+		i.Common.SSHPort = args.SSHPort
+	}
+	if args.LogLevel != DefaultLogLevel {
+		i.Common.LogLevel = args.LogLevel
 	}
 	if args.NoColor {
-		client.TermColorsEnable = false
+		i.Client.TermColorsEnable = false
 	}
-
-	if args.LogLevel != "" {
-		common.LogLevel = args.LogLevel
-	} else if sourceProcess == source.Client && args.ServersStr == "" && args.Discovery == "" {
-		// We are in serverless mode. Default log level is WARN.
-		common.LogLevel = "WARN"
+	if args.LogDir != "" {
+		i.Common.LogDir = args.LogDir
+	}
+	if args.Logger != "" {
+		i.Common.Logger = args.Logger
 	}
 
-	if args.SSHPort != DefaultSSHPort {
-		common.SSHPort = args.SSHPort
+	// Setup log directory.
+	if strings.Contains(i.Common.LogDir, "~/") {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			panic(err)
+		}
+		i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
 	}
 
+	// Serverless mode.
 	if args.Discovery == "" && (args.ServersStr == "" ||
 		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
+		i.Common.LogLevel = "WARN"
 	}
 
-	if sourceProcess == source.HealthCheck {
-		args.TrustAllHosts = true
-		if !args.Serverless && strings.ToLower(args.ServersStr) == "" {
-			args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort)
+	// Source type specific transormations.
+	sourceCb(i, args, additionalArgs)
+
+	// Spartan mode.
+	if args.Spartan {
+		args.Quiet = true
+		args.NoColor = true
+		i.Client.TermColorsEnable = false
+		if args.LogLevel == "" {
+			args.LogLevel = "ERROR"
+			i.Common.LogLevel = "ERROR"
 		}
 	}
-
-	// Interpret additional args as file list.
+	// Interpret additional args as file list or as query.
 	if args.What == "" {
 		var files []string
-		for _, file := range flag.Args() {
-			files = append(files, file)
+		for _, arg := range flag.Args() {
+			if args.QueryStr == "" && strings.Contains(strings.ToLower(arg), "select ") {
+				args.QueryStr = arg
+				continue
+			}
+			files = append(files, arg)
 		}
 		args.What = strings.Join(files, ",")
 	}
 
-	return client, server, common
+	return nil
+}
+
+func transformClient(i *initializer, args *Args, additionalArgs []string) error {
+	return nil
+}
+
+func transformServer(i *initializer, args *Args, additionalArgs []string) error {
+	return nil
+}
+
+func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error {
+	args.TrustAllHosts = true
+	if !args.Serverless && args.ServersStr == "" {
+		args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort)
+	}
+	return nil
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 2ae3b04..f3774ba 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -33,7 +33,7 @@ var mutex sync.Mutex
 var started bool
 
 // Start logger(s).
-func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source, logLevel string) {
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
@@ -41,27 +41,18 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Common.FatalPanic("Logger already started")
 	}
 
-	strategy := loggers.GetStrategy(config.Common.LogStrategy)
-	level := newLevel(logLevel)
-
 	switch sourceProcess {
 	case source.Client:
-		// This is a DTail client process running.
-		impl := loggers.FOUT
-		Client = New(source.Client, source.Client, level, impl, strategy)
-		Server = New(source.Client, source.Server, level, impl, strategy)
+		Client = New(source.Client, source.Client)
+		Server = New(source.Client, source.Server)
 		Common = Client
 	case source.Server:
-		// This is a DTail server process running.
-		impl := loggers.FILE
-		Client = New(source.Server, source.Client, level, impl, strategy)
-		Server = New(source.Server, source.Server, level, impl, strategy)
+		Client = New(source.Server, source.Client)
+		Server = New(source.Server, source.Server)
 		Common = Server
 	case source.HealthCheck:
-		// Health check isn't logging anything.
-		impl := loggers.NONE
-		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
-		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
+		Client = New(source.HealthCheck, source.Client)
+		Server = New(source.HealthCheck, source.Server)
 		Common = Client
 	}
 
@@ -93,16 +84,20 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source.Source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
+func New(sourceProcess, sourcePackage source.Source) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
 	}
+	strategy := loggers.GetStrategy(config.Common.LogStrategy)
+	loggerName := config.Common.Logger
+	level := newLevel(config.Common.LogLevel)
+
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), impl, strategy),
+		logger:        loggers.Factory(sourceProcess.String(), loggerName, strategy),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
-		maxLevel:      maxLevel,
+		maxLevel:      level,
 		hostname:      hostname,
 	}
 }
@@ -168,7 +163,7 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 }
 
 func (d *DLog) FatalPanic(args ...interface{}) {
-	d.log(FATAL, args)
+	d.log(Fatal, args)
 	d.Flush()
 
 	var sb strings.Builder
@@ -177,37 +172,37 @@ func (d *DLog) FatalPanic(args ...interface{}) {
 }
 
 func (d *DLog) Fatal(args ...interface{}) string {
-	return d.log(FATAL, args)
+	return d.log(Fatal, args)
 }
 
 func (d *DLog) Error(args ...interface{}) string {
-	return d.log(ERROR, args)
+	return d.log(Error, args)
 }
 
 func (d *DLog) Warn(args ...interface{}) string {
-	return d.log(WARN, args)
+	return d.log(Warn, args)
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	return d.log(INFO, args)
+	return d.log(Info, args)
 }
 
 func (d *DLog) Verbose(args ...interface{}) string {
-	return d.log(VERBOSE, args)
+	return d.log(Verbose, args)
 }
 
 func (d *DLog) Debug(args ...interface{}) string {
-	return d.log(DEBUG, args)
+	return d.log(Debug, args)
 }
 
 func (d *DLog) Trace(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
-	return d.log(TRACE, args)
+	return d.log(Trace, args)
 }
 
 func (d *DLog) Devel(args ...interface{}) string {
-	return d.log(DEVEL, args)
+	return d.log(Devel, args)
 }
 
 func (d *DLog) Raw(message string) string {
@@ -260,7 +255,7 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 		args[i] = fmt.Sprintf("%s=%v", k, v)
 		i++
 	}
-	return d.log(INFO, args)
+	return d.log(Info, args)
 }
 
 func (d *DLog) Flush()  { d.logger.Flush() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 84550f0..248ad83 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -8,80 +8,69 @@ import (
 type level int
 
 const (
-	FATAL   level = iota
-	ERROR   level = iota
-	WARN    level = iota
-	INFO    level = iota
-	DEFAULT level = iota
-	VERBOSE level = iota
-	DEBUG   level = iota
-	DEVEL   level = iota
-	TRACE   level = iota
-	ALL     level = iota
+	Fatal   level = iota
+	Error   level = iota
+	Warn    level = iota
+	Info    level = iota
+	Default level = iota
+	Verbose level = iota
+	Debug   level = iota
+	Devel   level = iota
+	Trace   level = iota
+	All     level = iota
 )
 
-var allLevels = []level{
-	FATAL,
-	ERROR,
-	WARN,
-	INFO,
-	DEFAULT,
-	VERBOSE,
-	DEBUG,
-	DEVEL,
-	TRACE,
-	ALL,
-}
+var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug, Devel, Trace, All}
 
 func newLevel(l string) level {
-	switch strings.ToUpper(l) {
-	case "FATAL":
-		return FATAL
-	case "ERROR":
-		return ERROR
-	case "WARN":
-		return WARN
-	case "INFO":
-		return INFO
+	switch strings.ToLower(l) {
+	case "fatal":
+		return Fatal
+	case "error":
+		return Error
+	case "warn":
+		return Warn
+	case "info":
+		return Info
 	case "":
 		fallthrough
-	case "DEFAULT":
-		return DEFAULT
-	case "VERBOSE":
-		return VERBOSE
-	case "DEBUG":
-		return DEBUG
-	case "DEVEL":
-		return DEVEL
-	case "TRACE":
-		return TRACE
-	case "ALL":
-		return ALL
+	case "default":
+		return Default
+	case "verbose":
+		return Verbose
+	case "debug":
+		return Debug
+	case "devel":
+		return Devel
+	case "trace":
+		return Trace
+	case "all":
+		return All
 	}
 	panic(fmt.Sprintf("Unknown log level %s, must be one of: %v", l, allLevels))
 }
 
 func (l level) String() string {
 	switch l {
-	case FATAL:
+	case Fatal:
 		return "FATAL"
-	case ERROR:
+	case Error:
 		return "ERROR"
-	case WARN:
+	case Warn:
 		return "WARN"
-	case INFO:
+	case Info:
 		return "INFO"
-	case DEFAULT:
+	case Default:
 		return "DEFAULT"
-	case VERBOSE:
+	case Verbose:
 		return "VERBOSE"
-	case DEBUG:
+	case Debug:
 		return "DEBUG"
-	case DEVEL:
+	case Devel:
 		return "DEVEL"
-	case TRACE:
+	case Trace:
 		return "TRACE"
-	case ALL:
+	case All:
 		return "ALL"
 	}
 
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 8697dc4..dda3ee6 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -2,45 +2,38 @@ package loggers
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 )
 
-type Impl int
-
-const (
-	NONE   Impl = iota
-	STDOUT Impl = iota
-	FILE   Impl = iota
-	FOUT   Impl = iota
-)
-
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
-func Factory(name string, impl Impl, strategy Strategy) Logger {
+func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("name:%s,fileBase:%s,impl:%v", name, strategy.FileBase, impl)
-
+	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName, rotationStrategy.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
 
 	singleton, ok := factoryMap[id]
 	if !ok {
-		switch impl {
-		case NONE:
+		switch strings.ToLower(loggerName) {
+		case "none":
 			singleton = none{}
-		case STDOUT:
+		case "stdout":
 			singleton = newStdout()
 			factoryMap[id] = singleton
-		case FILE:
-			singleton = newFile(strategy)
+		case "file":
+			singleton = newFile(rotationStrategy)
 			factoryMap[id] = singleton
-		case FOUT:
-			singleton = newFout(strategy)
+		case "fout":
+			singleton = newFout(rotationStrategy)
 			factoryMap[id] = singleton
+		default:
+			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
 		}
 	}
 
@@ -53,8 +46,7 @@ func FactoryRotate() {
 	if factoryMap == nil {
 		return
 	}
-
-	for _, impl := range factoryMap {
-		impl.Rotate()
+	for _, logger := range factoryMap {
+		logger.Rotate()
 	}
 }
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index 02c03a3..a777156 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -32,7 +32,7 @@ func TestDefaultLogFormat(t *testing.T) {
 		if val, ok := fields["$severity"]; !ok {
 			t.Errorf("Expected field '$severity', but no such field there in '%s'\n", input)
 		} else if val != "INFO" {
-			t.Errorf("Expected 'INFO' stored in field '$severity', but got '%s' in '%s'\n", val, input)
+			t.Errorf("Expected 'Info' stored in field '$severity', but got '%s' in '%s'\n", val, input)
 		}
 
 		if val, ok := fields["$time"]; !ok {
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index 87c8889..5f84afc 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -71,8 +71,8 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 
 	args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(job.Name))
 
-	query := fmt.Sprintf("%s outfile %s", job.Query, outfile)
-	client, err := clients.NewMaprClient(args, query, clients.NonCumulativeMode)
+	args.QueryStr = fmt.Sprintf("%s outfile %s", job.Query, outfile)
+	client, err := clients.NewMaprClient(args, clients.NonCumulativeMode)
 	if err != nil {
 		dlog.Server.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
 		return
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 4fa8f00..f73f82e 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -252,7 +252,7 @@ func (h *baseHandler) flush() {
 
 	for i := 0; i < 10; i++ {
 		if numUnsentMessages() == 0 {
-			dlog.Server.Debug(h.user, "All lines sent", fmt.Sprintf("%p", h))
+			dlog.Server.Debug(h.user, "ALL lines sent", fmt.Sprintf("%p", h))
 			return
 		}
 		dlog.Server.Debug(h.user, "Still lines to be sent")
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index 64e6573..ccb2225 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -82,8 +82,8 @@ func (s *scheduler) runJobs(ctx context.Context) {
 
 		args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(job.Name))
 
-		query := fmt.Sprintf("%s outfile %s", job.Query, outfile)
-		client, err := clients.NewMaprClient(args, query, clients.CumulativeMode)
+		args.QueryStr = fmt.Sprintf("%s outfile %s", job.Query, outfile)
+		client, err := clients.NewMaprClient(args, clients.CumulativeMode)
 		if err != nil {
 			dlog.Server.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
 			continue
-- 
cgit v1.2.3


From 7563abe9d5beaa18fa1eab0f65668f5dfcf79052 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 9 Oct 2021 16:44:28 +0300
Subject: add dtail health check unit test.

---
 internal/config/initializer.go    | 29 ++++++++++++++++----------
 internal/io/dlog/dlog.go          | 43 ++++++++++++++++-----------------------
 internal/io/dlog/loggers/file.go  | 10 +++++----
 internal/mapr/server/aggregate.go |  6 +++---
 4 files changed, 45 insertions(+), 43 deletions(-)

(limited to 'internal')

diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 5247699..e4cbeaf 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -93,6 +93,9 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA
 	if args.Logger != "" {
 		i.Common.Logger = args.Logger
 	}
+	if args.ConnectionsPerCPU == 0 {
+		args.ConnectionsPerCPU = DefaultConnectionsPerCPU
+	}
 
 	// Setup log directory.
 	if strings.Contains(i.Common.LogDir, "~/") {
@@ -103,14 +106,6 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA
 		i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
 	}
 
-	// Serverless mode.
-	if args.Discovery == "" && (args.ServersStr == "" ||
-		strings.ToLower(args.ServersStr) == "serverless") {
-		// We are not connecting to any servers.
-		args.Serverless = true
-		i.Common.LogLevel = "WARN"
-	}
-
 	// Source type specific transormations.
 	sourceCb(i, args, additionalArgs)
 
@@ -141,6 +136,14 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA
 }
 
 func transformClient(i *initializer, args *Args, additionalArgs []string) error {
+	// Serverless mode.
+	if args.Discovery == "" && (args.ServersStr == "" ||
+		strings.ToLower(args.ServersStr) == "serverless") {
+		// We are not connecting to any servers.
+		args.Serverless = true
+		i.Common.LogLevel = "warn"
+	}
+
 	return nil
 }
 
@@ -149,9 +152,13 @@ func transformServer(i *initializer, args *Args, additionalArgs []string) error
 }
 
 func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error {
-	args.TrustAllHosts = true
-	if !args.Serverless && args.ServersStr == "" {
-		args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort)
+	// Serverless mode.
+	if args.Discovery == "" && (args.ServersStr == "" ||
+		strings.ToLower(args.ServersStr) == "serverless") {
+		// We are not connecting to any servers.
+		args.Serverless = true
+		i.Common.LogLevel = "warn"
 	}
+	args.TrustAllHosts = true
 	return nil
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index f3774ba..28e6882 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -41,32 +41,25 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source)
 		Common.FatalPanic("Logger already started")
 	}
 
-	switch sourceProcess {
-	case source.Client:
-		Client = New(source.Client, source.Client)
-		Server = New(source.Client, source.Server)
-		Common = Client
-	case source.Server:
-		Client = New(source.Server, source.Client)
-		Server = New(source.Server, source.Server)
+	Client = new(sourceProcess, source.Client)
+	Server = new(sourceProcess, source.Server)
+	Common = Client
+	if sourceProcess == source.Server {
 		Common = Server
-	case source.HealthCheck:
-		Client = New(source.HealthCheck, source.Client)
-		Server = New(source.HealthCheck, source.Server)
-		Common = Client
 	}
 
 	var wg2 sync.WaitGroup
 	wg2.Add(2)
-	Client.start(ctx, &wg2)
-	Server.start(ctx, &wg2)
-	started = true
+	go Client.start(ctx, &wg2)
+	go Server.start(ctx, &wg2)
 
 	go rotation(ctx)
 	go func() {
 		wg2.Wait()
 		wg.Done()
 	}()
+
+	started = true
 }
 
 // DLog is the DTail logger.
@@ -83,8 +76,8 @@ type DLog struct {
 	hostname string
 }
 
-// New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source.Source) *DLog {
+// new creates a new DTail logger.
+func new(sourceProcess, sourcePackage source.Source) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
@@ -103,14 +96,12 @@ func New(sourceProcess, sourcePackage source.Source) *DLog {
 }
 
 func (d *DLog) start(ctx context.Context, wg *sync.WaitGroup) {
-	go func() {
-		defer wg.Done()
-		var wg2 sync.WaitGroup
-		wg2.Add(1)
-		d.logger.Start(ctx, &wg2)
-		<-ctx.Done()
-		wg2.Wait()
-	}()
+	defer wg.Done()
+	var wg2 sync.WaitGroup
+	wg2.Add(1)
+	d.logger.Start(ctx, &wg2)
+	<-ctx.Done()
+	wg2.Wait()
 }
 
 func (d *DLog) log(level level, args []interface{}) string {
@@ -202,6 +193,8 @@ func (d *DLog) Trace(args ...interface{}) string {
 }
 
 func (d *DLog) Devel(args ...interface{}) string {
+	_, file, line, _ := runtime.Caller(1)
+	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Devel, args)
 }
 
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 6e692a3..87280fd 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -126,7 +126,6 @@ func (f *file) getWriter(name string) *bufio.Writer {
 	if f.lastFileName == name {
 		return f.writer
 	}
-
 	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
 		if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil {
 			panic(err)
@@ -144,7 +143,7 @@ func (f *file) getWriter(name string) *bufio.Writer {
 		f.writer.Flush()
 		f.fd.Close()
 	}
-
+	// Set new writer.
 	f.fd = newFd
 	f.writer = bufio.NewWriterSize(f.fd, 1)
 	f.lastFileName = name
@@ -153,8 +152,11 @@ func (f *file) getWriter(name string) *bufio.Writer {
 }
 
 func (f *file) flush() {
-	defer f.writer.Flush()
-
+	defer func() {
+		if f.writer != nil {
+			f.writer.Flush()
+		}
+	}()
 	for {
 		select {
 		case m := <-f.bufferCh:
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 767aada..1f5d1c3 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -8,9 +8,8 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/dlog"
-	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/logformat"
 	"github.com/mimecast/dtail/internal/protocol"
@@ -148,7 +147,8 @@ func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]strin
 
 				maprLine := strings.TrimSpace(line.Content.String())
 				fields, err := a.parser.MakeFields(maprLine)
-				pool.RecycleBytesBuffer(line.Content)
+				// Can not recycle here for some rason.
+				//pool.RecycleBytesBuffer(line.Content)
 
 				if err != nil {
 					// Should fields be ignored anyway?
-- 
cgit v1.2.3


From ea1de3044e129d419f4e807f2624a009343a128f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 9 Oct 2021 21:10:29 +0300
Subject: vetting and linting and some code restyling

---
 internal/clients/baseclient.go                    |   7 +-
 internal/clients/catclient.go                     |   2 -
 internal/clients/connectors/serverconnection.go   |  48 +++++----
 internal/clients/connectors/serverless.go         |  16 +--
 internal/clients/grepclient.go                    |   5 +-
 internal/clients/handlers/healthhandler.go        |   3 +-
 internal/clients/handlers/maprhandler.go          |  13 ++-
 internal/clients/healthclient.go                  |  17 +--
 internal/clients/maprclient.go                    |  13 +--
 internal/clients/stats.go                         |  11 +-
 internal/clients/tailclient.go                    |   3 -
 internal/color/brush/brush.go                     |   8 +-
 internal/color/color.go                           |  11 +-
 internal/color/paint.go                           |  19 ++--
 internal/color/table.go                           |  19 +++-
 internal/config/args.go                           |   6 +-
 internal/config/client.go                         |  12 +--
 internal/config/config.go                         |   6 +-
 internal/config/initializer.go                    |  59 +++++-----
 internal/config/server.go                         |   7 +-
 internal/discovery/discovery.go                   |  13 +--
 internal/done.go                                  |   1 -
 internal/io/dlog/dlog.go                          |  22 +++-
 internal/io/dlog/level.go                         |   5 +-
 internal/io/dlog/loggers/factory.go               |   6 +-
 internal/io/dlog/loggers/file.go                  |  17 ++-
 internal/io/dlog/loggers/logger.go                |   1 +
 internal/io/dlog/loggers/strategy.go              |  11 +-
 internal/io/fs/catfile.go                         |   4 +-
 internal/io/fs/filereader.go                      |   3 +-
 internal/io/fs/permissions/permission_linuxacl.go |   2 +-
 internal/io/fs/readfile.go                        |  29 ++---
 internal/io/fs/tailfile.go                        |   4 +-
 internal/io/pool/builder.go                       |   3 +
 internal/io/pool/bytesbuffer.go                   |   3 +
 internal/io/prompt/prompt.go                      |   7 +-
 internal/io/signal/signal.go                      |   3 -
 internal/mapr/aggregateset.go                     |   4 -
 internal/mapr/client/aggregate.go                 |   9 +-
 internal/mapr/funcs/function.go                   |   7 +-
 internal/mapr/funcs/function_test.go              |  21 ++--
 internal/mapr/funcs/maskdigits.go                 |   2 -
 internal/mapr/globalgroupset.go                   |   8 --
 internal/mapr/groupset.go                         |   9 +-
 internal/mapr/logformat/default.go                |   5 +-
 internal/mapr/logformat/default_test.go           |  24 +++--
 internal/mapr/logformat/generickv.go              |   2 +-
 internal/mapr/logformat/parser.go                 |  12 +--
 internal/mapr/query.go                            |  17 ++-
 internal/mapr/query_test.go                       | 125 ++++++++++++++--------
 internal/mapr/selectcondition.go                  |  12 +--
 internal/mapr/server/aggregate.go                 |  30 +++---
 internal/mapr/setclause.go                        |   2 -
 internal/mapr/setcondition.go                     |  15 ++-
 internal/mapr/token.go                            |  11 +-
 internal/mapr/whereclause.go                      |  10 +-
 internal/mapr/wherecondition.go                   |  24 ++---
 internal/protocol/protocol.go                     |   1 -
 internal/regex/regex.go                           |   9 +-
 internal/regex/regex_test.go                      |  17 +--
 internal/server/continuous.go                     |   9 +-
 internal/server/handlers/basehandler.go           |  30 ++----
 internal/server/handlers/healthhandler.go         |  11 +-
 internal/server/handlers/mapcommand.go            |   7 +-
 internal/server/handlers/readcommand.go           |  41 ++++---
 internal/server/handlers/serverhandler.go         |  20 ++--
 internal/server/scheduler.go                      |   9 +-
 internal/server/server.go                         |  47 ++++----
 internal/server/stats.go                          |  11 +-
 internal/source/source.go                         |  14 ++-
 internal/ssh/client/authmethods.go                |  41 ++++---
 internal/ssh/client/customkeycallback.go          |   3 +-
 internal/ssh/client/knownhostscallback.go         |  19 +---
 internal/ssh/server/hostkey.go                    |   3 +-
 internal/ssh/server/publickeycallback.go          |  27 +++--
 internal/ssh/ssh.go                               |   4 -
 internal/user/name.go                             |   3 -
 internal/user/server/user.go                      |  30 +++---
 internal/version/version.go                       |   6 +-
 79 files changed, 584 insertions(+), 546 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index d5d7c2c..4a7bd84 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -64,7 +64,8 @@ func (c *baseClient) makeConnections(maker maker) {
 
 	discoveryService := discovery.New(c.Discovery, c.ServersStr, discovery.Shuffle)
 	for _, server := range discoveryService.ServerList() {
-		c.connections = append(c.connections, c.makeConnection(server, c.sshAuthMethods, c.hostKeyCallback))
+		c.connections = append(c.connections, c.makeConnection(server,
+			c.sshAuthMethods, c.hostKeyCallback))
 	}
 
 	c.stats = newTailStats(len(c.connections))
@@ -100,7 +101,9 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	return
 }
 
-func (c *baseClient) startConnection(ctx context.Context, i int, conn connectors.Connector) (status int) {
+func (c *baseClient) startConnection(ctx context.Context, i int,
+	conn connectors.Connector) (status int) {
+
 	for {
 		connCtx, cancel := context.WithCancel(ctx)
 		defer cancel()
diff --git a/internal/clients/catclient.go b/internal/clients/catclient.go
index 2726e7e..bd65560 100644
--- a/internal/clients/catclient.go
+++ b/internal/clients/catclient.go
@@ -22,7 +22,6 @@ func NewCatClient(args config.Args) (*CatClient, error) {
 	if args.RegexStr != "" {
 		return nil, errors.New("Can't use regex with 'cat' operating mode")
 	}
-
 	args.Mode = omode.CatClient
 
 	c := CatClient{
@@ -35,7 +34,6 @@ func NewCatClient(args config.Args) (*CatClient, error) {
 
 	c.init()
 	c.makeConnections(c)
-
 	return &c, nil
 }
 
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 1666a79..2d7b45a 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -16,7 +16,8 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-// ServerConnection represents a connection to a single remote dtail server via SSH protocol.
+// ServerConnection represents a connection to a single remote dtail server via
+// SSH protocol.
 type ServerConnection struct {
 	server          string
 	port            int
@@ -28,9 +29,11 @@ type ServerConnection struct {
 }
 
 // NewServerConnection returns a new DTail SSH server connection.
-func NewServerConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback, handler handlers.Handler, commands []string) *ServerConnection {
-	dlog.Client.Debug(server, "Creating new connection", server, handler, commands)
+func NewServerConnection(server string, userName string,
+	authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback,
+	handler handlers.Handler, commands []string) *ServerConnection {
 
+	dlog.Client.Debug(server, "Creating new connection", server, handler, commands)
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
 		server:          server,
@@ -48,10 +51,12 @@ func NewServerConnection(server string, userName string, authMethods []ssh.AuthM
 	return &c
 }
 
+// Server returns the server hostname connected to.
 func (c *ServerConnection) Server() string {
 	return c.server
 }
 
+// Handler returns the handler used for the connection.
 func (c *ServerConnection) Handler() handlers.Handler {
 	return c.handler
 }
@@ -72,23 +77,29 @@ func (c *ServerConnection) initServerPort() {
 	}
 }
 
-func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+// Start the connection to the server.
+func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
+	throttleCh, statsCh chan struct{}) {
+
 	// Throttle how many connections can be established concurrently (based on ch length)
 	dlog.Client.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
 
 	select {
 	case throttleCh <- struct{}{}:
 	case <-ctx.Done():
-		dlog.Client.Debug(c.server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
+		dlog.Client.Debug(c.server, "Not establishing connection as context is done",
+			len(throttleCh), cap(throttleCh))
 		return
 	}
 
-	dlog.Client.Debug(c.server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
+	dlog.Client.Debug(c.server, "Throttling says that the connection can be established",
+		len(throttleCh), cap(throttleCh))
 
 	go func() {
 		defer func() {
 			if !c.throttlingDone {
-				dlog.Client.Debug(c.server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
+				dlog.Client.Debug(c.server, "Unthrottling connection (1)",
+					len(throttleCh), cap(throttleCh))
 				c.throttlingDone = true
 				<-throttleCh
 			}
@@ -107,7 +118,9 @@ func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
 }
 
 // Dail into a new SSH connection. Close connection in case of an error.
-func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) error {
+func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
+	throttleCh, statsCh chan struct{}) error {
+
 	dlog.Client.Debug(c.server, "Incrementing connection stats")
 	statsCh <- struct{}{}
 	defer func() {
@@ -128,31 +141,30 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 }
 
 // Create the SSH session. Close the session in case of an error.
-func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
-	dlog.Client.Debug(c.server, "Creating SSH session")
+func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc,
+	client *ssh.Client, throttleCh chan struct{}) error {
 
+	dlog.Client.Debug(c.server, "Creating SSH session")
 	session, err := client.NewSession()
 	if err != nil {
 		return err
 	}
 	defer session.Close()
-
 	return c.handle(ctx, cancel, session, throttleCh)
 }
 
-func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
-	dlog.Client.Debug(c.server, "Creating handler for SSH session")
+func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc,
+	session *ssh.Session, throttleCh chan struct{}) error {
 
+	dlog.Client.Debug(c.server, "Creating handler for SSH session")
 	stdinPipe, err := session.StdinPipe()
 	if err != nil {
 		return err
 	}
-
 	stdoutPipe, err := session.StdoutPipe()
 	if err != nil {
 		return err
 	}
-
 	if err := session.Shell(); err != nil {
 		return err
 	}
@@ -161,12 +173,10 @@ func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc
 		io.Copy(stdinPipe, c.handler)
 		cancel()
 	}()
-
 	go func() {
 		io.Copy(c.handler, stdoutPipe)
 		cancel()
 	}()
-
 	go func() {
 		select {
 		case <-c.handler.Done():
@@ -182,13 +192,13 @@ func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc
 	}
 
 	if !c.throttlingDone {
-		dlog.Client.Debug(c.server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
+		dlog.Client.Debug(c.server, "Unthrottling connection (2)",
+			len(throttleCh), cap(throttleCh))
 		c.throttlingDone = true
 		<-throttleCh
 	}
 
 	<-ctx.Done()
 	c.handler.Shutdown()
-
 	return nil
 }
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index 768a5ce..2ff490a 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -18,8 +18,10 @@ type Serverless struct {
 	userName string
 }
 
-// NewServerConnection returns a new connection.
-func NewServerless(userName string, handler handlers.Handler, commands []string) *Serverless {
+// NewServerless starts a new serverless session.
+func NewServerless(userName string, handler handlers.Handler,
+	commands []string) *Serverless {
+
 	dlog.Client.Debug("Creating new serverless connector", handler, commands)
 	return &Serverless{
 		userName: userName,
@@ -28,15 +30,20 @@ func NewServerless(userName string, handler handlers.Handler, commands []string)
 	}
 }
 
+// Server returns serverless server indicator.
 func (s *Serverless) Server() string {
 	return "local(serverless)"
 }
 
+// Handler returns the handler used for the serverless connection.
 func (s *Serverless) Handler() handlers.Handler {
 	return s.handler
 }
 
-func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+// Start the serverless connection.
+func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc,
+	throttleCh, statsCh chan struct{}) {
+
 	dlog.Client.Debug("Starting serverless connector")
 	go func() {
 		defer cancel()
@@ -81,13 +88,11 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 		dlog.Client.Trace("io.Copy(serverHandler, s.handler) => done")
 		terminate()
 	}()
-
 	go func() {
 		io.Copy(s.handler, serverHandler)
 		dlog.Client.Trace("io.Copy(s.handler, serverHandler) => done")
 		terminate()
 	}()
-
 	go func() {
 		select {
 		case <-s.handler.Done():
@@ -107,6 +112,5 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 	<-ctx.Done()
 	dlog.Client.Trace("s.handler.Shutdown()")
 	s.handler.Shutdown()
-
 	return nil
 }
diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index ae21ff2..7521c67 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -12,7 +12,8 @@ import (
 	"github.com/mimecast/dtail/internal/omode"
 )
 
-// GrepClient searches a remote file for all lines matching a regular expression. Only the matching lines are displayed.
+// GrepClient searches a remote file for all lines matching a regular
+// expression. Only the matching lines are displayed.
 type GrepClient struct {
 	baseClient
 }
@@ -34,7 +35,6 @@ func NewGrepClient(args config.Args) (*GrepClient, error) {
 
 	c.init()
 	c.makeConnections(c)
-
 	return &c, nil
 }
 
@@ -51,6 +51,5 @@ func (c GrepClient) makeCommands() (commands []string) {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
 			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
-
 	return
 }
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 10ba1f7..47b594e 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -8,7 +8,8 @@ import (
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// HealthHandler is the handler used on the client side for running mapreduce aggregations.
+// HealthHandler is the handler used on the client side for running mapreduce
+// aggregations.
 type HealthHandler struct {
 	baseHandler
 }
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index d1acfbd..8718b35 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -10,7 +10,8 @@ import (
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// MaprHandler is the handler used on the client side for running mapreduce aggregations.
+// MaprHandler is the handler used on the client side for running mapreduce
+// aggregations.
 type MaprHandler struct {
 	baseHandler
 	aggregate *client.Aggregate
@@ -18,7 +19,9 @@ type MaprHandler struct {
 }
 
 // NewMaprHandler returns a new mapreduce client handler.
-func NewMaprHandler(server string, query *mapr.Query, globalGroup *mapr.GlobalGroupSet) *MaprHandler {
+func NewMaprHandler(server string, query *mapr.Query,
+	globalGroup *mapr.GlobalGroupSet) *MaprHandler {
+
 	return &MaprHandler{
 		baseHandler: baseHandler{
 			server:       server,
@@ -55,12 +58,12 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-// Handle a message received from server including mapr aggregation
-// related data.
+// Handle a message received from server including mapr aggregation related data.
 func (h *MaprHandler) handleAggregateMessage(message string) {
 	parts := strings.SplitN(message, protocol.FieldDelimiter, 3)
 	if len(parts) != 3 {
-		dlog.Client.Error("Unable to aggregate data", h.server, message, parts, len(parts), "expected 3 parts")
+		dlog.Client.Error("Unable to aggregate data", h.server, message, parts,
+			len(parts), "expected 3 parts")
 		return
 	}
 	if err := h.aggregate.Aggregate(parts[2]); err != nil {
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index ac1dc20..1a02827 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -32,7 +32,6 @@ func NewHealthClient(args config.Args) (*HealthClient, error) {
 	c.init()
 	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.HealthUser))
 	c.makeConnections(c)
-
 	return &c, nil
 }
 
@@ -45,28 +44,34 @@ func (c HealthClient) makeCommands() (commands []string) {
 	return
 }
 
+// Start the health client.
 func (c *HealthClient) Start(ctx context.Context, statsCh <-chan string) int {
 	status := c.baseClient.Start(ctx, statsCh)
 
 	switch status {
 	case 0:
 		if c.Serverless {
-			fmt.Printf("WARNING: All seems fine but the check only run in serverless mode, please specify a remote server via --server hostname:port\n")
+			fmt.Printf("WARNING: All seems fine but the check only run in serverless mode" +
+				", please specify a remote server via --server hostname:port\n")
 			return 1
 		}
 		fmt.Printf("OK: All fine at %s :-)\n", c.ServersStr)
 	case 2:
 		if c.Serverless {
-			fmt.Printf("CRITICAL: DTail server not operating properly (using serverless connction)!\n")
+			fmt.Printf("CRITICAL: DTail server not operating properly (using " +
+				"serverless connction)!\n")
 			return 2
 		}
-		fmt.Printf("CRITICAL: DTail server not operating properly at %s!\n", c.ServersStr)
+		fmt.Printf("CRITICAL: DTail server not operating properly at %s!\n",
+			c.ServersStr)
 	default:
 		if c.Serverless {
-			fmt.Printf("UNKNOWN: Received unknown status code %d (using serverless connection)\n", status)
+			fmt.Printf("UNKNOWN: Received unknown status code %d (using serverless "+
+				"connection)\n", status)
 			return status
 		}
-		fmt.Printf("UNKNOWN: Received unknown status code %d from %s!\n", status, c.ServersStr)
+		fmt.Printf("UNKNOWN: Received unknown status code %d from %s!\n",
+			status, c.ServersStr)
 	}
 
 	return status
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 412a219..04f258d 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -107,15 +107,14 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	return
 }
 
-// NEXT: Make this a callback function rather trying to use polymorphism to call this.
-// This applies to all clients.
+// NEXT: Make this a callback function rather trying to use polymorphism to call
+// this. This applies to all clients.
 func (c MaprClient) makeHandler(server string) handlers.Handler {
 	return handlers.NewMaprHandler(server, c.query, c.globalGroup)
 }
 
 func (c MaprClient) makeCommands() (commands []string) {
 	commands = append(commands, fmt.Sprintf("map %s", c.query.RawQuery))
-
 	modeStr := "cat"
 	if c.Mode == omode.TailClient {
 		modeStr = "tail"
@@ -134,7 +133,6 @@ func (c MaprClient) makeCommands() (commands []string) {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
 			modeStr, c.Args.SerializeOptions(), file, regex))
 	}
-
 	return
 }
 
@@ -155,7 +153,6 @@ func (c *MaprClient) reportResults() {
 		c.writeResultsToOutfile()
 		return
 	}
-
 	c.printResults()
 }
 
@@ -176,7 +173,6 @@ func (c *MaprClient) printResults() {
 	} else {
 		result, numRows, err = c.globalGroup.SwapOut().Result(c.query, rowsLimit)
 	}
-
 	if err != nil {
 		dlog.Client.FatalPanic(err)
 	}
@@ -202,8 +198,8 @@ func (c *MaprClient) printResults() {
 	dlog.Client.Raw(rawQuery)
 
 	if rowsLimit > 0 && numRows > rowsLimit {
-		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited terminal output to %d rows! Use 'limit' clause to override!",
-			numRows, rowsLimit))
+		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited terminal output "+
+			"to %d rows! Use 'limit' clause to override!", numRows, rowsLimit))
 	}
 	dlog.Client.Raw(result)
 }
@@ -215,7 +211,6 @@ func (c *MaprClient) writeResultsToOutfile() {
 		}
 		return
 	}
-
 	if err := c.globalGroup.SwapOut().WriteResult(c.query); err != nil {
 		dlog.Client.FatalPanic(err)
 	}
diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index fbef572..1315aea 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -36,9 +36,10 @@ func newTailStats(servers int) *stats {
 
 // Start starts printing client connection stats every time a signal is recieved or
 // connection count has changed.
-func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string, quiet bool) {
-	var connectedLast int
+func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{},
+	statsCh <-chan string, quiet bool) {
 
+	var connectedLast int
 	for {
 		var force bool
 		var messages []string
@@ -94,7 +95,9 @@ func (s *stats) printStatsDueInterrupt(messages []string) {
 	dlog.Client.Resume()
 }
 
-func (s *stats) statsData(connected, newConnections int, throttle int) map[string]interface{} {
+func (s *stats) statsData(connected, newConnections int,
+	throttle int) map[string]interface{} {
+
 	percConnected := percentOf(float64(s.servers), float64(connected))
 
 	data := make(map[string]interface{})
@@ -112,7 +115,6 @@ func (s *stats) statsData(connected, newConnections int, throttle int) map[strin
 
 func (s *stats) statsLine(connected, newConnections int, throttle int) string {
 	sb := strings.Builder{}
-
 	i := 0
 	for k, v := range s.statsData(connected, newConnections, throttle) {
 		if i > 0 {
@@ -123,7 +125,6 @@ func (s *stats) statsLine(connected, newConnections int, throttle int) string {
 		sb.WriteString(fmt.Sprintf("%v", v))
 		i++
 	}
-
 	return sb.String()
 }
 
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index d42a0e4..35c01d4 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -19,7 +19,6 @@ type TailClient struct {
 // NewTailClient returns a new TailClient.
 func NewTailClient(args config.Args) (*TailClient, error) {
 	args.Mode = omode.TailClient
-
 	c := TailClient{
 		baseClient: baseClient{
 			Args:       args,
@@ -30,7 +29,6 @@ func NewTailClient(args config.Args) (*TailClient, error) {
 
 	c.init()
 	c.makeConnections(c)
-
 	return &c, nil
 }
 
@@ -48,6 +46,5 @@ func (c TailClient) makeCommands() (commands []string) {
 			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
 	dlog.Client.Debug(commands)
-
 	return
 }
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 82fa410..63d63d8 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -32,7 +32,6 @@ func paintSeverity(sb *strings.Builder, text string) bool {
 	default:
 		return false
 	}
-
 	return true
 }
 
@@ -87,9 +86,9 @@ func paintRemote(sb *strings.Builder, line string) {
 		config.Client.TermColors.Remote.DelimiterAttr)
 
 	color.PaintWithAttr(sb, splitted[4],
-		config.Client.TermColors.Remote.IdFg,
-		config.Client.TermColors.Remote.IdBg,
-		config.Client.TermColors.Remote.IdAttr)
+		config.Client.TermColors.Remote.IDFg,
+		config.Client.TermColors.Remote.IDBg,
+		config.Client.TermColors.Remote.IDAttr)
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
 		config.Client.TermColors.Remote.DelimiterFg,
 		config.Client.TermColors.Remote.DelimiterBg,
@@ -191,6 +190,5 @@ func Colorfy(line string) string {
 			color.BgDefault,
 			color.AttrNone)
 	}
-
 	return sb.String()
 }
diff --git a/internal/color/color.go b/internal/color/color.go
index 5c25855..9d0bc2e 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -52,15 +52,19 @@ const (
 	AttrHidden     Attribute = escape + "[8m"
 )
 
+// ColorNames is the list of all supported terminal colors.
 var ColorNames = []string{
 	"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
 }
 
+// AttributeNames is the list of all supported terminal text attributes.
 var AttributeNames = []string{
-	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "Hidden", "None",
+	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink",
+	"Reverse", "Hidden", "None",
 }
 
-// ToFgColor converts a given string (e.g. from a config file) into a foreground color code.
+// ToFgColor converts a given string (e.g. from a config file) into a foreground
+// color code.
 func ToFgColor(s string) (FgColor, error) {
 	switch strings.ToLower(s) {
 	case "black":
@@ -86,7 +90,8 @@ func ToFgColor(s string) (FgColor, error) {
 	}
 }
 
-// ToBgColor converts a given string (e.g. from a config file) into a background color code.
+// ToBgColor converts a given string (e.g. from a config file) into a background
+// color code.
 func ToBgColor(s string) (BgColor, error) {
 	switch strings.ToLower(s) {
 	case "black":
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 53c9abb..7735d87 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -10,12 +10,14 @@ func PaintStr(text string, fg FgColor, bg BgColor) string {
 	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
 }
 
-// PaintStrWithAttr paints a given text in a given foreground/background/attribute combination
+// PaintStrWithAttr paints a given text in a given foreground/background/attribute
+// combination
 func PaintStrWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
 	if attr == AttrNone {
 		return PaintStr(text, fg, bg)
 	}
-	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
+	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset,
+		BgDefault, FgDefault)
 }
 
 // PaintStrFg paints a given text in a given foreground color.
@@ -48,8 +50,11 @@ func Reset(sb *strings.Builder) {
 	sb.WriteString(string(FgDefault))
 }
 
-// PaintWithAttr starts painting a given text in a given foreground/background/attribute combination.
-func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, attr Attribute) {
+// PaintWithAttr starts painting a given text in a given foreground/background/
+// attribute combination.
+func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor,
+	attr Attribute) {
+
 	if attr == AttrNone {
 		Paint(sb, text, fg, bg)
 		return
@@ -63,8 +68,10 @@ func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, att
 	sb.WriteString(string(FgDefault))
 }
 
-// PaintWithAttrs is similar to PaintWithAttr, but it takes multiple text attributes.
-func PaintWithAttrs(sb *strings.Builder, text string, fg FgColor, bg BgColor, attrs []Attribute) {
+// PaintWithAttrs is similar to PaintWithAttr, but it takes multiple attributes.
+func PaintWithAttrs(sb *strings.Builder, text string, fg FgColor, bg BgColor,
+	attrs []Attribute) {
+
 	sb.WriteString(string(fg))
 	sb.WriteString(string(bg))
 	for _, attr := range attrs {
diff --git a/internal/color/table.go b/internal/color/table.go
index 2115edf..e0e4946 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -5,8 +5,19 @@ import (
 	"os"
 )
 
-const sampleParagraph string = "Mimecast is Making Email Safer for Business. We believe that securely operating a business in the cloud requires new levels of IT preparedness, centered around cyber resilience. This is why we unify the delivery and management of security, continuity and data protection for email via one, simple-to-use cloud platform. Thousands of organizations trust us to increase their cyber resilience preparedness, streamline compliance, reduce IT complexity and keep their business running. We give employees fast and secure access to sensitive business information, and ensure email keeps flowing in the event of an outage. Mimecast will remain committed to protecting your IT assets through constant innovation and focus on your success."
+const sampleParagraph string = "Mimecast is Making Email Safer for Business. " +
+	"We believe that securely operating a business in the cloud requires new " +
+	"levels of IT preparedness, centered around cyber resilience. This is why " +
+	"we unify the delivery and management of security, continuity and data " +
+	"protection for email via one, simple-to-use cloud platform. Thousands of " +
+	"organizations trust us to increase their cyber resilience preparedness, " +
+	"streamline compliance, reduce IT complexity and keep their business running. " +
+	"We give employees fast and secure access to sensitive business information, " +
+	"and ensure email keeps flowing in the event of an outage. Mimecast will " +
+	"remain committed to protecting your IT assets through constant innovation " +
+	"and focus on your success."
 
+// TablePrintAndExit prints the color table and then exits the process.
 func TablePrintAndExit(displaySampleParagraph bool) {
 	for _, attr := range AttributeNames {
 		if attr == "Hidden" || attr == "SlowBlink" {
@@ -24,11 +35,13 @@ func printColorTable(attr string, displaySampleParagraph bool) {
 			if fg == bg {
 				continue
 			}
+
 			bgColor, _ := ToBgColor(bg)
 			attribute, _ := ToAttribute(attr)
-
-			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
+			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ",
+				fg, bg, attr)
 			fmt.Print(PaintStrWithAttr(text, fgColor, bgColor, attribute))
+
 			if displaySampleParagraph {
 				fmt.Print("\n")
 				fmt.Print(PaintStrWithAttr(sampleParagraph, fgColor, bgColor, attribute))
diff --git a/internal/config/args.go b/internal/config/args.go
index a671ae3..f721390 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -74,13 +74,13 @@ func (a *Args) String() string {
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
 func (a *Args) SerializeOptions() string {
-	return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, a.Serverless)
+	return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan,
+		a.Serverless)
 }
 
 // DeserializeOptions deserializes the options, but into a map.
 func DeserializeOptions(opts []string) (map[string]string, error) {
 	options := make(map[string]string, len(opts))
-
 	for _, o := range opts {
 		kv := strings.SplitN(o, "=", 2)
 		if len(kv) != 2 {
@@ -97,9 +97,7 @@ func DeserializeOptions(opts []string) (map[string]string, error) {
 			}
 			val = string(decoded)
 		}
-
 		options[key] = val
 	}
-
 	return options, nil
 }
diff --git a/internal/config/client.go b/internal/config/client.go
index ecd05c5..8227c68 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -15,9 +15,9 @@ type remoteTermColors struct {
 	HostnameAttr  color.Attribute
 	HostnameBg    color.BgColor
 	HostnameFg    color.FgColor
-	IdAttr        color.Attribute
-	IdBg          color.BgColor
-	IdFg          color.FgColor
+	IDAttr        color.Attribute
+	IDBg          color.BgColor
+	IDFg          color.FgColor
 	StatsOkAttr   color.Attribute
 	StatsOkBg     color.BgColor
 	StatsOkFg     color.FgColor
@@ -124,9 +124,9 @@ func newDefaultClientConfig() *ClientConfig {
 				HostnameAttr:  color.AttrBold,
 				HostnameBg:    color.BgBlue,
 				HostnameFg:    color.FgWhite,
-				IdAttr:        color.AttrDim,
-				IdBg:          color.BgBlue,
-				IdFg:          color.FgWhite,
+				IDAttr:        color.AttrDim,
+				IDBg:          color.BgBlue,
+				IDFg:          color.FgWhite,
 				StatsOkAttr:   color.AttrNone,
 				StatsOkBg:     color.BgGreen,
 				StatsOkFg:     color.FgBlack,
diff --git a/internal/config/config.go b/internal/config/config.go
index 077a658..b99b22b 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -9,11 +9,11 @@ const (
 	ScheduleUser string = "DTAIL-SCHEDULE"
 	// ContinuousUser is used for non-interactive continuous mapreduce queries.
 	ContinuousUser string = "DTAIL-CONTINUOUS"
-	// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
+	// InterruptTimeoutS specifies the Ctrl+C log pause interval.
 	InterruptTimeoutS int = 3
-	// ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start)
+	// DefaultConnectionsPerCPU controls how many connections are established concurrently.
 	DefaultConnectionsPerCPU int = 10
-	// DTailSSHServerDefaultPort is the default DServer port.
+	// DefaultSSHPort is the default DServer port.
 	DefaultSSHPort int = 2222
 	// DefaultLogLevel specifies the default log level (obviously)
 	DefaultLogLevel string = "INFO"
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index e4cbeaf..0a913db 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -20,13 +20,13 @@ type initializer struct {
 
 type transformCb func(*initializer, *Args, []string) error
 
-func (c *initializer) parseConfig(args *Args) error {
+func (in *initializer) parseConfig(args *Args) error {
 	if strings.ToUpper(args.ConfigFile) == "NONE" {
 		return nil
 	}
 
 	if args.ConfigFile != "" {
-		return c.parseSpecificConfig(args.ConfigFile)
+		return in.parseSpecificConfig(args.ConfigFile)
 	}
 
 	if homeDir, err := os.UserHomeDir(); err != nil {
@@ -35,7 +35,7 @@ func (c *initializer) parseConfig(args *Args) error {
 		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
 		for _, configPath := range paths {
 			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
-				c.parseSpecificConfig(configPath)
+				in.parseSpecificConfig(configPath)
 			}
 		}
 	}
@@ -43,7 +43,7 @@ func (c *initializer) parseConfig(args *Args) error {
 	return nil
 }
 
-func (c *initializer) parseSpecificConfig(configFile string) error {
+func (in *initializer) parseSpecificConfig(configFile string) error {
 	fd, err := os.Open(configFile)
 	if err != nil {
 		return fmt.Errorf("Unable to read config file: %v", err)
@@ -55,68 +55,74 @@ func (c *initializer) parseSpecificConfig(configFile string) error {
 		return fmt.Errorf("Unable to read config file %s: %v", configFile, err)
 	}
 
-	if err := json.Unmarshal([]byte(cfgBytes), c); err != nil {
+	if err := json.Unmarshal([]byte(cfgBytes), in); err != nil {
 		return fmt.Errorf("Unable to parse config file %s: %v", configFile, err)
 	}
 
 	return nil
 }
 
-func (i *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string) error {
+func (in *initializer) transformConfig(sourceProcess source.Source, args *Args,
+	additionalArgs []string) error {
 
 	switch sourceProcess {
 	case source.Server:
-		return i.optimusPrime(transformServer, args, additionalArgs)
+		return in.optimusPrime(transformServer, args, additionalArgs)
 	case source.Client:
-		return i.optimusPrime(transformClient, args, additionalArgs)
+		return in.optimusPrime(transformClient, args, additionalArgs)
 	case source.HealthCheck:
-		return i.optimusPrime(transformHealthCheck, args, additionalArgs)
+		return in.optimusPrime(transformHealthCheck, args, additionalArgs)
 	default:
-		return fmt.Errorf("Unable to transform config, unknown source '%s'", sourceProcess)
+		return fmt.Errorf("Unable to transform config, unknown source '%s'",
+			sourceProcess)
 	}
 }
 
-func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalArgs []string) error {
+func (in *initializer) optimusPrime(sourceCb transformCb, args *Args,
+	additionalArgs []string) error {
+
 	// Copy args to config objects.
+	// NEXT: Maybe unify args and config structs?
 	if args.SSHPort != DefaultSSHPort {
-		i.Common.SSHPort = args.SSHPort
+		in.Common.SSHPort = args.SSHPort
 	}
 	if args.LogLevel != DefaultLogLevel {
-		i.Common.LogLevel = args.LogLevel
+		in.Common.LogLevel = args.LogLevel
 	}
 	if args.NoColor {
-		i.Client.TermColorsEnable = false
+		in.Client.TermColorsEnable = false
 	}
 	if args.LogDir != "" {
-		i.Common.LogDir = args.LogDir
+		in.Common.LogDir = args.LogDir
 	}
 	if args.Logger != "" {
-		i.Common.Logger = args.Logger
+		in.Common.Logger = args.Logger
 	}
 	if args.ConnectionsPerCPU == 0 {
 		args.ConnectionsPerCPU = DefaultConnectionsPerCPU
 	}
 
 	// Setup log directory.
-	if strings.Contains(i.Common.LogDir, "~/") {
+	if strings.Contains(in.Common.LogDir, "~/") {
 		homeDir, err := os.UserHomeDir()
 		if err != nil {
 			panic(err)
 		}
-		i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
+		in.Common.LogDir = strings.ReplaceAll(in.Common.LogDir, "~/",
+			fmt.Sprintf("%s/", homeDir))
 	}
 
 	// Source type specific transormations.
-	sourceCb(i, args, additionalArgs)
+	sourceCb(in, args, additionalArgs)
 
 	// Spartan mode.
 	if args.Spartan {
 		args.Quiet = true
 		args.NoColor = true
-		i.Client.TermColorsEnable = false
+		in.Client.TermColorsEnable = false
 		if args.LogLevel == "" {
 			args.LogLevel = "ERROR"
-			i.Common.LogLevel = "ERROR"
+			in.Common.LogLevel = "ERROR"
 		}
 	}
 	// Interpret additional args as file list or as query.
@@ -135,29 +141,28 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA
 	return nil
 }
 
-func transformClient(i *initializer, args *Args, additionalArgs []string) error {
+func transformClient(in *initializer, args *Args, additionalArgs []string) error {
 	// Serverless mode.
 	if args.Discovery == "" && (args.ServersStr == "" ||
 		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
-		i.Common.LogLevel = "warn"
+		in.Common.LogLevel = "warn"
 	}
-
 	return nil
 }
 
-func transformServer(i *initializer, args *Args, additionalArgs []string) error {
+func transformServer(in *initializer, args *Args, additionalArgs []string) error {
 	return nil
 }
 
-func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error {
+func transformHealthCheck(in *initializer, args *Args, additionalArgs []string) error {
 	// Serverless mode.
 	if args.Discovery == "" && (args.ServersStr == "" ||
 		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
-		i.Common.LogLevel = "warn"
+		in.Common.LogLevel = "warn"
 	}
 	args.TrustAllHosts = true
 	return nil
diff --git a/internal/config/server.go b/internal/config/server.go
index 8bbb394..677f5ac 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -4,8 +4,8 @@ import (
 	"errors"
 )
 
-// Permissions map. Each SSH user has a list of permissions which
-// log files it is allowed to follow and which ones not.
+// Permissions map. Each SSH user has a list of permissions which log files it
+// is allowed to follow and which ones not.
 type Permissions struct {
 	// The default user permissions.
 	Default []string
@@ -68,7 +68,6 @@ var ServerRelaxedAuthEnable bool
 func newDefaultServerConfig() *ServerConfig {
 	defaultPermissions := []string{"^/.*"}
 	defaultBindAddress := "0.0.0.0"
-
 	return &ServerConfig{
 		SSHBindAddress:     defaultBindAddress,
 		MaxConnections:     10,
@@ -89,10 +88,8 @@ func ServerUserPermissions(userName string) (permissions []string, err error) {
 	if p, ok := Server.Permissions.Users[userName]; ok {
 		permissions = p
 	}
-
 	if len(permissions) == 0 {
 		err = errors.New("Empty set of permission, user won't be able to open any files")
 	}
-
 	return
 }
diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go
index 83ee95e..8bb1e85 100644
--- a/internal/discovery/discovery.go
+++ b/internal/discovery/discovery.go
@@ -73,7 +73,6 @@ func (d *Discovery) initRegex() {
 
 	regexStr := string(runes)
 	dlog.Common.Debug("Using filter regex", regexStr)
-
 	regex, err := regexp.Compile(regexStr)
 	if err != nil {
 		dlog.Common.FatalPanic("Could not compile regex", regexStr, err)
@@ -90,9 +89,7 @@ func (d *Discovery) ServerList() []string {
 	if d.regex != nil {
 		servers = d.filterList(servers)
 	}
-
 	servers = d.dedupList(servers)
-
 	if d.order == Shuffle {
 		servers = d.shuffleList(servers)
 	}
@@ -105,12 +102,10 @@ func (d *Discovery) serverListFromModule() []string {
 	if d.module != "" {
 		return d.serverListFromReflectedModule()
 	}
-
 	if _, err := os.Stat(d.server); err == nil {
 		// Appears to be a file name, now try to read from that file.
 		return d.ServerListFromFILE()
 	}
-
 	// Appears to be a list of FQDNs (or a single FQDN)
 	return d.ServerListFromCOMMA()
 }
@@ -120,18 +115,16 @@ func (d *Discovery) serverListFromModule() []string {
 // Discovery. Whereas MODULENAME must be a upeprcase string.
 func (d *Discovery) serverListFromReflectedModule() []string {
 	methodName := fmt.Sprintf("ServerListFrom%s", d.module)
-
+	// Now we are reflecting the serve discovery function by it's name.
 	rt := reflect.TypeOf(d)
 	reflectedMethod, ok := rt.MethodByName(methodName)
 	if !ok {
 		dlog.Common.FatalPanic("No such server discovery module", d.module, methodName)
 	}
-
 	inputValues := make([]reflect.Value, 1)
 	// Thist input value is method receiver.
 	inputValues[0] = reflect.ValueOf(d)
 	returnValues := reflectedMethod.Func.Call(inputValues)
-
 	// First return value is server list.
 	return returnValues[0].Interface().([]string)
 }
@@ -139,27 +132,23 @@ func (d *Discovery) serverListFromReflectedModule() []string {
 // Filter server list based on a regexp.
 func (d *Discovery) filterList(servers []string) (filtered []string) {
 	dlog.Common.Debug("Filtering server list")
-
 	for _, server := range servers {
 		if d.regex.MatchString(server) {
 			filtered = append(filtered, server)
 		}
 	}
-
 	return
 }
 
 // Deduplicate the server list.
 func (d *Discovery) dedupList(servers []string) (deduped []string) {
 	serverMap := make(map[string]struct{}, len(servers))
-
 	for _, server := range servers {
 		if _, ok := serverMap[server]; !ok {
 			serverMap[server] = struct{}{}
 			deduped = append(deduped, server)
 		}
 	}
-
 	dlog.Common.Debug("Deduped server list", len(servers), len(deduped))
 	return
 }
diff --git a/internal/done.go b/internal/done.go
index 5ea22a0..94f9289 100644
--- a/internal/done.go
+++ b/internal/done.go
@@ -35,7 +35,6 @@ func (d *Done) Done() <-chan struct{} {
 func (d *Done) Shutdown() {
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
-
 	select {
 	case <-d.ch:
 		return
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 28e6882..da67585 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -82,7 +82,7 @@ func new(sourceProcess, sourcePackage source.Source) *DLog {
 	if err != nil {
 		panic(err)
 	}
-	strategy := loggers.GetStrategy(config.Common.LogStrategy)
+	strategy := loggers.NewStrategy(config.Common.LogStrategy)
 	loggerName := config.Common.Logger
 	level := newLevel(config.Common.LogLevel)
 
@@ -153,6 +153,7 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 	}
 }
 
+// FatalPanic terminates the process with a fatal error.
 func (d *DLog) FatalPanic(args ...interface{}) {
 	d.log(Fatal, args)
 	d.Flush()
@@ -162,42 +163,51 @@ func (d *DLog) FatalPanic(args ...interface{}) {
 	panic(sb.String())
 }
 
+// Fatal logs a fatal error.
 func (d *DLog) Fatal(args ...interface{}) string {
 	return d.log(Fatal, args)
 }
 
+// Error logging.
 func (d *DLog) Error(args ...interface{}) string {
 	return d.log(Error, args)
 }
 
+// Warn logs a warning message.
 func (d *DLog) Warn(args ...interface{}) string {
 	return d.log(Warn, args)
 }
 
+// Info logging.
 func (d *DLog) Info(args ...interface{}) string {
 	return d.log(Info, args)
 }
 
+// Verbose logging.
 func (d *DLog) Verbose(args ...interface{}) string {
 	return d.log(Verbose, args)
 }
 
+// Debug logging.
 func (d *DLog) Debug(args ...interface{}) string {
 	return d.log(Debug, args)
 }
 
+// Trace logging.
 func (d *DLog) Trace(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Trace, args)
 }
 
+// Devel used for development purpose only logging (e.g. "print" debugging).
 func (d *DLog) Devel(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Devel, args)
 }
 
+// Raw message logging.
 func (d *DLog) Raw(message string) string {
 	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
 		d.logger.Log(time.Now(), message)
@@ -207,6 +217,7 @@ func (d *DLog) Raw(message string) string {
 	return message
 }
 
+// Mapreduce logging.
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
@@ -251,6 +262,11 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	return d.log(Info, args)
 }
 
-func (d *DLog) Flush()  { d.logger.Flush() }
-func (d *DLog) Pause()  { d.logger.Pause() }
+// Flush the log buffers.
+func (d *DLog) Flush() { d.logger.Flush() }
+
+// Pause the logging.
+func (d *DLog) Pause() { d.logger.Pause() }
+
+// Resume the logging.
 func (d *DLog) Resume() { d.logger.Resume() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 248ad83..0971094 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -7,6 +7,7 @@ import (
 
 type level int
 
+// Available log levels.
 const (
 	Fatal   level = iota
 	Error   level = iota
@@ -20,7 +21,8 @@ const (
 	All     level = iota
 )
 
-var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug, Devel, Trace, All}
+var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug,
+	Devel, Trace, All}
 
 func newLevel(l string) level {
 	switch strings.ToLower(l) {
@@ -73,6 +75,5 @@ func (l level) String() string {
 	case All:
 		return "ALL"
 	}
-
 	panic("Unknown log level " + fmt.Sprintf("%d", l))
 }
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index dda3ee6..415d7fb 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -9,11 +9,13 @@ import (
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
+// Factory is there to retrieve a logger based on various settings.
 func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName, rotationStrategy.FileBase, loggerName)
+	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName,
+		rotationStrategy.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
@@ -36,10 +38,10 @@ func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
 		}
 	}
-
 	return singleton
 }
 
+// FactoryRotate invokes a log rotation of all loggers.
 func FactoryRotate() {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 87280fd..94824fe 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -12,8 +12,7 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 )
 
-type fileWriter struct {
-}
+type fileWriter struct{}
 
 type fileMessageBuf struct {
 	now     time.Time
@@ -35,7 +34,7 @@ type file struct {
 }
 
 func newFile(strategy Strategy) *file {
-	f := file{
+	return &file{
 		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
@@ -43,16 +42,17 @@ func newFile(strategy Strategy) *file {
 		flushCh:  make(chan struct{}),
 		strategy: strategy,
 	}
-
-	return &f
 }
 
 func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 	f.mutex.Lock()
-	defer f.mutex.Unlock()
+	defer func() {
+		f.started = true
+		f.mutex.Unlock()
+	}()
 
-	// Logger already started from another Goroutine.
 	if f.started {
+		// Logger already started from another Goroutine.
 		wg.Done()
 		return
 	}
@@ -68,7 +68,6 @@ func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 
 	go func() {
 		defer wg.Done()
-
 		for {
 			select {
 			case m := <-f.bufferCh:
@@ -84,8 +83,6 @@ func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 			}
 		}
 	}()
-
-	f.started = true
 }
 
 func (f *file) Log(now time.Time, message string) {
diff --git a/internal/io/dlog/loggers/logger.go b/internal/io/dlog/loggers/logger.go
index c88900d..d4e85de 100644
--- a/internal/io/dlog/loggers/logger.go
+++ b/internal/io/dlog/loggers/logger.go
@@ -6,6 +6,7 @@ import (
 	"time"
 )
 
+// Logger is there to plug in your own log implementation.
 type Logger interface {
 	Log(now time.Time, message string)
 	LogWithColors(now time.Time, message, messageWithColors string)
diff --git a/internal/io/dlog/loggers/strategy.go b/internal/io/dlog/loggers/strategy.go
index a1b9355..25d10f0 100644
--- a/internal/io/dlog/loggers/strategy.go
+++ b/internal/io/dlog/loggers/strategy.go
@@ -5,19 +5,26 @@ import (
 	"path/filepath"
 )
 
+// Rotation is the actual strategy used for log rotation..
 type Rotation int
 
 const (
-	DailyRotation  Rotation = iota
+	// DailyRotation tells DTail to rotate its logs on a daily basis or on SIGHUP.
+	DailyRotation Rotation = iota
+	// SignalRotation tells DTail to rotate its logs only on SIGHUP.
 	SignalRotation Rotation = iota
 )
 
+// Strategy is a pair of the rotation and the file base.
 type Strategy struct {
+	// Rotation is the actual rotation strategy used.
 	Rotation Rotation
+	// FileBase can be a name (e.g. "dserver", "dmap") when signal rotation is used.
 	FileBase string
 }
 
-func GetStrategy(name string) Strategy {
+// NewStrategy returns the stratey based on its name.
+func NewStrategy(name string) Strategy {
 	switch name {
 	case "daily":
 		return Strategy{DailyRotation, ""}
diff --git a/internal/io/fs/catfile.go b/internal/io/fs/catfile.go
index 7f387bc..01c15ba 100644
--- a/internal/io/fs/catfile.go
+++ b/internal/io/fs/catfile.go
@@ -6,7 +6,9 @@ type CatFile struct {
 }
 
 // NewCatFile returns a new file catter.
-func NewCatFile(filePath string, globID string, serverMessages chan<- string, limiter chan struct{}) CatFile {
+func NewCatFile(filePath string, globID string, serverMessages chan<- string,
+	limiter chan struct{}) CatFile {
+
 	return CatFile{
 		readFile: readFile{
 			filePath:       filePath,
diff --git a/internal/io/fs/filereader.go b/internal/io/fs/filereader.go
index 0774837..7773142 100644
--- a/internal/io/fs/filereader.go
+++ b/internal/io/fs/filereader.go
@@ -7,7 +7,8 @@ import (
 	"github.com/mimecast/dtail/internal/regex"
 )
 
-// FileReader is the interface used on the dtail server to read/cat/grep/mapr... a file.
+// FileReader is the interface used on the dtail server to read/cat/grep/mapr...
+// a file.
 type FileReader interface {
 	Start(ctx context.Context, lines chan<- line.Line, re regex.Regex) error
 	FilePath() string
diff --git a/internal/io/fs/permissions/permission_linuxacl.go b/internal/io/fs/permissions/permission_linuxacl.go
index 7d2d7ca..904b90f 100644
--- a/internal/io/fs/permissions/permission_linuxacl.go
+++ b/internal/io/fs/permissions/permission_linuxacl.go
@@ -13,7 +13,7 @@ import (
 	"unsafe"
 )
 
-// ToRead checks whether user has Linux file system permissions to read a given file.
+// ToRead checks whether user has Linux file system permissions to read a file.
 func ToRead(user, filePath string) (bool, error) {
 	cUser := C.CString(user)
 	cFilePath := C.CString(filePath)
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index f128c07..92f85b6 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -42,7 +42,8 @@ type readFile struct {
 
 // String returns the string representation of the readFile
 func (f readFile) String() string {
-	return fmt.Sprintf("readFile(filePath:%s,globID:%s,retry:%v,canSkipLines:%v,seekEOF:%v)",
+	return fmt.Sprintf(
+		"readFile(filePath:%s,globID:%s,retry:%v,canSkipLines:%v,seekEOF:%v)",
 		f.filePath,
 		f.globID,
 		f.retry,
@@ -61,7 +62,9 @@ func (f readFile) Retry() bool {
 }
 
 // Start tailing a log file.
-func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Regex) error {
+func (f readFile) Start(ctx context.Context, lines chan<- line.Line,
+	re regex.Regex) error {
+
 	dlog.Common.Debug("readFile", f)
 	defer func() {
 		select {
@@ -74,7 +77,8 @@ func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Re
 	case f.limiter <- struct{}{}:
 	default:
 		select {
-		case f.serverMessages <- dlog.Common.Warn(f.filePath, f.globID, "Server limit reached. Queuing file..."):
+		case f.serverMessages <- dlog.Common.Warn(f.filePath, f.globID,
+			"Server limit reached. Queuing file..."):
 		case <-ctx.Done():
 			return nil
 		}
@@ -139,13 +143,11 @@ func (f readFile) makeReader(fd *os.File) (reader *bufio.Reader, err error) {
 	default:
 		reader = bufio.NewReader(fd)
 	}
-
 	return
 }
 
 func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Buffer, truncate <-chan struct{}) error {
 	var offset uint64
-
 	reader, err := f.makeReader(fd)
 	if err != nil {
 		return err
@@ -193,7 +195,8 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 		default:
 			if message.Len() >= lineLengthThreshold {
 				if !warnedAboutLongLine {
-					f.serverMessages <- dlog.Common.Warn(f.filePath, "Long log line, splitting into multiple lines")
+					f.serverMessages <- dlog.Common.Warn(f.filePath,
+						"Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
 				message.WriteString("\n")
@@ -210,9 +213,10 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 }
 
 // Filter log lines matching a given regular expression.
-func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
-	defer wg.Done()
+func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup,
+	rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
 
+	defer wg.Done()
 	for {
 		select {
 		case line, ok := <-rawLines:
@@ -231,9 +235,10 @@ func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-cha
 	}
 }
 
-func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int, re regex.Regex) (line.Line, bool) {
-	var read line.Line
+func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int,
+	re regex.Regex) (line.Line, bool) {
 
+	var read line.Line
 	if !re.Match(lineBytes.Bytes()) {
 		f.updateLineNotMatched()
 		f.updateLineNotTransmitted()
@@ -254,7 +259,6 @@ func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int, r
 		Count:           f.totalLineCount(),
 		TransmittedPerc: f.transmittedPerc(),
 	}
-
 	return read, true
 }
 
@@ -267,7 +271,6 @@ func (f readFile) truncated(fd *os.File) (bool, error) {
 	if err != nil {
 		return true, err
 	}
-
 	// Can not open file at original path.
 	pathFd, err := os.Open(f.filePath)
 	if err != nil {
@@ -280,10 +283,8 @@ func (f readFile) truncated(fd *os.File) (bool, error) {
 	if err != nil {
 		return true, err
 	}
-
 	if curPos > pathPos {
 		return true, errors.New("File got truncated")
 	}
-
 	return false, nil
 }
diff --git a/internal/io/fs/tailfile.go b/internal/io/fs/tailfile.go
index 14994e5..b03b45d 100644
--- a/internal/io/fs/tailfile.go
+++ b/internal/io/fs/tailfile.go
@@ -6,7 +6,9 @@ type TailFile struct {
 }
 
 // NewTailFile returns a new file tailer.
-func NewTailFile(filePath string, globID string, serverMessages chan<- string, limiter chan struct{}) TailFile {
+func NewTailFile(filePath string, globID string, serverMessages chan<- string,
+	limiter chan struct{}) TailFile {
+
 	return TailFile{
 		readFile: readFile{
 			filePath:       filePath,
diff --git a/internal/io/pool/builder.go b/internal/io/pool/builder.go
index c9dc221..89fcf81 100644
--- a/internal/io/pool/builder.go
+++ b/internal/io/pool/builder.go
@@ -5,6 +5,8 @@ import (
 	"sync"
 )
 
+// BuilderBuffer is there to optimize memory allocations (DTail allocates a lot
+// of memory while reading log data otherwise)
 var BuilderBuffer = sync.Pool{
 	New: func() interface{} {
 		sb := strings.Builder{}
@@ -12,6 +14,7 @@ var BuilderBuffer = sync.Pool{
 	},
 }
 
+// RecycleBuilderBuffer recycles the buffer again.
 func RecycleBuilderBuffer(sb *strings.Builder) {
 	sb.Reset()
 	BuilderBuffer.Put(sb)
diff --git a/internal/io/pool/bytesbuffer.go b/internal/io/pool/bytesbuffer.go
index 0a159f5..3d48f2c 100644
--- a/internal/io/pool/bytesbuffer.go
+++ b/internal/io/pool/bytesbuffer.go
@@ -5,6 +5,8 @@ import (
 	"sync"
 )
 
+// BytesBuffer is there to optimize memory allocations. DTail otherwise allocates
+// a lot of memory while reading logs.
 var BytesBuffer = sync.Pool{
 	New: func() interface{} {
 		b := bytes.Buffer{}
@@ -13,6 +15,7 @@ var BytesBuffer = sync.Pool{
 	},
 }
 
+// RecycleBytesBuffer recycles the buffer again.
 func RecycleBytesBuffer(b *bytes.Buffer) {
 	b.Reset()
 	BytesBuffer.Put(b)
diff --git a/internal/io/prompt/prompt.go b/internal/io/prompt/prompt.go
index 7c3cdb5..e82132d 100644
--- a/internal/io/prompt/prompt.go
+++ b/internal/io/prompt/prompt.go
@@ -19,7 +19,8 @@ type Answer struct {
 	Callback func()
 	// Runs after Callback and after logging resumes
 	EndCallback func()
-	AskAgain    bool
+	// AskAgain can be used to not to ask again about the question.
+	AskAgain bool
 }
 
 // Prompt used for interactive user input.
@@ -30,7 +31,6 @@ type Prompt struct {
 
 func (p *Prompt) askString() string {
 	var sb strings.Builder
-
 	sb.WriteString(p.question)
 	sb.WriteString("? (")
 
@@ -41,7 +41,6 @@ func (p *Prompt) askString() string {
 
 	sb.WriteString(strings.Join(ax, ","))
 	sb.WriteString("): ")
-
 	return sb.String()
 }
 
@@ -68,7 +67,6 @@ func (p *Prompt) Ask() {
 			if a.Callback != nil {
 				a.Callback()
 			}
-
 			if !a.AskAgain {
 				dlog.Common.Resume()
 				if a.EndCallback != nil {
@@ -90,6 +88,5 @@ func (p *Prompt) answer(answerStr string) (*Answer, bool) {
 		default:
 		}
 	}
-
 	return nil, false
 }
diff --git a/internal/io/signal/signal.go b/internal/io/signal/signal.go
index 14056c4..584b59c 100644
--- a/internal/io/signal/signal.go
+++ b/internal/io/signal/signal.go
@@ -14,10 +14,8 @@ import (
 func InterruptCh(ctx context.Context) <-chan string {
 	sigIntCh := make(chan os.Signal)
 	gosignal.Notify(sigIntCh, os.Interrupt)
-
 	sigOtherCh := make(chan os.Signal)
 	gosignal.Notify(sigOtherCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT)
-
 	statsCh := make(chan string)
 
 	go func() {
@@ -41,7 +39,6 @@ func InterruptCh(ctx context.Context) <-chan string {
 			}
 		}
 	}()
-
 	return statsCh
 }
 
diff --git a/internal/mapr/aggregateset.go b/internal/mapr/aggregateset.go
index 14e6943..c50c7a1 100644
--- a/internal/mapr/aggregateset.go
+++ b/internal/mapr/aggregateset.go
@@ -38,7 +38,6 @@ func (s *AggregateSet) String() string {
 func (s *AggregateSet) Merge(query *Query, set *AggregateSet) error {
 	s.Samples += set.Samples
 	//dlog.Common.Trace("Merge", set)
-
 	for _, sc := range query.Select {
 		storage := sc.FieldStorage
 		switch sc.Operation {
@@ -115,7 +114,6 @@ func (s *AggregateSet) addFloatMin(key string, value float64) {
 		s.FValues[key] = value
 		return
 	}
-
 	if f > value {
 		s.FValues[key] = value
 	}
@@ -128,7 +126,6 @@ func (s *AggregateSet) addFloatMax(key string, value float64) {
 		s.FValues[key] = value
 		return
 	}
-
 	if f < value {
 		s.FValues[key] = value
 	}
@@ -147,7 +144,6 @@ func (s *AggregateSet) setFloat(key string, value float64) {
 // Aggregate data to the aggregate set.
 func (s *AggregateSet) Aggregate(key string, agg AggregateOperation, value string, clientAggregation bool) (err error) {
 	var f float64
-
 	// First check if we can aggregate anything without converting value to float.
 	switch agg {
 	case Count:
diff --git a/internal/mapr/client/aggregate.go b/internal/mapr/client/aggregate.go
index d0c1d70..02a6a5a 100644
--- a/internal/mapr/client/aggregate.go
+++ b/internal/mapr/client/aggregate.go
@@ -23,7 +23,9 @@ type Aggregate struct {
 }
 
 // NewAggregate create new client aggregator.
-func NewAggregate(server string, query *mapr.Query, globalGroup *mapr.GlobalGroupSet) *Aggregate {
+func NewAggregate(server string, query *mapr.Query,
+	globalGroup *mapr.GlobalGroupSet) *Aggregate {
+
 	return &Aggregate{
 		query:       query,
 		group:       mapr.NewGroupSet(),
@@ -47,8 +49,8 @@ func (a *Aggregate) Aggregate(message string) error {
 
 	fields := a.makeFields(parts[2:])
 	set := a.group.GetSet(groupKey)
-
 	var addedSamples bool
+
 	for _, sc := range a.query.Select {
 		if val, ok := fields[sc.FieldStorage]; ok {
 			if err := set.Aggregate(sc.FieldStorage, sc.Operation, val, true); err != nil {
@@ -71,14 +73,12 @@ func (a *Aggregate) Aggregate(message string) error {
 		// Re-init local group (make it empty again).
 		a.group.InitSet()
 	}
-
 	return nil
 }
 
 // Create a map of key-value pairs from a part list such as ["foo=bar",  "bar=baz"].
 func (a *Aggregate) makeFields(parts []string) map[string]string {
 	fields := make(map[string]string, len(parts))
-
 	for _, part := range parts {
 		kv := strings.SplitN(part, protocol.AggregateKVDelimiter, 2)
 		if len(kv) != 2 {
@@ -86,6 +86,5 @@ func (a *Aggregate) makeFields(parts []string) map[string]string {
 		}
 		fields[kv[0]] = kv[1]
 	}
-
 	return fields
 }
diff --git a/internal/mapr/funcs/function.go b/internal/mapr/funcs/function.go
index 0433b9a..418d86f 100644
--- a/internal/mapr/funcs/function.go
+++ b/internal/mapr/funcs/function.go
@@ -19,13 +19,12 @@ type Function struct {
 // FunctionStack is a list of functions stacked each other
 type FunctionStack []Function
 
-// NewFunctionStack parses the input string, e.g. foo(bar("arg")) and returns a corresponding function stack.
+// NewFunctionStack parses the input string, e.g. foo(bar("arg")) and returns
+// a corresponding function stack.
 func NewFunctionStack(in string) (FunctionStack, string, error) {
 	var fs FunctionStack
-
 	getCallback := func(name string) (CallbackFunc, error) {
 		var cb CallbackFunc
-
 		switch name {
 		case "md5sum":
 			return Md5Sum, nil
@@ -51,7 +50,6 @@ func NewFunctionStack(in string) (FunctionStack, string, error) {
 		fs = append(fs, Function{name, call})
 		aux = aux[index+1 : len(aux)-1]
 	}
-
 	return fs, aux, nil
 }
 
@@ -62,6 +60,5 @@ func (fs FunctionStack) Call(str string) string {
 		str = fs[i].call(str)
 		//dlog.Common.Debug("Call.result", fs[i].Name, str)
 	}
-
 	return str
 }
diff --git a/internal/mapr/funcs/function_test.go b/internal/mapr/funcs/function_test.go
index 415683c..8b5d8b7 100644
--- a/internal/mapr/funcs/function_test.go
+++ b/internal/mapr/funcs/function_test.go
@@ -6,16 +6,19 @@ func TestFunction(t *testing.T) {
 	input := "md5sum($line)"
 	fs, arg, err := NewFunctionStack(input)
 	if err != nil {
-		t.Errorf("error parsing function input '%s': %s (%v)\n", input, err.Error(), fs)
+		t.Errorf("error parsing function input '%s': %s (%v)\n",
+			input, err.Error(), fs)
 	}
 	if arg != "$line" {
-		t.Errorf("error parsing function input '%s': expected argument '$line' but got '%s' (%v)\n", input, arg, fs)
+		t.Errorf("error parsing function input '%s': expected argument '$line' but "+
+			"got '%s' (%v)\n", input, arg, fs)
 	}
 	t.Log(input, fs, arg)
 
 	result := fs.Call(input)
 	if result != "b38699013d79e50d9d122433753959c1" {
-		t.Errorf("error executing function stack '%s': expected result 'b38699013d79e50d9d122433753959c1' but got '%s' (%v)\n", input, result, fs)
+		t.Errorf("error executing function stack '%s': expected result "+
+			"'b38699013d79e50d9d122433753959c1' but got '%s' (%v)\n", input, result, fs)
 	}
 
 	input = "maskdigits(md5sum(maskdigits($line)))"
@@ -24,22 +27,26 @@ func TestFunction(t *testing.T) {
 		t.Errorf("error parsing function input '%s': %s (%v)\n", input, err.Error(), fs)
 	}
 	if arg != "$line" {
-		t.Errorf("error parsing function input '%s': expected argument '$line' but got '%s' (%v)\n", input, arg, fs)
+		t.Errorf("error parsing function input '%s': expected argument '$line' but "+
+			"got '%s' (%v)\n", input, arg, fs)
 	}
 	t.Log(input, fs, arg)
 
 	result = fs.Call(input)
 	if result != ".fac.bbe..bb.........d...a.c..b." {
-		t.Errorf("error executing function stack '%s': expected result '.fac.bbe..bb.........d...a.c..b.' but got '%s' (%v)\n", input, result, fs)
+		t.Errorf("error executing function stack '%s': expected result "+
+			"'.fac.bbe..bb.........d...a.c..b.' but got '%s' (%v)\n", input, result, fs)
 	}
 
 	input = "md5sum$line)"
 	if fs, _, err := NewFunctionStack(input); err == nil {
-		t.Errorf("Expected error parsing function input '%s' (%v) but got no error\n", input, fs)
+		t.Errorf("Expected error parsing function input '%s' (%v) but got no error\n",
+			input, fs)
 	}
 
 	input = "md5sum(makedigits$line))"
 	if fs, _, err := NewFunctionStack(input); err == nil {
-		t.Errorf("Expected error parsing function input '%s' (%v) but got no error\n", input, fs)
+		t.Errorf("Expected error parsing function input '%s' (%v) but got no error\n",
+			input, fs)
 	}
 }
diff --git a/internal/mapr/funcs/maskdigits.go b/internal/mapr/funcs/maskdigits.go
index d51f3d8..925ec4d 100644
--- a/internal/mapr/funcs/maskdigits.go
+++ b/internal/mapr/funcs/maskdigits.go
@@ -3,12 +3,10 @@ package funcs
 // MaskDigits masks all digits (replaces them with .)
 func MaskDigits(input string) string {
 	s := []byte(input)
-
 	for i, b := range s {
 		if '0' <= b && b <= '9' {
 			s[i] = '.'
 		}
 	}
-
 	return string(s)
 }
diff --git a/internal/mapr/globalgroupset.go b/internal/mapr/globalgroupset.go
index 50bac37..2d7f10b 100644
--- a/internal/mapr/globalgroupset.go
+++ b/internal/mapr/globalgroupset.go
@@ -17,7 +17,6 @@ func NewGlobalGroupSet() *GlobalGroupSet {
 		semaphore: make(chan struct{}, 1),
 	}
 	g.InitSet()
-
 	return &g
 }
 
@@ -30,7 +29,6 @@ func (g *GlobalGroupSet) String() string {
 func (g *GlobalGroupSet) Merge(query *Query, group *GroupSet) error {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
-
 	return g.merge(query, group)
 }
 
@@ -48,14 +46,12 @@ func (g *GlobalGroupSet) MergeNoblock(query *Query, group *GroupSet) (bool, erro
 
 // Merge a group set into the global group set.
 func (g *GlobalGroupSet) merge(query *Query, group *GroupSet) error {
-
 	for groupKey, set := range group.sets {
 		s := g.GetSet(groupKey)
 		if err := s.Merge(query, set); err != nil {
 			return err
 		}
 	}
-
 	return nil
 }
 
@@ -68,7 +64,6 @@ func (g *GlobalGroupSet) IsEmpty() bool {
 func (g *GlobalGroupSet) NumSets() int {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
-
 	return len(g.sets)
 }
 
@@ -80,7 +75,6 @@ func (g *GlobalGroupSet) SwapOut() *GroupSet {
 
 	set := &GroupSet{sets: g.sets}
 	g.InitSet()
-
 	return set
 }
 
@@ -88,7 +82,6 @@ func (g *GlobalGroupSet) SwapOut() *GroupSet {
 func (g *GlobalGroupSet) WriteResult(query *Query) error {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
-
 	return g.GroupSet.WriteResult(query)
 }
 
@@ -96,6 +89,5 @@ func (g *GlobalGroupSet) WriteResult(query *Query) error {
 func (g *GlobalGroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
-
 	return g.GroupSet.Result(query, rowsLimit)
 }
diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index ce7630d..6ffc8b9 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -73,7 +73,6 @@ func (g *GroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	if err != nil {
 		return "", 0, err
 	}
-
 	if query.Limit != -1 {
 		rowsLimit = query.Limit
 	}
@@ -91,12 +90,14 @@ func (g *GroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 			if sc.FieldStorage == query.OrderBy {
 				attrs = append(attrs, config.Client.TermColors.MaprTable.HeaderSortKeyAttr)
 			}
+
 			for _, groupBy := range query.GroupBy {
 				if sc.FieldStorage == groupBy {
 					attrs = append(attrs, config.Client.TermColors.MaprTable.HeaderGroupKeyAttr)
 					break
 				}
 			}
+
 			color.PaintWithAttrs(sb, str,
 				config.Client.TermColors.MaprTable.HeaderFg,
 				config.Client.TermColors.MaprTable.HeaderBg,
@@ -191,7 +192,6 @@ func (*GroupSet) writeQueryFile(query *Query) error {
 
 	fd.WriteString(query.RawQuery)
 	os.Rename(tmpQueryFile, queryFile)
-
 	return nil
 }
 
@@ -256,7 +256,6 @@ func (g *GroupSet) WriteResult(query *Query) error {
 func (g *GroupSet) result(query *Query, gatherWidths bool) ([]result, []int, error) {
 	var rows []result
 	widths := make([]int, len(query.Select))
-
 	var valueStr string
 	var value float64
 
@@ -284,7 +283,8 @@ func (g *GroupSet) result(query *Query, gatherWidths bool) ([]result, []int, err
 				value = set.FValues[sc.FieldStorage] / float64(set.Samples)
 				valueStr = fmt.Sprintf("%f", value)
 			default:
-				return rows, widths, fmt.Errorf("Unknown aggregation method '%v'", sc.Operation)
+				return rows, widths, fmt.Errorf("Unknown aggregation method '%v'",
+					sc.Operation)
 			}
 
 			if sc.FieldStorage == query.OrderBy {
@@ -302,7 +302,6 @@ func (g *GroupSet) result(query *Query, gatherWidths bool) ([]result, []int, err
 				widths[i] = len(valueStr)
 			}
 		}
-
 		rows = append(rows, r)
 	}
 
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 8016667..9b6c855 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -11,9 +11,10 @@ import (
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
 
-	if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") {
+	if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") ||
+		!strings.HasPrefix(splitted[0], "INFO") {
 		// Not a DTail mapreduce log line.
-		return nil, IgnoreFieldsErr
+		return nil, ErrIgnoreFields
 	}
 
 	fields := make(map[string]string, len(splitted)+8)
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index a777156..28e1acc 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -32,49 +32,57 @@ func TestDefaultLogFormat(t *testing.T) {
 		if val, ok := fields["$severity"]; !ok {
 			t.Errorf("Expected field '$severity', but no such field there in '%s'\n", input)
 		} else if val != "INFO" {
-			t.Errorf("Expected 'Info' stored in field '$severity', but got '%s' in '%s'\n", val, input)
+			t.Errorf("Expected 'Info' stored in field '$severity', but got '%s' in '%s'\n",
+				val, input)
 		}
 
 		if val, ok := fields["$time"]; !ok {
 			t.Errorf("Expected field '$time', but no such field there in '%s'\n", input)
 		} else if val != time {
-			t.Errorf("Expected '%s' stored in field '$time', but got '%s' in '%s'\n", time, val, input)
+			t.Errorf("Expected '%s' stored in field '$time', but got '%s' in '%s'\n",
+				time, val, input)
 		}
 
 		if val, ok := fields["$date"]; !ok {
 			t.Errorf("Expected field '$date', but no such field there in '%s'\n", input)
 		} else if val != date {
-			t.Errorf("Expected '%s' stored in field '$date', but got '%s' in '%s'\n", date, val, input)
+			t.Errorf("Expected '%s' stored in field '$date', but got '%s' in '%s'\n",
+				date, val, input)
 		}
 
 		if val, ok := fields["$hour"]; !ok {
 			t.Errorf("Expected field '$hour', but no such field there in '%s'\n", input)
 		} else if val != hour {
-			t.Errorf("Expected '%s' stored in field '$hour', but got '%s' in '%s'\n", hour, val, input)
+			t.Errorf("Expected '%s' stored in field '$hour', but got '%s' in '%s'\n",
+				hour, val, input)
 		}
 
 		if val, ok := fields["$minute"]; !ok {
 			t.Errorf("Expected field '$minute', but no such field there in '%s'\n", input)
 		} else if val != minute {
-			t.Errorf("Expected '%s' stored in field '$minute', but got '%s' in '%s'\n", minute, val, input)
+			t.Errorf("Expected '%s' stored in field '$minute', but got '%s' in '%s'\n",
+				minute, val, input)
 		}
 
 		if val, ok := fields["$second"]; !ok {
 			t.Errorf("Expected field '$second', but no such field there in '%s'\n", input)
 		} else if val != second {
-			t.Errorf("Expected '%s' stored in field '$second', but got '%s' in '%s'\n", second, val, input)
+			t.Errorf("Expected '%s' stored in field '$second', but got '%s' in '%s'\n",
+				second, val, input)
 		}
 
 		if val, ok := fields["foo"]; !ok {
 			t.Errorf("Expected field 'foo', but no such field there in '%s'\n", input)
 		} else if val != "bar" {
-			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n", val, input)
+			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n",
+				val, input)
 		}
 
 		if val, ok := fields["bar"]; !ok {
 			t.Errorf("Expected field 'bar', but no such field there in '%s'\n", input)
 		} else if val != "foo" {
-			t.Errorf("Expected 'foo' stored in field 'bar', but got '%s' in '%s'\n", val, input)
+			t.Errorf("Expected 'foo' stored in field 'bar', but got '%s' in '%s'\n",
+				val, input)
 		}
 	}
 
diff --git a/internal/mapr/logformat/generickv.go b/internal/mapr/logformat/generickv.go
index 3769c22..433eb5f 100644
--- a/internal/mapr/logformat/generickv.go
+++ b/internal/mapr/logformat/generickv.go
@@ -6,7 +6,7 @@ import (
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// MakeFieldsGENERICKV is the generic key-value logfile parser.
+// MakeFieldsGENERIGKV is the generic key-value logfile parser.
 func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error) {
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
 	fields := make(map[string]string, len(splitted))
diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go
index a352580..129081d 100644
--- a/internal/mapr/logformat/parser.go
+++ b/internal/mapr/logformat/parser.go
@@ -11,7 +11,8 @@ import (
 	"github.com/mimecast/dtail/internal/mapr"
 )
 
-var IgnoreFieldsErr error = errors.New("Ignore this field set")
+// ErrIgnoreFields indicates that the fields should be ignored.
+var ErrIgnoreFields error = errors.New("Ignore this field set")
 
 // Parser is used to parse the mapreduce information from the server log files.
 type Parser struct {
@@ -26,11 +27,9 @@ type Parser struct {
 // NewParser returns a new log parser.
 func NewParser(logFormatName string, query *mapr.Query) (*Parser, error) {
 	hostname, err := os.Hostname()
-
 	if err != nil {
 		return nil, err
 	}
-
 	now := time.Now()
 	zone, offset := now.Zone()
 
@@ -44,7 +43,6 @@ func NewParser(logFormatName string, query *mapr.Query) (*Parser, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	return &p, nil
 }
 
@@ -53,7 +51,6 @@ func NewParser(logFormatName string, query *mapr.Query) (*Parser, error) {
 // Parser. Whereas MODULENAME must be a upeprcase string.
 func (p *Parser) reflectLogFormat(logFormatName string) error {
 	methodName := fmt.Sprintf("MakeFields%s", strings.ToUpper(logFormatName))
-
 	rt := reflect.TypeOf(p)
 	method, ok := rt.MethodByName(methodName)
 	if !ok {
@@ -62,7 +59,6 @@ func (p *Parser) reflectLogFormat(logFormatName string) error {
 
 	p.makeFieldsFunc = method.Func
 	p.makeFieldsReceiver = reflect.ValueOf(p)
-
 	return nil
 }
 
@@ -70,15 +66,11 @@ func (p *Parser) reflectLogFormat(logFormatName string) error {
 func (p *Parser) MakeFields(maprLine string) (fields map[string]string, err error) {
 	inputValues := []reflect.Value{p.makeFieldsReceiver, reflect.ValueOf(maprLine)}
 	returnValues := p.makeFieldsFunc.Call(inputValues)
-
 	errInterface := returnValues[1].Interface()
-
 	if errInterface == nil {
 		fields, err = returnValues[0].Interface().(map[string]string), nil
 		return
 	}
-
 	fields, err = returnValues[0].Interface().(map[string]string), errInterface.(error)
-
 	return
 }
diff --git a/internal/mapr/query.go b/internal/mapr/query.go
index 6c1d849..d7c32bd 100644
--- a/internal/mapr/query.go
+++ b/internal/mapr/query.go
@@ -32,7 +32,9 @@ type Query struct {
 }
 
 func (q Query) String() string {
-	return fmt.Sprintf("Query(Select:%v,Table:%s,Where:%v,Set:%vGroupBy:%v,GroupKey:%s,OrderBy:%v,ReverseOrder:%v,Interval:%v,Limit:%d,Outfile:%s,RawQuery:%s,tokens:%v,LogFormat:%s)",
+	return fmt.Sprintf("Query(Select:%v,Table:%s,Where:%v,Set:%vGroupBy:%v,"+
+		"GroupKey:%s,OrderBy:%v,ReverseOrder:%v,Interval:%v,Limit:%d,Outfile:%s,"+
+		"RawQuery:%s,tokens:%v,LogFormat:%s)",
 		q.Select,
 		q.Table,
 		q.Where,
@@ -54,18 +56,14 @@ func NewQuery(queryStr string) (*Query, error) {
 	if queryStr == "" {
 		return nil, nil
 	}
-
 	tokens := tokenize(queryStr)
-
 	q := Query{
 		RawQuery: queryStr,
 		tokens:   tokens,
 		Interval: time.Second * 5,
 		Limit:    -1,
 	}
-
-	err := q.parse(tokens)
-	return &q, err
+	return &q, q.parse(tokens)
 }
 
 // HasOutfile returns true if query result will be written to a CVS output file.
@@ -174,13 +172,13 @@ func (q *Query) parse(tokens []token) error {
 	}
 
 	if len(q.Select) < 1 {
-		return errors.New(invalidQuery + "Expected at least one field in 'select' clause but got none")
+		return errors.New(invalidQuery + "Expected at least one field in 'select' " +
+			"clause but got none")
 	}
 	if len(q.GroupBy) == 0 {
 		field := q.Select[0].Field
 		q.GroupBy = append(q.GroupBy, field)
 	}
-
 	if q.OrderBy != "" {
 		var orderFieldIsValid bool
 		for _, sc := range q.Select {
@@ -190,7 +188,8 @@ func (q *Query) parse(tokens []token) error {
 			}
 		}
 		if !orderFieldIsValid {
-			return errors.New(invalidQuery + fmt.Sprintf("Can not '(r)order by' '%s', must be present in 'select' clause", q.OrderBy))
+			return errors.New(invalidQuery + fmt.Sprintf("Can not '(r)order by' '%s',"+
+				"must be present in 'select' clause", q.OrderBy))
 		}
 	}
 
diff --git a/internal/mapr/query_test.go b/internal/mapr/query_test.go
index b0b6c3a..88f7387 100644
--- a/internal/mapr/query_test.go
+++ b/internal/mapr/query_test.go
@@ -13,18 +13,25 @@ func TestParseQuerySimple(t *testing.T) {
 		"select foo from bar where baz <",
 		"select foo from bar where baz < 100 bay eq 12 group",
 		"select foo from bar where baz < 100 bay eq 12 group by foo order by",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit set foo = bar;",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit set foo = bar;",
 	}
 	okQueries := []string{"select foo from bar",
 		"select foo from bar where",
 		"select foo from bar where baz < 100 bay eq 12",
 		"select foo from bar where baz < 100, bay eq 12",
 		"select foo from bar where baz < 100 and bay eq 12",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit 23",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit 23 outfile \"result.csv\"",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit 23 outfile \"result.csv\" set $foo = maskdigits(bar), $baz = 12, $bay = $foo;",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit 23",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit 23 outfile \"result.csv\"",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit 23 outfile \"result.csv\" " +
+			"set $foo = maskdigits(bar), $baz = 12, $bay = $foo;",
 	}
 
 	for _, queryStr := range errorQueries {
@@ -46,8 +53,13 @@ func TestParseQuerySimple(t *testing.T) {
 
 func TestParseQueryDeep(t *testing.T) {
 	dialects := []string{
-		"select s1, `from`, count(s3) from table where w1 == 2 and w2 eq \"free beer\" group by g1, g2 order by count(s3) interval 10 limit 23 set $foo = maskdigits(bar), $baz = 12, $bay = $foo logformat generic",
-		"SELECT s1, `from`, COUNT(s3) FROM table WHERE w1 == 2 AND w2 eq \"free beer\" GROUP g1, g2 ORDER count(s3) INTERVAL 10 LIMIT 23 SET $foo = maskdigits(bar), $baz = 12, $bay = $foo logformat generic",
+		"select s1, `from`, count(s3) from table where w1 == 2 and w2 eq " +
+			"\"free beer\" group by g1, g2 order by count(s3) interval 10 limit 23 " +
+			"set $foo = maskdigits(bar), $baz = 12, $bay = $foo logformat generic",
+
+		"SELECT s1, `from`, COUNT(s3) FROM table WHERE w1 == 2 AND w2 eq " +
+			"\"free beer\" GROUP g1, g2 ORDER count(s3) INTERVAL 10 LIMIT 23 " +
+			"SET $foo = maskdigits(bar), $baz = 12, $bay = $foo logformat generic",
 	}
 
 	for _, queryStr := range dialects {
@@ -55,119 +67,144 @@ func TestParseQueryDeep(t *testing.T) {
 		if err != nil {
 			t.Errorf("%s: %s", err.Error(), queryStr)
 		}
-
 		t.Log(q)
 
 		// 'select' clause
 		if len(q.Select) != 3 {
-			t.Errorf("Expected three elements in 'select' clause but got '%v': %s\n%v", q.Select, queryStr, q)
+			t.Errorf("Expected three elements in 'select' clause but got '%v': %s\n%v",
+				q.Select, queryStr, q)
 		}
-
 		if q.Select[0].Field != "s1" {
-			t.Errorf("Expected 's1' as first element in 'select' clause but got '%v': %s\n%v", q.Select[0].Field, queryStr, q)
+			t.Errorf("Expected 's1' as first element in 'select' clause but got '%v': %s\n%v",
+				q.Select[0].Field, queryStr, q)
 		}
 		if q.Select[0].Operation != Last {
-			t.Errorf("Expected 'last' as aggregation function of first element in 'select' clause but got '%v': %s\n%v", q.Select[0].Operation, queryStr, q)
+			t.Errorf("Expected 'last' as aggregation function of first element in "+
+				"'select' clause but got '%v': %s\n%v", q.Select[0].Operation, queryStr, q)
 		}
-
 		if q.Select[1].Field != "from" {
-			t.Errorf("Expected 'from' as second element in 'select' clause but got '%v': %s\n%v", q.Select[1].Field, queryStr, q)
+			t.Errorf("Expected 'from' as second element in 'select' clause but got "+
+				"'%v': %s\n%v", q.Select[1].Field, queryStr, q)
 		}
 		if q.Select[1].Operation != Last {
-			t.Errorf("Expected 'last' as aggregation function of second element in 'select' clause but got '%v': %s\n%v", q.Select[1].Operation, queryStr, q)
+			t.Errorf("Expected 'last' as aggregation function of second element in "+
+				"'select' clause but got '%v': %s\n%v", q.Select[1].Operation, queryStr, q)
 		}
-
 		if q.Select[2].Field != "s3" {
-			t.Errorf("Expected 's3' as third element in 'select' clause but got '%v': %s\n%v", q.Select[2].Field, queryStr, q)
+			t.Errorf("Expected 's3' as third element in 'select' clause but got "+
+				"'%v': %s\n%v", q.Select[2].Field, queryStr, q)
 		}
 		if q.Select[2].Operation != Count {
-			t.Errorf("Expected 'count' as aggregation function of third  element in 'select' clause but got '%v': %s\n%v", q.Select[2].Operation, queryStr, q)
+			t.Errorf("Expected 'count' as aggregation function of third  element in "+
+				"'select' clause but got '%v': %s\n%v", q.Select[2].Operation, queryStr, q)
 		}
 		if q.Select[2].FieldStorage != "count(s3)" {
-			t.Errorf("Expected 'count(s3)' as third element's storage in 'select' clause but got '%v': %s\n%v", q.Select[2].FieldStorage, queryStr, q)
+			t.Errorf("Expected 'count(s3)' as third element's storage in 'select' "+
+				"clause but got '%v': %s\n%v", q.Select[2].FieldStorage, queryStr, q)
 		}
 
 		// 'from' clause
 		if q.Table != "TABLE" {
-			t.Errorf("Expected 'TABLE' in 'from' clause but got '%v': %s\n%v", q.Table, queryStr, q)
+			t.Errorf("Expected 'TABLE' in 'from' clause but got '%v': %s\n%v",
+				q.Table, queryStr, q)
 		}
 
 		// 'where' clause
 		if len(q.Where) != 2 {
-			t.Errorf("Expected two elements in 'where' clause but got '%v': %s\n%v", q.Where, queryStr, q)
+			t.Errorf("Expected two elements in 'where' clause but got '%v': %s\n%v",
+				q.Where, queryStr, q)
 		}
 		if q.Where[0].lString != "w1" {
-			t.Errorf("Expected w1 as first element in 'where' clause but got '%v': %s\n%v", q.Where[0].lString, queryStr, q)
+			t.Errorf("Expected w1 as first element in 'where' clause but got '%v': %s\n%v",
+				q.Where[0].lString, queryStr, q)
 		}
 		if q.Where[0].Operation != FloatEq {
-			t.Errorf("Expected FloatEq operation in first 'where' condition but got '%v': %s\n%v", q.Where[0].Operation, queryStr, q)
+			t.Errorf("Expected FloatEq operation in first 'where' condition but got "+
+				"'%v': %s\n%v", q.Where[0].Operation, queryStr, q)
 		}
 		if q.Where[0].rFloat != 2 {
-			t.Errorf("Expected '2' as float argument in first 'where' condition but got '%v': %s\n%v", q.Where[0].rFloat, queryStr, q)
+			t.Errorf("Expected '2' as float argument in first 'where' condition but "+
+				"got '%v': %s\n%v", q.Where[0].rFloat, queryStr, q)
 		}
 		if q.Where[1].lString != "w2" {
-			t.Errorf("Expected w2 as second element in 'where' clause but got '%v': %s\n%v", q.Where[1].lString, queryStr, q)
+			t.Errorf("Expected w2 as second element in 'where' clause but got '%v': "+
+				"%s\n%v", q.Where[1].lString, queryStr, q)
 		}
 		if q.Where[1].Operation != StringEq {
-			t.Errorf("Expected StringEq operation in second 'where' condition but got '%v': %s\n%v", q.Where[0].Operation, queryStr, q)
+			t.Errorf("Expected StringEq operation in second 'where' condition but got "+
+				"'%v': %s\n%v", q.Where[0].Operation, queryStr, q)
 		}
 		if q.Where[1].rString != "free beer" {
-			t.Errorf("Expected 'free beer' as string argument in second 'where' condition but got '%v': %s\n%v", q.Where[0].rString, queryStr, q)
+			t.Errorf("Expected 'free beer' as string argument in second 'where' "+
+				"condition but got '%v': %s\n%v", q.Where[0].rString, queryStr, q)
 		}
 
 		// 'group by' clause
 		if len(q.GroupBy) != 2 {
-			t.Errorf("Expected two elements in 'group by' clause but got '%v': %s\n%v", q.GroupBy, queryStr, q)
+			t.Errorf("Expected two elements in 'group by' clause but got '%v': %s\n%v",
+				q.GroupBy, queryStr, q)
 		}
 		if q.GroupBy[0] != "g1" {
-			t.Errorf("Expected 'g1' as first element in 'group by' clause but got '%v': %s\n%v", q.GroupBy[0], queryStr, q)
+			t.Errorf("Expected 'g1' as first element in 'group by' clause but got "+
+				"'%v': %s\n%v", q.GroupBy[0], queryStr, q)
 		}
 		if q.GroupBy[1] != "g2" {
-			t.Errorf("Expected 'g2' as second element in 'group by' clause but got '%v': %s\n%v", q.GroupBy[1], queryStr, q)
+			t.Errorf("Expected 'g2' as second element in 'group by' clause but got "+
+				"'%v': %s\n%v", q.GroupBy[1], queryStr, q)
 		}
 		if q.GroupKey != "g1,g2" {
-			t.Errorf("Expected 'g1,g2' as group key in 'group by' clause but got '%v': %s\n%v", q.GroupKey, queryStr, q)
+			t.Errorf("Expected 'g1,g2' as group key in 'group by' clause but got "+
+				"'%v': %s\n%v", q.GroupKey, queryStr, q)
 		}
 
 		// 'order by' clause
 		if q.OrderBy != "count(s3)" {
-			t.Errorf("Expected 'count(s3)' as element in 'order by' clause but got '%v': %s\n%v", q.OrderBy, queryStr, q)
+			t.Errorf("Expected 'count(s3)' as element in 'order by' clause but got "+
+				"'%v': %s\n%v", q.OrderBy, queryStr, q)
 		}
 
 		// 'interval' clause
 		if q.Interval != time.Second*time.Duration(10) {
-			t.Errorf("Expected '10s' as duration 'interval' clause but got '%v': %s\n%v", q.Interval, queryStr, q)
+			t.Errorf("Expected '10s' as duration 'interval' clause but got '%v': %s\n%v",
+				q.Interval, queryStr, q)
 		}
 
 		// 'limit' clause
 		if q.Limit != 23 {
-			t.Errorf("Expected '23' as limit in 'limit' clause but got '%v': %s\n%v", q.Limit, queryStr, q)
+			t.Errorf("Expected '23' as limit in 'limit' clause but got '%v': %s\n%v",
+				q.Limit, queryStr, q)
 		}
 
 		// 'set' clause
 		if q.Set[0].lString != "$foo" {
-			t.Errorf("Expected '$foo' lvalue in first 'set' condition clause but got '%v': %s\n%v", q.Set[0].lString, queryStr, q)
+			t.Errorf("Expected '$foo' lvalue in first 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[0].lString, queryStr, q)
 		}
 		if q.Set[0].rString != "bar" {
-			t.Errorf("Expected 'bar' rvalue in first 'set' condition clause but got '%v': %s\n%v", q.Set[0].rString, queryStr, q)
+			t.Errorf("Expected 'bar' rvalue in first 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[0].rString, queryStr, q)
 		}
-
 		if q.Set[1].lString != "$baz" {
-			t.Errorf("Expected '$baz' lvalue in second 'set' condition clause but got '%v': %s\n%v", q.Set[1].lString, queryStr, q)
+			t.Errorf("Expected '$baz' lvalue in second 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[1].lString, queryStr, q)
 		}
 		if q.Set[1].rString != "12" {
-			t.Errorf("Expected '12' rvalue in second 'set' condition clause but got '%v': %s\n%v", q.Set[1].rString, queryStr, q)
+			t.Errorf("Expected '12' rvalue in second 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[1].rString, queryStr, q)
 		}
-
 		if q.Set[2].lString != "$bay" {
-			t.Errorf("Expected '$bay' lvalue in third 'set' condition clause but got '%v': %s\n%v", q.Set[2].lString, queryStr, q)
+			t.Errorf("Expected '$bay' lvalue in third 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[2].lString, queryStr, q)
 		}
 		if q.Set[2].rString != "$foo" {
-			t.Errorf("Expected '$foo' rvalue in third 'set' condition clause but got '%v': %s\n%v", q.Set[2].rString, queryStr, q)
+			t.Errorf("Expected '$foo' rvalue in third 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[2].rString, queryStr, q)
 		}
 
+		// 'logformat' clause
 		if q.LogFormat != "generic" {
-			t.Errorf("Expected 'generic' logformat got '%v': %s\n%v", q.LogFormat, queryStr, q)
+			t.Errorf("Expected 'generic' logformat got '%v': %s\n%v",
+				q.LogFormat, queryStr, q)
 		}
 	}
 }
diff --git a/internal/mapr/selectcondition.go b/internal/mapr/selectcondition.go
index d6aa0d4..5cfb8c7 100644
--- a/internal/mapr/selectcondition.go
+++ b/internal/mapr/selectcondition.go
@@ -37,7 +37,6 @@ func (sc selectCondition) String() string {
 
 func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 	var sel []selectCondition
-
 	// Parse select aggregation, e.g. sum(foo)
 	parse := func(token token) (selectCondition, error) {
 		var sc selectCondition
@@ -52,13 +51,15 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 
 		a := strings.Split(tokenStr, "(")
 		if len(a) != 2 {
-			return sc, errors.New(invalidQuery + "Can't parse 'select' aggregation: " + token.str)
+			return sc, errors.New(invalidQuery + "Can't parse 'select' aggregation: " +
+				token.str)
 		}
 		agg := a[0] // Aggregation, e.g. 'sum'
 
 		b := strings.Split(a[1], ")")
 		if len(b) != 2 {
-			return sc, errors.New(invalidQuery + "Can't parse 'select' field name from aggregation: " + token.str)
+			return sc, errors.New(invalidQuery + "Can't parse 'select' field name " +
+				"from aggregation: " + token.str)
 		}
 		sc.Field = b[0]            // Field name, e.g. 'foo'
 		sc.FieldStorage = tokenStr // e.g. 'sum(foo)'
@@ -79,9 +80,9 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 		case "len":
 			sc.Operation = Len
 		default:
-			return sc, errors.New(invalidQuery + "Unknown aggregation in 'select' clause: " + agg)
+			return sc, errors.New(invalidQuery +
+				"Unknown aggregation in 'select' clause: " + agg)
 		}
-
 		return sc, nil
 	}
 
@@ -92,6 +93,5 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 		}
 		sel = append(sel, sc)
 	}
-
 	return sel, nil
 }
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 1f5d1c3..97fee11 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -63,16 +63,14 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 		}
 	}
 
-	a := Aggregate{
+	return &Aggregate{
 		done:        internal.NewDone(),
 		NextLinesCh: make(chan chan line.Line, 10),
 		serialize:   make(chan struct{}),
 		hostname:    s[0],
 		query:       query,
 		parser:      logParser,
-	}
-
-	return &a, nil
+	}, nil
 }
 
 // Shutdown the aggregation engine.
@@ -95,12 +93,10 @@ func (a *Aggregate) Start(ctx context.Context, maprMessages chan<- string) {
 	}()
 
 	fieldsCh := a.fieldsFromLines(myCtx)
-
 	// Add fields (e.g. via 'set' clause)
 	if len(a.query.Set) > 0 {
 		fieldsCh = a.setAdditionalFields(myCtx, fieldsCh)
 	}
-
 	// Periodically pre-aggregate data every a.query.Interval seconds.
 	go a.aggregateTimer(myCtx)
 	a.aggregateAndSerialize(myCtx, fieldsCh, maprMessages)
@@ -147,17 +143,18 @@ func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]strin
 
 				maprLine := strings.TrimSpace(line.Content.String())
 				fields, err := a.parser.MakeFields(maprLine)
-				// Can not recycle here for some rason.
+				// Can't recycle it here yet, as field slices are still
+				// TODO: Add unit test reading from multiple mapreduce files lines.
+				// TODO: Add capability to recycle this bytes buffer.
 				//pool.RecycleBytesBuffer(line.Content)
 
 				if err != nil {
 					// Should fields be ignored anyway?
-					if err != logformat.IgnoreFieldsErr {
+					if err != logformat.ErrIgnoreFields {
 						dlog.Common.Error(fields, err)
 					}
 					continue
 				}
-
 				if !a.query.WhereClause(fields) {
 					continue
 				}
@@ -175,12 +172,12 @@ func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]strin
 	return fieldsCh
 }
 
-func (a *Aggregate) setAdditionalFields(ctx context.Context, fieldsCh <-chan map[string]string) <-chan map[string]string {
-	newFieldsCh := make(chan map[string]string)
+func (a *Aggregate) setAdditionalFields(ctx context.Context,
+	fieldsCh <-chan map[string]string) <-chan map[string]string {
 
+	newFieldsCh := make(chan map[string]string)
 	go func() {
 		defer close(newFieldsCh)
-
 		for {
 			fields, ok := <-fieldsCh
 			if !ok {
@@ -196,19 +193,18 @@ func (a *Aggregate) setAdditionalFields(ctx context.Context, fieldsCh <-chan map
 			}
 		}
 	}()
-
 	return newFieldsCh
 }
 
-func (a *Aggregate) aggregateAndSerialize(ctx context.Context, fieldsCh <-chan map[string]string, maprMessages chan<- string) {
-	group := mapr.NewGroupSet()
+func (a *Aggregate) aggregateAndSerialize(ctx context.Context,
+	fieldsCh <-chan map[string]string, maprMessages chan<- string) {
 
+	group := mapr.NewGroupSet()
 	serialize := func() {
 		dlog.Common.Info("Serializing mapreduce result")
 		group.Serialize(ctx, maprMessages)
 		group = mapr.NewGroupSet()
 	}
-
 	for {
 		select {
 		case fields, ok := <-fieldsCh:
@@ -227,7 +223,6 @@ func (a *Aggregate) aggregateAndSerialize(ctx context.Context, fieldsCh <-chan m
 
 func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 	var sb strings.Builder
-
 	for i, field := range a.query.GroupBy {
 		if i > 0 {
 			sb.WriteString(protocol.AggregateGroupKeyCombinator)
@@ -254,7 +249,6 @@ func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 		set.Samples++
 		return
 	}
-
 	dlog.Common.Trace("Aggregated data locally without adding new samples")
 }
 
diff --git a/internal/mapr/setclause.go b/internal/mapr/setclause.go
index b4c2f73..1843d31 100644
--- a/internal/mapr/setclause.go
+++ b/internal/mapr/setclause.go
@@ -7,7 +7,6 @@ func (q *Query) SetClause(fields map[string]string) error {
 		if !ok {
 			continue
 		}
-
 		switch sc.rType {
 		case FunctionStack:
 			fields[sc.lString] = sc.functionStack.Call(value)
@@ -15,6 +14,5 @@ func (q *Query) SetClause(fields map[string]string) error {
 			fields[sc.lString] = value
 		}
 	}
-
 	return nil
 }
diff --git a/internal/mapr/setcondition.go b/internal/mapr/setcondition.go
index 8c5cfc9..92b21f4 100644
--- a/internal/mapr/setcondition.go
+++ b/internal/mapr/setcondition.go
@@ -39,20 +39,22 @@ func makeSetConditions(tokens []token) (set []setCondition, err error) {
 		switch setOp {
 		case "=":
 		default:
-			return sc, nil, errors.New(invalidQuery + "Unknown operation in 'set' clause: " + setOp)
+			return sc, nil, errors.New(invalidQuery + "Unknown operation in 'set' " +
+				"clause: " + setOp)
 		}
 
 		if !tokens[0].isBareword {
-			return sc, nil, errors.New(invalidQuery + "Expected bareword at 'set' clause's lValue: " + tokens[0].str)
+			return sc, nil, errors.New(invalidQuery + "Expected bareword at 'set' " +
+				"clause's lValue: " + tokens[0].str)
 		}
-
 		sc.lString = tokens[0].str
 		if !strings.HasPrefix(sc.lString, "$") {
-			return sc, nil, errors.New(invalidQuery + "Expected field variable name (starting with $) at 'set' clause's lValue: " + tokens[0].str)
+			return sc, nil, errors.New(invalidQuery + "Expected field variable name " +
+				"(starting with $) at 'set' clause's lValue: " + tokens[0].str)
 		}
 		sc.rType = Field
-
 		rString := tokens[2].str
+
 		// Seems like a function call?
 		if strings.HasSuffix(rString, ")") {
 			functionStack, functionArg, err := funcs.NewFunctionStack(tokens[2].str)
@@ -72,7 +74,6 @@ func makeSetConditions(tokens []token) (set []setCondition, err error) {
 		} else {
 			sc.rType = Field
 		}
-
 		return sc, tokens[3:], nil
 	}
 
@@ -84,10 +85,8 @@ func makeSetConditions(tokens []token) (set []setCondition, err error) {
 		if err != nil {
 			return nil, err
 		}
-
 		set = append(set, sc)
 		tokens = tokensConsumeOptional(tokens, ",")
 	}
-
 	return
 }
diff --git a/internal/mapr/token.go b/internal/mapr/token.go
index 7c6578b..6ac7631 100644
--- a/internal/mapr/token.go
+++ b/internal/mapr/token.go
@@ -4,7 +4,8 @@ import (
 	"strings"
 )
 
-var keywords = [...]string{"select", "from", "where", "set", "group", "rorder", "order", "interval", "limit", "outfile", "logformat"}
+var keywords = [...]string{"select", "from", "where", "set", "group", "rorder",
+	"order", "interval", "limit", "outfile", "logformat"}
 
 // Represents a parsed token, used to parse the mapr query.
 type token struct {
@@ -16,13 +17,11 @@ func (t token) isKeyword() bool {
 	if !t.isBareword {
 		return false
 	}
-
 	for _, keyword := range keywords {
 		if strings.ToLower(t.str) == keyword {
 			return true
 		}
 	}
-
 	return false
 }
 
@@ -32,7 +31,6 @@ func (t token) String() string {
 
 func tokenize(queryStr string) []token {
 	var tokens []token
-
 	for i, part := range strings.Split(queryStr, "\"") {
 		// Even i, means that it is not a quoted string
 		if i%2 == 0 {
@@ -53,14 +51,12 @@ func tokenize(queryStr string) []token {
 		}
 		tokens = append(tokens, token)
 	}
-
 	return tokens
 }
 
 func tokensConsume(tokens []token) ([]token, []token) {
 	//dlog.Common.Trace("=====================")
 	var consumed []token
-
 	for i, t := range tokens {
 		if t.isKeyword() {
 			//dlog.Common.Trace("keyword", t)
@@ -84,7 +80,6 @@ func tokensConsume(tokens []token) ([]token, []token) {
 		//dlog.Common.Trace("bare", token)
 		consumed = append(consumed, t)
 	}
-
 	//dlog.Common.Trace("result", consumed)
 	return nil, consumed
 }
@@ -95,7 +90,6 @@ func tokensConsumeStr(tokens []token) ([]token, []string) {
 	for _, token := range found {
 		strings = append(strings, token.str)
 	}
-
 	return tokens, strings
 }
 
@@ -106,6 +100,5 @@ func tokensConsumeOptional(tokens []token, optional string) []token {
 	if strings.ToLower(tokens[0].str) == strings.ToLower(optional) {
 		return tokens[1:]
 	}
-
 	return tokens
 }
diff --git a/internal/mapr/whereclause.go b/internal/mapr/whereclause.go
index 6356d94..d9f32eb 100644
--- a/internal/mapr/whereclause.go
+++ b/internal/mapr/whereclause.go
@@ -10,7 +10,6 @@ import (
 func (q *Query) WhereClause(fields map[string]string) bool {
 	for _, wc := range q.Where {
 		var ok bool
-
 		if wc.Operation > FloatOperation {
 			var lValue, rValue float64
 			if lValue, ok = whereClauseFloatValue(fields, wc.lString, wc.lFloat, wc.lType); !ok {
@@ -36,11 +35,12 @@ func (q *Query) WhereClause(fields map[string]string) bool {
 			return false
 		}
 	}
-
 	return true
 }
 
-func whereClauseFloatValue(fields map[string]string, str string, float float64, t fieldType) (float64, bool) {
+func whereClauseFloatValue(fields map[string]string, str string, float float64,
+	t fieldType) (float64, bool) {
+
 	switch t {
 	case Float:
 		return float, true
@@ -60,7 +60,9 @@ func whereClauseFloatValue(fields map[string]string, str string, float float64,
 	}
 }
 
-func whereClauseStringValue(fields map[string]string, str string, t fieldType) (string, bool) {
+func whereClauseStringValue(fields map[string]string, str string,
+	t fieldType) (string, bool) {
+
 	switch t {
 	case Field:
 		value, ok := fields[str]
diff --git a/internal/mapr/wherecondition.go b/internal/mapr/wherecondition.go
index c60c0a5..280dcfb 100644
--- a/internal/mapr/wherecondition.go
+++ b/internal/mapr/wherecondition.go
@@ -46,15 +46,18 @@ type whereCondition struct {
 }
 
 func (wc *whereCondition) String() string {
-	return fmt.Sprintf("whereCondition(Operation:%v,lString:%s,lFloat:%v,lType:%s,rString:%s,rFloat:%v,rType:%s)",
-		wc.Operation, wc.lString, wc.lFloat, wc.lType.String(), wc.rString, wc.rFloat, wc.rType.String())
+	return fmt.Sprintf("whereCondition(Operation:%v,lString:%s,lFloat:%v,"+
+		"lType:%s,rString:%s,rFloat:%v,rType:%s)",
+		wc.Operation, wc.lString, wc.lFloat, wc.lType.String(), wc.rString,
+		wc.rFloat, wc.rType.String())
 }
 
 func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 	parse := func(tokens []token) (whereCondition, []token, error) {
 		var wc whereCondition
 		if len(tokens) < 3 {
-			return wc, nil, errors.New(invalidQuery + "Not enough arguments in 'where' clause")
+			err := errors.New(invalidQuery + "Not enough arguments in 'where' clause")
+			return wc, nil, err
 		}
 
 		whereOp := strings.ToLower(tokens[1].str)
@@ -94,7 +97,8 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 		case "nhassuffix":
 			wc.Operation = StringNotHasSuffix
 		default:
-			return wc, nil, errors.New(invalidQuery + "Unknown operation in 'where' clause: " + whereOp)
+			return wc, nil, errors.New(invalidQuery +
+				"Unknown operation in 'where' clause: " + whereOp)
 		}
 
 		wc.lString = tokens[0].str
@@ -102,7 +106,8 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 
 		if wc.Operation > FloatOperation {
 			if !tokens[0].isBareword {
-				return wc, nil, errors.New(invalidQuery + "Expected bareword at 'where' clause's lValue: " + tokens[0].str)
+				return wc, nil, errors.New(invalidQuery +
+					"Expected bareword at 'where' clause's lValue: " + tokens[0].str)
 			}
 			if f, err := strconv.ParseFloat(wc.lString, 64); err == nil {
 				wc.lFloat = f
@@ -112,7 +117,8 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 			}
 
 			if !tokens[2].isBareword {
-				return wc, nil, errors.New(invalidQuery + "Expected bareword at 'where' clause's rValue: " + tokens[2].str)
+				return wc, nil, errors.New(invalidQuery +
+					"Expected bareword at 'where' clause's rValue: " + tokens[2].str)
 			}
 			if f, err := strconv.ParseFloat(wc.rString, 64); err == nil {
 				wc.rFloat = f
@@ -133,23 +139,19 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 		} else {
 			wc.rType = String
 		}
-
 		return wc, tokens[3:], nil
 	}
 
 	for len(tokens) > 0 {
 		var wc whereCondition
 		var err error
-
 		wc, tokens, err = parse(tokens)
 		if err != nil {
 			return nil, err
 		}
-
 		where = append(where, wc)
 		tokens = tokensConsumeOptional(tokens, "and")
 	}
-
 	return
 }
 
@@ -170,7 +172,6 @@ func (wc *whereCondition) floatClause(lValue float64, rValue float64) bool {
 	default:
 		dlog.Common.Error("Unknown float operation", lValue, wc.Operation, rValue)
 	}
-
 	return false
 }
 
@@ -195,6 +196,5 @@ func (wc *whereCondition) stringClause(lValue string, rValue string) bool {
 	default:
 		dlog.Common.Error("Unknown string operation", lValue, wc.Operation, rValue)
 	}
-
 	return false
 }
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index 8c9e861..d29706c 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -13,7 +13,6 @@ const (
 	AggregateKVDelimiter string = "≔"
 	// AggregateDelimiter delimits parts of an aggregation message.
 	AggregateDelimiter string = "∥"
-	// AggregateDelimiter string = "⦀"
 	// AggregateGroupKeyCombinator combines the group set keys.
 	AggregateGroupKeyCombinator string = ","
 )
diff --git a/internal/regex/regex.go b/internal/regex/regex.go
index 352ffd6..eb6e1b3 100644
--- a/internal/regex/regex.go
+++ b/internal/regex/regex.go
@@ -48,9 +48,7 @@ func new(regexStr string, flags []Flag) (Regex, error) {
 		regexStr: regexStr,
 		flags:    flags,
 	}
-
 	re, err := regexp.Compile(regexStr)
-
 	if err != nil {
 		return r, err
 	}
@@ -94,11 +92,9 @@ func (r Regex) Serialize() (string, error) {
 	for _, flag := range r.flags {
 		flags = append(flags, flag.String())
 	}
-
 	if !r.initialized {
 		return "", fmt.Errorf("Unable to serialize regex as not initialized properly: %v", r)
 	}
-
 	return fmt.Sprintf("regex:%s %s", strings.Join(flags, ","), r.regexStr), nil
 }
 
@@ -109,12 +105,12 @@ func Deserialize(str string) (Regex, error) {
 	if len(s) < 2 {
 		return NewNoop(), nil
 	}
-
 	flagsStr := s[0]
 	regexStr := s[1]
 
 	if !strings.HasPrefix(flagsStr, "regex") {
-		return Regex{}, fmt.Errorf("unable to deserialize regex '%s': should start with string 'regex'", str)
+		return Regex{}, fmt.Errorf("unable to deserialize regex '%s': should start "+
+			"with string 'regex'", str)
 	}
 
 	// Parse regex flags, e.g. "regex:flag1,flag2,flag3..."
@@ -129,6 +125,5 @@ func Deserialize(str string) (Regex, error) {
 			flags = append(flags, flag)
 		}
 	}
-
 	return new(regexStr, flags)
 }
diff --git a/internal/regex/regex_test.go b/internal/regex/regex_test.go
index 2ce49ac..033a286 100644
--- a/internal/regex/regex_test.go
+++ b/internal/regex/regex_test.go
@@ -9,7 +9,8 @@ func TestRegex(t *testing.T) {
 
 	r := NewNoop()
 	if !r.MatchString(input) {
-		t.Errorf("expected to match string '%s' with noop regex '%v' but didn't\n", input, r)
+		t.Errorf("expected to match string '%s' with noop regex '%v' but didn't\n",
+			input, r)
 	}
 
 	r, err := New(".hello", Default)
@@ -17,7 +18,8 @@ func TestRegex(t *testing.T) {
 		t.Errorf("unable to create regex: %v\n", err)
 	}
 	if r.MatchString(input) {
-		t.Errorf("expected to match string '%s' with regex '%v' but didn't\n", input, r)
+		t.Errorf("expected to match string '%s' with regex '%v' but didn't\n",
+			input, r)
 	}
 
 	serialized, err := r.Serialize()
@@ -29,8 +31,8 @@ func TestRegex(t *testing.T) {
 		t.Errorf("unable to serialize deserialized regex: %v: %v\n", serialized, err)
 	}
 	if r.String() != r2.String() {
-		t.Errorf("regex should be the same after deserialize(serialize(..)), got '%s' but expected '%s'.\n",
-			r2.String(), r.String())
+		t.Errorf("regex should be the same after deserialize(serialize(..)), got "+
+			"'%s' but expected '%s'.\n", r2.String(), r.String())
 	}
 
 	r, err = New(".hello", Invert)
@@ -38,7 +40,8 @@ func TestRegex(t *testing.T) {
 		t.Errorf("unable to create regex: %v\n", err)
 	}
 	if !r.MatchString(input) {
-		t.Errorf("expected to not match string '%s' with regex '%v' but matched\n", input, r)
+		t.Errorf("expected to not match string '%s' with regex '%v' but matched\n",
+			input, r)
 	}
 
 	serialized, err = r.Serialize()
@@ -50,7 +53,7 @@ func TestRegex(t *testing.T) {
 		t.Errorf("unable to serialize deserialized regex: %v: %v\n", serialized, err)
 	}
 	if r.String() != r2.String() {
-		t.Errorf("regex should be the same after deserialize(serialize(..)), got '%s' but expected '%s'.\n",
-			r2.String(), r.String())
+		t.Errorf("regex should be the same after deserialize(serialize(..)), got "+
+			"'%s' but expected '%s'.\n", r2.String(), r.String())
 	}
 }
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index 5f84afc..93b3fcb 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -13,8 +13,7 @@ import (
 	gossh "golang.org/x/crypto/ssh"
 )
 
-type continuous struct {
-}
+type continuous struct{}
 
 func newContinuous() *continuous {
 	return &continuous{}
@@ -23,7 +22,6 @@ func newContinuous() *continuous {
 func (c *continuous) start(ctx context.Context) {
 	dlog.Server.Info("Starting continuous job runner after 10s")
 	time.Sleep(time.Second * 10)
-
 	c.runJobs(ctx)
 }
 
@@ -33,7 +31,6 @@ func (c *continuous) runJobs(ctx context.Context) {
 			dlog.Server.Debug(job.Name, "Not running job as not enabled")
 			continue
 		}
-
 		go func(job config.Continuous) {
 			c.runJob(ctx, job)
 			for {
@@ -54,7 +51,6 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 
 	files := fillDates(job.Files)
 	outfile := fillDates(job.Outfile)
-
 	servers := strings.Join(job.Servers, ",")
 	if servers == "" {
 		servers = config.Server.SSHBindAddress
@@ -70,7 +66,6 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	}
 
 	args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(job.Name))
-
 	args.QueryStr = fmt.Sprintf("%s outfile %s", job.Query, outfile)
 	client, err := clients.NewMaprClient(args, clients.NonCumulativeMode)
 	if err != nil {
@@ -80,7 +75,6 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 
 	jobCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
-
 	if job.RestartOnDayChange {
 		go func() {
 			if c.waitForDayChange(ctx) {
@@ -93,7 +87,6 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	dlog.Server.Info(fmt.Sprintf("Starting job %s", job.Name))
 	status := client.Start(jobCtx, make(chan string))
 	logMessage := fmt.Sprintf("Job exited with status %d", status)
-
 	if status != 0 {
 		dlog.Server.Warn(logMessage)
 		return
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index f73f82e..847e8f9 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -37,7 +37,7 @@ type baseHandler struct {
 	activeCommands   int32
 	quiet            bool
 	spartan          bool
-	serverless       bool
+	serverless       int32
 	readBuf          bytes.Buffer
 	writeBuf         bytes.Buffer
 }
@@ -59,16 +59,14 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	select {
 	case message := <-h.serverMessages:
 		if message[0] == '.' {
-			// Handle hidden message (don't display to the user, interpreted by dtail client)
+			// Handle hidden message (don't display to the user)
 			h.readBuf.WriteString(message)
 			h.readBuf.WriteByte(protocol.MessageDelimiter)
 			n = copy(p, h.readBuf.Bytes())
 			return
 		}
 
-		if h.serverless {
-			// In serverless mode we have logged the server message already via the
-			// dlog logger, no need to send the message again to the client part.
+		if h.serverless > 0 {
 			return
 		}
 
@@ -132,7 +130,6 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 			h.writeBuf.WriteByte(b)
 		}
 	}
-
 	n = len(p)
 	return
 }
@@ -145,13 +142,11 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err)+add)
 		return
 	}
-
 	args, argc, err = h.handleBase64(args, argc)
 	if err != nil {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
 	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
 		<-h.done.Done()
@@ -160,7 +155,6 @@ func (h *baseHandler) handleCommand(commandStr string) {
 
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
-
 	options, err := config.DeserializeOptions(splitted[1:])
 	if err != nil {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
@@ -191,8 +185,8 @@ func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, strin
 		if clientCompat > serverCompat {
 			toUpdate = "server"
 		}
-
-		err := fmt.Errorf("DTail server protocol version '%s' does not match client protocol version '%s', please update DTail %s!",
+		err := fmt.Errorf("the DTail server protocol version '%s' does not match "+
+			"client protocol version '%s', please update DTail %s",
 			protocol.ProtocolCompat, args[1], toUpdate)
 		return args, argc, add, err
 	}
@@ -201,8 +195,8 @@ func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, strin
 }
 
 func (h *baseHandler) handleBase64(args []string, argc int) ([]string, int, error) {
-	err := errors.New("Unable to decode client message, DTail server and client versions may not be compatible")
-
+	err := errors.New("unable to decode client message, DTail server and client " +
+		"versions may not be compatible")
 	if argc != 2 || args[0] != "base64" {
 		return args, argc, err
 	}
@@ -215,7 +209,8 @@ func (h *baseHandler) handleBase64(args []string, argc int) ([]string, int, erro
 
 	args = strings.Split(decodedStr, " ")
 	argc = len(decodedStr)
-	dlog.Server.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
+	dlog.Server.Trace(h.user, "Base64 decoded received command",
+		decodedStr, argc, args)
 
 	return args, argc, nil
 }
@@ -223,7 +218,8 @@ func (h *baseHandler) handleBase64(args []string, argc int) ([]string, int, erro
 func (h *baseHandler) handleAckCommand(argc int, args []string) {
 	if argc < 3 {
 		if !h.quiet {
-			h.send(h.serverMessages, dlog.Server.Warn(h.user, "Unable to parse command", args, argc))
+			h.send(h.serverMessages, dlog.Server.Warn(h.user,
+				"Unable to parse command", args, argc))
 		}
 		return
 	}
@@ -245,11 +241,9 @@ func (h *baseHandler) send(ch chan<- string, message string) {
 
 func (h *baseHandler) flush() {
 	dlog.Server.Trace(h.user, "flush()")
-
 	numUnsentMessages := func() int {
 		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
 	}
-
 	for i := 0; i < 10; i++ {
 		if numUnsentMessages() == 0 {
 			dlog.Server.Debug(h.user, "ALL lines sent", fmt.Sprintf("%p", h))
@@ -258,7 +252,6 @@ func (h *baseHandler) flush() {
 		dlog.Server.Debug(h.user, "Still lines to be sent")
 		time.Sleep(time.Millisecond * 10)
 	}
-
 	dlog.Server.Warn(h.user, "Some lines remain unsent", numUnsentMessages())
 }
 
@@ -279,7 +272,6 @@ func (h *baseHandler) shutdown() {
 		dlog.Server.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
 	case <-h.done.Done():
 	}
-
 	h.done.Shutdown()
 }
 
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
index 347ff66..8d6c400 100644
--- a/internal/server/handlers/healthhandler.go
+++ b/internal/server/handlers/healthhandler.go
@@ -35,24 +35,23 @@ func NewHealthHandler(user *user.User) *HealthHandler {
 	if err != nil {
 		dlog.Server.FatalPanic(err)
 	}
-
 	s := strings.Split(fqdn, ".")
 	h.hostname = s[0]
-
 	return &h
 }
 
-func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int, args []string,
-	commandName string, options map[string]string) {
-	dlog.Server.Debug(h.user, "Handling health command", argc, args)
+func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int,
+	args []string, commandName string, options map[string]string) {
 
+	dlog.Server.Debug(h.user, "Handling health command", argc, args)
 	switch commandName {
 	case "health":
 		h.send(h.serverMessages, "OK")
 	case ".ack":
 		h.handleAckCommand(argc, args)
 	default:
-		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown health command", commandName, argc, args))
+		h.send(h.serverMessages, dlog.Server.Error(h.user,
+			"Received unknown health command", commandName, argc, args))
 	}
 	h.shutdown()
 }
diff --git a/internal/server/handlers/mapcommand.go b/internal/server/handlers/mapcommand.go
index c3e600e..65e0ed8 100644
--- a/internal/server/handlers/mapcommand.go
+++ b/internal/server/handlers/mapcommand.go
@@ -14,18 +14,17 @@ type mapCommand struct {
 }
 
 // NewMapCommand returns a new server side mapreduce command.
-func newMapCommand(serverHandler *ServerHandler, argc int, args []string) (mapCommand, *server.Aggregate, error) {
-	m := mapCommand{server: serverHandler}
+func newMapCommand(serverHandler *ServerHandler, argc int,
+	args []string) (mapCommand, *server.Aggregate, error) {
 
+	m := mapCommand{server: serverHandler}
 	queryStr := strings.Join(args[1:], " ")
 	aggregate, err := server.NewAggregate(queryStr)
 	if err != nil {
 		return m, nil, err
 	}
-
 	m.aggregate = aggregate
 	return m, aggregate, nil
-
 }
 
 func (m mapCommand) Start(ctx context.Context, aggregatedMessages chan<- string) {
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index abc44c7..384e966 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -26,25 +26,30 @@ func newReadCommand(server *ServerHandler, mode omode.Mode) *readCommand {
 	}
 }
 
-func (r *readCommand) Start(ctx context.Context, argc int, args []string, retries int) {
-	re := regex.NewNoop()
+func (r *readCommand) Start(ctx context.Context, argc int, args []string,
+	retries int) {
 
+	re := regex.NewNoop()
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user, "Unable to parse command", err))
+			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user,
+				"Unable to parse command", err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to parse command", args, argc))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user,
+			"Unable to parse command", args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re, retries)
 }
 
-func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex, retries int) {
+func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
+	retries int) {
+
 	retryInterval := time.Second * 5
 	glob = filepath.Clean(glob)
 
@@ -58,7 +63,8 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 
 		if numPaths := len(paths); numPaths == 0 {
 			dlog.Server.Error(r.server.user, "No such file(s) to read", glob)
-			r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
+			r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user,
+				"Unable to read file(s), check server logs"))
 			select {
 			case <-ctx.Done():
 				return
@@ -72,31 +78,33 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 		return
 	}
 
-	r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Giving up to read file(s)"))
+	r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user,
+		"Giving up to read file(s)"))
 	return
 }
 
-func (r *readCommand) readFiles(ctx context.Context, paths []string, glob string, re regex.Regex, retryInterval time.Duration) {
+func (r *readCommand) readFiles(ctx context.Context, paths []string, glob string,
+	re regex.Regex, retryInterval time.Duration) {
+
 	var wg sync.WaitGroup
 	wg.Add(len(paths))
-
 	for _, path := range paths {
 		go r.readFileIfPermissions(ctx, &wg, path, glob, re)
 	}
-
 	wg.Wait()
 }
 
-func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGroup, path, glob string, re regex.Regex) {
+func (r *readCommand) readFileIfPermissions(ctx context.Context,
+	wg *sync.WaitGroup, path, glob string, re regex.Regex) {
+
 	defer wg.Done()
 	globID := r.makeGlobID(path, glob)
-
 	if !r.server.user.HasFilePermission(path, "readfiles") {
 		dlog.Server.Error(r.server.user, "No permission to read file", path, globID)
-		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user,
+			"Unable to read file(s), check server logs"))
 		return
 	}
-
 	r.readFile(ctx, path, globID, re)
 }
 
@@ -137,7 +145,6 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 				return
 			}
 		}
-
 		time.Sleep(time.Second * 2)
 		dlog.Server.Info(path, globID, "Reading file again")
 	}
@@ -156,11 +163,11 @@ func (r *readCommand) makeGlobID(path, glob string) string {
 	if len(idParts) > 0 {
 		return strings.Join(idParts, "/")
 	}
-
 	if len(pathParts) > 0 {
 		return pathParts[len(pathParts)-1]
 	}
 
-	r.server.send(r.server.serverMessages, dlog.Server.Warn("Empty file path given?", path, glob))
+	r.server.send(r.server.serverMessages,
+		dlog.Server.Warn("Empty file path given?", path, glob))
 	return ""
 }
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index aed8956..f12d590 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"strings"
+	"sync/atomic"
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/dlog"
@@ -23,7 +24,9 @@ type ServerHandler struct {
 }
 
 // NewServerHandler returns the server handler.
-func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
+func NewServerHandler(user *user.User, catLimiter,
+	tailLimiter chan struct{}) *ServerHandler {
+
 	dlog.Server.Debug(user, "Creating new server handler")
 	h := ServerHandler{
 		baseHandler: baseHandler{
@@ -51,11 +54,10 @@ func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *S
 	return &h
 }
 
-func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string,
-	commandName string, options map[string]string) {
+func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
+	args []string, commandName string, options map[string]string) {
 
 	dlog.Server.Debug(h.user, "Handling user command", argc, args)
-
 	h.incrementActiveCommands()
 	commandFinished := func() {
 		if h.decrementActiveCommands() == 0 {
@@ -73,7 +75,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 	}
 	if serverless, _ := options["serverless"]; serverless == "true" {
 		dlog.Server.Debug(h.user, "Enabling serverless mode")
-		h.serverless = true
+		atomic.AddInt32(&h.serverless, 1)
 	}
 
 	switch commandName {
@@ -83,14 +85,12 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			command.Start(ctx, argc, args, 1)
 			commandFinished()
 		}()
-
 	case "tail":
 		command := newReadCommand(h, omode.TailClient)
 		go func() {
 			command.Start(ctx, argc, args, 10)
 			commandFinished()
 		}()
-
 	case "map":
 		command, aggregate, err := newMapCommand(h, argc, args)
 		if err != nil {
@@ -99,19 +99,17 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			commandFinished()
 			return
 		}
-
 		h.aggregate = aggregate
 		go func() {
 			command.Start(ctx, h.maprMessages)
 			commandFinished()
 		}()
-
 	case ".ack":
 		h.handleAckCommand(argc, args)
 		commandFinished()
-
 	default:
-		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown user command", commandName, argc, args, options))
+		h.send(h.serverMessages, dlog.Server.Error(h.user,
+			"Received unknown user command", commandName, argc, args, options))
 		commandFinished()
 	}
 }
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index ccb2225..0ba65f7 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -16,8 +16,7 @@ import (
 	gossh "golang.org/x/crypto/ssh"
 )
 
-type scheduler struct {
-}
+type scheduler struct{}
 
 func newScheduler() *scheduler {
 	return &scheduler{}
@@ -28,7 +27,6 @@ func (s *scheduler) start(ctx context.Context) {
 	// First run after just 10s!
 	time.Sleep(time.Second * 10)
 	s.runJobs(ctx)
-
 	for {
 		select {
 		case <-time.After(time.Minute):
@@ -45,13 +43,11 @@ func (s *scheduler) runJobs(ctx context.Context) {
 			dlog.Server.Debug(job.Name, "Not running job as not enabled")
 			continue
 		}
-
 		hour, err := strconv.Atoi(time.Now().Format("15"))
 		if err != nil {
 			dlog.Server.Error(job.Name, "Unable to create job", err)
 			continue
 		}
-
 		if hour < job.TimeRange[0] || hour >= job.TimeRange[1] {
 			dlog.Server.Debug(job.Name, "Not running job out of time range")
 			continue
@@ -59,7 +55,6 @@ func (s *scheduler) runJobs(ctx context.Context) {
 
 		files := fillDates(job.Files)
 		outfile := fillDates(job.Outfile)
-
 		_, err = os.Stat(outfile)
 		if !os.IsNotExist(err) {
 			dlog.Server.Debug(job.Name, "Not running job as outfile already exists", outfile)
@@ -70,7 +65,6 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		if servers == "" {
 			servers = config.Server.SSHBindAddress
 		}
-
 		args := config.Args{
 			ConnectionsPerCPU: config.DefaultConnectionsPerCPU,
 			Discovery:         job.Discovery,
@@ -81,7 +75,6 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		}
 
 		args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(job.Name))
-
 		args.QueryStr = fmt.Sprintf("%s outfile %s", job.Query, outfile)
 		client, err := clients.NewMaprClient(args, clients.CumulativeMode)
 		if err != nil {
diff --git a/internal/server/server.go b/internal/server/server.go
index b3d4bff..0cb5e27 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -24,9 +24,9 @@ type Server struct {
 	stats stats
 	// SSH server configuration.
 	sshServerConfig *gossh.ServerConfig
-	// To control the max amount of concurrent cats (which can cause a lot of I/O on the server)
+	// To control the max amount of concurrent cats.
 	catLimiter chan struct{}
-	// To control the max amount of concurrent tails
+	// To control the max amount of concurrent tails.
 	tailLimiter chan struct{}
 	// To run scheduled tasks (if configured)
 	sched *scheduler
@@ -61,7 +61,6 @@ func New() *Server {
 // Start the server.
 func (s *Server) Start(ctx context.Context) int {
 	dlog.Server.Info("Starting server")
-
 	bindAt := fmt.Sprintf("%s:%d", config.Server.SSHBindAddress, config.Common.SSHPort)
 	dlog.Server.Info("Binding server", bindAt)
 
@@ -76,14 +75,12 @@ func (s *Server) Start(ctx context.Context) int {
 	go s.listenerLoop(ctx, listener)
 
 	<-ctx.Done()
-
 	// For future use.
 	return 0
 }
 
 func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 	dlog.Server.Debug("Starting listener loop")
-
 	for {
 		conn, err := listener.Accept() // Blocking
 		if err != nil {
@@ -101,7 +98,6 @@ func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 			conn.Close()
 			continue
 		}
-
 		go s.handleConnection(ctx, conn)
 	}
 }
@@ -116,22 +112,23 @@ func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
 	}
 
 	s.stats.incrementConnections()
-
 	go gossh.DiscardRequests(reqs)
 	for newChannel := range chans {
 		go s.handleChannel(ctx, sshConn, newChannel)
 	}
 }
 
-func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChannel gossh.NewChannel) {
+func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn,
+	newChannel gossh.NewChannel) {
+
 	user, err := user.New(sshConn.User(), sshConn.RemoteAddr().String())
 	if err != nil {
 		dlog.Server.Error(user, err)
 		newChannel.Reject(gossh.Prohibited, err.Error())
 		return
 	}
-	dlog.Server.Info(user, "Invoking channel handler")
 
+	dlog.Server.Info(user, "Invoking channel handler")
 	if newChannel.ChannelType() != "session" {
 		err := errors.New("Don'w allow other channel types than session")
 		dlog.Server.Error(user, err)
@@ -151,9 +148,10 @@ func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChann
 	}
 }
 
-func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-chan *gossh.Request, channel gossh.Channel, user *user.User) error {
-	dlog.Server.Info(user, "Invoking request handler")
+func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn,
+	in <-chan *gossh.Request, channel gossh.Channel, user *user.User) error {
 
+	dlog.Server.Info(user, "Invoking request handler")
 	for req := range in {
 		var payload = struct{ Value string }{}
 		gossh.Unmarshal(req.Payload, &payload)
@@ -167,7 +165,6 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 			default:
 				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter)
 			}
-
 			terminate := func() {
 				handler.Shutdown()
 				sshConn.Close()
@@ -178,13 +175,11 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 				io.Copy(channel, handler)
 				terminate()
 			}()
-
 			go func() {
 				// Broken pipe, cancel
 				io.Copy(handler, channel)
 				terminate()
 			}()
-
 			go func() {
 				select {
 				case <-ctx.Done():
@@ -192,7 +187,6 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 				}
 				terminate()
 			}()
-
 			go func() {
 				if err := sshConn.Wait(); err != nil && err != io.EOF {
 					dlog.Server.Error(user, err)
@@ -204,20 +198,19 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 
 			// Only serving shell type
 			req.Reply(true, nil)
-
 		default:
 			req.Reply(false, nil)
-
 			return fmt.Errorf("Closing SSH connection as unknown request recieved|%s|%v",
 				req.Type, payload.Value)
 		}
 	}
-
 	return nil
 }
 
 // Callback for SSH authentication.
-func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Permissions, error) {
+func (s *Server) Callback(c gossh.ConnMetadata,
+	authPayload []byte) (*gossh.Permissions, error) {
+
 	user, err := user.New(c.User(), c.RemoteAddr().String())
 	if err != nil {
 		return nil, err
@@ -229,7 +222,6 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	}
 
 	authInfo := string(authPayload)
-
 	splitted := strings.Split(c.RemoteAddr().String(), ":")
 	remoteIP := splitted[0]
 
@@ -259,23 +251,26 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	return nil, fmt.Errorf("user %s not authorized", user)
 }
 
-func (s *Server) backgroundCanSSH(user *user.User, jobName, remoteIP, allowedJobName string, allowFrom []string) bool {
-	dlog.Server.Debug("backgroundCanSSH", user, jobName, remoteIP, allowedJobName, allowFrom)
+func (s *Server) backgroundCanSSH(user *user.User, jobName, remoteIP,
+	allowedJobName string, allowFrom []string) bool {
 
+	dlog.Server.Debug("backgroundCanSSH", user, jobName, remoteIP, allowedJobName, allowFrom)
 	if jobName != allowedJobName {
-		dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Job name does not match, skipping to next one...", allowedJobName)
+		dlog.Server.Debug(user, jobName, "backgroundCanSSH",
+			"Job name does not match, skipping to next one...", allowedJobName)
 		return false
 	}
 
 	for _, myAddr := range allowFrom {
 		ips, err := net.LookupIP(myAddr)
 		if err != nil {
-			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Unable to lookup IP address for allowed hosts lookup, skipping to next one...", myAddr, err)
+			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Unable to lookup IP "+
+				"address for allowed hosts lookup, skipping to next one...", myAddr, err)
 			continue
 		}
-
 		for _, ip := range ips {
-			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Comparing IP addresses", remoteIP, ip.String())
+			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Comparing IP addresses",
+				remoteIP, ip.String())
 			if remoteIP == ip.String() {
 				return true
 			}
diff --git a/internal/server/stats.go b/internal/server/stats.go
index c07634d..99a644a 100644
--- a/internal/server/stats.go
+++ b/internal/server/stats.go
@@ -19,7 +19,6 @@ type stats struct {
 
 func (s *stats) incrementConnections() {
 	defer s.logServerStats()
-
 	s.mutex.Lock()
 	s.currentConnections++
 	s.lifetimeConnections++
@@ -28,7 +27,6 @@ func (s *stats) incrementConnections() {
 
 func (s *stats) decrementConnections() {
 	defer s.logServerStats()
-
 	s.mutex.Lock()
 	s.currentConnections--
 	s.mutex.Unlock()
@@ -40,8 +38,8 @@ func (s *stats) hasConnections() bool {
 	s.mutex.Unlock()
 
 	has := currentConnections > 0
-	dlog.Server.Info("stats", "Server with open connections?", has, currentConnections)
-
+	dlog.Server.Info("stats", "Server with open connections?",
+		has, currentConnections)
 	return has
 }
 
@@ -52,7 +50,6 @@ func (s *stats) logServerStats() {
 	data := make(map[string]interface{})
 	data["currentConnections"] = s.currentConnections
 	data["lifetimeConnections"] = s.lifetimeConnections
-
 	dlog.Server.Mapreduce("STATS", data)
 }
 
@@ -61,9 +58,9 @@ func (s *stats) serverLimitExceeded() error {
 	defer s.mutex.Unlock()
 
 	if s.currentConnections >= config.Server.MaxConnections {
-		return fmt.Errorf("Exceeded max allowed concurrent connections of %d", config.Server.MaxConnections)
+		return fmt.Errorf("Exceeded max allowed concurrent connections of %d",
+			config.Server.MaxConnections)
 	}
-
 	return nil
 }
 
diff --git a/internal/source/source.go b/internal/source/source.go
index be7aecd..4bb0784 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -1,10 +1,19 @@
 package source
 
+// Source specifies the origin of either the current process (dtail is a client
+// process, dserver is a server process) or the source code package (e.g.
+// dserver server side code or dtail client side code). Notice that dtail client
+// may also executes server code directly (e.g. via serverless mode) and that
+// the dserver may also executes client code (e.g. via scheduled server side
+// mapreduce queries).
 type Source int
 
 const (
-	Client      Source = iota
-	Server      Source = iota
+	// Client process or source code package.
+	Client Source = iota
+	// Server process or source code package.
+	Server Source = iota
+	// HealthCheck process or client source code package.
 	HealthCheck Source = iota
 )
 
@@ -17,6 +26,5 @@ func (s Source) String() string {
 	case HealthCheck:
 		return "HEALTHCHECK"
 	}
-
 	panic("Unknown source type")
 }
diff --git a/internal/ssh/client/authmethods.go b/internal/ssh/client/authmethods.go
index 4508319..ced1fb9 100644
--- a/internal/ssh/client/authmethods.go
+++ b/internal/ssh/client/authmethods.go
@@ -11,7 +11,10 @@ import (
 )
 
 // InitSSHAuthMethods initialises all known SSH auth methods on the client side.
-func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod, hostKeyCallback gossh.HostKeyCallback, trustAllHosts bool, throttleCh chan struct{}, privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
+func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod,
+	hostKeyCallback gossh.HostKeyCallback, trustAllHosts bool, throttleCh chan struct{},
+	privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
+
 	if len(sshAuthMethods) > 0 {
 		simpleCallback, err := NewSimpleCallback()
 		if err != nil {
@@ -19,20 +22,21 @@ func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod, hostKeyCallback gossh
 		}
 		return sshAuthMethods, simpleCallback
 	}
-
 	return initKnownHostsAuthMethods(trustAllHosts, throttleCh, privateKeyPath)
 }
 
-func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
-	var sshAuthMethods []gossh.AuthMethod
+func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{},
+	privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
 
+	var sshAuthMethods []gossh.AuthMethod
 	knownHostsPath := os.Getenv("HOME") + "/.ssh/known_hosts"
-	knownHostsCallback, err := NewKnownHostsCallback(knownHostsPath, trustAllHosts, throttleCh)
+	knownHostsCallback, err := NewKnownHostsCallback(knownHostsPath, trustAllHosts,
+		throttleCh)
 	if err != nil {
 		dlog.Common.FatalPanic(knownHostsPath, err)
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
-
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path",
+		knownHostsPath)
 	if config.Common.ExperimentalFeaturesEnable {
 		sshAuthMethods = append(sshAuthMethods, gossh.Password("experimental feature test"))
 		dlog.Common.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
@@ -43,7 +47,9 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, pri
 		authMethod, err := ssh.PrivateKey(privateKeyPath)
 		if err == nil {
 			sshAuthMethods = append(sshAuthMethods, authMethod)
-			dlog.Common.Debug("initKnownHostsAuthMethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+			dlog.Common.Debug("initKnownHostsAuthMethods",
+				"Added path to list of auth methods, not adding further methods",
+				privateKeyPath)
 			return sshAuthMethods, knownHostsCallback
 		}
 		dlog.Common.FatalPanic("Unable to use private SSH key", privateKeyPath, err)
@@ -53,30 +59,35 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, pri
 	authMethod, err := ssh.Agent()
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK) to list of auth methods, not adding further methods")
+		dlog.Common.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK)"+
+			"to list of auth methods, not adding further methods")
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err)
+	dlog.Common.Debug("initKnownHostsAuthMethods",
+		"Unable to init SSH Agent auth method", err)
 
 	// Third, try Linux/UNIX default key paths
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_rsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Common.Debug("initKnownHostsAuthmethods",
+			"Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key",
+		privateKeyPath, err)
 
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_dsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Common.Debug("initKnownHostsAuthmethods",
+			"Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
-
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key",
+		privateKeyPath, err)
 	dlog.Common.FatalPanic("Unable to find private SSH key information")
 
 	// Never reach this point.
diff --git a/internal/ssh/client/customkeycallback.go b/internal/ssh/client/customkeycallback.go
index 73e5289..53b8e3c 100644
--- a/internal/ssh/client/customkeycallback.go
+++ b/internal/ssh/client/customkeycallback.go
@@ -7,8 +7,7 @@ import (
 )
 
 // CustomCallback is a custom host key callback wrapper.
-type CustomCallback struct {
-}
+type CustomCallback struct{}
 
 // NewCustomCallback returns a new wrapper.
 func NewCustomCallback() (*CustomCallback, error) {
diff --git a/internal/ssh/client/knownhostscallback.go b/internal/ssh/client/knownhostscallback.go
index a73d612..65a590a 100644
--- a/internal/ssh/client/knownhostscallback.go
+++ b/internal/ssh/client/knownhostscallback.go
@@ -46,8 +46,9 @@ type KnownHostsCallback struct {
 }
 
 // NewKnownHostsCallback returns a new wrapper.
-func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool, throttleCh chan struct{}) (HostKeyCallback, error) {
-	// Ensure file exists
+func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool,
+	throttleCh chan struct{}) (HostKeyCallback, error) {
+
 	os.OpenFile(knownHostsPath, os.O_RDONLY|os.O_CREATE, 0666)
 	untrustedHosts := make(map[string]bool)
 
@@ -59,11 +60,9 @@ func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool, throttleCh
 		untrustedHosts:  untrustedHosts,
 		mutex:           &sync.Mutex{},
 	}
-
 	if trustAllHosts {
 		close(c.trustAllHostsCh)
 	}
-
 	return c, nil
 }
 
@@ -75,14 +74,12 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 		if err != nil {
 			return err
 		}
-
 		// Check for valid entry in known_hosts file
 		err = knownHostsCb(server, remote, key)
 		if err == nil {
 			// OK
 			return nil
 		}
-
 		// Make sure that interactive user callback does not interfere with
 		// SSH connection throttler.
 		<-c.throttleCh
@@ -96,11 +93,9 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 			ipLine:     knownhosts.Line([]string{remote.String()}, key),
 			responseCh: make(chan response),
 		}
-
 		dlog.Common.Warn("Encountered unknown host", unknown)
 		// Notify user that there is an unknown host
 		c.unknownCh <- unknown
-
 		// Wait for user input.
 		switch <-unknown.responseCh {
 		case trustHost:
@@ -112,7 +107,6 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 		c.mutex.Lock()
 		defer c.mutex.Unlock()
 		c.untrustedHosts[server] = true
-
 		return err
 	}
 }
@@ -121,7 +115,6 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 // be added to the known hosts or not.
 func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 	var hosts []unknownHost
-
 	for {
 		// Check whether there is a unknown host
 		select {
@@ -147,7 +140,6 @@ func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 
 func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 	var servers []string
-
 	for _, host := range hosts {
 		servers = append(servers, host.server)
 	}
@@ -165,7 +157,6 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 		strings.Join(servers, ","),
 		"Do you want to trust these hosts?",
 	)
-
 	p := prompt.New(question)
 
 	a := prompt.Answer{
@@ -223,7 +214,6 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 
 func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 	tmpKnownHostsPath := fmt.Sprintf("%s.tmp", c.knownHostsPath)
-
 	newFd, err := os.OpenFile(tmpKnownHostsPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 	if err != nil {
 		panic(fmt.Sprintf("%s: %s", tmpKnownHostsPath, err.Error()))
@@ -232,7 +222,6 @@ func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 
 	// Newly trusted hosts in normalized form
 	addresses := make(map[string]struct{})
-
 	// First write to new known hosts file, and keep track of addresses
 	for _, unknown := range hosts {
 		unknown.responseCh <- trustHost
@@ -255,7 +244,6 @@ func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 	defer oldFd.Close()
 
 	scanner := bufio.NewScanner(oldFd)
-
 	// Now, append all still valid old entries to the new host file
 	for scanner.Scan() {
 		line := scanner.Text()
@@ -283,6 +271,5 @@ func (c KnownHostsCallback) Untrusted(server string) bool {
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
 	_, ok := c.untrustedHosts[server]
-
 	return ok
 }
diff --git a/internal/ssh/server/hostkey.go b/internal/ssh/server/hostkey.go
index 20de1f0..33bd4e8 100644
--- a/internal/ssh/server/hostkey.go
+++ b/internal/ssh/server/hostkey.go
@@ -24,7 +24,8 @@ func PrivateHostKey() []byte {
 
 		pem := ssh.EncodePrivateKeyToPEM(privateKey)
 		if err := ioutil.WriteFile(hostKeyFile, pem, 0600); err != nil {
-			dlog.Common.Error("Unable to write private server RSA host key to file", hostKeyFile, err)
+			dlog.Common.Error("Unable to write private server RSA host key to file",
+				hostKeyFile, err)
 		}
 		return pem
 	}
diff --git a/internal/ssh/server/publickeycallback.go b/internal/ssh/server/publickeycallback.go
index 59d1f31..ebc428a 100644
--- a/internal/ssh/server/publickeycallback.go
+++ b/internal/ssh/server/publickeycallback.go
@@ -13,25 +13,28 @@ import (
 	gossh "golang.org/x/crypto/ssh"
 )
 
-// PublicKeyCallback is for the server to check whether a public SSH key is authorized ot not.
-func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
+// PublicKeyCallback is for the server to check whether a public SSH key is
+// authorized ot not.
+func PublicKeyCallback(c gossh.ConnMetadata,
+	offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
+
 	user, err := user.New(c.User(), c.RemoteAddr().String())
 	if err != nil {
 		return nil, err
 	}
-	dlog.Common.Info(user, "Incoming authorization")
 
+	dlog.Common.Info(user, "Incoming authorization")
 	cwd, err := os.Getwd()
 	if err != nil {
 		return nil, fmt.Errorf("Unable to get current working directory|%s|", err.Error())
 	}
-
 	if config.ServerRelaxedAuthEnable {
 		dlog.Common.Fatal(user, "Granting permissions via relaxed-auth")
 		return nil, nil
 	}
 
-	authorizedKeysFile := fmt.Sprintf("%s/%s/%s.authorized_keys", cwd, config.Common.CacheDir, user.Name)
+	authorizedKeysFile := fmt.Sprintf("%s/%s/%s.authorized_keys", cwd,
+		config.Common.CacheDir, user.Name)
 	if _, err := os.Stat(authorizedKeysFile); os.IsNotExist(err) {
 		user, err := osUser.Lookup(user.Name)
 		if err != nil {
@@ -44,23 +47,25 @@ func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*go
 	dlog.Common.Info(user, "Reading", authorizedKeysFile)
 	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysFile)
 	if err != nil {
-		return nil, fmt.Errorf("Unable to read authorized keys file|%s|%s|%s", authorizedKeysFile, user, err.Error())
+		return nil, fmt.Errorf("Unable to read authorized keys file|%s|%s|%s",
+			authorizedKeysFile, user, err.Error())
 	}
 
 	authorizedKeysMap := map[string]bool{}
 	for len(authorizedKeysBytes) > 0 {
 		authorizedPubKey, _, _, restBytes, err := gossh.ParseAuthorizedKey(authorizedKeysBytes)
 		if err != nil {
-			return nil, fmt.Errorf("Unable to parse authorized keys bytes|%s|%s", user, err.Error())
+			return nil, fmt.Errorf("Unable to parse authorized keys bytes|%s|%s",
+				user, err.Error())
 		}
 		authorizedKeysMap[string(authorizedPubKey.Marshal())] = true
 		authorizedKeysBytes = restBytes
-
-		dlog.Common.Debug(user, "Authorized public key fingerprint", gossh.FingerprintSHA256(authorizedPubKey))
+		dlog.Common.Debug(user, "Authorized public key fingerprint",
+			gossh.FingerprintSHA256(authorizedPubKey))
 	}
 
-	dlog.Common.Debug(user, "Offered public key fingerprint", gossh.FingerprintSHA256(offeredPubKey))
-
+	dlog.Common.Debug(user, "Offered public key fingerprint",
+		gossh.FingerprintSHA256(offeredPubKey))
 	if authorizedKeysMap[string(offeredPubKey.Marshal())] {
 		return &gossh.Permissions{
 			Extensions: map[string]string{
diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go
index 56494a7..db5aaf1 100644
--- a/internal/ssh/ssh.go
+++ b/internal/ssh/ssh.go
@@ -24,12 +24,10 @@ func GeneratePrivateRSAKey(size int) (*rsa.PrivateKey, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	err = privateKey.Validate()
 	if err != nil {
 		return nil, err
 	}
-
 	return privateKey, nil
 }
 
@@ -42,7 +40,6 @@ func EncodePrivateKeyToPEM(privateKey *rsa.PrivateKey) []byte {
 		Headers: nil,
 		Bytes:   derFormat,
 	}
-
 	return pem.EncodeToMemory(&block)
 }
 
@@ -80,7 +77,6 @@ func KeyFile(keyFile string) (gossh.AuthMethod, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	key, err := gossh.ParsePrivateKey(buffer)
 	if err != nil {
 		return nil, err
diff --git a/internal/user/name.go b/internal/user/name.go
index 28ab0a4..cd11907 100644
--- a/internal/user/name.go
+++ b/internal/user/name.go
@@ -10,11 +10,9 @@ func NoRootCheck() {
 	if err != nil {
 		panic(err)
 	}
-
 	if user.Uid == "0" {
 		panic("Not allowed to run as UID 0")
 	}
-
 	if user.Gid == "0" {
 		panic("Not allowed to run as GID 0")
 	}
@@ -26,6 +24,5 @@ func Name() string {
 	if err != nil {
 		panic(err)
 	}
-
 	return user.Username
 }
diff --git a/internal/user/server/user.go b/internal/user/server/user.go
index 70ead1c..aa7f8b1 100644
--- a/internal/user/server/user.go
+++ b/internal/user/server/user.go
@@ -49,7 +49,6 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 		dlog.Server.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
 		return true
 	}
-
 	if u.Name == config.ScheduleUser || u.Name == config.ContinuousUser {
 		// Background user has same permissions as dtail process itself.
 		return true
@@ -57,27 +56,29 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 
 	cleanPath, err := filepath.EvalSymlinks(filePath)
 	if err != nil {
-		dlog.Server.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
+		dlog.Server.Error(u, filePath, permissionType,
+			"Unable to evaluate symlinks", err)
 		hasPermission = false
 		return
 	}
 
 	cleanPath, err = filepath.Abs(cleanPath)
 	if err != nil {
-		dlog.Server.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
+		dlog.Server.Error(u, cleanPath, permissionType,
+			"Unable to make file path absolute", err)
 		hasPermission = false
 		return
 	}
 
 	if cleanPath != filePath {
-		dlog.Server.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
+		dlog.Server.Info(u, filePath, cleanPath, permissionType,
+			"Calculated new clean path from original file path (possibly symlink)")
 	}
 
 	hasPermission, err = u.hasFilePermission(cleanPath, permissionType)
 	if err != nil {
 		dlog.Server.Warn(u, cleanPath, err)
 	}
-
 	return
 }
 
@@ -86,18 +87,17 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 	if _, err := permissions.ToRead(u.Name, cleanPath); err != nil {
 		return false, fmt.Errorf("User without OS file system permissions to read path: '%v'", err)
 	}
-	dlog.Server.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
+	dlog.Server.Info(u, cleanPath, permissionType,
+		"User with OS file system permissions to path")
 
 	// Only allow to follow regular files or symlinks.
 	info, err := os.Lstat(cleanPath)
 	if err != nil {
 		return false, fmt.Errorf("Unable to determine file type: '%v'", err)
 	}
-
 	if !info.Mode().IsRegular() {
 		return false, fmt.Errorf("Can only open regular files or follow symlinks")
 	}
-
 	hasPermission, err := u.iteratePaths(cleanPath, permissionType)
 	if err != nil {
 		return false, err
@@ -109,10 +109,8 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 	// By default assume no permissions
 	hasPermission := false
-
 	for _, permission := range u.permissions {
 		typeStr := "readfiles" // Assume ReadFiles by default.
-
 		var regexStr string
 		var negate bool
 
@@ -123,7 +121,6 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 		}
 
 		dlog.Server.Debug(u, cleanPath, typeStr, permission)
-
 		if typeStr != permissionType {
 			continue
 		}
@@ -136,16 +133,17 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 
 		re, err := regexp.Compile(regexStr)
 		if err != nil {
-			return false, fmt.Errorf("Permission test failed, can't compile regex '%s': '%v'", regexStr, err)
+			return false, fmt.Errorf("Permission test failed, can't compile regex "+
+				"'%s': '%v'", regexStr, err)
 		}
-
 		if negate && re.MatchString(cleanPath) {
-			dlog.Server.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test failed partially, "+
+				"matching negative pattern '%s'", permission)
 			hasPermission = false
 		}
-
 		if !negate && re.MatchString(cleanPath) {
-			dlog.Server.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test passed partially, "+
+				"matching positive pattern", permission)
 			hasPermission = true
 		}
 	}
diff --git a/internal/version/version.go b/internal/version/version.go
index 4ff6eae..68b9e6e 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -20,7 +20,8 @@ const (
 
 // String representation of the DTail version.
 func String() string {
-	return fmt.Sprintf("%s %v Protocol %s %s", Name, Version, protocol.ProtocolCompat, Additional)
+	return fmt.Sprintf("%s %v Protocol %s %s", Name, Version,
+		protocol.ProtocolCompat, Additional)
 }
 
 // PaintedString is a prettier string representation of the DTail version.
@@ -31,13 +32,10 @@ func PaintedString() string {
 
 	name := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Name),
 		color.FgYellow, color.BgBlue, color.AttrBold)
-
 	version := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Version),
 		color.FgBlue, color.BgYellow, color.AttrBold)
-
 	protocol := color.PaintStr(fmt.Sprintf(" Protocol %s ", protocol.ProtocolCompat),
 		color.FgBlack, color.BgGreen)
-
 	additional := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Additional),
 		color.FgWhite, color.BgMagenta, color.AttrUnderline)
 
-- 
cgit v1.2.3


From 18b70fa2cf381ff27397c264394e3d825e660624 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 10 Oct 2021 14:02:12 +0300
Subject: add another dmap test - reading 100 source files at once

fix a data race when reading multiple files on one server from the same session at once
---
 internal/clients/maprclient.go            |  2 +-
 internal/config/config.go                 |  2 +-
 internal/config/initializer.go            |  4 +++-
 internal/io/dlog/level.go                 |  5 ++++
 internal/io/fs/permissions/permission.go  |  2 +-
 internal/server/handlers/basehandler.go   | 40 ++++++++++++++++++++++++++-----
 internal/server/handlers/healthhandler.go |  2 +-
 internal/server/handlers/serverhandler.go | 18 ++------------
 8 files changed, 48 insertions(+), 27 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 04f258d..074494c 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -108,7 +108,7 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 }
 
 // NEXT: Make this a callback function rather trying to use polymorphism to call
-// this. This applies to all clients.
+// this. This applies to all clients. It will make the code easier to read.
 func (c MaprClient) makeHandler(server string) handlers.Handler {
 	return handlers.NewMaprHandler(server, c.query, c.globalGroup)
 }
diff --git a/internal/config/config.go b/internal/config/config.go
index b99b22b..ee23829 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -16,7 +16,7 @@ const (
 	// DefaultSSHPort is the default DServer port.
 	DefaultSSHPort int = 2222
 	// DefaultLogLevel specifies the default log level (obviously)
-	DefaultLogLevel string = "INFO"
+	DefaultLogLevel string = "info"
 	// DefaultClientLogger specifies the default logger for the client commands.
 	DefaultClientLogger string = "fout"
 	// DefaultServerLogger specifies the default logger for dtail server.
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 0a913db..0c6dfdf 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -147,7 +147,9 @@ func transformClient(in *initializer, args *Args, additionalArgs []string) error
 		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
-		in.Common.LogLevel = "warn"
+		if args.LogLevel == DefaultLogLevel {
+			in.Common.LogLevel = "warn"
+		}
 	}
 	return nil
 }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 0971094..05d9ed9 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -9,6 +9,7 @@ type level int
 
 // Available log levels.
 const (
+	None    level = iota
 	Fatal   level = iota
 	Error   level = iota
 	Warn    level = iota
@@ -26,6 +27,8 @@ var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug,
 
 func newLevel(l string) level {
 	switch strings.ToLower(l) {
+	case "none":
+		return None
 	case "fatal":
 		return Fatal
 	case "error":
@@ -54,6 +57,8 @@ func newLevel(l string) level {
 
 func (l level) String() string {
 	switch l {
+	case None:
+		return "NONE"
 	case Fatal:
 		return "FATAL"
 	case Error:
diff --git a/internal/io/fs/permissions/permission.go b/internal/io/fs/permissions/permission.go
index e80dbb2..d621c09 100644
--- a/internal/io/fs/permissions/permission.go
+++ b/internal/io/fs/permissions/permission.go
@@ -9,6 +9,6 @@ import (
 // ToRead is to check whether user has read permissions to a given file.
 func ToRead(user, filePath string) (bool, error) {
 	// Only implemented for Linux, always expect true
-	dlog.Common.Warn(user, filePath, "Not performing ACL check as not compiled in")
+	dlog.Common.Debug(user, filePath, "Not performing ACL check as not compiled in")
 	return true, nil
 }
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 847e8f9..d814cc9 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -9,6 +9,7 @@ import (
 	"io"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -22,7 +23,7 @@ import (
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
-type handleCommandCb func(context.Context, int, []string, string, map[string]string)
+type handleCommandCb func(context.Context, int, []string, string)
 
 type baseHandler struct {
 	done             *internal.Done
@@ -35,11 +36,15 @@ type baseHandler struct {
 	user             *user.User
 	ackCloseReceived chan struct{}
 	activeCommands   int32
-	quiet            bool
-	spartan          bool
-	serverless       int32
 	readBuf          bytes.Buffer
 	writeBuf         bytes.Buffer
+
+	// Some global options + sync primitives required.
+	once       sync.Once
+	mutex      sync.Mutex
+	quiet      bool
+	spartan    bool
+	serverless bool
 }
 
 // Shutdown the handler.
@@ -66,7 +71,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 			return
 		}
 
-		if h.serverless > 0 {
+		if h.serverless {
 			return
 		}
 
@@ -160,8 +165,9 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
 	}
+	h.setOptions(options)
 
-	h.handleCommandCb(ctx, argc, args, commandName, options)
+	h.handleCommandCb(ctx, argc, args, commandName)
 }
 
 func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
@@ -232,6 +238,28 @@ func (h *baseHandler) handleAckCommand(argc int, args []string) {
 	}
 }
 
+func (h *baseHandler) setOptions(options map[string]string) {
+	// We have to make sure that this block is executed only once.
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+	// We can read the options only once, will cause a data race otherwise if
+	// changed multiple times for multiple incoming commands.
+	h.once.Do(func() {
+		if quiet, _ := options["quiet"]; quiet == "true" {
+			dlog.Server.Debug(h.user, "Enabling quiet mode")
+			h.quiet = true
+		}
+		if spartan, _ := options["spartan"]; spartan == "true" {
+			dlog.Server.Debug(h.user, "Enabling spartan mode")
+			h.spartan = true
+		}
+		if serverless, _ := options["serverless"]; serverless == "true" {
+			dlog.Server.Debug(h.user, "Enabling serverless mode")
+			h.serverless = true
+		}
+	})
+}
+
 func (h *baseHandler) send(ch chan<- string, message string) {
 	select {
 	case ch <- message:
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
index 8d6c400..0425696 100644
--- a/internal/server/handlers/healthhandler.go
+++ b/internal/server/handlers/healthhandler.go
@@ -41,7 +41,7 @@ func NewHealthHandler(user *user.User) *HealthHandler {
 }
 
 func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int,
-	args []string, commandName string, options map[string]string) {
+	args []string, commandName string) {
 
 	dlog.Server.Debug(h.user, "Handling health command", argc, args)
 	switch commandName {
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index f12d590..52c4570 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"os"
 	"strings"
-	"sync/atomic"
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/dlog"
@@ -55,7 +54,7 @@ func NewServerHandler(user *user.User, catLimiter,
 }
 
 func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
-	args []string, commandName string, options map[string]string) {
+	args []string, commandName string) {
 
 	dlog.Server.Debug(h.user, "Handling user command", argc, args)
 	h.incrementActiveCommands()
@@ -65,19 +64,6 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
 		}
 	}
 
-	if quiet, _ := options["quiet"]; quiet == "true" {
-		dlog.Server.Debug(h.user, "Enabling quiet mode")
-		h.quiet = true
-	}
-	if spartan, _ := options["spartan"]; spartan == "true" {
-		dlog.Server.Debug(h.user, "Enabling spartan mode")
-		h.spartan = true
-	}
-	if serverless, _ := options["serverless"]; serverless == "true" {
-		dlog.Server.Debug(h.user, "Enabling serverless mode")
-		atomic.AddInt32(&h.serverless, 1)
-	}
-
 	switch commandName {
 	case "grep", "cat":
 		command := newReadCommand(h, omode.CatClient)
@@ -109,7 +95,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
 		commandFinished()
 	default:
 		h.send(h.serverMessages, dlog.Server.Error(h.user,
-			"Received unknown user command", commandName, argc, args, options))
+			"Received unknown user command", commandName, argc, args))
 		commandFinished()
 	}
 }
-- 
cgit v1.2.3


From c9b7064a0cc4273e4e44c1a545f4d2a207a31b55 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 10 Oct 2021 19:42:48 +0300
Subject: refactor

---
 internal/clients/connectors/serverconnection.go | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 2d7b45a..2737ede 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -37,6 +37,7 @@ func NewServerConnection(server string, userName string,
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
 		server:          server,
+		port:            config.Common.SSHPort,
 		handler:         handler,
 		commands:        commands,
 		config: &ssh.ClientConfig{
@@ -47,25 +48,20 @@ func NewServerConnection(server string, userName string,
 		},
 	}
 
+	// TODO: After reconnecting the port is wrong! Due to string slicing?
 	c.initServerPort()
 	return &c
 }
 
 // Server returns the server hostname connected to.
-func (c *ServerConnection) Server() string {
-	return c.server
-}
+func (c *ServerConnection) Server() string { return c.server }
 
 // Handler returns the handler used for the connection.
-func (c *ServerConnection) Handler() handlers.Handler {
-	return c.handler
-}
+func (c *ServerConnection) Handler() handlers.Handler { return c.handler }
 
 // Attempt to parse the server port address from the provided server FQDN.
 func (c *ServerConnection) initServerPort() {
-	c.port = config.Common.SSHPort
 	parts := strings.Split(c.server, ":")
-
 	if len(parts) == 2 {
 		dlog.Client.Debug("Parsing port from hostname", parts)
 		port, err := strconv.Atoi(parts[1])
-- 
cgit v1.2.3


From a1185614fc0fd488381f8011079232d532d9b73e Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 11 Oct 2021 11:51:30 +0300
Subject: add dtail integration test

---
 internal/clients/connectors/serverconnection.go | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 2737ede..1df4d73 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -19,7 +19,11 @@ import (
 // ServerConnection represents a connection to a single remote dtail server via
 // SSH protocol.
 type ServerConnection struct {
-	server          string
+	// The full server string as received from the server discovery (can be with port number)
+	server string
+	// Only the hostname or FQDN (without the port number)
+	hostname string
+	// Only the port number.
 	port            int
 	config          *ssh.ClientConfig
 	handler         handlers.Handler
@@ -37,7 +41,6 @@ func NewServerConnection(server string, userName string,
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
 		server:          server,
-		port:            config.Common.SSHPort,
 		handler:         handler,
 		commands:        commands,
 		config: &ssh.ClientConfig{
@@ -48,7 +51,6 @@ func NewServerConnection(server string, userName string,
 		},
 	}
 
-	// TODO: After reconnecting the port is wrong! Due to string slicing?
 	c.initServerPort()
 	return &c
 }
@@ -61,6 +63,7 @@ func (c *ServerConnection) Handler() handlers.Handler { return c.handler }
 
 // Attempt to parse the server port address from the provided server FQDN.
 func (c *ServerConnection) initServerPort() {
+	c.port = config.Common.SSHPort
 	parts := strings.Split(c.server, ":")
 	if len(parts) == 2 {
 		dlog.Client.Debug("Parsing port from hostname", parts)
@@ -68,7 +71,7 @@ func (c *ServerConnection) initServerPort() {
 		if err != nil {
 			dlog.Client.FatalPanic("Unable to parse client port", c.server, parts, err)
 		}
-		c.server = parts[0]
+		c.hostname = parts[0]
 		c.port = port
 	}
 }
@@ -103,8 +106,8 @@ func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
 		}()
 
 		if err := c.dial(ctx, cancel, throttleCh, statsCh); err != nil {
-			dlog.Client.Warn(c.server, c.port, err)
-			if c.hostKeyCallback.Untrusted(fmt.Sprintf("%s:%d", c.server, c.port)) {
+			dlog.Client.Warn(c.server, err)
+			if c.hostKeyCallback.Untrusted(c.server) {
 				dlog.Client.Debug(c.server, "Not trusting host")
 			}
 		}
@@ -125,7 +128,7 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 	}()
 
 	dlog.Client.Debug(c.server, "Dialing into the connection")
-	address := fmt.Sprintf("%s:%d", c.server, c.port)
+	address := fmt.Sprintf("%s:%d", c.hostname, c.port)
 
 	client, err := ssh.Dial("tcp", address, c.config)
 	if err != nil {
-- 
cgit v1.2.3


From f654855286d81afdd56372b4548ba11d596da78d Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 11 Oct 2021 17:42:37 +0300
Subject: Update JSON-schema to reflect all recent config file changes.

---
 internal/config/client.go           | 2 +-
 internal/config/common.go           | 9 +++------
 internal/config/server.go           | 2 +-
 internal/io/dlog/dlog.go            | 4 ++--
 internal/io/dlog/loggers/factory.go | 8 ++++----
 5 files changed, 11 insertions(+), 14 deletions(-)

(limited to 'internal')

diff --git a/internal/config/client.go b/internal/config/client.go
index 8227c68..9f4df97 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -102,7 +102,7 @@ type termColors struct {
 // ClientConfig represents a DTail client configuration (empty as of now as there
 // are no available config options yet, but that may changes in the future).
 type ClientConfig struct {
-	TermColorsEnable bool
+	TermColorsEnable bool       `json:",omitempty"`
 	TermColors       termColors `json:",omitempty"`
 }
 
diff --git a/internal/config/common.go b/internal/config/common.go
index 9d22c95..7a72cfe 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -12,12 +12,10 @@ type CommonConfig struct {
 	Logger string
 	// LogLevel defines how much is logged.
 	LogLevel string `json:",omitempty"`
-	// LogStrategy defines the log rotation strategy.
-	LogStrategy string
+	// LogRotation strategy to be used.
+	LogRotation string
 	// The cache directory
 	CacheDir string
-	// The temp directory
-	TmpDir string `json:",omitempty"`
 }
 
 // Create a new default configuration.
@@ -28,8 +26,7 @@ func newDefaultCommonConfig() *CommonConfig {
 		LogDir:                     "log",
 		Logger:                     "stdout",
 		LogLevel:                   DefaultLogLevel,
-		LogStrategy:                "daily",
+		LogRotation:                "daily",
 		CacheDir:                   "cache",
-		TmpDir:                     "/tmp",
 	}
 }
diff --git a/internal/config/server.go b/internal/config/server.go
index 677f5ac..254ea0c 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -47,7 +47,7 @@ type ServerConfig struct {
 	MaxConcurrentCats int
 	// The max amount of concurrent tails per server.
 	MaxConcurrentTails int
-	// The user permissions.
+	// The user permissions. TODO: Add to JSON schema
 	Permissions Permissions `json:",omitempty"`
 	// The mapr log format
 	MapreduceLogFormat string `json:",omitempty"`
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index da67585..5e0c3a1 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -82,12 +82,12 @@ func new(sourceProcess, sourcePackage source.Source) *DLog {
 	if err != nil {
 		panic(err)
 	}
-	strategy := loggers.NewStrategy(config.Common.LogStrategy)
+	logRotation := loggers.NewStrategy(config.Common.LogRotation)
 	loggerName := config.Common.Logger
 	level := newLevel(config.Common.LogLevel)
 
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), loggerName, strategy),
+		logger:        loggers.Factory(sourceProcess.String(), loggerName, logRotation),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      level,
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 415d7fb..a5cc7cf 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -10,12 +10,12 @@ var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
 // Factory is there to retrieve a logger based on various settings.
-func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
+func Factory(sourceName, loggerName string, logRotation Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
 	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName,
-		rotationStrategy.FileBase, loggerName)
+		logRotation.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
@@ -29,10 +29,10 @@ func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 			singleton = newStdout()
 			factoryMap[id] = singleton
 		case "file":
-			singleton = newFile(rotationStrategy)
+			singleton = newFile(logRotation)
 			factoryMap[id] = singleton
 		case "fout":
-			singleton = newFout(rotationStrategy)
+			singleton = newFout(logRotation)
 			factoryMap[id] = singleton
 		default:
 			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
-- 
cgit v1.2.3


From 67af621613e8794d47f0265287b7b9d82e19c6bb Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 13 Oct 2021 09:00:03 +0300
Subject: add another dcat integration test - catting 100 files at once

---
 internal/server/handlers/basehandler.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'internal')

diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index d814cc9..6bc8268 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -63,7 +63,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 
 	select {
 	case message := <-h.serverMessages:
-		if message[0] == '.' {
+		if len(message) > 0 && message[0] == '.' {
 			// Handle hidden message (don't display to the user)
 			h.readBuf.WriteString(message)
 			h.readBuf.WriteByte(protocol.MessageDelimiter)
-- 
cgit v1.2.3


From 7eb36937fdc700f56ea294c83fb9b8e11958a328 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 13 Oct 2021 21:10:28 +0300
Subject: Merging grep context from master

---
 internal/config/args.go                   |  73 +++++++++++++--
 internal/io/fs/filereader.go              |   4 +-
 internal/io/fs/readfile.go                | 149 ++++++++++++++++++++++++++++--
 internal/lcontext/lcontext.go             |  22 +++++
 internal/server/handlers/basehandler.go   |  12 +--
 internal/server/handlers/healthhandler.go |   5 +-
 internal/server/handlers/readcommand.go   |  30 +++---
 internal/server/handlers/serverhandler.go |   9 +-
 8 files changed, 262 insertions(+), 42 deletions(-)
 create mode 100644 internal/lcontext/lcontext.go

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index f721390..f94e0a9 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -3,8 +3,10 @@ package config
 import (
 	"encoding/base64"
 	"fmt"
+	"strconv"
 	"strings"
 
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/omode"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -12,6 +14,7 @@ import (
 
 // Args is a helper struct to summarize common client arguments.
 type Args struct {
+	lcontext.LContext
 	Arguments          []string
 	ConfigFile         string
 	ConnectionsPerCPU  int
@@ -74,17 +77,50 @@ func (a *Args) String() string {
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
 func (a *Args) SerializeOptions() string {
-	return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan,
-		a.Serverless)
+	options := make(map[string]string)
+
+	if a.Quiet {
+		options["quiet"] = fmt.Sprintf("%v", a.Quiet)
+	}
+	if a.Spartan {
+		options["spartan"] = fmt.Sprintf("%v", a.Spartan)
+	}
+	if a.Serverless {
+		options["serverless"] = fmt.Sprintf("%v", a.Serverless)
+	}
+	if a.LContext.MaxCount != 0 {
+		options["max"] = fmt.Sprintf("%d", a.LContext.MaxCount)
+	}
+	if a.LContext.BeforeContext != 0 {
+		options["before"] = fmt.Sprintf("%d", a.LContext.BeforeContext)
+	}
+	if a.LContext.AfterContext != 0 {
+		options["after"] = fmt.Sprintf("%d", a.LContext.AfterContext)
+	}
+
+	var sb strings.Builder
+	var i int
+	for k, v := range options {
+		if i > 0 {
+			sb.WriteString(":")
+		}
+		sb.WriteString(k)
+		sb.WriteString("=")
+		sb.WriteString(v)
+		i++
+	}
+	return sb.String()
 }
 
 // DeserializeOptions deserializes the options, but into a map.
-func DeserializeOptions(opts []string) (map[string]string, error) {
+func DeserializeOptions(opts []string) (map[string]string, lcontext.LContext, error) {
 	options := make(map[string]string, len(opts))
+	var ltx lcontext.LContext
+
 	for _, o := range opts {
 		kv := strings.SplitN(o, "=", 2)
 		if len(kv) != 2 {
-			return options, fmt.Errorf("Unable to parse options: %v", kv)
+			return options, ltx, fmt.Errorf("Unable to parse options: %v", kv)
 		}
 		key := kv[0]
 		val := kv[1]
@@ -93,11 +129,34 @@ func DeserializeOptions(opts []string) (map[string]string, error) {
 			s := strings.SplitN(val, "%", 2)
 			decoded, err := base64.StdEncoding.DecodeString(s[1])
 			if err != nil {
-				return options, err
+				return options, ltx, err
 			}
 			val = string(decoded)
 		}
-		options[key] = val
+
+		switch key {
+		case "before":
+			iVal, err := strconv.Atoi(val)
+			if err != nil {
+				return options, ltx, err
+			}
+			ltx.BeforeContext = iVal
+		case "after":
+			iVal, err := strconv.Atoi(val)
+			if err != nil {
+				return options, ltx, err
+			}
+			ltx.AfterContext = iVal
+		case "max":
+			iVal, err := strconv.Atoi(val)
+			if err != nil {
+				return options, ltx, err
+			}
+			ltx.MaxCount = iVal
+		default:
+			options[key] = val
+		}
 	}
-	return options, nil
+
+	return options, ltx, nil
 }
diff --git a/internal/io/fs/filereader.go b/internal/io/fs/filereader.go
index 7773142..b05fd39 100644
--- a/internal/io/fs/filereader.go
+++ b/internal/io/fs/filereader.go
@@ -4,13 +4,15 @@ import (
 	"context"
 
 	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/regex"
 )
 
 // FileReader is the interface used on the dtail server to read/cat/grep/mapr...
 // a file.
 type FileReader interface {
-	Start(ctx context.Context, lines chan<- line.Line, re regex.Regex) error
+	Start(ctx context.Context, ltx lcontext.LContext, lines chan<- line.Line,
+		re regex.Regex) error
 	FilePath() string
 	Retry() bool
 }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 92f85b6..88d467e 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -16,6 +16,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/regex"
 
 	"github.com/DataDog/zstd"
@@ -62,8 +63,8 @@ func (f readFile) Retry() bool {
 }
 
 // Start tailing a log file.
-func (f readFile) Start(ctx context.Context, lines chan<- line.Line,
-	re regex.Regex) error {
+func (f readFile) Start(ctx context.Context, ltx lcontext.LContext,
+	lines chan<- line.Line, re regex.Regex) error {
 
 	dlog.Common.Debug("readFile", f)
 	defer func() {
@@ -102,7 +103,7 @@ func (f readFile) Start(ctx context.Context, lines chan<- line.Line,
 	wg.Add(1)
 
 	go f.periodicTruncateCheck(ctx, truncate)
-	go f.filter(ctx, &wg, rawLines, lines, re)
+	go f.filter(ctx, ltx, &wg, rawLines, lines, re)
 
 	err = f.read(ctx, fd, rawLines, truncate)
 	close(rawLines)
@@ -213,10 +214,27 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 }
 
 // Filter log lines matching a given regular expression.
-func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup,
-	rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
+func (f readFile) filter(ctx context.Context, ltx lcontext.LContext,
+	wg *sync.WaitGroup, rawLines <-chan *bytes.Buffer, lines chan<- line.Line,
+	re regex.Regex) {
 
 	defer wg.Done()
+	// Do we have any kind of local context settings? If so then run the more complex
+	// filterWithLContext method.
+	if ltx.Has() {
+		// We can not skip transmitting any lines to the client with a local
+		// grep context specified.
+		f.canSkipLines = false
+		f.filterWithLContext(ctx, ltx, rawLines, lines, re)
+		return
+	}
+
+	f.filterWithoutLContext(ctx, rawLines, lines, re)
+}
+
+func (f readFile) filterWithoutLContext(ctx context.Context, rawLines <-chan *bytes.Buffer,
+	lines chan<- line.Line, re regex.Regex) {
+
 	for {
 		select {
 		case line, ok := <-rawLines:
@@ -235,11 +253,126 @@ func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup,
 	}
 }
 
-func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int,
+// Filter log lines matching a given regular expression, however with local grep context.
+func (f readFile) filterWithLContext(ctx context.Context, ltx lcontext.LContext,
+	rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
+
+	// Scenario 1: Finish once maxCount hits found
+	maxCount := ltx.MaxCount
+	processMaxCount := maxCount > 0
+	maxReached := false
+
+	// Scenario 2: Print prev. N lines when current line matches.
+	before := ltx.BeforeContext
+	processBefore := before > 0
+	var beforeBuf chan *bytes.Buffer
+	if processBefore {
+		beforeBuf = make(chan *bytes.Buffer, before)
+	}
+
+	// Screnario 3: Print next N lines when current line matches.
+	after := 0
+	processAfter := ltx.AfterContext > 0
+
+	for lineBytesBuffer := range rawLines {
+		f.updatePosition()
+
+		if !re.Match(lineBytesBuffer.Bytes()) {
+			f.updateLineNotMatched()
+
+			if processAfter && after > 0 {
+				after--
+				myLine := line.Line{
+					Content:         lineBytesBuffer,
+					SourceID:        f.globID,
+					Count:           f.totalLineCount(),
+					TransmittedPerc: 100,
+				}
+
+				select {
+				case lines <- myLine:
+				case <-ctx.Done():
+					return
+				}
+
+			} else if processBefore {
+				// Keep last num BeforeContext raw messages.
+				select {
+				case beforeBuf <- lineBytesBuffer:
+				default:
+					pool.RecycleBytesBuffer(<-beforeBuf)
+					beforeBuf <- lineBytesBuffer
+				}
+			}
+			continue
+		}
+
+		f.updateLineMatched()
+
+		if processAfter {
+			if maxReached {
+				return
+			}
+			after = ltx.AfterContext
+		}
+
+		if processBefore {
+			i := uint64(len(beforeBuf))
+			for {
+				select {
+				case lineBytesBuffer := <-beforeBuf:
+					myLine := line.Line{
+						Content:         lineBytesBuffer,
+						SourceID:        f.globID,
+						Count:           f.totalLineCount() - i,
+						TransmittedPerc: 100,
+					}
+					i--
+
+					select {
+					case lines <- myLine:
+					case <-ctx.Done():
+						return
+					}
+				default:
+					// beforeBuf is now empty.
+				}
+				if len(beforeBuf) == 0 {
+					break
+				}
+			}
+		}
+
+		line := line.Line{
+			Content:         lineBytesBuffer,
+			SourceID:        f.globID,
+			Count:           f.totalLineCount(),
+			TransmittedPerc: 100,
+		}
+
+		select {
+		case lines <- line:
+			if processMaxCount {
+				maxCount--
+				if maxCount == 0 {
+					if !processAfter || after == 0 {
+						return
+					}
+					// Unfortunatley we have to continue filter, as there might be more lines to print
+					maxReached = true
+				}
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (f readFile) transmittable(lineBytesBuffer *bytes.Buffer, length, capacity int,
 	re regex.Regex) (line.Line, bool) {
 
 	var read line.Line
-	if !re.Match(lineBytes.Bytes()) {
+	if !re.Match(lineBytesBuffer.Bytes()) {
 		f.updateLineNotMatched()
 		f.updateLineNotTransmitted()
 		return read, false
@@ -254,7 +387,7 @@ func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int,
 	f.updateLineTransmitted()
 
 	read = line.Line{
-		Content:         lineBytes,
+		Content:         lineBytesBuffer,
 		SourceID:        f.globID,
 		Count:           f.totalLineCount(),
 		TransmittedPerc: f.transmittedPerc(),
diff --git a/internal/lcontext/lcontext.go b/internal/lcontext/lcontext.go
new file mode 100644
index 0000000..183ceb5
--- /dev/null
+++ b/internal/lcontext/lcontext.go
@@ -0,0 +1,22 @@
+package lcontext
+
+// LContext stands for line context (used by context aware grep queries e.g.)
+type LContext struct {
+	AfterContext  int
+	BeforeContext int
+	MaxCount      int
+}
+
+// Has returns true if it has any parameter set.
+func (c LContext) Has() bool {
+	if c.AfterContext > 0 {
+		return true
+	}
+	if c.BeforeContext > 0 {
+		return true
+	}
+	if c.MaxCount > 0 {
+		return true
+	}
+	return false
+}
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 6bc8268..c25f85a 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -18,12 +18,13 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
-type handleCommandCb func(context.Context, int, []string, string)
+type handleCommandCb func(context.Context, lcontext.LContext, int, []string, string)
 
 type baseHandler struct {
 	done             *internal.Done
@@ -160,14 +161,13 @@ func (h *baseHandler) handleCommand(commandStr string) {
 
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
-	options, err := config.DeserializeOptions(splitted[1:])
+	options, ltx, err := config.DeserializeOptions(splitted[1:])
 	if err != nil {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
 	}
-	h.setOptions(options)
-
-	h.handleCommandCb(ctx, argc, args, commandName)
+	h.handleOptions(options)
+	h.handleCommandCb(ctx, ltx, argc, args, commandName)
 }
 
 func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
@@ -238,7 +238,7 @@ func (h *baseHandler) handleAckCommand(argc int, args []string) {
 	}
 }
 
-func (h *baseHandler) setOptions(options map[string]string) {
+func (h *baseHandler) handleOptions(options map[string]string) {
 	// We have to make sure that this block is executed only once.
 	h.mutex.Lock()
 	defer h.mutex.Unlock()
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
index 0425696..6dd9872 100644
--- a/internal/server/handlers/healthhandler.go
+++ b/internal/server/handlers/healthhandler.go
@@ -8,6 +8,7 @@ import (
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/lcontext"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
@@ -40,8 +41,8 @@ func NewHealthHandler(user *user.User) *HealthHandler {
 	return &h
 }
 
-func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int,
-	args []string, commandName string) {
+func (h *HealthHandler) handleHealthCommand(ctx context.Context,
+	ltx lcontext.LContext, argc int, args []string, commandName string) {
 
 	dlog.Server.Debug(h.user, "Handling health command", argc, args)
 	switch commandName {
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 384e966..4728a55 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -10,6 +10,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/fs"
 	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 )
@@ -26,8 +27,8 @@ func newReadCommand(server *ServerHandler, mode omode.Mode) *readCommand {
 	}
 }
 
-func (r *readCommand) Start(ctx context.Context, argc int, args []string,
-	retries int) {
+func (r *readCommand) Start(ctx context.Context, ltx lcontext.LContext,
+	argc int, args []string, retries int) {
 
 	re := regex.NewNoop()
 	if argc >= 4 {
@@ -44,11 +45,11 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string,
 			"Unable to parse command", args, argc))
 		return
 	}
-	r.readGlob(ctx, args[1], re, retries)
+	r.readGlob(ctx, ltx, args[1], re, retries)
 }
 
-func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
-	retries int) {
+func (r *readCommand) readGlob(ctx context.Context, ltx lcontext.LContext,
+	glob string, re regex.Regex, retries int) {
 
 	retryInterval := time.Second * 5
 	glob = filepath.Clean(glob)
@@ -74,7 +75,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 			continue
 		}
 
-		r.readFiles(ctx, paths, glob, re, retryInterval)
+		r.readFiles(ctx, ltx, paths, glob, re, retryInterval)
 		return
 	}
 
@@ -83,18 +84,18 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 	return
 }
 
-func (r *readCommand) readFiles(ctx context.Context, paths []string, glob string,
-	re regex.Regex, retryInterval time.Duration) {
+func (r *readCommand) readFiles(ctx context.Context, ltx lcontext.LContext,
+	paths []string, glob string, re regex.Regex, retryInterval time.Duration) {
 
 	var wg sync.WaitGroup
 	wg.Add(len(paths))
 	for _, path := range paths {
-		go r.readFileIfPermissions(ctx, &wg, path, glob, re)
+		go r.readFileIfPermissions(ctx, ltx, &wg, path, glob, re)
 	}
 	wg.Wait()
 }
 
-func (r *readCommand) readFileIfPermissions(ctx context.Context,
+func (r *readCommand) readFileIfPermissions(ctx context.Context, ltx lcontext.LContext,
 	wg *sync.WaitGroup, path, glob string, re regex.Regex) {
 
 	defer wg.Done()
@@ -105,12 +106,13 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context,
 			"Unable to read file(s), check server logs"))
 		return
 	}
-	r.readFile(ctx, path, globID, re)
+	r.readFile(ctx, ltx, path, globID, re)
 }
 
-func (r *readCommand) readFile(ctx context.Context, path, globID string, re regex.Regex) {
-	dlog.Server.Info(r.server.user, "Start reading file", path, globID)
+func (r *readCommand) readFile(ctx context.Context, ltx lcontext.LContext,
+	path, globID string, re regex.Regex) {
 
+	dlog.Server.Info(r.server.user, "Start reading file", path, globID)
 	var reader fs.FileReader
 	switch r.mode {
 	case omode.TailClient:
@@ -129,7 +131,7 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 			lines = make(chan line.Line, 100)
 			aggregate.NextLinesCh <- lines
 		}
-		if err := reader.Start(ctx, lines, re); err != nil {
+		if err := reader.Start(ctx, ltx, lines, re); err != nil {
 			dlog.Server.Error(r.server.user, path, globID, err)
 		}
 		if aggregate != nil {
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 52c4570..36574a9 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -8,6 +8,7 @@ import (
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/omode"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
@@ -53,8 +54,8 @@ func NewServerHandler(user *user.User, catLimiter,
 	return &h
 }
 
-func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
-	args []string, commandName string) {
+func (h *ServerHandler) handleUserCommand(ctx context.Context, ltx lcontext.LContext,
+	argc int, args []string, commandName string) {
 
 	dlog.Server.Debug(h.user, "Handling user command", argc, args)
 	h.incrementActiveCommands()
@@ -68,13 +69,13 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
 	case "grep", "cat":
 		command := newReadCommand(h, omode.CatClient)
 		go func() {
-			command.Start(ctx, argc, args, 1)
+			command.Start(ctx, ltx, argc, args, 1)
 			commandFinished()
 		}()
 	case "tail":
 		command := newReadCommand(h, omode.TailClient)
 		go func() {
-			command.Start(ctx, argc, args, 10)
+			command.Start(ctx, ltx, argc, args, 10)
 			commandFinished()
 		}()
 	case "map":
-- 
cgit v1.2.3


From 1b2a158ed5fa3fa8fee2bf0881d694a893165f18 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Thu, 14 Oct 2021 20:10:55 +0300
Subject: add dgrep context integration tests

---
 internal/io/fs/readfile.go              | 23 +++++++++++++++--------
 internal/server/handlers/basehandler.go |  2 ++
 2 files changed, 17 insertions(+), 8 deletions(-)

(limited to 'internal')

diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 88d467e..28cbe58 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -99,15 +99,24 @@ func (f readFile) Start(ctx context.Context, ltx lcontext.LContext,
 	rawLines := make(chan *bytes.Buffer, 100)
 	truncate := make(chan struct{})
 
-	var wg sync.WaitGroup
-	wg.Add(1)
+	readCtx, readCancel := context.WithCancel(ctx)
+	var filterWg sync.WaitGroup
+	filterWg.Add(1)
 
 	go f.periodicTruncateCheck(ctx, truncate)
-	go f.filter(ctx, ltx, &wg, rawLines, lines, re)
+	go func() {
+		f.filter(ctx, ltx, rawLines, lines, re)
+		filterWg.Done()
+		// If the filter stopped, make the reader stop too, no need to read
+		// more data if there is nothing more the filter wants to filter for!
+		// E.g. it could be that we only want to filter N matches but not more.
+		readCancel()
+	}()
 
-	err = f.read(ctx, fd, rawLines, truncate)
+	err = f.read(readCtx, fd, rawLines, truncate)
 	close(rawLines)
-	wg.Wait()
+	// Filter may sends some data still. So wait until it is done here.
+	filterWg.Wait()
 
 	return err
 }
@@ -215,10 +224,8 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 
 // Filter log lines matching a given regular expression.
 func (f readFile) filter(ctx context.Context, ltx lcontext.LContext,
-	wg *sync.WaitGroup, rawLines <-chan *bytes.Buffer, lines chan<- line.Line,
-	re regex.Regex) {
+	rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
 
-	defer wg.Done()
 	// Do we have any kind of local context settings? If so then run the more complex
 	// filterWithLContext method.
 	if ltx.Has() {
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index c25f85a..934f2bc 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -159,6 +159,8 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		cancel()
 	}()
 
+	dlog.Server.Trace(args)
+	dlog.Server.Trace(args[0])
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
 	options, ltx, err := config.DeserializeOptions(splitted[1:])
-- 
cgit v1.2.3


From c20d77a574aacf6e7118d03e85021c119b934a21 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Thu, 14 Oct 2021 20:55:35 +0300
Subject: refactor

---
 internal/server/handlers/basehandler.go | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'internal')

diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 934f2bc..6d10d17 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -159,11 +159,16 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		cancel()
 	}()
 
-	dlog.Server.Trace(args)
-	dlog.Server.Trace(args[0])
-	splitted := strings.Split(args[0], ":")
-	commandName := splitted[0]
-	options, ltx, err := config.DeserializeOptions(splitted[1:])
+	parts := strings.Split(args[0], ":")
+	commandName := parts[0]
+
+	// Either no options or empty options provided.
+	if len(parts) == 1 || len(parts[1]) == 0 {
+		h.handleCommandCb(ctx, lcontext.LContext{}, argc, args, commandName)
+		return
+	}
+
+	options, ltx, err := config.DeserializeOptions(parts[1:])
 	if err != nil {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
-- 
cgit v1.2.3