From 4ebff5064251d24c7d80873074c9a614311b027d Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Fri, 30 Jul 2021 08:51:05 +0300
Subject: initial color config support

---
 internal/color/color.go   |  3 +--
 internal/config/client.go | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index 0736199..7309544 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -26,8 +26,7 @@ const (
 	Yellow    Color = escape + "[36m"
 	LightGray Color = escape + "[37m"
 
-	BgGray      Color = escape + "[40m"
-	BgRed       Color = escape + "[41m"
+	BgGray      Color = escape + "[40m" BgRed       Color = escape + "[41m"
 	BgGreen     Color = escape + "[42m"
 	BgOrange    Color = escape + "[43m"
 	BgBlue      Color = escape + "[44m"
diff --git a/internal/config/client.go b/internal/config/client.go
index 1515aae..3d8c45a 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -1,8 +1,25 @@
 package config
 
+// ClientColorConfig allows to override the default terminal color codes.
+type ClientColorConfig struct {
+	OkBgColor Color
+}
+
+/*
+	splitted := strings.Split(line, "|")
+	if splitted[2] == "100" {
+		splitted[2] = Paint(BgGreen, splitted[2])
+	} else {
+		splitted[2] = Paint(BgRed, splitted[2])
+	}
+	info := strings.Join(splitted[0:5], "|")
+	log := strings.Join(splitted[5:], "|")
+*/
+
 // ClientConfig represents a DTail client configuration (empty as of now as there
 // are no available config options yet, but that may changes in the future).
 type ClientConfig struct {
+	TerminalColors ClientColorConfig `json:",omitempty"`
 }
 
 // Create a new default client configuration.
-- 
cgit v1.2.3


From f46abf36e80a67d13d12dbe4ba8c899026e53961 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 31 Jul 2021 18:20:03 +0300
Subject: more on configurable colors

---
 internal/color/color.go      | 69 --------------------------------------------
 internal/color/colorfy.go    | 56 -----------------------------------
 internal/config/client.go    | 51 ++++++++++++++++++++++----------
 internal/io/logger/logger.go | 10 +++----
 internal/version/version.go  |  3 +-
 5 files changed, 42 insertions(+), 147 deletions(-)
 delete mode 100644 internal/color/color.go
 delete mode 100644 internal/color/colorfy.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
deleted file mode 100644
index 7309544..0000000
--- a/internal/color/color.go
+++ /dev/null
@@ -1,69 +0,0 @@
-// Package color is used to prettify console output via ANSII terminal colors.
-package color
-
-import (
-	"fmt"
-)
-
-// Color name.
-type Color string
-
-// Attribute of a color.
-type Attribute string
-
-// The possible color variations.
-const (
-	escape        = "\x1b"
-	reset         = escape + "[0m"
-	seq    string = "%s%s%s"
-
-	Gray      Color = escape + "[30m"
-	Red       Color = escape + "[31m"
-	Green     Color = escape + "[32m"
-	Orange    Color = escape + "[33m"
-	Blue      Color = escape + "[34m"
-	Magenta   Color = escape + "[35m"
-	Yellow    Color = escape + "[36m"
-	LightGray Color = escape + "[37m"
-
-	BgGray      Color = escape + "[40m" BgRed       Color = escape + "[41m"
-	BgGreen     Color = escape + "[42m"
-	BgOrange    Color = escape + "[43m"
-	BgBlue      Color = escape + "[44m"
-	BgMagenta   Color = escape + "[45m"
-	BgYellow    Color = escape + "[46m"
-	BgLightGray Color = escape + "[47m"
-
-	Bold         Attribute = escape + "[1m"
-	Italic       Attribute = escape + "[3m"
-	Underline    Attribute = escape + "[4m"
-	ReverseColor Attribute = escape + "[7m"
-
-	resetBold      = escape + "[22m"
-	resetItalic    = escape + "[23m"
-	resetUnderline = escape + "[24m"
-
-	Test     Color     = BgYellow
-	TestAttr Attribute = Bold
-)
-
-// Colored DTail client output enabled.
-var Colored bool
-
-// Paint a given string in a given color.
-func Paint(c Color, s string) string {
-	return fmt.Sprintf(seq, c, s, reset)
-}
-
-// Attr adds a given attribute to a given string, such as "bold" or "italic".
-func Attr(c Attribute, s string) string {
-	switch c {
-	case Bold:
-		return fmt.Sprintf(seq, Bold, s, resetBold)
-	case Italic:
-		return fmt.Sprintf(seq, Italic, s, resetItalic)
-	case Underline:
-		return fmt.Sprintf(seq, Underline, s, resetUnderline)
-	}
-	panic("Unknown attribute")
-}
diff --git a/internal/color/colorfy.go b/internal/color/colorfy.go
deleted file mode 100644
index a2beb7a..0000000
--- a/internal/color/colorfy.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package color
-
-import (
-	"fmt"
-	"strings"
-)
-
-// Add some color to log lines received from remote servers.
-func paintRemote(line string) string {
-	splitted := strings.Split(line, "|")
-	if splitted[2] == "100" {
-		splitted[2] = Paint(BgGreen, splitted[2])
-	} else {
-		splitted[2] = Paint(BgRed, splitted[2])
-	}
-	info := strings.Join(splitted[0:5], "|")
-	log := strings.Join(splitted[5:], "|")
-
-	if strings.HasPrefix(log, "WARN") {
-		log = Paint(BgYellow, log)
-	} else if strings.HasPrefix(log, "ERROR") {
-		log = Paint(BgRed, log)
-	} else if strings.HasPrefix(log, "FATAL") {
-		log = Attr(Bold, Paint(BgRed, log))
-	} else {
-		log = Paint(Blue, log)
-	}
-
-	return fmt.Sprintf("%s|%s", info, log)
-}
-
-// Add some color to stats generated by the client.
-func paintClientStats(line string) string {
-	splitted := strings.Split(line, "|")
-	first := strings.Join(splitted[0:4], "|")
-	connected := Paint(BgBlue, splitted[4])
-	last := strings.Join(splitted[5:], "|")
-
-	return fmt.Sprintf("%s|%s|%s", first, connected, last)
-}
-
-// Colorfy a given line based on the line's content.
-func Colorfy(line string) string {
-	switch {
-	case strings.HasPrefix(line, "REMOTE"):
-		return paintRemote(line)
-	case strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|"):
-		return paintClientStats(line)
-	case strings.Contains(line, "ERROR"):
-		return Paint(Magenta, line)
-	case strings.Contains(line, "WARN"):
-		return Paint(Magenta, line)
-	}
-
-	return line
-}
diff --git a/internal/config/client.go b/internal/config/client.go
index 3d8c45a..d6d106f 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -1,28 +1,47 @@
 package config
 
-// ClientColorConfig allows to override the default terminal color codes.
-type ClientColorConfig struct {
-	OkBgColor Color
-}
+import "github.com/mimecast/dtail/internal/color"
 
-/*
-	splitted := strings.Split(line, "|")
-	if splitted[2] == "100" {
-		splitted[2] = Paint(BgGreen, splitted[2])
-	} else {
-		splitted[2] = Paint(BgRed, splitted[2])
-	}
-	info := strings.Join(splitted[0:5], "|")
-	log := strings.Join(splitted[5:], "|")
-*/
+// ClientColorConfig allows to override the default terminal color color.
+type termColors struct {
+	RemoteTextFg      color.Color
+	RemoteStatsOkBg   color.Color
+	RemoteStatsWarnBg color.Color
+	RemoteTraceBg     color.Color
+	RemoteDebugBg     color.Color
+	RemoteWarnBg      color.Color
+	RemoteErrorBg     color.Color
+	RemoteFatalBg     color.Color
+	RemoteFatalAttr   color.Attribute
+	ClientStatsBg     color.Color
+	ClientWarnFg      color.Color
+	ClientErrorFg     color.Color
+}
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
 // are no available config options yet, but that may changes in the future).
 type ClientConfig struct {
-	TerminalColors ClientColorConfig `json:",omitempty"`
+	TermColorsEnabled bool
+	TermColors        termColors `json:",omitempty"`
 }
 
 // Create a new default client configuration.
 func newDefaultClientConfig() *ClientConfig {
-	return &ClientConfig{}
+	return &ClientConfig{
+		TermColorsEnabled: true,
+		TermColors: termColors{
+			RemoteTextFg:      color.LightGray,
+			RemoteStatsOkBg:   color.BgGreen,
+			RemoteStatsWarnBg: color.BgRed,
+			RemoteTraceBg:     color.BgYellow,
+			RemoteDebugBg:     color.BgYellow,
+			RemoteWarnBg:      color.BgYellow,
+			RemoteErrorBg:     color.BgRed,
+			RemoteFatalBg:     color.BgRed,
+			RemoteFatalAttr:   color.Bold,
+			ClientStatsBg:     color.BgBlue,
+			ClientWarnFg:      color.Purple,
+			ClientErrorFg:     color.BgRed,
+		},
+	}
 }
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 4254eef..bef5293 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -12,7 +12,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/color/brush"
 	"github.com/mimecast/dtail/internal/config"
 )
 
@@ -207,8 +207,8 @@ func write(what, severity, message string) {
 	if Mode.logToStdout {
 		line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message)
 
-		if color.Colored {
-			line = color.Colorfy(line)
+		if config.Client.TermColorsEnabled {
+			line = brush.Colorfy(line)
 		}
 
 		stdoutBufCh <- line
@@ -262,8 +262,8 @@ func Raw(message string) {
 	}
 
 	if Mode.logToStdout {
-		if color.Colored {
-			message = color.Colorfy(message)
+		if config.Client.TermColorsEnabled {
+			message = brush.Colorfy(message)
 		}
 		stdoutBufCh <- message
 	}
diff --git a/internal/version/version.go b/internal/version/version.go
index c0f349c..7c206b4 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
 )
 
 const (
@@ -25,7 +26,7 @@ func String() string {
 
 // PaintedString is a prettier string representation of the DTail version.
 func PaintedString() string {
-	if !color.Colored {
+	if !config.Client.TermColorsEnabled {
 		return String()
 	}
 	name := color.Paint(color.Yellow, Name)
-- 
cgit v1.2.3


From eefb05ae411edef4de293099c5f6611a2ec7e9fb Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 9 Aug 2021 10:22:54 +0300
Subject: we have all basic ansi text color codes and attributes now. we can
 also convert a string to a corresponding code (e.g. from a config file).

---
 internal/color/color.go | 164 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100644 internal/color/color.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
new file mode 100644
index 0000000..c6760d5
--- /dev/null
+++ b/internal/color/color.go
@@ -0,0 +1,164 @@
+// Package color contains all terminal color codes we know of.
+package color
+
+import (
+	"fmt"
+	"strings"
+)
+
+// FgColor is the text foreground color.
+type FgColor string
+
+// BgColor is the text background color.
+type BgColor string
+
+// Attribute of text.
+type Attribute string
+
+// The possible color variations.
+const (
+	escape        = "\x1b"
+	seq    string = "%s%s%s"
+
+	FgBlack   FgColor = escape + "[30m"
+	FgRed     FgColor = escape + "[31m"
+	FgGreen   FgColor = escape + "[32m"
+	FgYellow  FgColor = escape + "[33m"
+	FgBlue    FgColor = escape + "[34m"
+	FgMagenta FgColor = escape + "[35m"
+	FgCyan    FgColor = escape + "[36m"
+	FgWhite   FgColor = escape + "[37m"
+	FgDefault FgColor = escape + "[39m"
+
+	BgBlack   BgColor = escape + "[40m"
+	BgRed     BgColor = escape + "[41m"
+	BgGreen   BgColor = escape + "[42m"
+	BgYellow  BgColor = escape + "[43m"
+	BgBlue    BgColor = escape + "[44m"
+	BgMagenta BgColor = escape + "[45m"
+	BgCyan    BgColor = escape + "[46m"
+	BgWhite   BgColor = escape + "[47m"
+	BgDefault BgColor = escape + "[49m"
+
+	AttReset      Attribute = escape + "[0m"
+	AttBold       Attribute = escape + "[1m"
+	AttDim        Attribute = escape + "[2m"
+	AttItalic     Attribute = escape + "[3m"
+	AttUnderline  Attribute = escape + "[4m"
+	AttBlink      Attribute = escape + "[5m"
+	AttSlowBlink  Attribute = escape + "[5m"
+	AttRapidBlink Attribute = escape + "[6m"
+	AttReverse    Attribute = escape + "[7m"
+	AttHidden     Attribute = escape + "[8m"
+
+	// Internal (manual) testing.
+	FgTest  FgColor   = FgBlue
+	BgTest  BgColor   = BgYellow
+	AttTest Attribute = AttBold
+)
+
+// Colored DTail client output enabled.
+var Colored bool
+
+// Paint paints a given text in a given foreground/background color combination.
+func Paint(fg FgColor, bg FgColor, text string) string {
+	return fmt.Sprintf(seq, fg, bg, text, BgDefault, FgDefault)
+}
+
+// PaintWithAttr paints a given text in a given foreground/background/attribute combination
+func PaintWithAtt(fg FgColor, bg FgColor, att Attribute, text string) string {
+	return fmt.Sprintf(seq, fg, bg, att, text, AttReset, BgDefault, FgDefault)
+}
+
+// PaintFg paints a given text in a given foreground color.
+func PaintFg(fg FgColor, text string) string {
+	return fmt.Sprintf(seq, fg, text, FgDefault)
+}
+
+// PaintBg paints a given text in a given background color.
+func PaintBg(bg BgColor, text string) string {
+	return fmt.Sprintf(seq, bg, text, BgDefault)
+}
+
+// PaintAtt adds a given attribute to a given text, such as "bold" or "italic".
+func PaintAtt(att Attribute, text string) string {
+	return fmt.Sprintf(seq, att, text, AttReset)
+}
+
+// ToFgColor converts a given string (e.g. from a config file) into a foreground color code.
+func ToFgColor(s string) (FgColor, error) {
+	switch strings.ToLower(s) {
+	case "black":
+		return FgBlack, nil
+	case "red":
+		return FgRed, nil
+	case "green":
+		return FgGreen, nil
+	case "yellow":
+		return FgYellow, nil
+	case "blue":
+		return FgBlue, nil
+	case "magenta":
+		return FgMagenta, nil
+	case "cyan":
+		return FgCyan, nil
+	case "white":
+		return FgWhite, nil
+	case "default":
+		return FgDefault, nil
+	default:
+		return FgDefault, fmt.Errorf("unknown foreground text color '" + s + "'")
+	}
+}
+
+// ToBgColor converts a given string (e.g. from a config file) into a background color code.
+func ToBgColor(s string) (BgColor, error) {
+	switch strings.ToLower(s) {
+	case "black":
+		return BgBlack, nil
+	case "red":
+		return BgRed, nil
+	case "green":
+		return BgGreen, nil
+	case "yellow":
+		return BgYellow, nil
+	case "blue":
+		return BgBlue, nil
+	case "magenta":
+		return BgMagenta, nil
+	case "cyan":
+		return BgCyan, nil
+	case "white":
+		return BgWhite, nil
+	case "default":
+		return BgDefault, nil
+	default:
+		return BgDefault, fmt.Errorf("unknown background text color '" + s + "'")
+	}
+}
+
+// ToAttribute converts a given string (e.g. from a config file) into a text attribute.
+func ToAttColor(s string) (Attribute, error) {
+	switch strings.ToLower(s) {
+	case "bold":
+		return AttBold, nil
+	case "dim":
+		return AttDim, nil
+	case "italic":
+		return AttItalic, nil
+	case "underline":
+		return AttUnderline, nil
+	case "blink":
+		return AttBlink, nil
+	case "slowblink":
+		return AttSlowBlink, nil
+	case "rapidblink":
+		return AttRapidBlink, nil
+	case "reverse":
+		return AttReverse, nil
+	case "hidden":
+		return AttHidden, nil
+	default:
+		return AttReset, fmt.Errorf("unknown text attribute '" + s + "'")
+	}
+}
-- 
cgit v1.2.3


From 4a1fbb7ed3143e38e10884955839a87966c3255e Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 9 Aug 2021 10:33:57 +0300
Subject: change paint API

---
 internal/color/color.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index c6760d5..52a5752 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -61,27 +61,27 @@ const (
 var Colored bool
 
 // Paint paints a given text in a given foreground/background color combination.
-func Paint(fg FgColor, bg FgColor, text string) string {
+func Paint(text string, fg FgColor, bg FgColor) string {
 	return fmt.Sprintf(seq, fg, bg, text, BgDefault, FgDefault)
 }
 
 // PaintWithAttr paints a given text in a given foreground/background/attribute combination
-func PaintWithAtt(fg FgColor, bg FgColor, att Attribute, text string) string {
+func PaintWithAtt(text string, fg FgColor, bg FgColor, att Attribute) string {
 	return fmt.Sprintf(seq, fg, bg, att, text, AttReset, BgDefault, FgDefault)
 }
 
 // PaintFg paints a given text in a given foreground color.
-func PaintFg(fg FgColor, text string) string {
+func PaintFg(text string, fg FgColor) string {
 	return fmt.Sprintf(seq, fg, text, FgDefault)
 }
 
 // PaintBg paints a given text in a given background color.
-func PaintBg(bg BgColor, text string) string {
+func PaintBg(text string, bg BgColor) string {
 	return fmt.Sprintf(seq, bg, text, BgDefault)
 }
 
 // PaintAtt adds a given attribute to a given text, such as "bold" or "italic".
-func PaintAtt(att Attribute, text string) string {
+func PaintAtt(text string, att Attribute) string {
 	return fmt.Sprintf(seq, att, text, AttReset)
 }
 
-- 
cgit v1.2.3


From 03ec9f8fc1b30d0efb60584c90a027ff5816d153 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 10 Aug 2021 11:23:07 +0300
Subject: add color unit test

---
 internal/color/color.go      | 68 ++++++++++++++++++++------------------------
 internal/color/color_test.go | 57 +++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 37 deletions(-)
 create mode 100644 internal/color/color_test.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index 52a5752..965675f 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -17,8 +17,7 @@ type Attribute string
 
 // The possible color variations.
 const (
-	escape        = "\x1b"
-	seq    string = "%s%s%s"
+	escape = "\x1b"
 
 	FgBlack   FgColor = escape + "[30m"
 	FgRed     FgColor = escape + "[31m"
@@ -40,49 +39,44 @@ const (
 	BgWhite   BgColor = escape + "[47m"
 	BgDefault BgColor = escape + "[49m"
 
-	AttReset      Attribute = escape + "[0m"
-	AttBold       Attribute = escape + "[1m"
-	AttDim        Attribute = escape + "[2m"
-	AttItalic     Attribute = escape + "[3m"
-	AttUnderline  Attribute = escape + "[4m"
-	AttBlink      Attribute = escape + "[5m"
-	AttSlowBlink  Attribute = escape + "[5m"
-	AttRapidBlink Attribute = escape + "[6m"
-	AttReverse    Attribute = escape + "[7m"
-	AttHidden     Attribute = escape + "[8m"
-
-	// Internal (manual) testing.
-	FgTest  FgColor   = FgBlue
-	BgTest  BgColor   = BgYellow
-	AttTest Attribute = AttBold
+	AttrReset      Attribute = escape + "[0m"
+	AttrBold       Attribute = escape + "[1m"
+	AttrDim        Attribute = escape + "[2m"
+	AttrItalic     Attribute = escape + "[3m"
+	AttrUnderline  Attribute = escape + "[4m"
+	AttrBlink      Attribute = escape + "[5m"
+	AttrSlowBlink  Attribute = escape + "[5m"
+	AttrRapidBlink Attribute = escape + "[6m"
+	AttrReverse    Attribute = escape + "[7m"
+	AttrHidden     Attribute = escape + "[8m"
 )
 
 // Colored DTail client output enabled.
 var Colored bool
 
 // Paint paints a given text in a given foreground/background color combination.
-func Paint(text string, fg FgColor, bg FgColor) string {
-	return fmt.Sprintf(seq, fg, bg, text, BgDefault, FgDefault)
+func Paint(text string, fg FgColor, bg BgColor) string {
+	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
 }
 
 // PaintWithAttr paints a given text in a given foreground/background/attribute combination
-func PaintWithAtt(text string, fg FgColor, bg FgColor, att Attribute) string {
-	return fmt.Sprintf(seq, fg, bg, att, text, AttReset, BgDefault, FgDefault)
+func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
+	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
 }
 
 // PaintFg paints a given text in a given foreground color.
 func PaintFg(text string, fg FgColor) string {
-	return fmt.Sprintf(seq, fg, text, FgDefault)
+	return fmt.Sprintf("%s%s%s", fg, text, FgDefault)
 }
 
 // PaintBg paints a given text in a given background color.
 func PaintBg(text string, bg BgColor) string {
-	return fmt.Sprintf(seq, bg, text, BgDefault)
+	return fmt.Sprintf("%s%s%s", bg, text, BgDefault)
 }
 
-// PaintAtt adds a given attribute to a given text, such as "bold" or "italic".
-func PaintAtt(text string, att Attribute) string {
-	return fmt.Sprintf(seq, att, text, AttReset)
+// PaintAttr adds a given attribute to a given text, such as "bold" or "italic".
+func PaintAttr(text string, attr Attribute) string {
+	return fmt.Sprintf("%s%s%s", attr, text, AttrReset)
 }
 
 // ToFgColor converts a given string (e.g. from a config file) into a foreground color code.
@@ -138,27 +132,27 @@ func ToBgColor(s string) (BgColor, error) {
 }
 
 // ToAttribute converts a given string (e.g. from a config file) into a text attribute.
-func ToAttColor(s string) (Attribute, error) {
+func ToAttribute(s string) (Attribute, error) {
 	switch strings.ToLower(s) {
 	case "bold":
-		return AttBold, nil
+		return AttrBold, nil
 	case "dim":
-		return AttDim, nil
+		return AttrDim, nil
 	case "italic":
-		return AttItalic, nil
+		return AttrItalic, nil
 	case "underline":
-		return AttUnderline, nil
+		return AttrUnderline, nil
 	case "blink":
-		return AttBlink, nil
+		return AttrBlink, nil
 	case "slowblink":
-		return AttSlowBlink, nil
+		return AttrSlowBlink, nil
 	case "rapidblink":
-		return AttRapidBlink, nil
+		return AttrRapidBlink, nil
 	case "reverse":
-		return AttReverse, nil
+		return AttrReverse, nil
 	case "hidden":
-		return AttHidden, nil
+		return AttrHidden, nil
 	default:
-		return AttReset, fmt.Errorf("unknown text attribute '" + s + "'")
+		return AttrReset, fmt.Errorf("unknown text attribute '" + s + "'")
 	}
 }
diff --git a/internal/color/color_test.go b/internal/color/color_test.go
new file mode 100644
index 0000000..0024b7f
--- /dev/null
+++ b/internal/color/color_test.go
@@ -0,0 +1,57 @@
+package color
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestColors(t *testing.T) {
+	colors := []string{
+		"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
+	}
+
+	text := " Mimecast "
+	builder := strings.Builder{}
+
+	for _, color := range colors {
+		fgColor, err := ToFgColor(color)
+		if err != nil {
+			t.Errorf("unable to paint foreground : %s\n%v", text, err)
+		}
+		builder.WriteString(PaintFg(text, fgColor))
+
+		bgColor, err := ToBgColor(color)
+		if err != nil {
+			t.Errorf("unable to paint background: %s\n%v", text, err)
+		}
+		builder.WriteString(PaintBg(text, bgColor))
+	}
+
+	for _, color := range colors {
+		fgColor, _ := ToFgColor(color)
+		for _, color := range colors {
+			bgColor, _ := ToBgColor(color)
+			builder.WriteString(Paint(text, fgColor, bgColor))
+		}
+	}
+
+	t.Log(builder.String())
+}
+func TestAttributes(t *testing.T) {
+	attributes := []string{
+		"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "hidden",
+	}
+
+	text := " Mimecast "
+	builder := strings.Builder{}
+
+	for _, attribute := range attributes {
+		att, err := ToAttribute(attribute)
+		if err != nil {
+			t.Errorf("unable to paint attribute: %s\n%v", text, err)
+		}
+		builder.WriteString(PaintWithAttr(text, FgWhite, BgBlue, att))
+	}
+
+	t.Log(builder.String())
+}
-- 
cgit v1.2.3


From f3e08046badcbc852a6875b5553ee0a648021552 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 10 Aug 2021 12:05:40 +0300
Subject: can compile with new color codes

---
 internal/color/color.go     |   4 ++
 internal/config/client.go   | 108 ++++++++++++++++++++++++++++++++++----------
 internal/version/version.go |  21 ++++++---
 3 files changed, 104 insertions(+), 29 deletions(-)

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index 965675f..f444294 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -39,6 +39,7 @@ const (
 	BgWhite   BgColor = escape + "[47m"
 	BgDefault BgColor = escape + "[49m"
 
+	AttrNone       Attribute = ""
 	AttrReset      Attribute = escape + "[0m"
 	AttrBold       Attribute = escape + "[1m"
 	AttrDim        Attribute = escape + "[2m"
@@ -61,6 +62,9 @@ func Paint(text string, fg FgColor, bg BgColor) string {
 
 // PaintWithAttr paints a given text in a given foreground/background/attribute combination
 func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
+	if attr == AttrNone {
+		return Paint(text, fg, bg)
+	}
 	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
 }
 
diff --git a/internal/config/client.go b/internal/config/client.go
index d6d106f..84ad037 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -4,18 +4,49 @@ import "github.com/mimecast/dtail/internal/color"
 
 // ClientColorConfig allows to override the default terminal color color.
 type termColors struct {
-	RemoteTextFg      color.Color
-	RemoteStatsOkBg   color.Color
-	RemoteStatsWarnBg color.Color
-	RemoteTraceBg     color.Color
-	RemoteDebugBg     color.Color
-	RemoteWarnBg      color.Color
-	RemoteErrorBg     color.Color
-	RemoteFatalBg     color.Color
-	RemoteFatalAttr   color.Attribute
-	ClientStatsBg     color.Color
-	ClientWarnFg      color.Color
-	ClientErrorFg     color.Color
+	ClientErrorAttr color.Attribute
+	ClientErrorBg   color.BgColor
+	ClientErrorFg   color.FgColor
+
+	ClientStatsAttr color.Attribute
+	ClientStatsBg   color.BgColor
+	ClientStatsFg   color.FgColor
+
+	ClientWarnAttr color.Attribute
+	ClientWarnBg   color.BgColor
+	ClientWarnFg   color.FgColor
+
+	RemoteDebugAttr color.Attribute
+	RemoteDebugBg   color.BgColor
+	RemoteDebugFg   color.FgColor
+
+	RemoteErrorAttr color.Attribute
+	RemoteErrorBg   color.BgColor
+	RemoteErrorFg   color.FgColor
+
+	RemoteFatalAttr color.Attribute
+	RemoteFatalBg   color.BgColor
+	RemoteFatalFg   color.FgColor
+
+	RemoteStatsOkAttr color.Attribute
+	RemoteStatsOkBg   color.BgColor
+	RemoteStatsOkFg   color.FgColor
+
+	RemoteStatsWarnAttr color.Attribute
+	RemoteStatsWarnBg   color.BgColor
+	RemoteStatsWarnFg   color.FgColor
+
+	RemoteTextAttr color.Attribute
+	RemoteTextBg   color.BgColor
+	RemoteTextFg   color.FgColor
+
+	RemoteTraceAttr color.Attribute
+	RemoteTraceBg   color.BgColor
+	RemoteTraceFg   color.FgColor
+
+	RemoteWarnAttr color.Attribute
+	RemoteWarnBg   color.BgColor
+	RemoteWarnFg   color.FgColor
 }
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
@@ -30,18 +61,49 @@ func newDefaultClientConfig() *ClientConfig {
 	return &ClientConfig{
 		TermColorsEnabled: true,
 		TermColors: termColors{
-			RemoteTextFg:      color.LightGray,
+			ClientErrorAttr: color.AttrBold,
+			ClientErrorBg:   color.BgBlack,
+			ClientErrorFg:   color.FgRed,
+
+			ClientStatsAttr: color.AttrDim,
+			ClientStatsBg:   color.BgBlue,
+			ClientStatsFg:   color.FgWhite,
+
+			ClientWarnAttr: color.AttrNone,
+			ClientWarnBg:   color.BgBlack,
+			ClientWarnFg:   color.FgMagenta,
+
+			RemoteDebugAttr: color.AttrNone,
+			RemoteDebugBg:   color.BgGreen,
+			RemoteDebugFg:   color.FgWhite,
+
+			RemoteErrorAttr: color.AttrBold,
+			RemoteErrorBg:   color.BgRed,
+			RemoteErrorFg:   color.FgWhite,
+
+			RemoteFatalAttr: color.AttrBlink,
+			RemoteFatalBg:   color.BgRed,
+			RemoteFatalFg:   color.FgBlue,
+
+			RemoteStatsOkAttr: color.AttrNone,
 			RemoteStatsOkBg:   color.BgGreen,
-			RemoteStatsWarnBg: color.BgRed,
-			RemoteTraceBg:     color.BgYellow,
-			RemoteDebugBg:     color.BgYellow,
-			RemoteWarnBg:      color.BgYellow,
-			RemoteErrorBg:     color.BgRed,
-			RemoteFatalBg:     color.BgRed,
-			RemoteFatalAttr:   color.Bold,
-			ClientStatsBg:     color.BgBlue,
-			ClientWarnFg:      color.Purple,
-			ClientErrorFg:     color.BgRed,
+			RemoteStatsOkFg:   color.FgWhite,
+
+			RemoteStatsWarnAttr: color.AttrNone,
+			RemoteStatsWarnBg:   color.BgRed,
+			RemoteStatsWarnFg:   color.FgWhite,
+
+			RemoteTextAttr: color.AttrNone,
+			RemoteTextBg:   color.BgBlack,
+			RemoteTextFg:   color.FgWhite,
+
+			RemoteTraceAttr: color.AttrBold,
+			RemoteTraceBg:   color.BgGreen,
+			RemoteTraceFg:   color.FgWhite,
+
+			RemoteWarnAttr: color.AttrBold,
+			RemoteWarnBg:   color.BgYellow,
+			RemoteWarnFg:   color.FgWhite,
 		},
 	}
 }
diff --git a/internal/version/version.go b/internal/version/version.go
index 7c206b4..a513fdc 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,10 +13,10 @@ const (
 	Name string = "DTail"
 	// Version of DTail.
 	Version string = "3.2.0"
-	// Additional information for DTail
-	Additional string = ""
 	// ProtocolCompat -ibility version.
 	ProtocolCompat string = "3"
+	// Additional information for DTail
+	Additional string = "Have a lot of fun!"
 )
 
 // String representation of the DTail version.
@@ -29,11 +29,20 @@ func PaintedString() string {
 	if !config.Client.TermColorsEnabled {
 		return String()
 	}
-	name := color.Paint(color.Yellow, Name)
-	version := color.Paint(color.Blue, Version)
-	descr := color.Paint(color.Green, Additional)
 
-	return fmt.Sprintf("%s %v Protocol %s %s", name, version, ProtocolCompat, descr)
+	name := color.PaintWithAttr(Name,
+		color.FgYellow, color.BgBlue, color.AttrBold)
+
+	version := color.PaintWithAttr(fmt.Sprintf(" %s ", Version),
+		color.FgBlue, color.BgYellow, color.AttrBold)
+
+	protocol := color.Paint(fmt.Sprintf(" Protocol %s ", ProtocolCompat),
+		color.FgBlack, color.BgGreen)
+
+	additional := color.PaintWithAttr(fmt.Sprintf(" %s ", Additional),
+		color.FgWhite, color.BgMagenta, color.AttrBlink)
+
+	return fmt.Sprintf("%s%v%s%s", name, version, protocol, additional)
 }
 
 // PrintAndExit prints the program version and exists.
-- 
cgit v1.2.3


From 9bb96433be61d89f8e86f6f9ded8e35f74156a63 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 11 Aug 2021 09:32:13 +0300
Subject: add colorTable option

---
 internal/color/color.go      |  4 ++++
 internal/color/color_test.go | 23 +++++++++--------------
 internal/color/table.go      | 41 +++++++++++++++++++++++++++++++++++++++++
 internal/config/client.go    |  6 +++---
 internal/version/version.go  |  2 +-
 5 files changed, 58 insertions(+), 18 deletions(-)
 create mode 100644 internal/color/table.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index f444294..a2490be 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -156,6 +156,10 @@ func ToAttribute(s string) (Attribute, error) {
 		return AttrReverse, nil
 	case "hidden":
 		return AttrHidden, nil
+	case "none":
+		fallthrough
+	case "":
+		return AttrNone, nil
 	default:
 		return AttrReset, fmt.Errorf("unknown text attribute '" + s + "'")
 	}
diff --git a/internal/color/color_test.go b/internal/color/color_test.go
index 0024b7f..16b2f90 100644
--- a/internal/color/color_test.go
+++ b/internal/color/color_test.go
@@ -6,14 +6,10 @@ import (
 )
 
 func TestColors(t *testing.T) {
-	colors := []string{
-		"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
-	}
-
 	text := " Mimecast "
 	builder := strings.Builder{}
 
-	for _, color := range colors {
+	for _, color := range ColorNames {
 		fgColor, err := ToFgColor(color)
 		if err != nil {
 			t.Errorf("unable to paint foreground : %s\n%v", text, err)
@@ -27,10 +23,13 @@ func TestColors(t *testing.T) {
 		builder.WriteString(PaintBg(text, bgColor))
 	}
 
-	for _, color := range colors {
-		fgColor, _ := ToFgColor(color)
-		for _, color := range colors {
-			bgColor, _ := ToBgColor(color)
+	for _, fg := range ColorNames {
+		fgColor, _ := ToFgColor(fg)
+		for _, bg := range ColorNames {
+			if fg == bg {
+				continue
+			}
+			bgColor, _ := ToBgColor(bg)
 			builder.WriteString(Paint(text, fgColor, bgColor))
 		}
 	}
@@ -38,14 +37,10 @@ func TestColors(t *testing.T) {
 	t.Log(builder.String())
 }
 func TestAttributes(t *testing.T) {
-	attributes := []string{
-		"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "hidden",
-	}
-
 	text := " Mimecast "
 	builder := strings.Builder{}
 
-	for _, attribute := range attributes {
+	for _, attribute := range AttributeNames {
 		att, err := ToAttribute(attribute)
 		if err != nil {
 			t.Errorf("unable to paint attribute: %s\n%v", text, err)
diff --git a/internal/color/table.go b/internal/color/table.go
new file mode 100644
index 0000000..8e40a78
--- /dev/null
+++ b/internal/color/table.go
@@ -0,0 +1,41 @@
+package color
+
+import (
+	"fmt"
+	"os"
+)
+
+var ColorNames = []string{
+	"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
+}
+
+var AttributeNames = []string{
+	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "Hidden", "None",
+}
+
+func TablePrintAndExit() {
+	for _, attr := range AttributeNames {
+		if attr == "Hidden" || attr == "SlowBlink" {
+			continue
+		}
+		printColorTable(attr)
+	}
+	os.Exit(0)
+}
+
+func printColorTable(attr string) {
+	for _, fg := range ColorNames {
+		fgColor, _ := ToFgColor(fg)
+		for _, bg := range ColorNames {
+			if fg == bg {
+				continue
+			}
+			bgColor, _ := ToBgColor(bg)
+			attribute, _ := ToAttribute(attr)
+
+			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
+			fmt.Print(PaintWithAttr(text, fgColor, bgColor, attribute))
+			fmt.Print("\n")
+		}
+	}
+}
diff --git a/internal/config/client.go b/internal/config/client.go
index 84ad037..5386576 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -75,7 +75,7 @@ func newDefaultClientConfig() *ClientConfig {
 
 			RemoteDebugAttr: color.AttrNone,
 			RemoteDebugBg:   color.BgGreen,
-			RemoteDebugFg:   color.FgWhite,
+			RemoteDebugFg:   color.FgBlack,
 
 			RemoteErrorAttr: color.AttrBold,
 			RemoteErrorBg:   color.BgRed,
@@ -83,11 +83,11 @@ func newDefaultClientConfig() *ClientConfig {
 
 			RemoteFatalAttr: color.AttrBlink,
 			RemoteFatalBg:   color.BgRed,
-			RemoteFatalFg:   color.FgBlue,
+			RemoteFatalFg:   color.FgWhite,
 
 			RemoteStatsOkAttr: color.AttrNone,
 			RemoteStatsOkBg:   color.BgGreen,
-			RemoteStatsOkFg:   color.FgWhite,
+			RemoteStatsOkFg:   color.FgBlack,
 
 			RemoteStatsWarnAttr: color.AttrNone,
 			RemoteStatsWarnBg:   color.BgRed,
diff --git a/internal/version/version.go b/internal/version/version.go
index a513fdc..7cfe12c 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -30,7 +30,7 @@ func PaintedString() string {
 		return String()
 	}
 
-	name := color.PaintWithAttr(Name,
+	name := color.PaintWithAttr(fmt.Sprintf(" %s ", Name),
 		color.FgYellow, color.BgBlue, color.AttrBold)
 
 	version := color.PaintWithAttr(fmt.Sprintf(" %s ", Version),
-- 
cgit v1.2.3


From 1e00c256842e125a865a6cc3f9aa70a1a498f6dc Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 11 Aug 2021 09:42:51 +0300
Subject: add paint.go

---
 internal/color/color.go | 31 ++++---------------------------
 internal/color/paint.go | 31 +++++++++++++++++++++++++++++++
 internal/color/table.go |  8 --------
 3 files changed, 35 insertions(+), 35 deletions(-)
 create mode 100644 internal/color/paint.go

(limited to 'internal')

diff --git a/internal/color/color.go b/internal/color/color.go
index a2490be..692de51 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -52,35 +52,12 @@ const (
 	AttrHidden     Attribute = escape + "[8m"
 )
 
-// Colored DTail client output enabled.
-var Colored bool
-
-// Paint paints a given text in a given foreground/background color combination.
-func Paint(text string, fg FgColor, bg BgColor) string {
-	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
-}
-
-// PaintWithAttr paints a given text in a given foreground/background/attribute combination
-func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
-	if attr == AttrNone {
-		return Paint(text, fg, bg)
-	}
-	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
-}
-
-// PaintFg paints a given text in a given foreground color.
-func PaintFg(text string, fg FgColor) string {
-	return fmt.Sprintf("%s%s%s", fg, text, FgDefault)
-}
-
-// PaintBg paints a given text in a given background color.
-func PaintBg(text string, bg BgColor) string {
-	return fmt.Sprintf("%s%s%s", bg, text, BgDefault)
+var ColorNames = []string{
+	"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
 }
 
-// PaintAttr adds a given attribute to a given text, such as "bold" or "italic".
-func PaintAttr(text string, attr Attribute) string {
-	return fmt.Sprintf("%s%s%s", attr, text, AttrReset)
+var AttributeNames = []string{
+	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "Hidden", "None",
 }
 
 // ToFgColor converts a given string (e.g. from a config file) into a foreground color code.
diff --git a/internal/color/paint.go b/internal/color/paint.go
new file mode 100644
index 0000000..7862467
--- /dev/null
+++ b/internal/color/paint.go
@@ -0,0 +1,31 @@
+package color
+
+import "fmt"
+
+// Paint paints a given text in a given foreground/background color combination.
+func Paint(text string, fg FgColor, bg BgColor) string {
+	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
+}
+
+// PaintWithAttr paints a given text in a given foreground/background/attribute combination
+func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
+	if attr == AttrNone {
+		return Paint(text, fg, bg)
+	}
+	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
+}
+
+// PaintFg paints a given text in a given foreground color.
+func PaintFg(text string, fg FgColor) string {
+	return fmt.Sprintf("%s%s%s", fg, text, FgDefault)
+}
+
+// PaintBg paints a given text in a given background color.
+func PaintBg(text string, bg BgColor) string {
+	return fmt.Sprintf("%s%s%s", bg, text, BgDefault)
+}
+
+// PaintAttr adds a given attribute to a given text, such as "bold" or "italic".
+func PaintAttr(text string, attr Attribute) string {
+	return fmt.Sprintf("%s%s%s", attr, text, AttrReset)
+}
diff --git a/internal/color/table.go b/internal/color/table.go
index 8e40a78..8c36047 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -5,14 +5,6 @@ import (
 	"os"
 )
 
-var ColorNames = []string{
-	"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
-}
-
-var AttributeNames = []string{
-	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "Hidden", "None",
-}
-
 func TablePrintAndExit() {
 	for _, attr := range AttributeNames {
 		if attr == "Hidden" || attr == "SlowBlink" {
-- 
cgit v1.2.3


From 3cc8887885f24a3f0d607af24197bc364ab16b8d Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Thu, 12 Aug 2021 10:56:36 +0300
Subject: add missing brush and also add color client configs plus jsonschema

---
 internal/color/brush/brush.go | 104 ++++++++++++++++++++++++++++++++++++++++++
 internal/color/color.go       |   2 +-
 internal/color/color_test.go  |   1 +
 internal/config/client.go     |   6 +--
 internal/io/logger/logger.go  |   4 +-
 internal/version/version.go   |   9 +++-
 6 files changed, 118 insertions(+), 8 deletions(-)
 create mode 100644 internal/color/brush/brush.go

(limited to 'internal')

diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
new file mode 100644
index 0000000..c5efff6
--- /dev/null
+++ b/internal/color/brush/brush.go
@@ -0,0 +1,104 @@
+package brush
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
+)
+
+// Add some color to log lines received from remote servers.
+func paintRemote(line string) string {
+	splitted := strings.Split(line, "|")
+	if splitted[2] == "100" {
+		splitted[2] = color.Paint(splitted[2],
+			config.Client.TermColors.RemoteStatsOkFg,
+			config.Client.TermColors.RemoteStatsOkBg)
+	} else {
+		splitted[2] = color.Paint(splitted[2],
+			config.Client.TermColors.RemoteStatsWarnFg,
+			config.Client.TermColors.RemoteStatsWarnBg)
+	}
+
+	info := strings.Join(splitted[0:5], "|")
+	log := strings.Join(splitted[5:], "|")
+
+	switch {
+	case strings.HasPrefix(log, "WARN"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteWarnFg,
+			config.Client.TermColors.RemoteWarnBg,
+			config.Client.TermColors.RemoteWarnAttr)
+
+	case strings.HasPrefix(log, "ERROR"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteErrorFg,
+			config.Client.TermColors.RemoteErrorBg,
+			config.Client.TermColors.RemoteErrorAttr)
+
+	case strings.HasPrefix(log, "FATAL"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteFatalFg,
+			config.Client.TermColors.RemoteFatalBg,
+			config.Client.TermColors.RemoteFatalAttr)
+
+	case strings.HasPrefix(log, "DEBUG"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteDebugFg,
+			config.Client.TermColors.RemoteDebugBg,
+			config.Client.TermColors.RemoteDebugAttr)
+
+	case strings.HasPrefix(log, "TRACE"):
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteTraceFg,
+			config.Client.TermColors.RemoteTraceBg,
+			config.Client.TermColors.RemoteTraceAttr)
+
+	default:
+		log = color.PaintWithAttr(log,
+			config.Client.TermColors.RemoteTextFg,
+			config.Client.TermColors.RemoteTextBg,
+			config.Client.TermColors.RemoteTextAttr)
+	}
+
+	return fmt.Sprintf("%s|%s", info, log)
+}
+
+// Add some color to stats generated by the client.
+func paintClientStats(line string) string {
+	splitted := strings.Split(line, "|")
+	first := strings.Join(splitted[0:4], "|")
+	connected := color.PaintWithAttr(splitted[4],
+		config.Client.TermColors.ClientStatsFg,
+		config.Client.TermColors.ClientStatsBg,
+		config.Client.TermColors.ClientStatsAttr)
+	last := strings.Join(splitted[5:], "|")
+
+	return fmt.Sprintf("%s|%s|%s", first, connected, last)
+}
+
+// Colorfy a given line based on the line's content.
+func Colorfy(line string) string {
+	switch {
+	case strings.HasPrefix(line, "REMOTE"):
+		return paintRemote(line)
+
+	case strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|"):
+		return paintClientStats(line)
+
+	case strings.Contains(line, "ERROR"):
+		return color.PaintWithAttr(line,
+			config.Client.TermColors.ClientErrorFg,
+			config.Client.TermColors.ClientErrorBg,
+			config.Client.TermColors.ClientErrorAttr)
+
+	case strings.Contains(line, "WARN"):
+		return color.PaintWithAttr(line,
+			config.Client.TermColors.ClientWarnFg,
+			config.Client.TermColors.ClientWarnBg,
+			config.Client.TermColors.ClientWarnAttr)
+	}
+
+	return line
+}
diff --git a/internal/color/color.go b/internal/color/color.go
index 692de51..5c25855 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -138,6 +138,6 @@ func ToAttribute(s string) (Attribute, error) {
 	case "":
 		return AttrNone, nil
 	default:
-		return AttrReset, fmt.Errorf("unknown text attribute '" + s + "'")
+		return AttrNone, fmt.Errorf("unknown text attribute '" + s + "'")
 	}
 }
diff --git a/internal/color/color_test.go b/internal/color/color_test.go
index 16b2f90..bfc7c54 100644
--- a/internal/color/color_test.go
+++ b/internal/color/color_test.go
@@ -36,6 +36,7 @@ func TestColors(t *testing.T) {
 
 	t.Log(builder.String())
 }
+
 func TestAttributes(t *testing.T) {
 	text := " Mimecast "
 	builder := strings.Builder{}
diff --git a/internal/config/client.go b/internal/config/client.go
index 5386576..c1b488c 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -52,14 +52,14 @@ type termColors struct {
 // ClientConfig represents a DTail client configuration (empty as of now as there
 // are no available config options yet, but that may changes in the future).
 type ClientConfig struct {
-	TermColorsEnabled bool
-	TermColors        termColors `json:",omitempty"`
+	TermColorsEnable bool
+	TermColors       termColors `json:",omitempty"`
 }
 
 // Create a new default client configuration.
 func newDefaultClientConfig() *ClientConfig {
 	return &ClientConfig{
-		TermColorsEnabled: true,
+		TermColorsEnable: true,
 		TermColors: termColors{
 			ClientErrorAttr: color.AttrBold,
 			ClientErrorBg:   color.BgBlack,
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index bef5293..bb9dc02 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -207,7 +207,7 @@ func write(what, severity, message string) {
 	if Mode.logToStdout {
 		line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message)
 
-		if config.Client.TermColorsEnabled {
+		if config.Client.TermColorsEnable {
 			line = brush.Colorfy(line)
 		}
 
@@ -262,7 +262,7 @@ func Raw(message string) {
 	}
 
 	if Mode.logToStdout {
-		if config.Client.TermColorsEnabled {
+		if config.Client.TermColorsEnable {
 			message = brush.Colorfy(message)
 		}
 		stdoutBufCh <- message
diff --git a/internal/version/version.go b/internal/version/version.go
index 7cfe12c..3e683bd 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -26,7 +26,7 @@ func String() string {
 
 // PaintedString is a prettier string representation of the DTail version.
 func PaintedString() string {
-	if !config.Client.TermColorsEnabled {
+	if !config.Client.TermColorsEnable {
 		return String()
 	}
 
@@ -45,8 +45,13 @@ func PaintedString() string {
 	return fmt.Sprintf("%s%v%s%s", name, version, protocol, additional)
 }
 
+// Print the version.
+func Print() {
+	fmt.Println(PaintedString())
+}
+
 // PrintAndExit prints the program version and exists.
 func PrintAndExit() {
-	fmt.Println(PaintedString())
+	Print()
 	os.Exit(0)
 }
-- 
cgit v1.2.3


From c2522ffb59514443816a96386c16bb7527cbe57c Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 21 Aug 2021 14:54:24 +0300
Subject: read files bytewise for more control of whats happening - change
 transport protocol for more control over newlines

---
 internal/clients/handlers/basehandler.go  |  6 +--
 internal/clients/handlers/maprhandler.go  |  3 +-
 internal/io/fs/readfile.go                | 71 ++++++++++++-------------------
 internal/io/logger/logger.go              |  7 ++-
 internal/mapr/aggregateset.go             | 10 +++--
 internal/protocol/protocol.go             | 10 +++++
 internal/server/handlers/serverhandler.go | 15 ++++---
 internal/version/version.go               |  7 ++-
 8 files changed, 66 insertions(+), 63 deletions(-)
 create mode 100644 internal/protocol/protocol.go

(limited to 'internal')

diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index f07fd90..acafe0e 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/version"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 type baseHandler struct {
@@ -43,7 +43,7 @@ func (h *baseHandler) SendMessage(command string) error {
 	logger.Debug("Sending command", h.server, command, encoded)
 
 	select {
-	case h.commands <- fmt.Sprintf("protocol %s base64 %v;", version.ProtocolCompat, encoded):
+	case h.commands <- fmt.Sprintf("protocol %s base64 %v;", protocol.ProtocolCompat, encoded):
 	case <-time.After(time.Second * 5):
 		return fmt.Errorf("Timed out sending command '%s' (base64: '%s')", command, encoded)
 	case <-h.Done():
@@ -57,7 +57,7 @@ func (h *baseHandler) SendMessage(command string) error {
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		h.receiveBuf = append(h.receiveBuf, b)
-		if b == '\n' {
+		if b == protocol.MessageDelimiter {
 			if len(h.receiveBuf) == 0 {
 				continue
 			}
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index fb71c8f..7ac5895 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -7,6 +7,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/client"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // MaprHandler is the handler used on the client side for running mapreduce aggregations.
@@ -58,7 +59,7 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 // related data.
 func (h *MaprHandler) handleAggregateMessage(message string) {
 	h.count++
-	parts := strings.Split(message, "➔")
+	parts := strings.Split(message, protocol.AggregateDelimiter)
 
 	// Index 0 contains 'AGGREGATE', 1 contains server host.
 	// Aggregation data begins from index 2.
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 6757bd6..8a365a1 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/protocol"
 	"github.com/mimecast/dtail/internal/regex"
 
 	"github.com/DataDog/zstd"
@@ -148,80 +149,64 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan []byte, t
 	if err != nil {
 		return err
 	}
-	rawLine := make([]byte, 0, 512)
 
 	lineLengthThreshold := 1024 * 1024 // 1mb
-	longLineWarning := false
+	warnedAboutLongLine := false
+	message := make([]byte, 0, 512)
 
 	for {
 		select {
 		case <-ctx.Done():
 			return nil
-		default:
-		}
-
-		select {
 		case <-truncate:
 			if isTruncated, err := f.truncated(fd); isTruncated {
 				return err
 			}
-			logger.Info(f.filePath, "Current offset", offset)
 		default:
 		}
 
-		// Read some bytes (max 4k at once as of go 1.12). isPrefix will
-		// be set if line does not fit into 4k buffer.
-		bytes, isPrefix, err := reader.ReadLine()
+		b, err := reader.ReadByte()
 
 		if err != nil {
-			// If EOF, sleep a couple of ms and return with nil error.
-			// If other error, return with non-nil error.
 			if err != io.EOF {
 				return err
 			}
 			if !f.seekEOF {
-				logger.Debug(f.FilePath(), "End of file reached")
+				logger.Info(f.FilePath(), "End of file reached")
 				return nil
 			}
 			time.Sleep(time.Millisecond * 100)
 			continue
 		}
+		offset++
 
-		rawLine = append(rawLine, bytes...)
-		offset += uint64(len(bytes))
-
-		if !isPrefix {
-			// last LineRead call returned contend until end of line.
-			rawLine = append(rawLine, '\n')
-			select {
-			case rawLines <- rawLine:
-			case <-ctx.Done():
-				return nil
+		switch b {
+		case '\n':
+			if len(message) == 0 {
+				time.Sleep(time.Millisecond * 100)
+				continue
 			}
-			rawLine = make([]byte, 0, 512)
-			if longLineWarning {
-				longLineWarning = false
-			}
-			continue
-		}
-
-		// Last LineRead call could not read content until end of line, buffer
-		// was too small. Determine whether we exceed the max line length we
-		// want dtail to send to the client at once. Possibly split up log line
-		// into multiple log lines.
-		if len(rawLine) >= lineLengthThreshold {
-			if !longLineWarning {
-				f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
-				// Only print out one warning per long log line.
-				longLineWarning = true
-			}
-			rawLine = append(rawLine, '\n')
 			select {
-			case rawLines <- rawLine:
+			case rawLines <- append(message, protocol.MessageDelimiter):
+				message = make([]byte, 0, 512)
+				warnedAboutLongLine = false
 			case <-ctx.Done():
 				return nil
 			}
-			rawLine = make([]byte, 0, 512)
+		default:
+			if len(message) >= lineLengthThreshold {
+				if !warnedAboutLongLine {
+					f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
+					warnedAboutLongLine = true
+				}
+				select {
+				case <-ctx.Done():
+					return nil
+				case rawLines <- append(message, protocol.MessageDelimiter):
+					message = make([]byte, 0, 512)
+				}
+			}
+			message = append(message, b)
 		}
 	}
 }
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index bb9dc02..3a3935d 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -205,7 +205,7 @@ func Trace(args ...interface{}) string {
 // Write log line to buffer and/or log file.
 func write(what, severity, message string) {
 	if Mode.logToStdout {
-		line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message)
+		line := fmt.Sprintf("%s|%s|%s|%s", what, hostname, severity, message)
 
 		if config.Client.TermColorsEnable {
 			line = brush.Colorfy(line)
@@ -219,7 +219,7 @@ func write(what, severity, message string) {
 		timeStr := t.Format("20060102-150405")
 		fileLogBufCh <- buf{
 			time:    t,
-			message: fmt.Sprintf("%s|%s|%s|%s\n", severity, timeStr, what, message),
+			message: fmt.Sprintf("%s|%s|%s|%s", severity, timeStr, what, message),
 		}
 	}
 }
@@ -326,6 +326,7 @@ func Flush() {
 		select {
 		case message := <-stdoutBufCh:
 			stdoutWriter.WriteString(message)
+			stdoutWriter.WriteString("\n")
 		default:
 			stdoutWriter.Flush()
 			return
@@ -338,6 +339,7 @@ func writeToStdout(ctx context.Context) {
 		select {
 		case message := <-stdoutBufCh:
 			stdoutWriter.WriteString(message)
+			stdoutWriter.WriteString("\n")
 		case <-time.After(time.Millisecond * 100):
 			stdoutWriter.Flush()
 		case <-pauseCh:
@@ -365,6 +367,7 @@ func writeToFile(ctx context.Context) {
 			dateStr := buf.time.Format("20060102")
 			w := fileWriter(dateStr)
 			w.WriteString(buf.message)
+			w.WriteString("\n")
 		case <-pauseCh:
 		PAUSE:
 			for {
diff --git a/internal/mapr/aggregateset.go b/internal/mapr/aggregateset.go
index a6cc6eb..029d87b 100644
--- a/internal/mapr/aggregateset.go
+++ b/internal/mapr/aggregateset.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // AggregateSet represents aggregated key/value pairs from the
@@ -70,20 +72,20 @@ func (s *AggregateSet) Serialize(ctx context.Context, groupKey string, ch chan<-
 	var sb strings.Builder
 
 	sb.WriteString(groupKey)
-	sb.WriteString("➔")
-	sb.WriteString(fmt.Sprintf("%d➔", s.Samples))
+	sb.WriteString(protocol.AggregateDelimiter)
+	sb.WriteString(fmt.Sprintf("%d%s", s.Samples, protocol.AggregateDelimiter))
 
 	for k, v := range s.FValues {
 		sb.WriteString(k)
 		sb.WriteString("=")
-		sb.WriteString(fmt.Sprintf("%v➔", v))
+		sb.WriteString(fmt.Sprintf("%v%s", v, protocol.AggregateDelimiter))
 	}
 
 	for k, v := range s.SValues {
 		sb.WriteString(k)
 		sb.WriteString("=")
 		sb.WriteString(v)
-		sb.WriteString("➔")
+		sb.WriteString(protocol.AggregateDelimiter)
 	}
 
 	select {
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
new file mode 100644
index 0000000..2a570cd
--- /dev/null
+++ b/internal/protocol/protocol.go
@@ -0,0 +1,10 @@
+package protocol
+
+const (
+	// ProtocolCompat -ibility version
+	ProtocolCompat string = "4"
+	// MessageDelimiter delimits separate messages.
+	MessageDelimiter byte = '¬'
+	// AggregateDelimiter delimits parts of an aggregation message.
+	AggregateDelimiter string = "➔"
+)
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 185e7c2..23e3aeb 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -17,8 +17,8 @@ import (
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
+	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
-	"github.com/mimecast/dtail/internal/version"
 )
 
 const (
@@ -92,24 +92,27 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			}
 			if message[0] == '.' {
 				// Handle hidden message (don't display to the user, interpreted by dtail client)
-				wholePayload := []byte(fmt.Sprintf("%s\n", message))
+				wholePayload := []byte(fmt.Sprintf("%s%b", message, protocol.MessageDelimiter))
 				n = copy(p, wholePayload)
 				return
 			}
 
 			// Handle normal server message (display to the user)
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
+			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%b", h.hostname, message, protocol.MessageDelimiter))
 			n = copy(p, wholePayload)
 			return
 
 		case message := <-h.aggregatedMessages:
 			// Send mapreduce-aggregated data as a message.
-			data := fmt.Sprintf("AGGREGATE➔%s➔%s\n", h.hostname, message)
+			data := fmt.Sprintf("AGGREGATE%s%s%s%s%b",
+				protocol.AggregateDelimiter, h.hostname,
+				protocol.AggregateDelimiter, message, protocol.MessageDelimiter)
 			wholePayload := []byte(data)
 			n = copy(p, wholePayload)
 			return
 
 		case line := <-h.lines:
+			//fmt.Printf("<<<%d,%s>>>\n", len(line.Content), line.Content)
 			// Send normal file content data as a message.
 			serverInfo := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|",
 				h.hostname, line.TransmittedPerc, line.Count, line.SourceID))
@@ -182,8 +185,8 @@ func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, err
 		return args, argc, errors.New("unable to determine protocol version")
 	}
 
-	if args[1] != version.ProtocolCompat {
-		err := fmt.Errorf("server with protocol version '%s' but client with '%s', please update DTail", version.ProtocolCompat, args[1])
+	if args[1] != protocol.ProtocolCompat {
+		err := fmt.Errorf("server with protocol version '%s' but client with '%s', please update DTail", protocol.ProtocolCompat, args[1])
 		return args, argc, err
 	}
 
diff --git a/internal/version/version.go b/internal/version/version.go
index 3e683bd..7f07c83 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 const (
@@ -13,15 +14,13 @@ const (
 	Name string = "DTail"
 	// Version of DTail.
 	Version string = "3.2.0"
-	// ProtocolCompat -ibility version.
-	ProtocolCompat string = "3"
 	// Additional information for DTail
 	Additional string = "Have a lot of fun!"
 )
 
 // String representation of the DTail version.
 func String() string {
-	return fmt.Sprintf("%s %v Protocol %s %s", Name, Version, ProtocolCompat, Additional)
+	return fmt.Sprintf("%s %v Protocol %s %s", Name, Version, protocol.ProtocolCompat, Additional)
 }
 
 // PaintedString is a prettier string representation of the DTail version.
@@ -36,7 +35,7 @@ func PaintedString() string {
 	version := color.PaintWithAttr(fmt.Sprintf(" %s ", Version),
 		color.FgBlue, color.BgYellow, color.AttrBold)
 
-	protocol := color.Paint(fmt.Sprintf(" Protocol %s ", ProtocolCompat),
+	protocol := color.Paint(fmt.Sprintf(" Protocol %s ", protocol.ProtocolCompat),
 		color.FgBlack, color.BgGreen)
 
 	additional := color.PaintWithAttr(fmt.Sprintf(" %s ", Additional),
-- 
cgit v1.2.3


From 9883a190109623b64e6d311dc2b462a6eae68003 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 22 Aug 2021 10:07:00 +0300
Subject: introduces the protocol package

---
 internal/clients/handlers/basehandler.go   | 3 ++-
 internal/clients/handlers/healthhandler.go | 3 ++-
 internal/clients/handlers/maprhandler.go   | 2 +-
 internal/clients/maprclient.go             | 3 ++-
 internal/server/handlers/controlhandler.go | 3 ++-
 internal/server/handlers/serverhandler.go  | 6 +++---
 internal/ssh/ssh.go                        | 3 ++-
 7 files changed, 14 insertions(+), 9 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index acafe0e..fe83faa 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -56,13 +56,14 @@ func (h *baseHandler) SendMessage(command string) error {
 // Read data from the dtail server via Writer interface.
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		h.receiveBuf = append(h.receiveBuf, b)
 		if b == protocol.MessageDelimiter {
 			if len(h.receiveBuf) == 0 {
 				continue
 			}
 			message := string(h.receiveBuf)
 			h.handleMessageType(message)
+		} else {
+			h.receiveBuf = append(h.receiveBuf, b)
 		}
 	}
 
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 0440706..213748c 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // HealthHandler implements the handler required for health checks.
@@ -72,7 +73,7 @@ func (h *HealthHandler) SendMessage(command string) error {
 func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		h.receiveBuf = append(h.receiveBuf, b)
-		if b == '\n' {
+		if b == protocol.MessageDelimiter { // '\n' {
 			h.receive <- string(h.receiveBuf)
 			h.receiveBuf = h.receiveBuf[:0]
 		}
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index 7ac5895..afad507 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -37,7 +37,7 @@ func NewMaprHandler(server string, query *mapr.Query, globalGroup *mapr.GlobalGr
 func (h *MaprHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		h.baseHandler.receiveBuf = append(h.baseHandler.receiveBuf, b)
-		if b == '\n' {
+		if b == protocol.MessageDelimiter { // '\n' {
 			if len(h.baseHandler.receiveBuf) == 0 {
 				continue
 			}
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 1c0c2cc..77b674b 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -170,7 +170,8 @@ func (c *MaprClient) printResults() {
 		return
 	}
 
-	logger.Raw(fmt.Sprintf("%s\n", c.query.RawQuery))
+	//logger.Raw(fmt.Sprintf("%s\n", c.query.RawQuery))
+	logger.Raw(c.query.RawQuery)
 	logger.Raw(result)
 }
 
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
index 1e17c78..a217b40 100644
--- a/internal/server/handlers/controlhandler.go
+++ b/internal/server/handlers/controlhandler.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
@@ -56,7 +57,7 @@ func (h *ControlHandler) Read(p []byte) (n int, err error) {
 	for {
 		select {
 		case message := <-h.serverMessages:
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
+			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%b", h.hostname, message, protocol.MessageDelimiter))
 			n = copy(p, wholePayload)
 			return
 		case <-h.done.Done():
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 23e3aeb..9541a34 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -92,13 +92,13 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			}
 			if message[0] == '.' {
 				// Handle hidden message (don't display to the user, interpreted by dtail client)
-				wholePayload := []byte(fmt.Sprintf("%s%b", message, protocol.MessageDelimiter))
+				wholePayload := []byte(fmt.Sprintf("%s%s", message, string(protocol.MessageDelimiter)))
 				n = copy(p, wholePayload)
 				return
 			}
 
 			// Handle normal server message (display to the user)
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%b", h.hostname, message, protocol.MessageDelimiter))
+			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%s", h.hostname, message, string(protocol.MessageDelimiter)))
 			n = copy(p, wholePayload)
 			return
 
@@ -112,7 +112,7 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			return
 
 		case line := <-h.lines:
-			//fmt.Printf("<<<%d,%s>>>\n", len(line.Content), line.Content)
+			//fmt.Printf("\t<<<%d,%s>>>\n", len(line.Content), line.Content)
 			// Send normal file content data as a message.
 			serverInfo := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|",
 				h.hostname, line.TransmittedPerc, line.Count, line.SourceID))
diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go
index 3a2e416..78bf99e 100644
--- a/internal/ssh/ssh.go
+++ b/internal/ssh/ssh.go
@@ -6,12 +6,13 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"github.com/mimecast/dtail/internal/io/logger"
 	"io/ioutil"
 	"net"
 	"os"
 	"syscall"
 
+	"github.com/mimecast/dtail/internal/io/logger"
+
 	gossh "golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/crypto/ssh/terminal"
-- 
cgit v1.2.3


From 6d727b9bdbc387c8a5c34406a2c4de9140face38 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 28 Aug 2021 19:36:46 +0100
Subject: use a byte.Buffer in the file reader

---
 internal/io/fs/readfile.go                | 32 ++++++++++++++++++-------------
 internal/io/line/line.go                  |  5 +++--
 internal/io/pool/bytesbuffer.go           | 19 ++++++++++++++++++
 internal/mapr/server/aggregate.go         |  4 +++-
 internal/server/handlers/serverhandler.go |  9 +++++----
 5 files changed, 49 insertions(+), 20 deletions(-)
 create mode 100644 internal/io/pool/bytesbuffer.go

(limited to 'internal')

diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 8a365a1..e44f30e 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -2,6 +2,7 @@ package fs
 
 import (
 	"bufio"
+	"bytes"
 	"compress/gzip"
 	"context"
 	"errors"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 	"github.com/mimecast/dtail/internal/regex"
 
@@ -90,7 +92,7 @@ func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Re
 		fd.Seek(0, io.SeekEnd)
 	}
 
-	rawLines := make(chan []byte, 100)
+	rawLines := make(chan *bytes.Buffer, 100)
 	truncate := make(chan struct{})
 
 	var wg sync.WaitGroup
@@ -142,7 +144,7 @@ func (f readFile) makeReader(fd *os.File) (reader *bufio.Reader, err error) {
 	return
 }
 
-func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan []byte, truncate <-chan struct{}) error {
+func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Buffer, truncate <-chan struct{}) error {
 	var offset uint64
 
 	reader, err := f.makeReader(fd)
@@ -152,7 +154,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan []byte, t
 
 	lineLengthThreshold := 1024 * 1024 // 1mb
 	warnedAboutLongLine := false
-	message := make([]byte, 0, 512)
+	message := pool.BytesBuffer.Get().(*bytes.Buffer)
 
 	for {
 		select {
@@ -182,37 +184,41 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan []byte, t
 
 		switch b {
 		case '\n':
-			if len(message) == 0 {
+			if message.Len() == 0 {
 				time.Sleep(time.Millisecond * 100)
 				continue
 			}
+			message.WriteByte(protocol.MessageDelimiter)
 			select {
-			case rawLines <- append(message, protocol.MessageDelimiter):
-				message = make([]byte, 0, 512)
+			case rawLines <- message:
+				message = pool.BytesBuffer.Get().(*bytes.Buffer)
+				//fmt.Printf("%d %d %p\n", message.Len(), message.Cap(), message)
 				warnedAboutLongLine = false
 			case <-ctx.Done():
 				return nil
 			}
 		default:
-			if len(message) >= lineLengthThreshold {
+			if message.Len() >= lineLengthThreshold {
 				if !warnedAboutLongLine {
 					f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
+				message.WriteByte(protocol.MessageDelimiter)
 				select {
+				case rawLines <- message:
+					message = pool.BytesBuffer.Get().(*bytes.Buffer)
+					//fmt.Printf("%d %d %p\n", message.Len(), message.Cap(), message)
 				case <-ctx.Done():
 					return nil
-				case rawLines <- append(message, protocol.MessageDelimiter):
-					message = make([]byte, 0, 512)
 				}
 			}
-			message = append(message, b)
+			message.WriteByte(b)
 		}
 	}
 }
 
 // Filter log lines matching a given regular expression.
-func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-chan []byte, lines chan<- line.Line, re regex.Regex) {
+func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
 	defer wg.Done()
 
 	for {
@@ -233,10 +239,10 @@ func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-cha
 	}
 }
 
-func (f readFile) transmittable(lineBytes []byte, length, capacity int, re regex.Regex) (line.Line, bool) {
+func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int, re regex.Regex) (line.Line, bool) {
 	var read line.Line
 
-	if !re.Match(lineBytes) {
+	if !re.Match(lineBytes.Bytes()) {
 		f.updateLineNotMatched()
 		f.updateLineNotTransmitted()
 		return read, false
diff --git a/internal/io/line/line.go b/internal/io/line/line.go
index 715be34..d306c88 100644
--- a/internal/io/line/line.go
+++ b/internal/io/line/line.go
@@ -1,13 +1,14 @@
 package line
 
 import (
+	"bytes"
 	"fmt"
 )
 
 // Line represents a read log line.
 type Line struct {
 	// The content of the log line.
-	Content []byte
+	Content *bytes.Buffer
 	// Until now, how many log lines were processed?
 	Count uint64
 	// Sometimes we produce too many log lines so that the client
@@ -25,7 +26,7 @@ type Line struct {
 // Return a human readable representation of the followed line.
 func (l Line) String() string {
 	return fmt.Sprintf("Line(Content:%s,TransmittedPerc:%v,Count:%v,SourceID:%s)",
-		string(l.Content),
+		l.Content.String(),
 		l.TransmittedPerc,
 		l.Count,
 		l.SourceID)
diff --git a/internal/io/pool/bytesbuffer.go b/internal/io/pool/bytesbuffer.go
new file mode 100644
index 0000000..0a159f5
--- /dev/null
+++ b/internal/io/pool/bytesbuffer.go
@@ -0,0 +1,19 @@
+package pool
+
+import (
+	"bytes"
+	"sync"
+)
+
+var BytesBuffer = sync.Pool{
+	New: func() interface{} {
+		b := bytes.Buffer{}
+		b.Grow(128)
+		return &b
+	},
+}
+
+func RecycleBytesBuffer(b *bytes.Buffer) {
+	b.Reset()
+	BytesBuffer.Put(b)
+}
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 28bb074..9106f52 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -10,6 +10,7 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/logformat"
 )
@@ -136,7 +137,8 @@ func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 					return
 				}
 
-				maprLine := strings.TrimSpace(string(line.Content))
+				maprLine := strings.TrimSpace(line.Content.String())
+				pool.RecycleBytesBuffer(line.Content)
 				fields, err := a.parser.MakeFields(maprLine)
 				logger.Debug(fields, err)
 
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 9541a34..62f3c2b 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -15,6 +15,7 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/protocol"
@@ -114,10 +115,10 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		case line := <-h.lines:
 			//fmt.Printf("\t<<<%d,%s>>>\n", len(line.Content), line.Content)
 			// Send normal file content data as a message.
-			serverInfo := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|",
-				h.hostname, line.TransmittedPerc, line.Count, line.SourceID))
-			wholePayload := append(serverInfo, line.Content[:]...)
-			n = copy(p, wholePayload)
+			payload := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|%s",
+				h.hostname, line.TransmittedPerc, line.Count, line.SourceID, line.Content.String()))
+			n = copy(p, payload)
+			pool.RecycleBytesBuffer(line.Content)
 			return
 
 		case <-time.After(time.Second):
-- 
cgit v1.2.3


From 8c2e94030d0e31289c35fcfb56499707fd4a7ccd Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 28 Aug 2021 20:10:54 +0100
Subject: make use of more buffers on server side

---
 internal/protocol/protocol.go             |  2 ++
 internal/server/handlers/serverhandler.go | 40 +++++++++++++++++++++----------
 2 files changed, 29 insertions(+), 13 deletions(-)

(limited to 'internal')

diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index 2a570cd..43021a2 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -5,6 +5,8 @@ const (
 	ProtocolCompat string = "4"
 	// MessageDelimiter delimits separate messages.
 	MessageDelimiter byte = '¬'
+	// FieldDelimiter delimits aggregation fields.
+	FieldDelimiter byte = '|'
 	// AggregateDelimiter delimits parts of an aggregation message.
 	AggregateDelimiter string = "➔"
 )
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 62f3c2b..f5aefa2 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"bytes"
 	"context"
 	"encoding/base64"
 	"errors"
@@ -93,31 +94,44 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			}
 			if message[0] == '.' {
 				// Handle hidden message (don't display to the user, interpreted by dtail client)
-				wholePayload := []byte(fmt.Sprintf("%s%s", message, string(protocol.MessageDelimiter)))
-				n = copy(p, wholePayload)
+				payload := []byte(fmt.Sprintf("%s%s", message, string(protocol.MessageDelimiter)))
+				n = copy(p, payload)
 				return
 			}
 
 			// Handle normal server message (display to the user)
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%s", h.hostname, message, string(protocol.MessageDelimiter)))
-			n = copy(p, wholePayload)
+			payload := []byte(fmt.Sprintf("SERVER|%s|%s%s", h.hostname, message, string(protocol.MessageDelimiter)))
+			n = copy(p, payload)
 			return
 
 		case message := <-h.aggregatedMessages:
 			// Send mapreduce-aggregated data as a message.
-			data := fmt.Sprintf("AGGREGATE%s%s%s%s%b",
-				protocol.AggregateDelimiter, h.hostname,
-				protocol.AggregateDelimiter, message, protocol.MessageDelimiter)
-			wholePayload := []byte(data)
-			n = copy(p, wholePayload)
+			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
+			buf.WriteString("AGGREGATE")
+			buf.WriteString(protocol.AggregateDelimiter)
+			buf.WriteString(h.hostname)
+			buf.WriteString(protocol.AggregateDelimiter)
+			buf.WriteString(message)
+			buf.WriteByte(protocol.MessageDelimiter)
+			n = copy(p, buf.Bytes())
+			pool.RecycleBytesBuffer(buf)
 			return
 
 		case line := <-h.lines:
-			//fmt.Printf("\t<<<%d,%s>>>\n", len(line.Content), line.Content)
-			// Send normal file content data as a message.
-			payload := []byte(fmt.Sprintf("REMOTE|%s|%3d|%v|%s|%s",
-				h.hostname, line.TransmittedPerc, line.Count, line.SourceID, line.Content.String()))
+			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
+			buf.WriteString("REMOTE")
+			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(h.hostname)
+			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
+			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(fmt.Sprintf("%v", line.Count))
+			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(line.SourceID)
+			buf.WriteByte(protocol.FieldDelimiter)
+			payload := append(buf.Bytes(), line.Content.Bytes()...)
 			n = copy(p, payload)
+			pool.RecycleBytesBuffer(buf)
 			pool.RecycleBytesBuffer(line.Content)
 			return
 
-- 
cgit v1.2.3


From 23982f331c2154a66b86d596226c24454fd06be5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 28 Aug 2021 20:26:32 +0100
Subject: 1. Major performance gain by not checking for file truncation aftter 
   each bytes read. 2. Introduce field separator to the protocol package.

---
 internal/clients/healthclient.go          |  3 ++-
 internal/color/brush/brush.go             | 11 ++++++-----
 internal/io/fs/readfile.go                | 20 +++++++++-----------
 internal/mapr/logformat/default.go        |  4 +++-
 internal/protocol/protocol.go             |  2 +-
 internal/server/handlers/serverhandler.go | 10 +++++-----
 6 files changed, 26 insertions(+), 24 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index e93f6be..692464c 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -11,6 +11,7 @@ import (
 	"github.com/mimecast/dtail/internal/clients/remote"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/omode"
+	"github.com/mimecast/dtail/internal/protocol"
 
 	gossh "golang.org/x/crypto/ssh"
 )
@@ -57,7 +58,7 @@ func (c *HealthClient) Start(ctx context.Context) (status int) {
 		select {
 		case data := <-receive:
 			// Parse recieved data.
-			s := strings.Split(data, "|")
+			s := strings.Split(data, protocol.FieldDelimiter)
 			message := s[len(s)-1]
 			if strings.HasPrefix(message, "done;") {
 				return
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index c5efff6..87c02d0 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -6,11 +6,12 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // Add some color to log lines received from remote servers.
 func paintRemote(line string) string {
-	splitted := strings.Split(line, "|")
+	splitted := strings.Split(line, protocol.FieldDelimiter)
 	if splitted[2] == "100" {
 		splitted[2] = color.Paint(splitted[2],
 			config.Client.TermColors.RemoteStatsOkFg,
@@ -21,8 +22,8 @@ func paintRemote(line string) string {
 			config.Client.TermColors.RemoteStatsWarnBg)
 	}
 
-	info := strings.Join(splitted[0:5], "|")
-	log := strings.Join(splitted[5:], "|")
+	info := strings.Join(splitted[0:5], protocol.FieldDelimiter)
+	log := strings.Join(splitted[5:], protocol.FieldDelimiter)
 
 	switch {
 	case strings.HasPrefix(log, "WARN"):
@@ -67,8 +68,8 @@ func paintRemote(line string) string {
 
 // Add some color to stats generated by the client.
 func paintClientStats(line string) string {
-	splitted := strings.Split(line, "|")
-	first := strings.Join(splitted[0:4], "|")
+	splitted := strings.Split(line, protocol.FieldDelimiter)
+	first := strings.Join(splitted[0:4], protocol.FieldDelimiter)
 	connected := color.PaintWithAttr(splitted[4],
 		config.Client.TermColors.ClientStatsFg,
 		config.Client.TermColors.ClientStatsBg,
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index e44f30e..f2f672a 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -157,22 +157,21 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 	message := pool.BytesBuffer.Get().(*bytes.Buffer)
 
 	for {
-		select {
-		case <-ctx.Done():
-			return nil
-		case <-truncate:
-			if isTruncated, err := f.truncated(fd); isTruncated {
-				return err
-			}
-		default:
-		}
-
 		b, err := reader.ReadByte()
 
 		if err != nil {
 			if err != io.EOF {
 				return err
 			}
+			select {
+			case <-truncate:
+				if isTruncated, err := f.truncated(fd); isTruncated {
+					return err
+				}
+			case <-ctx.Done():
+				return nil
+			default:
+			}
 			if !f.seekEOF {
 				logger.Info(f.FilePath(), "End of file reached")
 				return nil
@@ -207,7 +206,6 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 				select {
 				case rawLines <- message:
 					message = pool.BytesBuffer.Get().(*bytes.Buffer)
-					//fmt.Printf("%d %d %p\n", message.Len(), message.Cap(), message)
 				case <-ctx.Done():
 					return nil
 				}
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 44bf558..32a34bd 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -3,12 +3,14 @@ package logformat
 import (
 	"errors"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // MakeFieldsDEFAULT is the default log file mapreduce parser.
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields := make(map[string]string, 20)
-	splitted := strings.Split(maprLine, "|")
+	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
 
 	fields["*"] = "*"
 	fields["$line"] = maprLine
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index 43021a2..d3035e4 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -6,7 +6,7 @@ const (
 	// MessageDelimiter delimits separate messages.
 	MessageDelimiter byte = '¬'
 	// FieldDelimiter delimits aggregation fields.
-	FieldDelimiter byte = '|'
+	FieldDelimiter string = "|"
 	// AggregateDelimiter delimits parts of an aggregation message.
 	AggregateDelimiter string = "➔"
 )
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index f5aefa2..14fc5d0 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -120,15 +120,15 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		case line := <-h.lines:
 			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
 			buf.WriteString("REMOTE")
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			buf.WriteString(h.hostname)
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			buf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			buf.WriteString(fmt.Sprintf("%v", line.Count))
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			buf.WriteString(line.SourceID)
-			buf.WriteByte(protocol.FieldDelimiter)
+			buf.WriteString(protocol.FieldDelimiter)
 			payload := append(buf.Bytes(), line.Content.Bytes()...)
 			n = copy(p, payload)
 			pool.RecycleBytesBuffer(buf)
-- 
cgit v1.2.3


From bfcb0f159a4835cc6a0326ff46de7934e0363aed Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 5 Sep 2021 15:51:26 +0300
Subject: finalize new default color schema

---
 internal/color/brush/brush.go | 110 ++++++++++++++++++----------
 internal/color/paint.go       |  59 +++++++++++----
 internal/color/table.go       |   2 +-
 internal/config/client.go     | 164 +++++++++++++++++++++---------------------
 internal/protocol/protocol.go |   4 +-
 internal/version/version.go   |   8 +--
 6 files changed, 212 insertions(+), 135 deletions(-)

(limited to 'internal')

diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 87c02d0..415028a 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -1,7 +1,6 @@
 package brush
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/mimecast/dtail/internal/color"
@@ -10,96 +9,133 @@ import (
 )
 
 // Add some color to log lines received from remote servers.
-func paintRemote(line string) string {
-	splitted := strings.Split(line, protocol.FieldDelimiter)
+func paintRemote(sb *strings.Builder, line string) {
+	splitted := strings.SplitN(line, protocol.FieldDelimiter, 6)
+
+	color.PaintWithAttr(sb, splitted[0],
+		config.Client.TermColors.RemoteStrFg,
+		config.Client.TermColors.RemoteStrBg,
+		config.Client.TermColors.RemoteStrAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[1],
+		config.Client.TermColors.RemoteServerFg,
+		config.Client.TermColors.RemoteServerBg,
+		config.Client.TermColors.RemoteServerAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
 	if splitted[2] == "100" {
-		splitted[2] = color.Paint(splitted[2],
+		color.PaintWithAttr(sb, splitted[2],
 			config.Client.TermColors.RemoteStatsOkFg,
-			config.Client.TermColors.RemoteStatsOkBg)
+			config.Client.TermColors.RemoteStatsOkBg,
+			config.Client.TermColors.RemoteStatsOkAttr)
 	} else {
-		splitted[2] = color.Paint(splitted[2],
+		color.PaintWithAttr(sb, splitted[2],
 			config.Client.TermColors.RemoteStatsWarnFg,
-			config.Client.TermColors.RemoteStatsWarnBg)
+			config.Client.TermColors.RemoteStatsWarnBg,
+			config.Client.TermColors.RemoteStatsWarnAttr)
 	}
 
-	info := strings.Join(splitted[0:5], protocol.FieldDelimiter)
-	log := strings.Join(splitted[5:], protocol.FieldDelimiter)
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[3],
+		config.Client.TermColors.RemoteCountFg,
+		config.Client.TermColors.RemoteCountBg,
+		config.Client.TermColors.RemoteCountAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[4],
+		config.Client.TermColors.RemoteIdFg,
+		config.Client.TermColors.RemoteIdBg,
+		config.Client.TermColors.RemoteIdAttr)
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.DelimiterFg,
+		config.Client.TermColors.DelimiterBg,
+		config.Client.TermColors.DelimiterAttr)
+
+	log := splitted[5]
 
 	switch {
 	case strings.HasPrefix(log, "WARN"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteWarnFg,
 			config.Client.TermColors.RemoteWarnBg,
 			config.Client.TermColors.RemoteWarnAttr)
 
 	case strings.HasPrefix(log, "ERROR"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteErrorFg,
 			config.Client.TermColors.RemoteErrorBg,
 			config.Client.TermColors.RemoteErrorAttr)
 
 	case strings.HasPrefix(log, "FATAL"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteFatalFg,
 			config.Client.TermColors.RemoteFatalBg,
 			config.Client.TermColors.RemoteFatalAttr)
 
 	case strings.HasPrefix(log, "DEBUG"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteDebugFg,
 			config.Client.TermColors.RemoteDebugBg,
 			config.Client.TermColors.RemoteDebugAttr)
 
 	case strings.HasPrefix(log, "TRACE"):
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteTraceFg,
 			config.Client.TermColors.RemoteTraceBg,
 			config.Client.TermColors.RemoteTraceAttr)
 
 	default:
-		log = color.PaintWithAttr(log,
+		color.PaintWithAttr(sb, log,
 			config.Client.TermColors.RemoteTextFg,
 			config.Client.TermColors.RemoteTextBg,
 			config.Client.TermColors.RemoteTextAttr)
 	}
-
-	return fmt.Sprintf("%s|%s", info, log)
-}
-
-// Add some color to stats generated by the client.
-func paintClientStats(line string) string {
-	splitted := strings.Split(line, protocol.FieldDelimiter)
-	first := strings.Join(splitted[0:4], protocol.FieldDelimiter)
-	connected := color.PaintWithAttr(splitted[4],
-		config.Client.TermColors.ClientStatsFg,
-		config.Client.TermColors.ClientStatsBg,
-		config.Client.TermColors.ClientStatsAttr)
-	last := strings.Join(splitted[5:], "|")
-
-	return fmt.Sprintf("%s|%s|%s", first, connected, last)
 }
 
 // Colorfy a given line based on the line's content.
 func Colorfy(line string) string {
+	sb := strings.Builder{}
+
 	switch {
 	case strings.HasPrefix(line, "REMOTE"):
-		return paintRemote(line)
-
-	case strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|"):
-		return paintClientStats(line)
+		paintRemote(&sb, line)
 
 	case strings.Contains(line, "ERROR"):
-		return color.PaintWithAttr(line,
+		color.PaintWithAttr(&sb, line,
 			config.Client.TermColors.ClientErrorFg,
 			config.Client.TermColors.ClientErrorBg,
 			config.Client.TermColors.ClientErrorAttr)
 
 	case strings.Contains(line, "WARN"):
-		return color.PaintWithAttr(line,
+		color.PaintWithAttr(&sb, line,
 			config.Client.TermColors.ClientWarnFg,
 			config.Client.TermColors.ClientWarnBg,
 			config.Client.TermColors.ClientWarnAttr)
+
+	default:
+		color.PaintWithAttr(&sb, line,
+			color.FgDefault,
+			color.BgDefault,
+			color.AttrNone)
 	}
 
-	return line
+	color.ResetWithAttr(&sb)
+	return sb.String()
 }
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 7862467..2798ff7 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -1,31 +1,66 @@
 package color
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
-// Paint paints a given text in a given foreground/background color combination.
-func Paint(text string, fg FgColor, bg BgColor) string {
+// PaintStr paints a given text in a given foreground/background color combination.
+func PaintStr(text string, fg FgColor, bg BgColor) string {
 	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
 }
 
-// PaintWithAttr paints a given text in a given foreground/background/attribute combination
-func PaintWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
+// PaintStrWithAttr paints a given text in a given foreground/background/attribute combination
+func PaintStrWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
 	if attr == AttrNone {
-		return Paint(text, fg, bg)
+		return PaintStr(text, fg, bg)
 	}
 	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
 }
 
-// PaintFg paints a given text in a given foreground color.
-func PaintFg(text string, fg FgColor) string {
+// PaintStrFg paints a given text in a given foreground color.
+func PaintStrFg(text string, fg FgColor) string {
 	return fmt.Sprintf("%s%s%s", fg, text, FgDefault)
 }
 
-// PaintBg paints a given text in a given background color.
-func PaintBg(text string, bg BgColor) string {
+// PaintStrBg paints a given text in a given background color.
+func PaintStrBg(text string, bg BgColor) string {
 	return fmt.Sprintf("%s%s%s", bg, text, BgDefault)
 }
 
-// PaintAttr adds a given attribute to a given text, such as "bold" or "italic".
-func PaintAttr(text string, attr Attribute) string {
+// PaintStrAttr adds a given attribute to a given text, such as "bold" or "italic".
+func PaintStrAttr(text string, attr Attribute) string {
 	return fmt.Sprintf("%s%s%s", attr, text, AttrReset)
 }
+
+// Paint paints a given text in a given foreground/background color combination.
+func Paint(sb *strings.Builder, text string, fg FgColor, bg BgColor) {
+	sb.WriteString(string(fg))
+	sb.WriteString(string(bg))
+	sb.WriteString(text)
+}
+
+// Reset background and foreground colors.
+func Reset(sb *strings.Builder) {
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
+}
+
+// PaintWithAttr starts painting a given text in a given foreground/background/attribute combination.
+func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, attr Attribute) {
+	if attr == AttrNone {
+		Paint(sb, text, fg, bg)
+		return
+	}
+	sb.WriteString(string(fg))
+	sb.WriteString(string(bg))
+	sb.WriteString(string(attr))
+	sb.WriteString(text)
+}
+
+// ResetWithAttr resets background, foreground and attributes.
+func ResetWithAttr(sb *strings.Builder) {
+	sb.WriteString(string(AttrReset))
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
+}
diff --git a/internal/color/table.go b/internal/color/table.go
index 8c36047..e2265e3 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -26,7 +26,7 @@ func printColorTable(attr string) {
 			attribute, _ := ToAttribute(attr)
 
 			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
-			fmt.Print(PaintWithAttr(text, fgColor, bgColor, attribute))
+			fmt.Print(PaintStrWithAttr(text, fgColor, bgColor, attribute))
 			fmt.Print("\n")
 		}
 	}
diff --git a/internal/config/client.go b/internal/config/client.go
index c1b488c..05e4f93 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -4,49 +4,51 @@ import "github.com/mimecast/dtail/internal/color"
 
 // ClientColorConfig allows to override the default terminal color color.
 type termColors struct {
-	ClientErrorAttr color.Attribute
-	ClientErrorBg   color.BgColor
-	ClientErrorFg   color.FgColor
-
-	ClientStatsAttr color.Attribute
-	ClientStatsBg   color.BgColor
-	ClientStatsFg   color.FgColor
-
-	ClientWarnAttr color.Attribute
-	ClientWarnBg   color.BgColor
-	ClientWarnFg   color.FgColor
-
-	RemoteDebugAttr color.Attribute
-	RemoteDebugBg   color.BgColor
-	RemoteDebugFg   color.FgColor
-
-	RemoteErrorAttr color.Attribute
-	RemoteErrorBg   color.BgColor
-	RemoteErrorFg   color.FgColor
-
-	RemoteFatalAttr color.Attribute
-	RemoteFatalBg   color.BgColor
-	RemoteFatalFg   color.FgColor
-
-	RemoteStatsOkAttr color.Attribute
-	RemoteStatsOkBg   color.BgColor
-	RemoteStatsOkFg   color.FgColor
-
+	ClientErrorAttr     color.Attribute
+	ClientErrorBg       color.BgColor
+	ClientErrorFg       color.FgColor
+	ClientWarnAttr      color.Attribute
+	ClientWarnBg        color.BgColor
+	ClientWarnFg        color.FgColor
+	DelimiterAttr       color.Attribute
+	DelimiterBg         color.BgColor
+	DelimiterFg         color.FgColor
+	RemoteCountAttr     color.Attribute
+	RemoteCountBg       color.BgColor
+	RemoteCountFg       color.FgColor
+	RemoteDebugAttr     color.Attribute
+	RemoteDebugBg       color.BgColor
+	RemoteDebugFg       color.FgColor
+	RemoteErrorAttr     color.Attribute
+	RemoteErrorBg       color.BgColor
+	RemoteErrorFg       color.FgColor
+	RemoteFatalAttr     color.Attribute
+	RemoteFatalBg       color.BgColor
+	RemoteFatalFg       color.FgColor
+	RemoteIdAttr        color.Attribute
+	RemoteIdBg          color.BgColor
+	RemoteIdFg          color.FgColor
+	RemoteServerAttr    color.Attribute
+	RemoteServerBg      color.BgColor
+	RemoteServerFg      color.FgColor
+	RemoteStatsOkAttr   color.Attribute
+	RemoteStatsOkBg     color.BgColor
+	RemoteStatsOkFg     color.FgColor
 	RemoteStatsWarnAttr color.Attribute
 	RemoteStatsWarnBg   color.BgColor
 	RemoteStatsWarnFg   color.FgColor
-
-	RemoteTextAttr color.Attribute
-	RemoteTextBg   color.BgColor
-	RemoteTextFg   color.FgColor
-
-	RemoteTraceAttr color.Attribute
-	RemoteTraceBg   color.BgColor
-	RemoteTraceFg   color.FgColor
-
-	RemoteWarnAttr color.Attribute
-	RemoteWarnBg   color.BgColor
-	RemoteWarnFg   color.FgColor
+	RemoteStrAttr       color.Attribute
+	RemoteStrBg         color.BgColor
+	RemoteStrFg         color.FgColor
+	RemoteTextAttr      color.Attribute
+	RemoteTextBg        color.BgColor
+	RemoteTextFg        color.FgColor
+	RemoteTraceAttr     color.Attribute
+	RemoteTraceBg       color.BgColor
+	RemoteTraceFg       color.FgColor
+	RemoteWarnAttr      color.Attribute
+	RemoteWarnBg        color.BgColor
+	RemoteWarnFg        color.FgColor
 }
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
@@ -61,49 +63,51 @@ func newDefaultClientConfig() *ClientConfig {
 	return &ClientConfig{
 		TermColorsEnable: true,
 		TermColors: termColors{
-			ClientErrorAttr: color.AttrBold,
-			ClientErrorBg:   color.BgBlack,
-			ClientErrorFg:   color.FgRed,
-
-			ClientStatsAttr: color.AttrDim,
-			ClientStatsBg:   color.BgBlue,
-			ClientStatsFg:   color.FgWhite,
-
-			ClientWarnAttr: color.AttrNone,
-			ClientWarnBg:   color.BgBlack,
-			ClientWarnFg:   color.FgMagenta,
-
-			RemoteDebugAttr: color.AttrNone,
-			RemoteDebugBg:   color.BgGreen,
-			RemoteDebugFg:   color.FgBlack,
-
-			RemoteErrorAttr: color.AttrBold,
-			RemoteErrorBg:   color.BgRed,
-			RemoteErrorFg:   color.FgWhite,
-
-			RemoteFatalAttr: color.AttrBlink,
-			RemoteFatalBg:   color.BgRed,
-			RemoteFatalFg:   color.FgWhite,
-
-			RemoteStatsOkAttr: color.AttrNone,
-			RemoteStatsOkBg:   color.BgGreen,
-			RemoteStatsOkFg:   color.FgBlack,
-
+			ClientErrorAttr:     color.AttrBold,
+			ClientErrorBg:       color.BgBlack,
+			ClientErrorFg:       color.FgRed,
+			ClientWarnAttr:      color.AttrNone,
+			ClientWarnBg:        color.BgBlack,
+			ClientWarnFg:        color.FgMagenta,
+			DelimiterAttr:       color.AttrDim,
+			DelimiterBg:         color.BgBlue,
+			DelimiterFg:         color.FgCyan,
+			RemoteCountAttr:     color.AttrDim,
+			RemoteCountBg:       color.BgBlue,
+			RemoteCountFg:       color.FgGreen,
+			RemoteDebugAttr:     color.AttrBold,
+			RemoteDebugBg:       color.BgBlack,
+			RemoteDebugFg:       color.FgGreen,
+			RemoteErrorAttr:     color.AttrBold,
+			RemoteErrorBg:       color.BgRed,
+			RemoteErrorFg:       color.FgWhite,
+			RemoteFatalAttr:     color.AttrBlink,
+			RemoteFatalBg:       color.BgRed,
+			RemoteFatalFg:       color.FgWhite,
+			RemoteIdAttr:        color.AttrDim,
+			RemoteIdBg:          color.BgBlue,
+			RemoteIdFg:          color.FgWhite,
+			RemoteServerAttr:    color.AttrBold,
+			RemoteServerBg:      color.BgBlue,
+			RemoteServerFg:      color.FgWhite,
+			RemoteStatsOkAttr:   color.AttrNone,
+			RemoteStatsOkBg:     color.BgGreen,
+			RemoteStatsOkFg:     color.FgBlue,
 			RemoteStatsWarnAttr: color.AttrNone,
 			RemoteStatsWarnBg:   color.BgRed,
 			RemoteStatsWarnFg:   color.FgWhite,
-
-			RemoteTextAttr: color.AttrNone,
-			RemoteTextBg:   color.BgBlack,
-			RemoteTextFg:   color.FgWhite,
-
-			RemoteTraceAttr: color.AttrBold,
-			RemoteTraceBg:   color.BgGreen,
-			RemoteTraceFg:   color.FgWhite,
-
-			RemoteWarnAttr: color.AttrBold,
-			RemoteWarnBg:   color.BgYellow,
-			RemoteWarnFg:   color.FgWhite,
+			RemoteStrAttr:       color.AttrDim,
+			RemoteStrBg:         color.BgBlue,
+			RemoteStrFg:         color.FgWhite,
+			RemoteTextAttr:      color.AttrNone,
+			RemoteTextBg:        color.BgBlack,
+			RemoteTextFg:        color.FgWhite,
+			RemoteTraceAttr:     color.AttrBold,
+			RemoteTraceBg:       color.BgGreen,
+			RemoteTraceFg:       color.FgWhite,
+			RemoteWarnAttr:      color.AttrBold,
+			RemoteWarnBg:        color.BgYellow,
+			RemoteWarnFg:        color.FgWhite,
 		},
 	}
 }
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index d3035e4..e92ec6a 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -7,6 +7,8 @@ const (
 	MessageDelimiter byte = '¬'
 	// FieldDelimiter delimits aggregation fields.
 	FieldDelimiter string = "|"
+	// Arrow for multiple purposes
+	Arrow string = "➔"
 	// AggregateDelimiter delimits parts of an aggregation message.
-	AggregateDelimiter string = "➔"
+	AggregateDelimiter string = Arrow
 )
diff --git a/internal/version/version.go b/internal/version/version.go
index 7f07c83..693192f 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -29,16 +29,16 @@ func PaintedString() string {
 		return String()
 	}
 
-	name := color.PaintWithAttr(fmt.Sprintf(" %s ", Name),
+	name := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Name),
 		color.FgYellow, color.BgBlue, color.AttrBold)
 
-	version := color.PaintWithAttr(fmt.Sprintf(" %s ", Version),
+	version := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Version),
 		color.FgBlue, color.BgYellow, color.AttrBold)
 
-	protocol := color.Paint(fmt.Sprintf(" Protocol %s ", protocol.ProtocolCompat),
+	protocol := color.PaintStr(fmt.Sprintf(" Protocol %s ", protocol.ProtocolCompat),
 		color.FgBlack, color.BgGreen)
 
-	additional := color.PaintWithAttr(fmt.Sprintf(" %s ", Additional),
+	additional := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Additional),
 		color.FgWhite, color.BgMagenta, color.AttrBlink)
 
 	return fmt.Sprintf("%s%v%s%s", name, version, protocol, additional)
-- 
cgit v1.2.3


From 64429c63054106e39d1a28c488d76db534ff2ed5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 5 Sep 2021 15:55:57 +0300
Subject: fix unit test

---
 internal/color/color_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'internal')

diff --git a/internal/color/color_test.go b/internal/color/color_test.go
index bfc7c54..7002052 100644
--- a/internal/color/color_test.go
+++ b/internal/color/color_test.go
@@ -14,13 +14,13 @@ func TestColors(t *testing.T) {
 		if err != nil {
 			t.Errorf("unable to paint foreground : %s\n%v", text, err)
 		}
-		builder.WriteString(PaintFg(text, fgColor))
+		builder.WriteString(PaintStrFg(text, fgColor))
 
 		bgColor, err := ToBgColor(color)
 		if err != nil {
 			t.Errorf("unable to paint background: %s\n%v", text, err)
 		}
-		builder.WriteString(PaintBg(text, bgColor))
+		builder.WriteString(PaintStrBg(text, bgColor))
 	}
 
 	for _, fg := range ColorNames {
@@ -30,7 +30,7 @@ func TestColors(t *testing.T) {
 				continue
 			}
 			bgColor, _ := ToBgColor(bg)
-			builder.WriteString(Paint(text, fgColor, bgColor))
+			builder.WriteString(PaintStr(text, fgColor, bgColor))
 		}
 	}
 
@@ -46,7 +46,7 @@ func TestAttributes(t *testing.T) {
 		if err != nil {
 			t.Errorf("unable to paint attribute: %s\n%v", text, err)
 		}
-		builder.WriteString(PaintWithAttr(text, FgWhite, BgBlue, att))
+		builder.WriteString(PaintStrWithAttr(text, FgWhite, BgBlue, att))
 	}
 
 	t.Log(builder.String())
-- 
cgit v1.2.3


From c895b3c8293ddbf46d66278061d7e0127adc57f7 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 5 Sep 2021 16:04:45 +0300
Subject: -colorTable in combination with -debug prints whole sample paragraphs
 in all color combinations.

---
 internal/color/table.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'internal')

diff --git a/internal/color/table.go b/internal/color/table.go
index e2265e3..7ecfbca 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -5,17 +5,19 @@ import (
 	"os"
 )
 
-func TablePrintAndExit() {
+const sampleParagraph string = "Mimecast is Making Email Safer for Business. We believe that securely operating a business in the cloud requires new levels of IT preparedness, centered around cyber resilience. This is why we unify the delivery and management of security, continuity and data protection for email via one, simple-to-use cloud platform. Thousands of organizations trust us to increase their cyber resilience preparedness, streamline compliance, reduce IT complexity and keep their business running. We give employees fast and secure access to sensitive business information, and ensure email keeps flowing in the event of an outage. Mimecast will remain committed to protecting your IT assets through constant innovation and focus on your success."
+
+func TablePrintAndExit(useSampleParagraph bool) {
 	for _, attr := range AttributeNames {
 		if attr == "Hidden" || attr == "SlowBlink" {
 			continue
 		}
-		printColorTable(attr)
+		printColorTable(attr, useSampleParagraph)
 	}
 	os.Exit(0)
 }
 
-func printColorTable(attr string) {
+func printColorTable(attr string, useSampleParagraph bool) {
 	for _, fg := range ColorNames {
 		fgColor, _ := ToFgColor(fg)
 		for _, bg := range ColorNames {
@@ -27,6 +29,11 @@ func printColorTable(attr string) {
 
 			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
 			fmt.Print(PaintStrWithAttr(text, fgColor, bgColor, attribute))
+			if useSampleParagraph {
+				fmt.Print("\n")
+				fmt.Print(PaintStrWithAttr(sampleParagraph, fgColor, bgColor, attribute))
+				fmt.Print("\n")
+			}
 			fmt.Print("\n")
 		}
 	}
-- 
cgit v1.2.3


From 2c1c70313bb03cf2b2d7e7afadb07a48ff6bb690 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 6 Sep 2021 09:22:21 +0300
Subject: REMOTE and CLIENT colors are brushed correctly too now

---
 internal/clients/stats.go     |  14 ++-
 internal/color/brush/brush.go | 231 +++++++++++++++++++++++++++---------------
 internal/config/client.go     | 229 ++++++++++++++++++++++++-----------------
 3 files changed, 297 insertions(+), 177 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index d8163d4..3f76e0f 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
 )
@@ -54,7 +55,6 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 		throttle := len(throttleCh)
 
 		newConnections := connected - connectedLast
-
 		if (connected == connectedLast || quiet) && !force {
 			continue
 		}
@@ -77,7 +77,15 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 
 func (s *stats) printStatsDueInterrupt(messages []string) {
 	logger.Pause()
-	for _, message := range messages {
+	for i, message := range messages {
+		if i > 0 && config.Client.TermColorsEnable {
+			fmt.Println(color.PaintStrWithAttr(message,
+				config.Client.TermColors.Client.TextFg,
+				config.Client.TermColors.Client.TextBg,
+				config.Client.TermColors.Client.TextAttr,
+			))
+			continue
+		}
 		fmt.Println(fmt.Sprintf(" %s", message))
 	}
 	time.Sleep(time.Second * time.Duration(config.InterruptTimeoutS))
@@ -99,7 +107,6 @@ func (s *stats) statsLine(connected, newConnections int, throttle int) string {
 func (s *stats) numConnected() int {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
-
 	return s.connected
 }
 
@@ -107,6 +114,5 @@ func percentOf(total float64, value float64) float64 {
 	if total == 0 || total == value {
 		return 100
 	}
-
 	return value / (total / 100.0)
 }
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 415028a..785a0d0 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -8,105 +8,165 @@ import (
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// Add some color to log lines received from remote servers.
+func paintSeverity(sb *strings.Builder, text string) bool {
+	switch {
+	case strings.HasPrefix(text, "WARN"):
+		color.PaintWithAttr(sb, text,
+			config.Client.TermColors.Common.SeverityWarnFg,
+			config.Client.TermColors.Common.SeverityWarnBg,
+			config.Client.TermColors.Common.SeverityWarnAttr)
+
+	case strings.HasPrefix(text, "ERROR"):
+		color.PaintWithAttr(sb, text,
+			config.Client.TermColors.Common.SeverityErrorFg,
+			config.Client.TermColors.Common.SeverityErrorBg,
+			config.Client.TermColors.Common.SeverityErrorAttr)
+
+	case strings.HasPrefix(text, "FATAL"):
+		color.PaintWithAttr(sb, text,
+			config.Client.TermColors.Common.SeverityFatalFg,
+			config.Client.TermColors.Common.SeverityFatalBg,
+			config.Client.TermColors.Common.SeverityFatalAttr)
+
+	default:
+		return false
+	}
+
+	return true
+}
+
 func paintRemote(sb *strings.Builder, line string) {
 	splitted := strings.SplitN(line, protocol.FieldDelimiter, 6)
 
 	color.PaintWithAttr(sb, splitted[0],
-		config.Client.TermColors.RemoteStrFg,
-		config.Client.TermColors.RemoteStrBg,
-		config.Client.TermColors.RemoteStrAttr)
+		config.Client.TermColors.Remote.RemoteFg,
+		config.Client.TermColors.Remote.RemoteBg,
+		config.Client.TermColors.Remote.RemoteAttr)
 
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
 	color.PaintWithAttr(sb, splitted[1],
-		config.Client.TermColors.RemoteServerFg,
-		config.Client.TermColors.RemoteServerBg,
-		config.Client.TermColors.RemoteServerAttr)
+		config.Client.TermColors.Remote.HostnameFg,
+		config.Client.TermColors.Remote.HostnameBg,
+		config.Client.TermColors.Remote.HostnameAttr)
 
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
 	if splitted[2] == "100" {
 		color.PaintWithAttr(sb, splitted[2],
-			config.Client.TermColors.RemoteStatsOkFg,
-			config.Client.TermColors.RemoteStatsOkBg,
-			config.Client.TermColors.RemoteStatsOkAttr)
+			config.Client.TermColors.Remote.StatsOkFg,
+			config.Client.TermColors.Remote.StatsOkBg,
+			config.Client.TermColors.Remote.StatsOkAttr)
 	} else {
 		color.PaintWithAttr(sb, splitted[2],
-			config.Client.TermColors.RemoteStatsWarnFg,
-			config.Client.TermColors.RemoteStatsWarnBg,
-			config.Client.TermColors.RemoteStatsWarnAttr)
+			config.Client.TermColors.Remote.StatsWarnFg,
+			config.Client.TermColors.Remote.StatsWarnBg,
+			config.Client.TermColors.Remote.StatsWarnAttr)
 	}
 
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
 	color.PaintWithAttr(sb, splitted[3],
-		config.Client.TermColors.RemoteCountFg,
-		config.Client.TermColors.RemoteCountBg,
-		config.Client.TermColors.RemoteCountAttr)
+		config.Client.TermColors.Remote.CountFg,
+		config.Client.TermColors.Remote.CountBg,
+		config.Client.TermColors.Remote.CountAttr)
 
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
 	color.PaintWithAttr(sb, splitted[4],
-		config.Client.TermColors.RemoteIdFg,
-		config.Client.TermColors.RemoteIdBg,
-		config.Client.TermColors.RemoteIdAttr)
+		config.Client.TermColors.Remote.IdFg,
+		config.Client.TermColors.Remote.IdBg,
+		config.Client.TermColors.Remote.IdAttr)
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
-		config.Client.TermColors.DelimiterFg,
-		config.Client.TermColors.DelimiterBg,
-		config.Client.TermColors.DelimiterAttr)
+		config.Client.TermColors.Remote.DelimiterFg,
+		config.Client.TermColors.Remote.DelimiterBg,
+		config.Client.TermColors.Remote.DelimiterAttr)
 
-	log := splitted[5]
+	if paintSeverity(sb, splitted[5]) {
+		return
+	}
+	color.PaintWithAttr(sb, splitted[5],
+		config.Client.TermColors.Remote.TextFg,
+		config.Client.TermColors.Remote.TextBg,
+		config.Client.TermColors.Remote.TextAttr)
+}
 
-	switch {
-	case strings.HasPrefix(log, "WARN"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteWarnFg,
-			config.Client.TermColors.RemoteWarnBg,
-			config.Client.TermColors.RemoteWarnAttr)
-
-	case strings.HasPrefix(log, "ERROR"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteErrorFg,
-			config.Client.TermColors.RemoteErrorBg,
-			config.Client.TermColors.RemoteErrorAttr)
-
-	case strings.HasPrefix(log, "FATAL"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteFatalFg,
-			config.Client.TermColors.RemoteFatalBg,
-			config.Client.TermColors.RemoteFatalAttr)
-
-	case strings.HasPrefix(log, "DEBUG"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteDebugFg,
-			config.Client.TermColors.RemoteDebugBg,
-			config.Client.TermColors.RemoteDebugAttr)
-
-	case strings.HasPrefix(log, "TRACE"):
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteTraceFg,
-			config.Client.TermColors.RemoteTraceBg,
-			config.Client.TermColors.RemoteTraceAttr)
+func paintClient(sb *strings.Builder, line string) {
+	splitted := strings.SplitN(line, protocol.FieldDelimiter, 3)
 
-	default:
-		color.PaintWithAttr(sb, log,
-			config.Client.TermColors.RemoteTextFg,
-			config.Client.TermColors.RemoteTextBg,
-			config.Client.TermColors.RemoteTextAttr)
+	color.PaintWithAttr(sb, splitted[0],
+		config.Client.TermColors.Client.ClientFg,
+		config.Client.TermColors.Client.ClientBg,
+		config.Client.TermColors.Client.ClientAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.Client.DelimiterFg,
+		config.Client.TermColors.Client.DelimiterBg,
+		config.Client.TermColors.Client.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[1],
+		config.Client.TermColors.Client.HostnameFg,
+		config.Client.TermColors.Client.HostnameBg,
+		config.Client.TermColors.Client.HostnameAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.Client.DelimiterFg,
+		config.Client.TermColors.Client.DelimiterBg,
+		config.Client.TermColors.Client.DelimiterAttr)
+
+	if paintSeverity(sb, splitted[2]) {
+		return
 	}
+
+	color.PaintWithAttr(sb, splitted[2],
+		config.Client.TermColors.Client.TextFg,
+		config.Client.TermColors.Client.TextBg,
+		config.Client.TermColors.Client.TextAttr)
+}
+
+func paintServer(sb *strings.Builder, line string) {
+	splitted := strings.SplitN(line, protocol.FieldDelimiter, 3)
+
+	color.PaintWithAttr(sb, splitted[0],
+		config.Client.TermColors.Server.ServerFg,
+		config.Client.TermColors.Server.ServerBg,
+		config.Client.TermColors.Server.ServerAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.Server.DelimiterFg,
+		config.Client.TermColors.Server.DelimiterBg,
+		config.Client.TermColors.Server.DelimiterAttr)
+
+	color.PaintWithAttr(sb, splitted[1],
+		config.Client.TermColors.Server.HostnameFg,
+		config.Client.TermColors.Server.HostnameBg,
+		config.Client.TermColors.Server.HostnameAttr)
+
+	color.PaintWithAttr(sb, protocol.FieldDelimiter,
+		config.Client.TermColors.Server.DelimiterFg,
+		config.Client.TermColors.Server.DelimiterBg,
+		config.Client.TermColors.Server.DelimiterAttr)
+
+	if paintSeverity(sb, splitted[2]) {
+		return
+	}
+
+	color.PaintWithAttr(sb, splitted[2],
+		config.Client.TermColors.Server.TextFg,
+		config.Client.TermColors.Server.TextBg,
+		config.Client.TermColors.Server.TextAttr)
 }
 
 // Colorfy a given line based on the line's content.
@@ -117,18 +177,25 @@ func Colorfy(line string) string {
 	case strings.HasPrefix(line, "REMOTE"):
 		paintRemote(&sb, line)
 
-	case strings.Contains(line, "ERROR"):
-		color.PaintWithAttr(&sb, line,
-			config.Client.TermColors.ClientErrorFg,
-			config.Client.TermColors.ClientErrorBg,
-			config.Client.TermColors.ClientErrorAttr)
-
-	case strings.Contains(line, "WARN"):
-		color.PaintWithAttr(&sb, line,
-			config.Client.TermColors.ClientWarnFg,
-			config.Client.TermColors.ClientWarnBg,
-			config.Client.TermColors.ClientWarnAttr)
-
+	case strings.HasPrefix(line, "CLIENT"):
+		paintClient(&sb, line)
+
+	case strings.HasPrefix(line, "SERVER"):
+		paintServer(&sb, line)
+
+		/*
+			case strings.Contains(line, "ERROR"):
+				color.PaintWithAttr(&sb, line,
+					config.Client.TermColors.ClientErrorFg,
+					config.Client.TermColors.ClientErrorBg,
+					config.Client.TermColors.ClientErrorAttr)
+
+			case strings.Contains(line, "WARN"):
+				color.PaintWithAttr(&sb, line,
+					config.Client.TermColors.ClientWarnFg,
+					config.Client.TermColors.ClientWarnBg,
+					config.Client.TermColors.ClientWarnAttr)
+		*/
 	default:
 		color.PaintWithAttr(&sb, line,
 			color.FgDefault,
diff --git a/internal/config/client.go b/internal/config/client.go
index 05e4f93..837c7d6 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -2,53 +2,80 @@ package config
 
 import "github.com/mimecast/dtail/internal/color"
 
-// ClientColorConfig allows to override the default terminal color color.
+type remoteTermColors struct {
+	DelimiterAttr color.Attribute
+	DelimiterBg   color.BgColor
+	DelimiterFg   color.FgColor
+	RemoteAttr    color.Attribute
+	RemoteBg      color.BgColor
+	RemoteFg      color.FgColor
+	CountAttr     color.Attribute
+	CountBg       color.BgColor
+	CountFg       color.FgColor
+	HostnameAttr  color.Attribute
+	HostnameBg    color.BgColor
+	HostnameFg    color.FgColor
+	IdAttr        color.Attribute
+	IdBg          color.BgColor
+	IdFg          color.FgColor
+	StatsOkAttr   color.Attribute
+	StatsOkBg     color.BgColor
+	StatsOkFg     color.FgColor
+	StatsWarnAttr color.Attribute
+	StatsWarnBg   color.BgColor
+	StatsWarnFg   color.FgColor
+	TextAttr      color.Attribute
+	TextBg        color.BgColor
+	TextFg        color.FgColor
+}
+
+type clientTermColors struct {
+	DelimiterAttr color.Attribute
+	DelimiterBg   color.BgColor
+	DelimiterFg   color.FgColor
+	ClientAttr    color.Attribute
+	ClientBg      color.BgColor
+	ClientFg      color.FgColor
+	HostnameAttr  color.Attribute
+	HostnameBg    color.BgColor
+	HostnameFg    color.FgColor
+	TextAttr      color.Attribute
+	TextBg        color.BgColor
+	TextFg        color.FgColor
+}
+
+type serverTermColors struct {
+	DelimiterAttr color.Attribute
+	DelimiterBg   color.BgColor
+	DelimiterFg   color.FgColor
+	ServerAttr    color.Attribute
+	ServerBg      color.BgColor
+	ServerFg      color.FgColor
+	HostnameAttr  color.Attribute
+	HostnameBg    color.BgColor
+	HostnameFg    color.FgColor
+	TextAttr      color.Attribute
+	TextBg        color.BgColor
+	TextFg        color.FgColor
+}
+
+type commonTermColors struct {
+	SeverityErrorAttr color.Attribute
+	SeverityErrorBg   color.BgColor
+	SeverityErrorFg   color.FgColor
+	SeverityFatalAttr color.Attribute
+	SeverityFatalBg   color.BgColor
+	SeverityFatalFg   color.FgColor
+	SeverityWarnAttr  color.Attribute
+	SeverityWarnBg    color.BgColor
+	SeverityWarnFg    color.FgColor
+}
+
 type termColors struct {
-	ClientErrorAttr     color.Attribute
-	ClientErrorBg       color.BgColor
-	ClientErrorFg       color.FgColor
-	ClientWarnAttr      color.Attribute
-	ClientWarnBg        color.BgColor
-	ClientWarnFg        color.FgColor
-	DelimiterAttr       color.Attribute
-	DelimiterBg         color.BgColor
-	DelimiterFg         color.FgColor
-	RemoteCountAttr     color.Attribute
-	RemoteCountBg       color.BgColor
-	RemoteCountFg       color.FgColor
-	RemoteDebugAttr     color.Attribute
-	RemoteDebugBg       color.BgColor
-	RemoteDebugFg       color.FgColor
-	RemoteErrorAttr     color.Attribute
-	RemoteErrorBg       color.BgColor
-	RemoteErrorFg       color.FgColor
-	RemoteFatalAttr     color.Attribute
-	RemoteFatalBg       color.BgColor
-	RemoteFatalFg       color.FgColor
-	RemoteIdAttr        color.Attribute
-	RemoteIdBg          color.BgColor
-	RemoteIdFg          color.FgColor
-	RemoteServerAttr    color.Attribute
-	RemoteServerBg      color.BgColor
-	RemoteServerFg      color.FgColor
-	RemoteStatsOkAttr   color.Attribute
-	RemoteStatsOkBg     color.BgColor
-	RemoteStatsOkFg     color.FgColor
-	RemoteStatsWarnAttr color.Attribute
-	RemoteStatsWarnBg   color.BgColor
-	RemoteStatsWarnFg   color.FgColor
-	RemoteStrAttr       color.Attribute
-	RemoteStrBg         color.BgColor
-	RemoteStrFg         color.FgColor
-	RemoteTextAttr      color.Attribute
-	RemoteTextBg        color.BgColor
-	RemoteTextFg        color.FgColor
-	RemoteTraceAttr     color.Attribute
-	RemoteTraceBg       color.BgColor
-	RemoteTraceFg       color.FgColor
-	RemoteWarnAttr      color.Attribute
-	RemoteWarnBg        color.BgColor
-	RemoteWarnFg        color.FgColor
+	Remote remoteTermColors
+	Client clientTermColors
+	Server serverTermColors
+	Common commonTermColors
 }
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
@@ -63,51 +90,71 @@ func newDefaultClientConfig() *ClientConfig {
 	return &ClientConfig{
 		TermColorsEnable: true,
 		TermColors: termColors{
-			ClientErrorAttr:     color.AttrBold,
-			ClientErrorBg:       color.BgBlack,
-			ClientErrorFg:       color.FgRed,
-			ClientWarnAttr:      color.AttrNone,
-			ClientWarnBg:        color.BgBlack,
-			ClientWarnFg:        color.FgMagenta,
-			DelimiterAttr:       color.AttrDim,
-			DelimiterBg:         color.BgBlue,
-			DelimiterFg:         color.FgCyan,
-			RemoteCountAttr:     color.AttrDim,
-			RemoteCountBg:       color.BgBlue,
-			RemoteCountFg:       color.FgGreen,
-			RemoteDebugAttr:     color.AttrBold,
-			RemoteDebugBg:       color.BgBlack,
-			RemoteDebugFg:       color.FgGreen,
-			RemoteErrorAttr:     color.AttrBold,
-			RemoteErrorBg:       color.BgRed,
-			RemoteErrorFg:       color.FgWhite,
-			RemoteFatalAttr:     color.AttrBlink,
-			RemoteFatalBg:       color.BgRed,
-			RemoteFatalFg:       color.FgWhite,
-			RemoteIdAttr:        color.AttrDim,
-			RemoteIdBg:          color.BgBlue,
-			RemoteIdFg:          color.FgWhite,
-			RemoteServerAttr:    color.AttrBold,
-			RemoteServerBg:      color.BgBlue,
-			RemoteServerFg:      color.FgWhite,
-			RemoteStatsOkAttr:   color.AttrNone,
-			RemoteStatsOkBg:     color.BgGreen,
-			RemoteStatsOkFg:     color.FgBlue,
-			RemoteStatsWarnAttr: color.AttrNone,
-			RemoteStatsWarnBg:   color.BgRed,
-			RemoteStatsWarnFg:   color.FgWhite,
-			RemoteStrAttr:       color.AttrDim,
-			RemoteStrBg:         color.BgBlue,
-			RemoteStrFg:         color.FgWhite,
-			RemoteTextAttr:      color.AttrNone,
-			RemoteTextBg:        color.BgBlack,
-			RemoteTextFg:        color.FgWhite,
-			RemoteTraceAttr:     color.AttrBold,
-			RemoteTraceBg:       color.BgGreen,
-			RemoteTraceFg:       color.FgWhite,
-			RemoteWarnAttr:      color.AttrBold,
-			RemoteWarnBg:        color.BgYellow,
-			RemoteWarnFg:        color.FgWhite,
+			Remote: remoteTermColors{
+				DelimiterAttr: color.AttrDim,
+				DelimiterBg:   color.BgBlue,
+				DelimiterFg:   color.FgCyan,
+				RemoteAttr:    color.AttrDim,
+				RemoteBg:      color.BgBlue,
+				RemoteFg:      color.FgWhite,
+				CountAttr:     color.AttrDim,
+				CountBg:       color.BgBlue,
+				CountFg:       color.FgGreen,
+				HostnameAttr:  color.AttrBold,
+				HostnameBg:    color.BgBlue,
+				HostnameFg:    color.FgWhite,
+				IdAttr:        color.AttrDim,
+				IdBg:          color.BgBlue,
+				IdFg:          color.FgWhite,
+				StatsOkAttr:   color.AttrNone,
+				StatsOkBg:     color.BgGreen,
+				StatsOkFg:     color.FgBlue,
+				StatsWarnAttr: color.AttrNone,
+				StatsWarnBg:   color.BgRed,
+				StatsWarnFg:   color.FgWhite,
+				TextAttr:      color.AttrNone,
+				TextBg:        color.BgBlack,
+				TextFg:        color.FgWhite,
+			},
+			Client: clientTermColors{
+				DelimiterAttr: color.AttrDim,
+				DelimiterBg:   color.BgYellow,
+				DelimiterFg:   color.FgBlack,
+				ClientAttr:    color.AttrDim,
+				ClientBg:      color.BgYellow,
+				ClientFg:      color.FgBlack,
+				HostnameAttr:  color.AttrDim,
+				HostnameBg:    color.BgYellow,
+				HostnameFg:    color.FgBlack,
+				TextAttr:      color.AttrNone,
+				TextBg:        color.BgYellow,
+				TextFg:        color.FgBlack,
+			},
+			Server: serverTermColors{
+				DelimiterAttr: color.AttrDim,
+				DelimiterBg:   color.BgCyan,
+				DelimiterFg:   color.FgBlack,
+				ServerAttr:    color.AttrDim,
+				ServerBg:      color.BgCyan,
+				ServerFg:      color.FgBlack,
+				HostnameAttr:  color.AttrBold,
+				HostnameBg:    color.BgCyan,
+				HostnameFg:    color.FgBlack,
+				TextAttr:      color.AttrNone,
+				TextBg:        color.BgCyan,
+				TextFg:        color.FgBlack,
+			},
+			Common: commonTermColors{
+				SeverityErrorAttr: color.AttrBold,
+				SeverityErrorBg:   color.BgRed,
+				SeverityErrorFg:   color.FgWhite,
+				SeverityFatalAttr: color.AttrBlink,
+				SeverityFatalBg:   color.BgRed,
+				SeverityFatalFg:   color.FgWhite,
+				SeverityWarnAttr:  color.AttrNone,
+				SeverityWarnBg:    color.BgBlack,
+				SeverityWarnFg:    color.FgRed,
+			},
 		},
 	}
 }
-- 
cgit v1.2.3


From cc89d3fb8be2465af276d7ef03ea2a8affd87b2e Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 6 Sep 2021 13:48:55 +0300
Subject: Print out client/server update notice even from dtail server 4 to
 dtail client 3.

---
 internal/clients/handlers/basehandler.go  |  2 +-
 internal/config/client.go                 |  8 ++++----
 internal/protocol/protocol.go             |  4 +---
 internal/server/handlers/serverhandler.go | 30 +++++++++++++++++++++++-------
 4 files changed, 29 insertions(+), 15 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index fe83faa..63ceaac 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -56,7 +56,7 @@ func (h *baseHandler) SendMessage(command string) error {
 // Read data from the dtail server via Writer interface.
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		if b == protocol.MessageDelimiter {
+		if b == protocol.MessageDelimiter || b == '\n' {
 			if len(h.receiveBuf) == 0 {
 				continue
 			}
diff --git a/internal/config/client.go b/internal/config/client.go
index 837c7d6..8bde7a4 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -127,8 +127,8 @@ func newDefaultClientConfig() *ClientConfig {
 				HostnameBg:    color.BgYellow,
 				HostnameFg:    color.FgBlack,
 				TextAttr:      color.AttrNone,
-				TextBg:        color.BgYellow,
-				TextFg:        color.FgBlack,
+				TextBg:        color.BgBlack,
+				TextFg:        color.FgWhite,
 			},
 			Server: serverTermColors{
 				DelimiterAttr: color.AttrDim,
@@ -151,9 +151,9 @@ func newDefaultClientConfig() *ClientConfig {
 				SeverityFatalAttr: color.AttrBlink,
 				SeverityFatalBg:   color.BgRed,
 				SeverityFatalFg:   color.FgWhite,
-				SeverityWarnAttr:  color.AttrNone,
+				SeverityWarnAttr:  color.AttrBold,
 				SeverityWarnBg:    color.BgBlack,
-				SeverityWarnFg:    color.FgRed,
+				SeverityWarnFg:    color.FgWhite,
 			},
 		},
 	}
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index e92ec6a..d3035e4 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -7,8 +7,6 @@ const (
 	MessageDelimiter byte = '¬'
 	// FieldDelimiter delimits aggregation fields.
 	FieldDelimiter string = "|"
-	// Arrow for multiple purposes
-	Arrow string = "➔"
 	// AggregateDelimiter delimits parts of an aggregation message.
-	AggregateDelimiter string = Arrow
+	AggregateDelimiter string = "➔"
 )
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 14fc5d0..14f46a3 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -167,9 +168,9 @@ func (h *ServerHandler) handleCommand(commandStr string) {
 	logger.Debug(h.user, commandStr)
 	ctx := context.Background()
 
-	args, argc, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
+	args, argc, add, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
 	if err != nil {
-		h.send(h.serverMessages, logger.Error(h.user, err))
+		h.send(h.serverMessages, logger.Error(h.user, err)+add)
 		return
 	}
 
@@ -193,19 +194,34 @@ func (h *ServerHandler) handleCommand(commandStr string) {
 	h.handleUserCommand(ctx, argc, args)
 }
 
-func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, error) {
+func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
 	argc := len(args)
+	var add string
 
 	if argc <= 2 || args[0] != "protocol" {
-		return args, argc, errors.New("unable to determine protocol version")
+		return args, argc, add, errors.New("unable to determine protocol version")
 	}
 
 	if args[1] != protocol.ProtocolCompat {
-		err := fmt.Errorf("server with protocol version '%s' but client with '%s', please update DTail", protocol.ProtocolCompat, args[1])
-		return args, argc, err
+		clientCompat, _ := strconv.Atoi(args[1])
+		serverCompat, _ := strconv.Atoi(protocol.ProtocolCompat)
+		if clientCompat <= 3 {
+			// Protocol version 3 or lower expect a newline as message separator
+			// One day (after 2 major versions) this exception may be removed!
+			add = "\n"
+		}
+
+		toUpdate := "client"
+		if clientCompat > serverCompat {
+			toUpdate = "server"
+		}
+
+		err := fmt.Errorf("DTail server protocol version '%s' does not match client protocol version '%s', please update DTail %s!",
+			protocol.ProtocolCompat, args[1], toUpdate)
+		return args, argc, add, err
 	}
 
-	return args[2:], argc - 2, nil
+	return args[2:], argc - 2, add, nil
 }
 
 func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, error) {
-- 
cgit v1.2.3


From 6ae75e8f106d3eee18ea61e6c4d6925c6f514460 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 6 Sep 2021 14:07:07 +0300
Subject: fine tweak color schema

---
 internal/clients/stats.go     |  6 +++---
 internal/color/brush/brush.go | 14 --------------
 internal/color/paint.go       |  5 +++++
 internal/config/client.go     |  4 ++--
 4 files changed, 10 insertions(+), 19 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index 3f76e0f..faeb9fb 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -80,9 +80,9 @@ func (s *stats) printStatsDueInterrupt(messages []string) {
 	for i, message := range messages {
 		if i > 0 && config.Client.TermColorsEnable {
 			fmt.Println(color.PaintStrWithAttr(message,
-				config.Client.TermColors.Client.TextFg,
-				config.Client.TermColors.Client.TextBg,
-				config.Client.TermColors.Client.TextAttr,
+				config.Client.TermColors.Client.ClientFg,
+				config.Client.TermColors.Client.ClientBg,
+				config.Client.TermColors.Client.ClientAttr,
 			))
 			continue
 		}
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 785a0d0..524829b 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -183,19 +183,6 @@ func Colorfy(line string) string {
 	case strings.HasPrefix(line, "SERVER"):
 		paintServer(&sb, line)
 
-		/*
-			case strings.Contains(line, "ERROR"):
-				color.PaintWithAttr(&sb, line,
-					config.Client.TermColors.ClientErrorFg,
-					config.Client.TermColors.ClientErrorBg,
-					config.Client.TermColors.ClientErrorAttr)
-
-			case strings.Contains(line, "WARN"):
-				color.PaintWithAttr(&sb, line,
-					config.Client.TermColors.ClientWarnFg,
-					config.Client.TermColors.ClientWarnBg,
-					config.Client.TermColors.ClientWarnAttr)
-		*/
 	default:
 		color.PaintWithAttr(&sb, line,
 			color.FgDefault,
@@ -203,6 +190,5 @@ func Colorfy(line string) string {
 			color.AttrNone)
 	}
 
-	color.ResetWithAttr(&sb)
 	return sb.String()
 }
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 2798ff7..5430acd 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -38,6 +38,8 @@ func Paint(sb *strings.Builder, text string, fg FgColor, bg BgColor) {
 	sb.WriteString(string(fg))
 	sb.WriteString(string(bg))
 	sb.WriteString(text)
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
 }
 
 // Reset background and foreground colors.
@@ -56,6 +58,9 @@ func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, att
 	sb.WriteString(string(bg))
 	sb.WriteString(string(attr))
 	sb.WriteString(text)
+	sb.WriteString(string(AttrReset))
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
 }
 
 // ResetWithAttr resets background, foreground and attributes.
diff --git a/internal/config/client.go b/internal/config/client.go
index 8bde7a4..3c2f7de 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -99,7 +99,7 @@ func newDefaultClientConfig() *ClientConfig {
 				RemoteFg:      color.FgWhite,
 				CountAttr:     color.AttrDim,
 				CountBg:       color.BgBlue,
-				CountFg:       color.FgGreen,
+				CountFg:       color.FgWhite,
 				HostnameAttr:  color.AttrBold,
 				HostnameBg:    color.BgBlue,
 				HostnameFg:    color.FgWhite,
@@ -108,7 +108,7 @@ func newDefaultClientConfig() *ClientConfig {
 				IdFg:          color.FgWhite,
 				StatsOkAttr:   color.AttrNone,
 				StatsOkBg:     color.BgGreen,
-				StatsOkFg:     color.FgBlue,
+				StatsOkFg:     color.FgBlack,
 				StatsWarnAttr: color.AttrNone,
 				StatsWarnBg:   color.BgRed,
 				StatsWarnFg:   color.FgWhite,
-- 
cgit v1.2.3


From f74a9e4b35feb8c07d8a70b5a581088a0a59889d Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 7 Sep 2021 10:01:32 +0300
Subject: Produce MAPREDUCE lines, can aggregate these via default log format

---
 internal/clients/stats.go               | 49 ++++++++++++++++++++++++---------
 internal/color/brush/brush.go           | 12 ++++----
 internal/io/logger/logger.go            | 40 +++++++++++++++++++++------
 internal/io/pool/builder.go             | 18 ++++++++++++
 internal/mapr/logformat/default.go      | 15 ++++++++--
 internal/mapr/logformat/default_test.go | 39 +++++++++++++++-----------
 internal/server/stats.go                | 12 +++++---
 internal/version/version.go             |  2 +-
 8 files changed, 137 insertions(+), 50 deletions(-)
 create mode 100644 internal/io/pool/builder.go

(limited to 'internal')

diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index faeb9fb..6da443c 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -11,12 +11,13 @@ import (
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // Used to collect and display various client stats.
 type stats struct {
 	// Total amount servers to connect to.
-	connectionsTotal int
+	servers int
 	// To keep track of what connected and disconnected
 	connectionsEstCh chan struct{}
 	// Amount of servers connections are established.
@@ -25,10 +26,10 @@ type stats struct {
 	mutex sync.Mutex
 }
 
-func newTailStats(connectionsTotal int) *stats {
+func newTailStats(servers int) *stats {
 	return &stats{
-		connectionsTotal: connectionsTotal,
-		connectionsEstCh: make(chan struct{}, connectionsTotal),
+		servers:          servers,
+		connectionsEstCh: make(chan struct{}, servers),
 		connected:        0,
 	}
 }
@@ -59,13 +60,14 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 			continue
 		}
 
-		stats := s.statsLine(connected, newConnections, throttle)
 		switch force {
 		case true:
+			stats := s.statsLine(connected, newConnections, throttle)
 			messages = append(messages, fmt.Sprintf("Connection stats: %s", stats))
 			s.printStatsDueInterrupt(messages)
 		default:
-			logger.Info(stats)
+			data := s.statsData(connected, newConnections, throttle)
+			logger.Mapreduce("STATS", data)
 		}
 
 		connectedLast = connected
@@ -92,16 +94,37 @@ func (s *stats) printStatsDueInterrupt(messages []string) {
 	logger.Resume()
 }
 
+func (s *stats) statsData(connected, newConnections int, throttle int) map[string]interface{} {
+	percConnected := percentOf(float64(s.servers), float64(connected))
+
+	data := make(map[string]interface{})
+	data["connected"] = connected
+	data["servers"] = s.servers
+	data["connected%"] = int(percConnected)
+	data["new"] = newConnections
+	data["throttle"] = throttle
+	data["goroutines"] = runtime.NumGoroutine()
+	data["cgocalls"] = runtime.NumCgoCall()
+	data["cpu"] = runtime.NumCPU()
+
+	return data
+}
+
 func (s *stats) statsLine(connected, newConnections int, throttle int) string {
-	percConnected := percentOf(float64(s.connectionsTotal), float64(connected))
+	sb := strings.Builder{}
 
-	var stats []string
-	stats = append(stats, fmt.Sprintf("connected=%d/%d(%d%%)", connected, s.connectionsTotal, int(percConnected)))
-	stats = append(stats, fmt.Sprintf("new=%d", newConnections))
-	stats = append(stats, fmt.Sprintf("throttle=%d", throttle))
-	stats = append(stats, fmt.Sprintf("cpus/goroutines=%d/%d", runtime.NumCPU(), runtime.NumGoroutine()))
+	i := 0
+	for k, v := range s.statsData(connected, newConnections, throttle) {
+		if i > 0 {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+		sb.WriteString(k)
+		sb.WriteByte('=')
+		sb.WriteString(fmt.Sprintf("%v", v))
+		i++
+	}
 
-	return strings.Join(stats, "|")
+	return sb.String()
 }
 
 func (s *stats) numConnected() int {
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 524829b..82fa410 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -171,20 +172,21 @@ func paintServer(sb *strings.Builder, line string) {
 
 // Colorfy a given line based on the line's content.
 func Colorfy(line string) string {
-	sb := strings.Builder{}
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
 
 	switch {
 	case strings.HasPrefix(line, "REMOTE"):
-		paintRemote(&sb, line)
+		paintRemote(sb, line)
 
 	case strings.HasPrefix(line, "CLIENT"):
-		paintClient(&sb, line)
+		paintClient(sb, line)
 
 	case strings.HasPrefix(line, "SERVER"):
-		paintServer(&sb, line)
+		paintServer(sb, line)
 
 	default:
-		color.PaintWithAttr(&sb, line,
+		color.PaintWithAttr(sb, line,
 			color.FgDefault,
 			color.BgDefault,
 			color.AttrNone)
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 3a3935d..6890201 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -14,6 +14,8 @@ import (
 
 	"github.com/mimecast/dtail/internal/color/brush"
 	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 const (
@@ -132,6 +134,24 @@ func Info(args ...interface{}) string {
 	return log(clientStr, infoStr, args)
 }
 
+// Mapreduce message logging.
+func Mapreduce(table string, data map[string]interface{}) string {
+	args := make([]interface{}, len(data)+1)
+
+	args[0] = fmt.Sprintf("MAPREDUCE:%s", strings.ToUpper(table))
+	i := 1
+	for k, v := range data {
+		args[i] = fmt.Sprintf("%s=%v", k, v)
+		i++
+	}
+
+	if Mode.Server {
+		return log(serverStr, infoStr, args)
+	}
+
+	return log(clientStr, infoStr, args)
+}
+
 // Warn message logging.
 func Warn(args ...interface{}) string {
 	if !Mode.Quiet {
@@ -230,24 +250,28 @@ func log(what string, severity string, args []interface{}) string {
 		return ""
 	}
 
-	messages := []string{}
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+
+	for i, arg := range args {
+		if i > 0 {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
 
-	for _, arg := range args {
 		switch v := arg.(type) {
 		case string:
-			messages = append(messages, v)
+			sb.WriteString(v)
 		case int:
-			messages = append(messages, fmt.Sprintf("%d", v))
+			sb.WriteString(fmt.Sprintf("%d", v))
 		case error:
-			messages = append(messages, v.Error())
+			sb.WriteString(v.Error())
 		default:
-			messages = append(messages, fmt.Sprintf("%v", v))
+			sb.WriteString(fmt.Sprintf("%v", v))
 		}
 	}
 
-	message := strings.Join(messages, "|")
+	message := sb.String()
+	pool.RecycleBuilderBuffer(sb)
 	write(what, severity, message)
-
 	return fmt.Sprintf("%s|%s", severity, message)
 }
 
diff --git a/internal/io/pool/builder.go b/internal/io/pool/builder.go
new file mode 100644
index 0000000..c9dc221
--- /dev/null
+++ b/internal/io/pool/builder.go
@@ -0,0 +1,18 @@
+package pool
+
+import (
+	"strings"
+	"sync"
+)
+
+var BuilderBuffer = sync.Pool{
+	New: func() interface{} {
+		sb := strings.Builder{}
+		return &sb
+	},
+}
+
+func RecycleBuilderBuffer(sb *strings.Builder) {
+	sb.Reset()
+	BuilderBuffer.Put(sb)
+}
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 32a34bd..2881047 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -9,8 +9,8 @@ import (
 
 // MakeFieldsDEFAULT is the default log file mapreduce parser.
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
-	fields := make(map[string]string, 20)
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
+	fields := make(map[string]string, len(splitted))
 
 	fields["*"] = "*"
 	fields["$line"] = maprLine
@@ -19,10 +19,19 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields["$timezone"] = p.timeZoneName
 	fields["$timeoffset"] = p.timeZoneOffset
 
-	for _, kv := range splitted {
+	kvStart := 0
+	// DTail mapreduce format
+	if len(splitted) > 3 && strings.HasPrefix(splitted[3], "MAPREDUCE:") {
+		fields["$severity"] = splitted[0]
+		// TODO: Parse time like we do at Mimecast
+		fields["$time"] = splitted[1]
+		kvStart = 4
+	}
+
+	for _, kv := range splitted[kvStart:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
-			return fields, errors.New("Error parsing mapr token: " + kv)
+			return fields, errors.New("Error parsing mapreduce token: " + kv)
 		}
 		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
 	}
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index d7a4da4..6284008 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -10,26 +10,33 @@ func TestDefaultLogFormat(t *testing.T) {
 		t.Errorf("Unable to create parser: %s", err.Error())
 	}
 
-	fields, err := parser.MakeFields("foo=bar|baz=bay")
-
-	if err != nil {
-		t.Errorf("Unable to parse: %s", err.Error())
+	inputs := []string{
+		"foo=bar|baz=bay",
+		"INFO|20210907-065632|SERVER|MAPREDUCE:TEST|foo=bar|baz=bay",
 	}
 
-	if bar, ok := fields["foo"]; !ok {
-		t.Errorf("Expected field 'foo', but no such field there\n")
-	} else if bar != "bar" {
-		t.Errorf("Expected 'bar' stored in field 'foo', but got '%s'\n", bar)
-	}
+	for _, input := range inputs {
+		fields, err := parser.MakeFields(input)
+
+		if err != nil {
+			t.Errorf("Parser unable to make fields: %s", err.Error())
+		}
+
+		if bar, ok := fields["foo"]; !ok {
+			t.Errorf("Expected field 'foo', but no such field there\n")
+		} else if bar != "bar" {
+			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s'\n", bar)
+		}
+
+		if bay, ok := fields["baz"]; !ok {
+			t.Errorf("Expected field 'baz', but no such field there\n")
+		} else if bay != "bay" {
+			t.Errorf("Expected 'bay' stored in field 'baz', but got '%s'\n", bay)
+		}
 
-	if bay, ok := fields["baz"]; !ok {
-		t.Errorf("Expected field 'baz', but no such field there\n")
-	} else if bay != "bay" {
-		t.Errorf("Expected 'bay' stored in field 'baz', but got '%s'\n", bay)
 	}
 
-	fields, err = parser.MakeFields("foo=bar|bazbay")
-	if err == nil {
-		t.Errorf("Expected error but didn't: %s", err.Error())
+	if _, err := parser.MakeFields("foo=bar|bazbay"); err == nil {
+		t.Errorf("Expected error but didn't")
 	}
 }
diff --git a/internal/server/stats.go b/internal/server/stats.go
index ac579ad..3e8c71d 100644
--- a/internal/server/stats.go
+++ b/internal/server/stats.go
@@ -50,10 +50,14 @@ func (s *stats) logServerStats() {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 
-	currentConnections := fmt.Sprintf("currentConnections=%d", s.currentConnections)
-	lifetimeConnections := fmt.Sprintf("lifetimeConnections=%d", s.lifetimeConnections)
-	goroutines := fmt.Sprintf("goroutines=%d", runtime.NumGoroutine())
-	logger.Info("stats", currentConnections, lifetimeConnections, goroutines)
+	data := make(map[string]interface{})
+	data["currentConnections"] = s.currentConnections
+	data["lifetimeConnections"] = s.lifetimeConnections
+	data["goroutines"] = runtime.NumGoroutine()
+	data["cgocalls"] = runtime.NumCgoCall()
+	data["cpu"] = runtime.NumCPU()
+
+	logger.Mapreduce("STATS", data)
 }
 
 func (s *stats) serverLimitExceeded() error {
diff --git a/internal/version/version.go b/internal/version/version.go
index 693192f..a54b560 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,7 +13,7 @@ const (
 	// Name of DTail.
 	Name string = "DTail"
 	// Version of DTail.
-	Version string = "3.2.0"
+	Version string = "4.0.0-RC1"
 	// Additional information for DTail
 	Additional string = "Have a lot of fun!"
 )
-- 
cgit v1.2.3


From c83c9e61a08c7ea1cb528bc26dfab25b46faa866 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 8 Sep 2021 08:39:41 +0300
Subject: Don't fail parsing whole line when one kv could not be parsed

---
 internal/mapr/logformat/default.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'internal')

diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 2881047..da976bd 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -1,9 +1,9 @@
 package logformat
 
 import (
-	"errors"
 	"strings"
 
+	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -31,7 +31,8 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	for _, kv := range splitted[kvStart:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
-			return fields, errors.New("Error parsing mapreduce token: " + kv)
+			logger.Debug("Unable to parse key-value token, ignoring it", kv)
+			continue
 		}
 		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
 	}
-- 
cgit v1.2.3


From 16dc57e1e1c28e9d762424e596223a980770e059 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 8 Sep 2021 19:10:50 +0300
Subject: mapreduce tables are in colors now too

---
 internal/clients/baseclient.go             |   1 +
 internal/clients/handlers/basehandler.go   |  29 ++--
 internal/clients/handlers/healthhandler.go |  11 +-
 internal/clients/handlers/maprhandler.go   |  42 ++---
 internal/clients/maprclient.go             |  31 +++-
 internal/color/paint.go                    |  13 ++
 internal/config/client.go                  |  48 +++++-
 internal/io/fs/readfile.go                 |   5 +-
 internal/mapr/aggregateset.go              |  17 +-
 internal/mapr/client/aggregate.go          |  18 +-
 internal/mapr/globalgroupset.go            |   1 +
 internal/mapr/groupset.go                  | 258 ++++++++++++++++++++---------
 internal/mapr/logformat/default.go         |  28 ++--
 internal/mapr/logformat/default_test.go    |  15 +-
 internal/mapr/logformat/generickv.go       |  31 ++++
 internal/mapr/logformat/parser.go          |   2 +
 internal/mapr/server/aggregate.go          |  17 +-
 internal/protocol/protocol.go              |  11 +-
 internal/server/handlers/controlhandler.go |   3 +-
 internal/server/handlers/serverhandler.go  | 118 +++++++------
 20 files changed, 469 insertions(+), 230 deletions(-)
 create mode 100644 internal/mapr/logformat/generickv.go

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index f20156f..de0c101 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -71,6 +71,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	go c.hostKeyCallback.PromptAddHosts(ctx)
 	// Print client stats every time something on statsCh is recieved.
 	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet)
+
 	// Keep count of active connections
 	active := make(chan struct{}, len(c.connections))
 
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 63ceaac..51f33c1 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"bytes"
 	"encoding/base64"
 	"fmt"
 	"io"
@@ -17,7 +18,7 @@ type baseHandler struct {
 	server       string
 	shellStarted bool
 	commands     chan string
-	receiveBuf   []byte
+	receiveBuf   bytes.Buffer
 	status       int
 }
 
@@ -56,14 +57,23 @@ func (h *baseHandler) SendMessage(command string) error {
 // Read data from the dtail server via Writer interface.
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		if b == protocol.MessageDelimiter || b == '\n' {
-			if len(h.receiveBuf) == 0 {
+		switch b {
+		/*
+			// TODO: Next DTail version make it so that '\n' gets ignored. For now
+			// leave it for compatibility with older DTail server + ability to display
+			// the protocol mismatch warn message.
+			case '\n' {
+				continue
+		*/
+		case '\n', protocol.MessageDelimiter:
+			message := h.receiveBuf.String()
+			if len(message) == 0 {
 				continue
 			}
-			message := string(h.receiveBuf)
 			h.handleMessageType(message)
-		} else {
-			h.receiveBuf = append(h.receiveBuf, b)
+			h.receiveBuf.Reset()
+		default:
+			h.receiveBuf.WriteByte(b)
 		}
 	}
 
@@ -78,25 +88,22 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	case <-h.Done():
 		return 0, io.EOF
 	}
-
 	return
 }
 
 // Handle various message types.
 func (h *baseHandler) handleMessageType(message string) {
-	if len(h.receiveBuf) == 0 {
+	if len(message) == 0 {
 		return
 	}
 
 	// Hidden server commands starti with a dot "."
-	if h.receiveBuf[0] == '.' {
+	if message[0] == '.' {
 		h.handleHiddenMessage(message)
-		h.receiveBuf = h.receiveBuf[:0]
 		return
 	}
 
 	logger.Raw(message)
-	h.receiveBuf = h.receiveBuf[:0]
 }
 
 // Handle messages received from server which are not meant to be displayed
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 213748c..eca0348 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"time"
@@ -13,7 +14,7 @@ import (
 type HealthHandler struct {
 	done *internal.Done
 	// Buffer of incoming data from server.
-	receiveBuf []byte
+	receiveBuf bytes.Buffer
 	// To send commands to the server.
 	commands chan string
 	// To receive messages from the server.
@@ -72,10 +73,10 @@ func (h *HealthHandler) SendMessage(command string) error {
 // Server writes byte stream to client.
 func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		h.receiveBuf = append(h.receiveBuf, b)
-		if b == protocol.MessageDelimiter { // '\n' {
-			h.receive <- string(h.receiveBuf)
-			h.receiveBuf = h.receiveBuf[:0]
+		h.receiveBuf.WriteByte(b)
+		if b == protocol.MessageDelimiter {
+			h.receive <- h.receiveBuf.String()
+			h.receiveBuf.Reset()
 		}
 	}
 
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index afad507..65b1454 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -15,7 +15,6 @@ type MaprHandler struct {
 	baseHandler
 	aggregate *client.Aggregate
 	query     *mapr.Query
-	count     uint64
 }
 
 // NewMaprHandler returns a new mapreduce client handler.
@@ -36,19 +35,20 @@ func NewMaprHandler(server string, query *mapr.Query, globalGroup *mapr.GlobalGr
 // Read data from the dtail server via Writer interface.
 func (h *MaprHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		h.baseHandler.receiveBuf = append(h.baseHandler.receiveBuf, b)
-		if b == protocol.MessageDelimiter { // '\n' {
-			if len(h.baseHandler.receiveBuf) == 0 {
-				continue
+		switch b {
+		case '\n':
+			continue
+		case protocol.MessageDelimiter:
+			message := h.baseHandler.receiveBuf.String()
+			logger.Debug(message)
+			if message[0] == 'A' {
+				h.handleAggregateMessage(message)
+			} else {
+				h.baseHandler.handleMessageType(message)
 			}
-			message := string(h.baseHandler.receiveBuf)
-
-			if h.baseHandler.receiveBuf[0] == 'A' {
-				h.handleAggregateMessage(strings.TrimSpace(message))
-				h.baseHandler.receiveBuf = h.baseHandler.receiveBuf[:0]
-				continue
-			}
-			h.baseHandler.handleMessageType(message)
+			h.baseHandler.receiveBuf.Reset()
+		default:
+			h.baseHandler.receiveBuf.WriteByte(b)
 		}
 	}
 
@@ -58,12 +58,12 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 // Handle a message received from server including mapr aggregation
 // related data.
 func (h *MaprHandler) handleAggregateMessage(message string) {
-	h.count++
-	parts := strings.Split(message, protocol.AggregateDelimiter)
-
-	// Index 0 contains 'AGGREGATE', 1 contains server host.
-	// Aggregation data begins from index 2.
-	logger.Debug("Received aggregate data", h.server, h.count, parts)
-	h.aggregate.Aggregate(parts[2:])
-	logger.Debug("Aggregated aggregate data", h.server, h.count)
+	parts := strings.SplitN(message, protocol.FieldDelimiter, 3)
+	if len(parts) != 3 {
+		logger.Error("Unable to aggregate data", h.server, message, parts, len(parts), "expected 3 parts")
+		return
+	}
+	if err := h.aggregate.Aggregate(parts[2]); err != nil {
+		logger.Error("Unable to aggregate data", h.server, message, err)
+	}
 }
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 77b674b..cab9a6c 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/omode"
@@ -37,6 +39,8 @@ type MaprClient struct {
 	query *mapr.Query
 	// Additative result or new result every interval run?
 	cumulative bool
+	// The last result string received
+	lastResult string
 }
 
 // NewMaprClient returns a new mapreduce client.
@@ -154,24 +158,37 @@ func (c *MaprClient) reportResults() {
 func (c *MaprClient) printResults() {
 	var result string
 	var err error
-	var numLines int
+	var numRows int
 
 	if c.cumulative {
-		result, numLines, err = c.globalGroup.Result(c.query)
+		result, numRows, err = c.globalGroup.Result(c.query)
 	} else {
-		result, numLines, err = c.globalGroup.SwapOut().Result(c.query)
+		result, numRows, err = c.globalGroup.SwapOut().Result(c.query)
 	}
+
 	if err != nil {
 		logger.FatalExit(err)
 	}
 
-	if numLines == 0 {
-		logger.Warn("Empty result set this time...")
+	if result == c.lastResult {
+		logger.Debug("Result hasn't changed compared to last time...")
 		return
 	}
+	c.lastResult = result
 
-	//logger.Raw(fmt.Sprintf("%s\n", c.query.RawQuery))
-	logger.Raw(c.query.RawQuery)
+	if numRows == 0 {
+		logger.Debug("Empty result set this time...")
+		return
+	}
+
+	rawQuery := c.query.RawQuery
+	if config.Client.TermColorsEnable {
+		rawQuery = color.PaintStrWithAttr(rawQuery,
+			config.Client.TermColors.MaprTable.RawQueryFg,
+			config.Client.TermColors.MaprTable.RawQueryBg,
+			config.Client.TermColors.MaprTable.RawQueryAttr)
+	}
+	logger.Raw(rawQuery)
 	logger.Raw(result)
 }
 
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 5430acd..53c9abb 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -63,6 +63,19 @@ func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, att
 	sb.WriteString(string(FgDefault))
 }
 
+// PaintWithAttrs is similar to PaintWithAttr, but it takes multiple text attributes.
+func PaintWithAttrs(sb *strings.Builder, text string, fg FgColor, bg BgColor, attrs []Attribute) {
+	sb.WriteString(string(fg))
+	sb.WriteString(string(bg))
+	for _, attr := range attrs {
+		sb.WriteString(string(attr))
+	}
+	sb.WriteString(text)
+	sb.WriteString(string(AttrReset))
+	sb.WriteString(string(BgDefault))
+	sb.WriteString(string(FgDefault))
+}
+
 // ResetWithAttr resets background, foreground and attributes.
 func ResetWithAttr(sb *strings.Builder) {
 	sb.WriteString(string(AttrReset))
diff --git a/internal/config/client.go b/internal/config/client.go
index 3c2f7de..795c4a4 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -71,11 +71,32 @@ type commonTermColors struct {
 	SeverityWarnFg    color.FgColor
 }
 
+type maprTableTermColors struct {
+	DataAttr            color.Attribute
+	DataBg              color.BgColor
+	DataFg              color.FgColor
+	DelimiterAttr       color.Attribute
+	DelimiterBg         color.BgColor
+	DelimiterFg         color.FgColor
+	HeaderAttr          color.Attribute
+	HeaderBg            color.BgColor
+	HeaderDelimiterAttr color.Attribute
+	HeaderDelimiterBg   color.BgColor
+	HeaderDelimiterFg   color.FgColor
+	HeaderFg            color.FgColor
+	HeaderGroupKeyAttr  color.Attribute
+	HeaderSortKeyAttr   color.Attribute
+	RawQueryAttr        color.Attribute
+	RawQueryBg          color.BgColor
+	RawQueryFg          color.FgColor
+}
+
 type termColors struct {
-	Remote remoteTermColors
-	Client clientTermColors
-	Server serverTermColors
-	Common commonTermColors
+	Remote    remoteTermColors
+	Client    clientTermColors
+	Server    serverTermColors
+	Common    commonTermColors
+	MaprTable maprTableTermColors
 }
 
 // ClientConfig represents a DTail client configuration (empty as of now as there
@@ -155,6 +176,25 @@ func newDefaultClientConfig() *ClientConfig {
 				SeverityWarnBg:    color.BgBlack,
 				SeverityWarnFg:    color.FgWhite,
 			},
+			MaprTable: maprTableTermColors{
+				DataAttr:            color.AttrNone,
+				DataBg:              color.BgBlue,
+				DataFg:              color.FgWhite,
+				DelimiterAttr:       color.AttrDim,
+				DelimiterBg:         color.BgBlue,
+				DelimiterFg:         color.FgWhite,
+				HeaderAttr:          color.AttrBold,
+				HeaderBg:            color.BgBlue,
+				HeaderFg:            color.FgWhite,
+				HeaderDelimiterAttr: color.AttrDim,
+				HeaderDelimiterBg:   color.BgBlue,
+				HeaderDelimiterFg:   color.FgWhite,
+				HeaderSortKeyAttr:   color.AttrUnderline,
+				HeaderGroupKeyAttr:  color.AttrReverse,
+				RawQueryAttr:        color.AttrDim,
+				RawQueryBg:          color.BgBlack,
+				RawQueryFg:          color.FgCyan,
+			},
 		},
 	}
 }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index f2f672a..c0d44dd 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -16,7 +16,6 @@ import (
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/io/pool"
-	"github.com/mimecast/dtail/internal/protocol"
 	"github.com/mimecast/dtail/internal/regex"
 
 	"github.com/DataDog/zstd"
@@ -187,7 +186,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 				time.Sleep(time.Millisecond * 100)
 				continue
 			}
-			message.WriteByte(protocol.MessageDelimiter)
+			message.WriteString("\n")
 			select {
 			case rawLines <- message:
 				message = pool.BytesBuffer.Get().(*bytes.Buffer)
@@ -202,7 +201,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 					f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
-				message.WriteByte(protocol.MessageDelimiter)
+				message.WriteString("\n")
 				select {
 				case rawLines <- message:
 					message = pool.BytesBuffer.Get().(*bytes.Buffer)
diff --git a/internal/mapr/aggregateset.go b/internal/mapr/aggregateset.go
index 029d87b..47f4925 100644
--- a/internal/mapr/aggregateset.go
+++ b/internal/mapr/aggregateset.go
@@ -6,6 +6,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -68,22 +70,25 @@ func (s *AggregateSet) Merge(query *Query, set *AggregateSet) error {
 
 // Serialize the aggregate set so it can be sent over the wire.
 func (s *AggregateSet) Serialize(ctx context.Context, groupKey string, ch chan<- string) {
-	//logger.Trace("Serialising mapr.AggregateSet", s)
-	var sb strings.Builder
+	logger.Trace("Serialising mapr.AggregateSet", s)
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
 
 	sb.WriteString(groupKey)
 	sb.WriteString(protocol.AggregateDelimiter)
-	sb.WriteString(fmt.Sprintf("%d%s", s.Samples, protocol.AggregateDelimiter))
+	sb.WriteString(fmt.Sprintf("%d", s.Samples))
+	sb.WriteString(protocol.AggregateDelimiter)
 
 	for k, v := range s.FValues {
 		sb.WriteString(k)
-		sb.WriteString("=")
-		sb.WriteString(fmt.Sprintf("%v%s", v, protocol.AggregateDelimiter))
+		sb.WriteString(protocol.AggregateKVDelimiter)
+		sb.WriteString(fmt.Sprintf("%v", v))
+		sb.WriteString(protocol.AggregateDelimiter)
 	}
 
 	for k, v := range s.SValues {
 		sb.WriteString(k)
-		sb.WriteString("=")
+		sb.WriteString(protocol.AggregateKVDelimiter)
 		sb.WriteString(v)
 		sb.WriteString(protocol.AggregateDelimiter)
 	}
diff --git a/internal/mapr/client/aggregate.go b/internal/mapr/client/aggregate.go
index 10b34d4..5cc09a1 100644
--- a/internal/mapr/client/aggregate.go
+++ b/internal/mapr/client/aggregate.go
@@ -1,11 +1,13 @@
 package client
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
 
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // Aggregate mapreduce data on the DTail client side.
@@ -31,12 +33,18 @@ func NewAggregate(server string, query *mapr.Query, globalGroup *mapr.GlobalGrou
 }
 
 // Aggregate data from mapr log line into local (and global) group sets.
-func (a *Aggregate) Aggregate(parts []string) {
+func (a *Aggregate) Aggregate(message string) error {
+	parts := strings.Split(message, protocol.AggregateDelimiter)
+	if len(parts) < 4 {
+		return fmt.Errorf("aggregate message without any real data")
+	}
+
 	groupKey := parts[0]
 	samples, err := strconv.Atoi(parts[1])
 	if err != nil {
-		logger.FatalExit("Unable to parse sample count", parts[1], err, parts)
+		return fmt.Errorf("unable to parse sample count '%s': %v", parts[1], err)
 	}
+
 	fields := a.makeFields(parts[2:])
 	set := a.group.GetSet(groupKey)
 
@@ -63,6 +71,8 @@ func (a *Aggregate) Aggregate(parts []string) {
 		// Re-init local group (make it empty again).
 		a.group.InitSet()
 	}
+
+	return nil
 }
 
 // Create a map of key-value pairs from a part list such as ["foo=bar",  "bar=baz"].
@@ -70,8 +80,8 @@ func (a *Aggregate) makeFields(parts []string) map[string]string {
 	fields := make(map[string]string, len(parts))
 
 	for _, part := range parts {
-		kv := strings.SplitN(part, "=", 2)
-		if len(kv) < 2 {
+		kv := strings.SplitN(part, protocol.AggregateKVDelimiter, 2)
+		if len(kv) != 2 {
 			continue
 		}
 		fields[kv[0]] = kv[1]
diff --git a/internal/mapr/globalgroupset.go b/internal/mapr/globalgroupset.go
index cfab506..21bf990 100644
--- a/internal/mapr/globalgroupset.go
+++ b/internal/mapr/globalgroupset.go
@@ -48,6 +48,7 @@ func (g *GlobalGroupSet) MergeNoblock(query *Query, group *GroupSet) (bool, erro
 
 // Merge a group set into the global group set.
 func (g *GlobalGroupSet) merge(query *Query, group *GroupSet) error {
+
 	for groupKey, set := range group.sets {
 		s := g.GetSet(groupKey)
 		if err := s.Merge(query, set); err != nil {
diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index 6ee2811..d29559a 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -4,13 +4,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"sort"
 	"strconv"
 	"strings"
 
+	"github.com/mimecast/dtail/internal/color"
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // GroupSet represents a map of aggregate sets. The group sets
@@ -22,6 +25,14 @@ type GroupSet struct {
 	sets map[string]*AggregateSet
 }
 
+// Internal helper type
+type result struct {
+	groupKey string
+	values   []string
+	widths   []int
+	orderBy  float64
+}
+
 // NewGroupSet returns a new empty group set.
 func NewGroupSet() *GroupSet {
 	g := GroupSet{}
@@ -58,17 +69,118 @@ func (g *GroupSet) Serialize(ctx context.Context, ch chan<- string) {
 
 // Result returns a nicely formated result of the query from the group set.
 func (g *GroupSet) Result(query *Query) (string, int, error) {
-	return g.limitedResult(query, query.Limit, "\t", " ", false)
+	rows, widths, err := g.result(query, true)
+	if err != nil {
+		return "", 0, err
+	}
+
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
+
+	// Generate header now
+	lastIndex := len(query.Select) - 1
+	for i, sc := range query.Select {
+		format := fmt.Sprintf(" %%%ds ", widths[i])
+		str := fmt.Sprintf(format, sc.FieldStorage)
+		if config.Client.TermColorsEnable {
+			attrs := []color.Attribute{config.Client.TermColors.MaprTable.HeaderAttr}
+			if sc.FieldStorage == query.OrderBy {
+				attrs = append(attrs, config.Client.TermColors.MaprTable.HeaderSortKeyAttr)
+			}
+			for _, groupBy := range query.GroupBy {
+				if sc.FieldStorage == groupBy {
+					attrs = append(attrs, config.Client.TermColors.MaprTable.HeaderGroupKeyAttr)
+					break
+				}
+			}
+			color.PaintWithAttrs(sb, str,
+				config.Client.TermColors.MaprTable.HeaderFg,
+				config.Client.TermColors.MaprTable.HeaderBg,
+				attrs)
+		} else {
+			sb.WriteString(str)
+		}
+
+		if i == lastIndex {
+			continue
+		}
+		if config.Client.TermColorsEnable {
+			color.PaintWithAttr(sb, protocol.FieldDelimiter,
+				config.Client.TermColors.MaprTable.HeaderDelimiterFg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterBg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterAttr)
+		} else {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+	}
+	sb.WriteString("\n")
+
+	for i := 0; i < len(query.Select); i++ {
+		str := fmt.Sprintf("-%s-", strings.Repeat("-", widths[i]))
+		if config.Client.TermColorsEnable {
+			color.PaintWithAttr(sb, str,
+				config.Client.TermColors.MaprTable.HeaderDelimiterFg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterBg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterAttr)
+		} else {
+			sb.WriteString(str)
+		}
+		if i == lastIndex {
+			continue
+		}
+		if config.Client.TermColorsEnable {
+			color.PaintWithAttr(sb, protocol.FieldDelimiter,
+				config.Client.TermColors.MaprTable.HeaderDelimiterFg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterBg,
+				config.Client.TermColors.MaprTable.HeaderDelimiterAttr)
+		} else {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+	}
+	sb.WriteString("\n")
+
+	// And now write the data
+	for i, r := range rows {
+		if i == query.Limit {
+			break
+		}
+		for j, value := range r.values {
+			format := fmt.Sprintf(" %%%ds ", widths[j])
+			str := fmt.Sprintf(format, value)
+			if config.Client.TermColorsEnable {
+				color.PaintWithAttr(sb, str,
+					config.Client.TermColors.MaprTable.DataFg,
+					config.Client.TermColors.MaprTable.DataBg,
+					config.Client.TermColors.MaprTable.DataAttr)
+			} else {
+				sb.WriteString(str)
+			}
+
+			if j == lastIndex {
+				continue
+			}
+			if config.Client.TermColorsEnable {
+				color.PaintWithAttr(sb, protocol.FieldDelimiter,
+					config.Client.TermColors.MaprTable.DelimiterFg,
+					config.Client.TermColors.MaprTable.DelimiterBg,
+					config.Client.TermColors.MaprTable.DelimiterAttr)
+			} else {
+				sb.WriteString(protocol.FieldDelimiter)
+			}
+		}
+		sb.WriteString("\n")
+	}
+
+	return sb.String(), len(rows), nil
 }
 
-// WriteResult writes the result to an outfile.
+// WriteResult writes the result to an CSV outfile.
 func (g *GroupSet) WriteResult(query *Query) error {
 	if !query.HasOutfile() {
 		return errors.New("No outfile specified")
 	}
 
-	// -1: Don't limit the result, include all data sets
-	result, _, err := g.limitedResult(query, -1, "", ",", true)
+	rows, _, err := g.result(query, false)
 	if err != nil {
 		return err
 	}
@@ -76,9 +188,37 @@ func (g *GroupSet) WriteResult(query *Query) error {
 	logger.Info("Writing outfile", query.Outfile)
 	tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile)
 
-	if err := ioutil.WriteFile(tmpOutfile, []byte(result), 0644); err != nil {
+	file, err := os.Create(tmpOutfile)
+	if err != nil {
 		return err
 	}
+	defer file.Close()
+
+	// Generate header now
+	lastIndex := len(query.Select) - 1
+	for i, sc := range query.Select {
+		file.WriteString(sc.FieldStorage)
+		if i == lastIndex {
+			continue
+		}
+		file.WriteString(protocol.CSVDelimiter)
+	}
+	file.WriteString("\n")
+
+	// And now write the data
+	for i, r := range rows {
+		if i == query.Limit {
+			break
+		}
+		for j, value := range r.values {
+			file.WriteString(value)
+			if j == lastIndex {
+				continue
+			}
+			file.WriteString(protocol.CSVDelimiter)
+		}
+		file.WriteString("\n")
+	}
 
 	if err := os.Rename(tmpOutfile, query.Outfile); err != nil {
 		os.Remove(tmpOutfile)
@@ -88,32 +228,22 @@ func (g *GroupSet) WriteResult(query *Query) error {
 	return nil
 }
 
-// Return a nicely formated result of the query from the group set.
-func (g *GroupSet) limitedResult(query *Query, limit int, lineStarter, fieldSeparator string, addHeader bool) (string, int, error) {
-	type result struct {
-		groupKey  string
-		resultStr string
-		orderBy   float64
-	}
+// Return a sorted result slice of the query from the group set.
+func (g *GroupSet) result(query *Query, gatherWidths bool) ([]result, []int, error) {
+	var rows []result
+	widths := make([]int, len(query.Select))
 
-	var resultSlice []result
+	var valueStr string
+	var value float64
 
 	for groupKey, set := range g.sets {
-		var sb strings.Builder
 		r := result{groupKey: groupKey}
 
-		lastIndex := len(query.Select) - 1
 		for i, sc := range query.Select {
-			storage := sc.FieldStorage
-			orderByThis := storage == query.OrderBy
-
 			switch sc.Operation {
 			case Count:
-				value := set.FValues[storage]
-				sb.WriteString(fmt.Sprintf("%d", int(value)))
-				if orderByThis {
-					r.orderBy = value
-				}
+				value = set.FValues[sc.FieldStorage]
+				valueStr = fmt.Sprintf("%d", int(value))
 			case Len:
 				fallthrough
 			case Sum:
@@ -121,74 +251,48 @@ func (g *GroupSet) limitedResult(query *Query, limit int, lineStarter, fieldSepa
 			case Min:
 				fallthrough
 			case Max:
-				value := set.FValues[storage]
-				sb.WriteString(fmt.Sprintf("%f", value))
-				if orderByThis {
-					r.orderBy = value
-				}
+				value = set.FValues[sc.FieldStorage]
+				valueStr = fmt.Sprintf("%f", value)
 			case Last:
-				value := set.SValues[storage]
-				if orderByThis {
-					f, err := strconv.ParseFloat(value, 64)
-					if err == nil {
-						r.orderBy = f
-					}
-				}
-				sb.WriteString(value)
+				valueStr = set.SValues[sc.FieldStorage]
+				value, _ = strconv.ParseFloat(valueStr, 64)
 			case Avg:
-				value := set.FValues[storage] / float64(set.Samples)
-				sb.WriteString(fmt.Sprintf("%f", value))
-				if orderByThis {
-					r.orderBy = value
-				}
+				value = set.FValues[sc.FieldStorage] / float64(set.Samples)
+				valueStr = fmt.Sprintf("%f", value)
 			default:
-				return "", 0, fmt.Errorf("Unknown aggregation method '%v'", sc.Operation)
+				return rows, widths, fmt.Errorf("Unknown aggregation method '%v'", sc.Operation)
 			}
-			if i != lastIndex {
-				sb.WriteString(fieldSeparator)
+
+			if sc.FieldStorage == query.OrderBy {
+				r.orderBy = value
+			}
+			r.values = append(r.values, valueStr)
+
+			if !gatherWidths {
+				continue
+			}
+			if widths[i] < len(sc.FieldStorage) {
+				widths[i] = len(sc.FieldStorage)
+			}
+			if widths[i] < len(valueStr) {
+				widths[i] = len(valueStr)
 			}
 		}
 
-		r.resultStr = sb.String()
-		resultSlice = append(resultSlice, r)
+		rows = append(rows, r)
 	}
 
 	if query.OrderBy != "" {
 		if query.ReverseOrder {
-			sort.SliceStable(resultSlice, func(i, j int) bool {
-				return resultSlice[i].orderBy < resultSlice[j].orderBy
+			sort.SliceStable(rows, func(i, j int) bool {
+				return rows[i].orderBy < rows[j].orderBy
 			})
 		} else {
-			sort.SliceStable(resultSlice, func(i, j int) bool {
-				return resultSlice[i].orderBy > resultSlice[j].orderBy
+			sort.SliceStable(rows, func(i, j int) bool {
+				return rows[i].orderBy > rows[j].orderBy
 			})
 		}
 	}
 
-	var sb strings.Builder
-
-	// Write header first
-	if addHeader {
-		lastIndex := len(query.Select) - 1
-		sb.WriteString(lineStarter)
-		for i, sc := range query.Select {
-			sb.WriteString(sc.FieldStorage)
-			if i != lastIndex {
-				sb.WriteString(fieldSeparator)
-			}
-		}
-		sb.WriteString("\n")
-	}
-
-	// And now write the data
-	for i, r := range resultSlice {
-		if i == limit {
-			break
-		}
-		sb.WriteString(lineStarter)
-		sb.WriteString(r.resultStr)
-		sb.WriteString("\n")
-	}
-
-	return sb.String(), len(resultSlice), nil
+	return rows, widths, nil
 }
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index da976bd..c67137e 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -1,16 +1,22 @@
 package logformat
 
 import (
+	"fmt"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// MakeFieldsDEFAULT is the default log file mapreduce parser.
+// MakeFieldsDEFAULT is the default DTail log file key-value parser.
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
-	fields := make(map[string]string, len(splitted))
+
+	if len(splitted) < 3 || !strings.HasPrefix(splitted[3], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") {
+		// Not a DTail mapreduce log line.
+		return nil, IgnoreFieldsErr
+	}
+
+	fields := make(map[string]string, len(splitted)+8)
 
 	fields["*"] = "*"
 	fields["$line"] = maprLine
@@ -19,20 +25,14 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields["$timezone"] = p.timeZoneName
 	fields["$timeoffset"] = p.timeZoneOffset
 
-	kvStart := 0
-	// DTail mapreduce format
-	if len(splitted) > 3 && strings.HasPrefix(splitted[3], "MAPREDUCE:") {
-		fields["$severity"] = splitted[0]
-		// TODO: Parse time like we do at Mimecast
-		fields["$time"] = splitted[1]
-		kvStart = 4
-	}
+	fields["$severity"] = splitted[0]
+	// TODO: Parse time like we do at Mimecast
+	fields["$time"] = splitted[1]
 
-	for _, kv := range splitted[kvStart:] {
+	for _, kv := range splitted[4:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
-			logger.Debug("Unable to parse key-value token, ignoring it", kv)
-			continue
+			return fields, fmt.Errorf("Unable to parse key-value token '%s'", kv)
 		}
 		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
 	}
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index 6284008..79911d1 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -11,8 +11,8 @@ func TestDefaultLogFormat(t *testing.T) {
 	}
 
 	inputs := []string{
-		"foo=bar|baz=bay",
 		"INFO|20210907-065632|SERVER|MAPREDUCE:TEST|foo=bar|baz=bay",
+		"INFO|20210907-065732|CLIENT|MAPREDUCE:YOMAN|baz=bay|foo=bar",
 	}
 
 	for _, input := range inputs {
@@ -23,20 +23,21 @@ func TestDefaultLogFormat(t *testing.T) {
 		}
 
 		if bar, ok := fields["foo"]; !ok {
-			t.Errorf("Expected field 'foo', but no such field there\n")
+			t.Errorf("Expected field 'foo', but no such field there in '%s'\n", input)
 		} else if bar != "bar" {
-			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s'\n", bar)
+			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n", bar, input)
 		}
 
 		if bay, ok := fields["baz"]; !ok {
-			t.Errorf("Expected field 'baz', but no such field there\n")
+			t.Errorf("Expected field 'baz', but no such field there in '%s'\n", input)
 		} else if bay != "bay" {
-			t.Errorf("Expected 'bay' stored in field 'baz', but got '%s'\n", bay)
+			t.Errorf("Expected 'bay' stored in field 'baz', but got '%s' in '%s'\n", bay, input)
 		}
 
 	}
 
-	if _, err := parser.MakeFields("foo=bar|bazbay"); err == nil {
-		t.Errorf("Expected error but didn't")
+	fields, err := parser.MakeFields("foozoo=bar|bazbay")
+	if _, ok := fields["foo"]; ok {
+		t.Errorf("Expected fiending field 'foo', but found it\n")
 	}
 }
diff --git a/internal/mapr/logformat/generickv.go b/internal/mapr/logformat/generickv.go
new file mode 100644
index 0000000..23d75cb
--- /dev/null
+++ b/internal/mapr/logformat/generickv.go
@@ -0,0 +1,31 @@
+package logformat
+
+import (
+	"strings"
+
+	"github.com/mimecast/dtail/internal/protocol"
+)
+
+// MakeFieldsGENERICKV is the generic key-value logfile parser.
+func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error) {
+	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
+	fields := make(map[string]string, len(splitted))
+
+	fields["*"] = "*"
+	fields["$line"] = maprLine
+	fields["$empty"] = ""
+	fields["$hostname"] = p.hostname
+	fields["$timezone"] = p.timeZoneName
+	fields["$timeoffset"] = p.timeZoneOffset
+
+	for _, kv := range splitted[0:] {
+		keyAndValue := strings.SplitN(kv, "=", 2)
+		if len(keyAndValue) != 2 {
+			//logger.Debug("Unable to parse key-value token, ignoring it", kv)
+			continue
+		}
+		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
+	}
+
+	return fields, nil
+}
diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go
index c53729a..6582b5f 100644
--- a/internal/mapr/logformat/parser.go
+++ b/internal/mapr/logformat/parser.go
@@ -12,6 +12,8 @@ import (
 	"github.com/mimecast/dtail/internal/mapr"
 )
 
+var IgnoreFieldsErr error = errors.New("Ignore this field set")
+
 // Parser is used to parse the mapreduce information from the server log files.
 type Parser struct {
 	hostname           string
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 9106f52..a6d6bb1 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -13,6 +13,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/logformat"
+	"github.com/mimecast/dtail/internal/protocol"
 )
 
 // Aggregate is for aggregating mapreduce data on the DTail server side.
@@ -89,7 +90,6 @@ func (a *Aggregate) Shutdown() {
 
 // Start an aggregation.
 func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
-
 	myCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -109,6 +109,7 @@ func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
 		fieldsCh = a.addFields(myCtx, fieldsCh)
 	}
 
+	// Periodically pre-aggregate data every a.query.Interval seconds.
 	go a.aggregateTimer(myCtx)
 	a.makeMaprLines(myCtx, fieldsCh, maprLines)
 }
@@ -139,13 +140,16 @@ func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 
 				maprLine := strings.TrimSpace(line.Content.String())
 				pool.RecycleBytesBuffer(line.Content)
-				fields, err := a.parser.MakeFields(maprLine)
-				logger.Debug(fields, err)
 
+				fields, err := a.parser.MakeFields(maprLine)
 				if err != nil {
-					logger.Error(err)
+					// Should fields be ignored anyway?
+					if err != logformat.IgnoreFieldsErr {
+						logger.Error(fields, err)
+					}
 					continue
 				}
+
 				if !a.query.WhereClause(fields) {
 					continue
 				}
@@ -170,7 +174,7 @@ func (a *Aggregate) addFields(ctx context.Context, fieldsCh <-chan map[string]st
 		defer close(ch)
 
 		for {
-			// fieldsCh will be closed via 'makeFields' if ctx is done
+			// fieldsCh will be closed via 'makeFields' when ctx is done
 			fields, ok := <-fieldsCh
 			if !ok {
 				return
@@ -219,12 +223,11 @@ func (a *Aggregate) makeMaprLines(ctx context.Context, fieldsCh <-chan map[strin
 }
 
 func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
-	//logger.Trace("Aggregating", group, fields)
 	var sb strings.Builder
 
 	for i, field := range a.query.GroupBy {
 		if i > 0 {
-			sb.WriteString(" ")
+			sb.WriteString(protocol.AggregateGroupKeyCombinator)
 		}
 		if val, ok := fields[field]; ok {
 			sb.WriteString(val)
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index d3035e4..8c9e861 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -5,8 +5,15 @@ const (
 	ProtocolCompat string = "4"
 	// MessageDelimiter delimits separate messages.
 	MessageDelimiter byte = '¬'
-	// FieldDelimiter delimits aggregation fields.
+	// FieldDelimiter delimits messagefields.
 	FieldDelimiter string = "|"
+	// CSVDelimiter delimits CSV file fields.kj:w
+	CSVDelimiter string = ","
+	// AggregateKVDelimiter delimits key-values of an aggregation message.
+	AggregateKVDelimiter string = "≔"
 	// AggregateDelimiter delimits parts of an aggregation message.
-	AggregateDelimiter string = "➔"
+	AggregateDelimiter string = "∥"
+	// AggregateDelimiter string = "⦀"
+	// AggregateGroupKeyCombinator combines the group set keys.
+	AggregateGroupKeyCombinator string = ","
 )
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
index a217b40..1e17c78 100644
--- a/internal/server/handlers/controlhandler.go
+++ b/internal/server/handlers/controlhandler.go
@@ -8,7 +8,6 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
@@ -57,7 +56,7 @@ func (h *ControlHandler) Read(p []byte) (n int, err error) {
 	for {
 		select {
 		case message := <-h.serverMessages:
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s%b", h.hostname, message, protocol.MessageDelimiter))
+			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
 			n = copy(p, wholePayload)
 			return
 		case <-h.done.Done():
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 14f46a3..e74e686 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -38,7 +38,6 @@ type ServerHandler struct {
 	aggregate          *server.Aggregate
 	aggregatedMessages chan string
 	serverMessages     chan string
-	payload            []byte
 	hostname           string
 	user               *user.User
 	catLimiter         chan struct{}
@@ -47,6 +46,8 @@ type ServerHandler struct {
 	activeCommands     int32
 	activeReaders      int32
 	quiet              bool
+	readBuf            bytes.Buffer
+	writeBuf           bytes.Buffer
 }
 
 // NewServerHandler returns the server handler.
@@ -86,77 +87,74 @@ func (h *ServerHandler) Done() <-chan struct{} {
 
 // Read is to send data to the dtail client via Reader interface.
 func (h *ServerHandler) Read(p []byte) (n int, err error) {
-	for {
-		select {
-		case message := <-h.serverMessages:
-			if len(message) == 0 {
-				logger.Warn(h.user, "Empty message recieved")
-				return
-			}
-			if message[0] == '.' {
-				// Handle hidden message (don't display to the user, interpreted by dtail client)
-				payload := []byte(fmt.Sprintf("%s%s", message, string(protocol.MessageDelimiter)))
-				n = copy(p, payload)
-				return
-			}
-
-			// Handle normal server message (display to the user)
-			payload := []byte(fmt.Sprintf("SERVER|%s|%s%s", h.hostname, message, string(protocol.MessageDelimiter)))
-			n = copy(p, payload)
-			return
+	defer h.readBuf.Reset()
 
-		case message := <-h.aggregatedMessages:
-			// Send mapreduce-aggregated data as a message.
-			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
-			buf.WriteString("AGGREGATE")
-			buf.WriteString(protocol.AggregateDelimiter)
-			buf.WriteString(h.hostname)
-			buf.WriteString(protocol.AggregateDelimiter)
-			buf.WriteString(message)
-			buf.WriteByte(protocol.MessageDelimiter)
-			n = copy(p, buf.Bytes())
-			pool.RecycleBytesBuffer(buf)
+	select {
+	case message := <-h.serverMessages:
+		if message[0] == '.' {
+			// Handle hidden message (don't display to the user, interpreted by dtail client)
+			h.readBuf.WriteString(message)
+			h.readBuf.WriteByte(protocol.MessageDelimiter)
+			n = copy(p, h.readBuf.Bytes())
 			return
+		}
 
-		case line := <-h.lines:
-			buf := pool.BytesBuffer.Get().(*bytes.Buffer)
-			buf.WriteString("REMOTE")
-			buf.WriteString(protocol.FieldDelimiter)
-			buf.WriteString(h.hostname)
-			buf.WriteString(protocol.FieldDelimiter)
-			buf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
-			buf.WriteString(protocol.FieldDelimiter)
-			buf.WriteString(fmt.Sprintf("%v", line.Count))
-			buf.WriteString(protocol.FieldDelimiter)
-			buf.WriteString(line.SourceID)
-			buf.WriteString(protocol.FieldDelimiter)
-			payload := append(buf.Bytes(), line.Content.Bytes()...)
-			n = copy(p, payload)
-			pool.RecycleBytesBuffer(buf)
-			pool.RecycleBytesBuffer(line.Content)
+		// Handle normal server message (display to the user)
+		h.readBuf.WriteString("SERVER")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(message)
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+
+	case message := <-h.aggregatedMessages:
+		// Send mapreduce-aggregated data as a message.
+		h.readBuf.WriteString("AGGREGATE")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(message)
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+
+	case line := <-h.lines:
+		h.readBuf.WriteString("REMOTE")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(line.SourceID)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(line.Content.String())
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+		pool.RecycleBytesBuffer(line.Content)
+
+	case <-time.After(time.Second):
+		// Once in a while check whether we are done.
+		select {
+		case <-h.done.Done():
+			err = io.EOF
 			return
-
-		case <-time.After(time.Second):
-			// Once in a while check whether we are done.
-			select {
-			case <-h.done.Done():
-				return 0, io.EOF
-			default:
-			}
+		default:
 		}
 	}
+	return
 }
 
 // Write is to receive data from the dtail client via Writer interface.
 func (h *ServerHandler) Write(p []byte) (n int, err error) {
-	for _, c := range p {
-		switch c {
+	for _, b := range p {
+		switch b {
 		case ';':
-			commandStr := strings.TrimSpace(string(h.payload))
-			h.handleCommand(commandStr)
-			h.payload = nil
+			h.handleCommand(string(h.writeBuf.Bytes()))
+			h.writeBuf.Reset()
 		default:
-			h.payload = append(h.payload, c)
+			h.writeBuf.WriteByte(b)
 		}
 	}
 
-- 
cgit v1.2.3


From 842fd5800000bb68d6306a9ecad80a98ed762a2f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 12 Sep 2021 19:01:34 +0300
Subject: limit mapreduce table output to 10 rows by default

---
 internal/clients/maprclient.go  | 17 ++++++++++++++---
 internal/mapr/globalgroupset.go |  4 ++--
 internal/mapr/groupset.go       |  8 ++++++--
 3 files changed, 22 insertions(+), 7 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index cab9a6c..2cad15d 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -158,12 +158,18 @@ func (c *MaprClient) reportResults() {
 func (c *MaprClient) printResults() {
 	var result string
 	var err error
-	var numRows int
+	var numRows, rowsLimit int
+
+	if c.query.Limit == -1 {
+		// Limit output to 10 rows when the result is printed to stdout.
+		// This can be overriden with the limit clause though.
+		rowsLimit = 10
+	}
 
 	if c.cumulative {
-		result, numRows, err = c.globalGroup.Result(c.query)
+		result, numRows, err = c.globalGroup.Result(c.query, rowsLimit)
 	} else {
-		result, numRows, err = c.globalGroup.SwapOut().Result(c.query)
+		result, numRows, err = c.globalGroup.SwapOut().Result(c.query, rowsLimit)
 	}
 
 	if err != nil {
@@ -189,6 +195,11 @@ func (c *MaprClient) printResults() {
 			config.Client.TermColors.MaprTable.RawQueryAttr)
 	}
 	logger.Raw(rawQuery)
+
+	if rowsLimit > 0 && numRows > rowsLimit {
+		logger.Warn(fmt.Sprintf("Got %d results but limited output to %d rows! Use 'limit' clause to override!",
+			numRows, rowsLimit))
+	}
 	logger.Raw(result)
 }
 
diff --git a/internal/mapr/globalgroupset.go b/internal/mapr/globalgroupset.go
index 21bf990..50bac37 100644
--- a/internal/mapr/globalgroupset.go
+++ b/internal/mapr/globalgroupset.go
@@ -93,9 +93,9 @@ func (g *GlobalGroupSet) WriteResult(query *Query) error {
 }
 
 // Result returns the result of the mapreduce aggregation as a string.
-func (g *GlobalGroupSet) Result(query *Query) (string, int, error) {
+func (g *GlobalGroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
 
-	return g.GroupSet.Result(query)
+	return g.GroupSet.Result(query, rowsLimit)
 }
diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index d29559a..9bff790 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -68,12 +68,16 @@ func (g *GroupSet) Serialize(ctx context.Context, ch chan<- string) {
 }
 
 // Result returns a nicely formated result of the query from the group set.
-func (g *GroupSet) Result(query *Query) (string, int, error) {
+func (g *GroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	rows, widths, err := g.result(query, true)
 	if err != nil {
 		return "", 0, err
 	}
 
+	if query.Limit != -1 {
+		rowsLimit = query.Limit
+	}
+
 	sb := pool.BuilderBuffer.Get().(*strings.Builder)
 	defer pool.RecycleBuilderBuffer(sb)
 
@@ -141,7 +145,7 @@ func (g *GroupSet) Result(query *Query) (string, int, error) {
 
 	// And now write the data
 	for i, r := range rows {
-		if i == query.Limit {
+		if i == rowsLimit {
 			break
 		}
 		for j, value := range r.values {
-- 
cgit v1.2.3


From 2ebe7e9d63ba62c6f19749c39fe0a577d86ca775 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 12 Sep 2021 19:04:42 +0300
Subject: bugfix: dmap skipped the last couple of mapreduce lines

---
 internal/clients/maprclient.go            |   3 +-
 internal/mapr/server/aggregate.go         | 102 ++++++++++++------------------
 internal/server/handlers/readcommand.go   |  15 +++--
 internal/server/handlers/serverhandler.go |  86 +++++++++----------------
 4 files changed, 84 insertions(+), 122 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 2cad15d..68352ea 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -158,7 +158,8 @@ func (c *MaprClient) reportResults() {
 func (c *MaprClient) printResults() {
 	var result string
 	var err error
-	var numRows, rowsLimit int
+	var numRows int
+	rowsLimit := -1
 
 	if c.query.Limit == -1 {
 		// Limit output to 10 rows when the result is printed to stdout.
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index a6d6bb1..d11ed7d 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -19,16 +19,12 @@ import (
 // Aggregate is for aggregating mapreduce data on the DTail server side.
 type Aggregate struct {
 	done *internal.Done
-	// Log lines to process (parsing MAPREDUCE lines).
-	Lines chan line.Line
+	// NextLinesCh can be used to use a new line ch.
+	NextLinesCh chan chan line.Line
 	// Hostname of the current server (used to populate $hostname field).
 	hostname string
 	// Signals to serialize data.
 	serialize chan struct{}
-	// Signals to flush data.
-	flush chan struct{}
-	// Signals that data has been flushed
-	flushed chan struct{}
 	// The mapr query
 	query *mapr.Query
 	// The mapr log format parser
@@ -69,14 +65,12 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 	}
 
 	a := Aggregate{
-		done:      internal.NewDone(),
-		Lines:     make(chan line.Line, 100),
-		serialize: make(chan struct{}),
-		flush:     make(chan struct{}),
-		flushed:   make(chan struct{}),
-		hostname:  s[0],
-		query:     query,
-		parser:    logParser,
+		done:        internal.NewDone(),
+		NextLinesCh: make(chan chan line.Line, 10),
+		serialize:   make(chan struct{}),
+		hostname:    s[0],
+		query:       query,
+		parser:      logParser,
 	}
 
 	return &a, nil
@@ -84,12 +78,11 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 
 // Shutdown the aggregation engine.
 func (a *Aggregate) Shutdown() {
-	a.Flush()
 	a.done.Shutdown()
 }
 
 // Start an aggregation.
-func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
+func (a *Aggregate) Start(ctx context.Context, maprMessages chan<- string) {
 	myCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -102,16 +95,16 @@ func (a *Aggregate) Start(ctx context.Context, maprLines chan<- string) {
 		}
 	}()
 
-	fieldsCh := a.makeFields(myCtx)
+	fieldsCh := a.fieldsFromLines(myCtx)
 
 	// Add fields (e.g. via 'set' clause)
 	if len(a.query.Set) > 0 {
-		fieldsCh = a.addFields(myCtx, fieldsCh)
+		fieldsCh = a.setAdditionalFields(myCtx, fieldsCh)
 	}
 
 	// Periodically pre-aggregate data every a.query.Interval seconds.
 	go a.aggregateTimer(myCtx)
-	a.makeMaprLines(myCtx, fieldsCh, maprLines)
+	a.aggregateAndSerialize(myCtx, fieldsCh, maprMessages)
 }
 
 func (a *Aggregate) aggregateTimer(ctx context.Context) {
@@ -125,23 +118,38 @@ func (a *Aggregate) aggregateTimer(ctx context.Context) {
 	}
 }
 
-func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
-	ch := make(chan map[string]string)
+func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]string {
+	fieldsCh := make(chan map[string]string)
 
 	go func() {
-		defer close(ch)
+		defer close(fieldsCh)
+		var lines chan line.Line
+
+		// Gather first lines channel (first input file)
+		select {
+		case lines = <-a.NextLinesCh:
+		case <-ctx.Done():
+			return
+		}
 
 		for {
 			select {
-			case line, ok := <-a.Lines:
+			case line, ok := <-lines:
 				if !ok {
-					return
+					select {
+					case lines = <-a.NextLinesCh:
+						// Have a new lines channel (e.g. new input file)
+					case <-ctx.Done():
+					default:
+						// No new lines channel found.
+						return
+					}
 				}
 
 				maprLine := strings.TrimSpace(line.Content.String())
+				fields, err := a.parser.MakeFields(maprLine)
 				pool.RecycleBytesBuffer(line.Content)
 
-				fields, err := a.parser.MakeFields(maprLine)
 				if err != nil {
 					// Should fields be ignored anyway?
 					if err != logformat.IgnoreFieldsErr {
@@ -155,7 +163,7 @@ func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 				}
 
 				select {
-				case ch <- fields:
+				case fieldsCh <- fields:
 				case <-ctx.Done():
 				}
 			case <-ctx.Done():
@@ -164,17 +172,16 @@ func (a *Aggregate) makeFields(ctx context.Context) <-chan map[string]string {
 		}
 	}()
 
-	return ch
+	return fieldsCh
 }
 
-func (a *Aggregate) addFields(ctx context.Context, fieldsCh <-chan map[string]string) <-chan map[string]string {
-	ch := make(chan map[string]string)
+func (a *Aggregate) setAdditionalFields(ctx context.Context, fieldsCh <-chan map[string]string) <-chan map[string]string {
+	newFieldsCh := make(chan map[string]string)
 
 	go func() {
-		defer close(ch)
+		defer close(newFieldsCh)
 
 		for {
-			// fieldsCh will be closed via 'makeFields' when ctx is done
 			fields, ok := <-fieldsCh
 			if !ok {
 				return
@@ -184,23 +191,22 @@ func (a *Aggregate) addFields(ctx context.Context, fieldsCh <-chan map[string]st
 			}
 
 			select {
-			case ch <- fields:
+			case newFieldsCh <- fields:
 			case <-ctx.Done():
 			}
 		}
 	}()
 
-	return ch
+	return newFieldsCh
 }
 
-func (a *Aggregate) makeMaprLines(ctx context.Context, fieldsCh <-chan map[string]string, maprLines chan<- string) {
+func (a *Aggregate) aggregateAndSerialize(ctx context.Context, fieldsCh <-chan map[string]string, maprMessages chan<- string) {
 	group := mapr.NewGroupSet()
 
 	serialize := func() {
 		logger.Info("Serializing mapreduce result")
-		group.Serialize(ctx, maprLines)
+		group.Serialize(ctx, maprMessages)
 		group = mapr.NewGroupSet()
-		logger.Info("Done serializing mapreduce result")
 	}
 
 	for {
@@ -213,9 +219,6 @@ func (a *Aggregate) makeMaprLines(ctx context.Context, fieldsCh <-chan map[strin
 			a.aggregate(group, fields)
 		case <-a.serialize:
 			serialize()
-		case <-a.flush:
-			serialize()
-			a.flushed <- struct{}{}
 		case <-ctx.Done():
 			return
 		}
@@ -264,24 +267,3 @@ func (a *Aggregate) Serialize(ctx context.Context) {
 	case <-ctx.Done():
 	}
 }
-
-// Flush all data.
-func (a *Aggregate) Flush() {
-	select {
-	case a.flush <- struct{}{}:
-		logger.Info("Flushing mapreduce data")
-	case <-time.After(time.Minute):
-		logger.Warn("Starting to flush mapreduce data takes over a minute")
-		return
-	case <-a.done.Done():
-		return
-	}
-
-	select {
-	case <-a.flushed:
-		logger.Info("Done flushing")
-	case <-time.After(time.Minute):
-		logger.Warn("Waiting for data to be flushed takes over a minute")
-	case <-a.done.Done():
-	}
-}
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 5bab26f..69dd4a5 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/io/fs"
+	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
@@ -113,16 +114,20 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 	}
 
 	lines := r.server.lines
-
-	// Plug in mappreduce engine
-	if r.server.aggregate != nil {
-		lines = r.server.aggregate.Lines
-	}
+	aggregate := r.server.aggregate
 
 	for {
+		if aggregate != nil {
+			lines = make(chan line.Line, 100)
+			aggregate.NextLinesCh <- lines
+		}
 		if err := reader.Start(ctx, lines, re); err != nil {
 			logger.Error(r.server.user, path, globID, err)
 		}
+		if aggregate != nil {
+			// Also makes aggregate to Flush
+			close(lines)
+		}
 
 		select {
 		case <-ctx.Done():
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index e74e686..ed19412 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -32,36 +32,35 @@ const (
 // the Bi-directional communication between SSH client and server.
 // This handler implements the handler of the SSH server.
 type ServerHandler struct {
-	done               *internal.Done
-	lines              chan line.Line
-	regex              string
-	aggregate          *server.Aggregate
-	aggregatedMessages chan string
-	serverMessages     chan string
-	hostname           string
-	user               *user.User
-	catLimiter         chan struct{}
-	tailLimiter        chan struct{}
-	ackCloseReceived   chan struct{}
-	activeCommands     int32
-	activeReaders      int32
-	quiet              bool
-	readBuf            bytes.Buffer
-	writeBuf           bytes.Buffer
+	done             *internal.Done
+	lines            chan line.Line
+	regex            string
+	aggregate        *server.Aggregate
+	maprMessages     chan string
+	serverMessages   chan string
+	hostname         string
+	user             *user.User
+	catLimiter       chan struct{}
+	tailLimiter      chan struct{}
+	ackCloseReceived chan struct{}
+	activeCommands   int32
+	quiet            bool
+	readBuf          bytes.Buffer
+	writeBuf         bytes.Buffer
 }
 
 // NewServerHandler returns the server handler.
 func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
 	h := ServerHandler{
-		done:               internal.NewDone(),
-		lines:              make(chan line.Line, 100),
-		serverMessages:     make(chan string, 10),
-		aggregatedMessages: make(chan string, 10),
-		ackCloseReceived:   make(chan struct{}),
-		catLimiter:         catLimiter,
-		tailLimiter:        tailLimiter,
-		regex:              ".",
-		user:               user,
+		done:             internal.NewDone(),
+		lines:            make(chan line.Line, 100),
+		serverMessages:   make(chan string, 10),
+		maprMessages:     make(chan string, 10),
+		ackCloseReceived: make(chan struct{}),
+		catLimiter:       catLimiter,
+		tailLimiter:      tailLimiter,
+		regex:            ".",
+		user:             user,
 	}
 
 	fqdn, err := os.Hostname()
@@ -108,7 +107,7 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		h.readBuf.WriteByte(protocol.MessageDelimiter)
 		n = copy(p, h.readBuf.Bytes())
 
-	case message := <-h.aggregatedMessages:
+	case message := <-h.maprMessages:
 		// Send mapreduce-aggregated data as a message.
 		h.readBuf.WriteString("AGGREGATE")
 		h.readBuf.WriteString(protocol.FieldDelimiter)
@@ -260,14 +259,6 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			h.shutdown()
 		}
 	}
-	readerFinished := func() {
-		if h.decrementActiveReaders() == 0 {
-			if h.aggregate == nil {
-				return
-			}
-			h.aggregate.Shutdown()
-		}
-	}
 
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
@@ -289,18 +280,14 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 	case "grep", "cat":
 		command := newReadCommand(h, omode.CatClient)
 		go func() {
-			h.incrementActiveReaders()
 			command.Start(ctx, argc, args, 1)
-			readerFinished()
 			commandFinished()
 		}()
 
 	case "tail":
 		command := newReadCommand(h, omode.TailClient)
 		go func() {
-			h.incrementActiveReaders()
 			command.Start(ctx, argc, args, 10)
-			readerFinished()
 			commandFinished()
 		}()
 
@@ -315,7 +302,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 
 		h.aggregate = aggregate
 		go func() {
-			command.Start(ctx, h.aggregatedMessages)
+			command.Start(ctx, h.maprMessages)
 			commandFinished()
 		}()
 
@@ -361,15 +348,11 @@ func (h *ServerHandler) serverMessageC() chan<- string {
 	return h.serverMessages
 }
 
-func (h *ServerHandler) flush() {
-	logger.Debug(h.user, "flush()")
-
-	if h.aggregate != nil {
-		h.aggregate.Flush()
-	}
+func (h *ServerHandler) flushMessages() {
+	logger.Debug(h.user, "flushMessages()")
 
 	unsentMessages := func() int {
-		return len(h.lines) + len(h.serverMessages) + len(h.aggregatedMessages)
+		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
 	}
 	for i := 0; i < 3; i++ {
 		if unsentMessages() == 0 {
@@ -385,7 +368,7 @@ func (h *ServerHandler) flush() {
 
 func (h *ServerHandler) shutdown() {
 	logger.Debug(h.user, "shutdown()")
-	h.flush()
+	h.flushMessages()
 
 	go func() {
 		select {
@@ -413,15 +396,6 @@ func (h *ServerHandler) decrementActiveCommands() int32 {
 	return atomic.LoadInt32(&h.activeCommands)
 }
 
-func (h *ServerHandler) incrementActiveReaders() {
-	atomic.AddInt32(&h.activeReaders, 1)
-}
-
-func (h *ServerHandler) decrementActiveReaders() int32 {
-	atomic.AddInt32(&h.activeReaders, -1)
-	return atomic.LoadInt32(&h.activeReaders)
-}
-
 func readOptions(opts []string) (map[string]string, error) {
 	options := make(map[string]string, len(opts))
 
-- 
cgit v1.2.3


From 5e717af91e8012c72ec7dc0204420dea46f187db Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 18 Sep 2021 11:57:09 +0300
Subject: new docker test cases - also change default FATAL bg color to magenta

---
 internal/clients/args.go                 | 46 +++++++++++++++++++++++++++++++-
 internal/clients/baseclient.go           |  3 +++
 internal/clients/handlers/basehandler.go |  7 -----
 internal/config/client.go                |  4 +--
 4 files changed, 50 insertions(+), 10 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
index 7f782f1..7ce1634 100644
--- a/internal/clients/args.go
+++ b/internal/clients/args.go
@@ -1,6 +1,13 @@
 package clients
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/omode"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -22,5 +29,42 @@ type Args struct {
 	SSHAuthMethods     []gossh.AuthMethod
 	SSHHostKeyCallback gossh.HostKeyCallback
 	PrivateKeyPathFile string
-	Quiet            bool
+	Quiet              bool
+}
+
+// When no servers are given, connect to localhost!
+func (a *Args) handleEmptyServer() {
+	if a.Discovery != "" || a.ServersStr != "" {
+		return
+	}
+
+	fqdn, err := os.Hostname()
+	if err != nil {
+		logger.FatalExit(err)
+	}
+	a.ServersStr = fmt.Sprintf("%s:%d", fqdn, config.Common.SSHPort)
+	// I am trusting my own hostname.
+	a.TrustAllHosts = true
+	logger.Debug("Will connect to local server", a.ServersStr)
+
+	cleanPath := func(dirtyPath string) string {
+		cleanPath, err := filepath.EvalSymlinks(dirtyPath)
+		if err != nil {
+			logger.FatalExit("Unable to evaluate symlinks", dirtyPath, err)
+		}
+		cleanPath, err = filepath.Abs(cleanPath)
+		if err != nil {
+			logger.FatalExit("Unable to make file path absolute", dirtyPath, cleanPath, err)
+		}
+		return cleanPath
+	}
+
+	logger.Debug("Dirty file paths", a.What)
+	var filePaths []string
+	for _, dirtyPath := range strings.Split(a.What, ",") {
+		filePaths = append(filePaths, cleanPath(dirtyPath))
+	}
+
+	a.What = strings.Join(filePaths, ",")
+	logger.Debug("Clean file paths", a.What)
 }
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index de0c101..455174e 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -41,6 +41,9 @@ type baseClient struct {
 func (c *baseClient) init() {
 	logger.Debug("Initiating base client")
 
+	// When empty server list provided, connect to localhost by default.
+	c.Args.handleEmptyServer()
+
 	flag := regex.Default
 	if c.Args.RegexInvert {
 		flag = regex.Invert
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 51f33c1..74559e9 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -112,12 +112,5 @@ func (h *baseHandler) handleHiddenMessage(message string) {
 	switch {
 	case strings.HasPrefix(message, ".syn close connection"):
 		h.SendMessage(".ack close connection")
-		select {
-		case <-time.After(time.Second * 5):
-			logger.Debug("Shutting down client after timeout and sending ack to server")
-			h.Shutdown()
-		case <-h.Done():
-			return
-		}
 	}
 }
diff --git a/internal/config/client.go b/internal/config/client.go
index 795c4a4..152ab15 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -169,8 +169,8 @@ func newDefaultClientConfig() *ClientConfig {
 				SeverityErrorAttr: color.AttrBold,
 				SeverityErrorBg:   color.BgRed,
 				SeverityErrorFg:   color.FgWhite,
-				SeverityFatalAttr: color.AttrBlink,
-				SeverityFatalBg:   color.BgRed,
+				SeverityFatalAttr: color.AttrBold,
+				SeverityFatalBg:   color.BgMagenta,
 				SeverityFatalFg:   color.FgWhite,
 				SeverityWarnAttr:  color.AttrBold,
 				SeverityWarnBg:    color.BgBlack,
-- 
cgit v1.2.3


From 6506e20f6c80f4acb7434eb9dd14f784a67189cd Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 18 Sep 2021 14:41:25 +0300
Subject: add spartan mode

---
 internal/clients/args.go                  | 34 +++++++++++++++++++++++++++----
 internal/clients/baseclient.go            |  3 ---
 internal/clients/catclient.go             |  6 ++++--
 internal/clients/grepclient.go            |  7 +++++--
 internal/clients/handlers/basehandler.go  | 15 +++++++-------
 internal/clients/maprclient.go            |  7 +++++--
 internal/clients/tailclient.go            |  7 +++++--
 internal/config/read.go                   |  5 ++++-
 internal/io/fs/readfile.go                | 13 +++++++-----
 internal/server/handlers/serverhandler.go | 31 +++++++++++++++++++---------
 10 files changed, 89 insertions(+), 39 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
index 7ce1634..081b911 100644
--- a/internal/clients/args.go
+++ b/internal/clients/args.go
@@ -1,6 +1,7 @@
 package clients
 
 import (
+	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -30,14 +31,35 @@ type Args struct {
 	SSHHostKeyCallback gossh.HostKeyCallback
 	PrivateKeyPathFile string
 	Quiet              bool
+	Spartan            bool
+	NoColor            bool
 }
 
-// When no servers are given, connect to localhost!
-func (a *Args) handleEmptyServer() {
-	if a.Discovery != "" || a.ServersStr != "" {
-		return
+// Transform the arguments based on certain conditions.
+func (a *Args) Transform(args []string) {
+	// Interpret additional args as file list.
+	if a.What == "" {
+		var files []string
+		for _, file := range flag.Args() {
+			files = append(files, file)
+		}
+		a.What = strings.Join(files, ",")
+	}
+
+	if a.Spartan {
+		a.Quiet = true
+		a.NoColor = true
 	}
+}
 
+// TransformAfterConfigFile same as Transform, but after the config file has been read.
+func (a *Args) TransformAfterConfigFile() {
+	if a.Discovery == "" && a.ServersStr == "" {
+		a.handleEmptyServer()
+	}
+}
+
+func (a *Args) handleEmptyServer() {
 	fqdn, err := os.Hostname()
 	if err != nil {
 		logger.FatalExit(err)
@@ -68,3 +90,7 @@ func (a *Args) handleEmptyServer() {
 	a.What = strings.Join(filePaths, ",")
 	logger.Debug("Clean file paths", a.What)
 }
+
+func (a *Args) SerializeOptions() string {
+	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
+}
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 455174e..de0c101 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -41,9 +41,6 @@ type baseClient struct {
 func (c *baseClient) init() {
 	logger.Debug("Initiating base client")
 
-	// When empty server list provided, connect to localhost by default.
-	c.Args.handleEmptyServer()
-
 	flag := regex.Default
 	if c.Args.RegexInvert {
 		flag = regex.Invert
diff --git a/internal/clients/catclient.go b/internal/clients/catclient.go
index b7b6131..d14cdcc 100644
--- a/internal/clients/catclient.go
+++ b/internal/clients/catclient.go
@@ -42,9 +42,11 @@ func (c CatClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c CatClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
+			c.Mode.String(),
+			c.Args.SerializeOptions(),
+			file, c.Regex.Serialize()))
 	}
 	return
 }
diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index 652c31b..ea5022b 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -41,9 +41,12 @@ func (c GrepClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c GrepClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
+			c.Mode.String(),
+			c.Args.SerializeOptions(),
+			file,
+			c.Regex.Serialize()))
 	}
 
 	return
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 74559e9..0f2d1b5 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -67,9 +67,12 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 		*/
 		case '\n', protocol.MessageDelimiter:
 			message := h.receiveBuf.String()
-			if len(message) == 0 {
-				continue
-			}
+			/*
+				// dcat/grep should actually display empty lines.
+					if len(message) == 0 {
+						continue
+					}
+			*/
 			h.handleMessageType(message)
 			h.receiveBuf.Reset()
 		default:
@@ -93,12 +96,8 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 
 // Handle various message types.
 func (h *baseHandler) handleMessageType(message string) {
-	if len(message) == 0 {
-		return
-	}
-
 	// Hidden server commands starti with a dot "."
-	if message[0] == '.' {
+	if len(message) > 0 && message[0] == '.' {
 		h.handleHiddenMessage(message)
 		return
 	}
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 68352ea..e6ab96b 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -116,7 +116,6 @@ func (c MaprClient) makeHandler(server string) handlers.Handler {
 
 func (c MaprClient) makeCommands() (commands []string) {
 	commands = append(commands, fmt.Sprintf("map %s", c.query.RawQuery))
-	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 
 	modeStr := "cat"
 	if c.Mode == omode.TailClient {
@@ -128,7 +127,11 @@ func (c MaprClient) makeCommands() (commands []string) {
 			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout, modeStr, file, c.Regex.Serialize()))
 			continue
 		}
-		commands = append(commands, fmt.Sprintf("%s:%s %s %s", modeStr, options, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
+			modeStr,
+			c.Args.SerializeOptions(),
+			file,
+			c.Regex.Serialize()))
 	}
 
 	return
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index cefbaa7..360354b 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -38,9 +38,12 @@ func (c TailClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c TailClient) makeCommands() (commands []string) {
-	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
+			c.Mode.String(),
+			c.Args.SerializeOptions(),
+			file,
+			c.Regex.Serialize()))
 	}
 	logger.Debug(commands)
 
diff --git a/internal/config/read.go b/internal/config/read.go
index a4e605b..ea358f8 100644
--- a/internal/config/read.go
+++ b/internal/config/read.go
@@ -5,7 +5,7 @@ import (
 )
 
 // Read the DTail configuration.
-func Read(configFile string, sshPort int) {
+func Read(configFile string, sshPort int, noColor bool) {
 	initializer := configInitializer{
 		Common: newDefaultCommonConfig(),
 		Server: newDefaultServerConfig(),
@@ -34,4 +34,7 @@ func Read(configFile string, sshPort int) {
 	if sshPort != 2222 {
 		Common.SSHPort = sshPort
 	}
+	if noColor {
+		Client.TermColorsEnable = false
+	}
 }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index c0d44dd..ec33c60 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -182,11 +182,14 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 
 		switch b {
 		case '\n':
-			if message.Len() == 0 {
-				time.Sleep(time.Millisecond * 100)
-				continue
-			}
-			message.WriteString("\n")
+			/*
+				// dcat/dgrep should actually transfer empty lines
+					if message.Len() == 0 {
+						time.Sleep(time.Millisecond * 100)
+						continue
+					}
+			*/
+			//message.WriteString("\n")
 			select {
 			case rawLines <- message:
 				message = pool.BytesBuffer.Get().(*bytes.Buffer)
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index ed19412..2f3b73b 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -45,6 +45,7 @@ type ServerHandler struct {
 	ackCloseReceived chan struct{}
 	activeCommands   int32
 	quiet            bool
+	spartan          bool
 	readBuf          bytes.Buffer
 	writeBuf         bytes.Buffer
 }
@@ -118,16 +119,18 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 		n = copy(p, h.readBuf.Bytes())
 
 	case line := <-h.lines:
-		h.readBuf.WriteString("REMOTE")
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(h.hostname)
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(line.SourceID)
-		h.readBuf.WriteString(protocol.FieldDelimiter)
+		if !h.spartan {
+			h.readBuf.WriteString("REMOTE")
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(h.hostname)
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(line.SourceID)
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+		}
 		h.readBuf.WriteString(line.Content.String())
 		h.readBuf.WriteByte(protocol.MessageDelimiter)
 		n = copy(p, h.readBuf.Bytes())
@@ -275,6 +278,12 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			h.quiet = true
 		}
 	}
+	if spartan, ok := options["spartan"]; ok {
+		if spartan == "true" {
+			logger.Debug(h.user, "Enabling spartan mode")
+			h.spartan = true
+		}
+	}
 
 	switch commandName {
 	case "grep", "cat":
@@ -397,6 +406,7 @@ func (h *ServerHandler) decrementActiveCommands() int32 {
 }
 
 func readOptions(opts []string) (map[string]string, error) {
+	logger.Debug("Parsing options", opts)
 	options := make(map[string]string, len(opts))
 
 	for _, o := range opts {
@@ -416,6 +426,7 @@ func readOptions(opts []string) (map[string]string, error) {
 			val = string(decoded)
 		}
 
+		logger.Debug("Setting option", key, val)
 		options[key] = val
 	}
 
-- 
cgit v1.2.3


From 69b88a1cae0a61bd22530c384f40166b37b9f1ea Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 18 Sep 2021 18:43:19 +0300
Subject: remote connector is now an interface

---
 internal/clients/baseclient.go                  |  24 +--
 internal/clients/connectors/connector.go        |  17 ++
 internal/clients/connectors/serverconnection.go | 225 ++++++++++++++++++++++++
 internal/clients/healthclient.go                |  12 +-
 internal/clients/remote/connection.go           | 212 ----------------------
 5 files changed, 259 insertions(+), 231 deletions(-)
 create mode 100644 internal/clients/connectors/connector.go
 create mode 100644 internal/clients/connectors/serverconnection.go
 delete mode 100644 internal/clients/remote/connection.go

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index de0c101..d0631fc 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mimecast/dtail/internal/clients/remote"
+	"github.com/mimecast/dtail/internal/clients/connectors"
 	"github.com/mimecast/dtail/internal/discovery"
 	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/omode"
@@ -23,7 +23,7 @@ type baseClient struct {
 	// List of remote servers to connect to.
 	servers []string
 	// We have one connection per remote server.
-	connections []*remote.Connection
+	connections []connectors.Connector
 	// SSH auth methods to use to connect to the remote servers.
 	sshAuthMethods []gossh.AuthMethod
 	// To deal with SSH host keys
@@ -77,7 +77,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 
 	var mutex sync.Mutex
 	for i, conn := range c.connections {
-		go func(i int, conn *remote.Connection) {
+		go func(i int, conn connectors.Connector) {
 			connStatus := c.start(ctx, active, i, conn)
 
 			// Update global status.
@@ -93,7 +93,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	return
 }
 
-func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, conn *remote.Connection) (status int) {
+func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, conn connectors.Connector) (status int) {
 	// Increment connection count
 	active <- struct{}{}
 	// Derement connection count
@@ -105,26 +105,20 @@ func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, con
 
 		conn.Start(connCtx, cancel, c.throttleCh, c.stats.connectionsEstCh)
 		// Retrieve status code from handler (dtail client will exit with that status)
-		status = conn.Handler.Status()
+		status = conn.Handler().Status()
 
 		if !c.retry {
 			return
 		}
 
 		time.Sleep(time.Second * 2)
-		logger.Debug(conn.Server, "Reconnecting")
-
-		conn = c.makeConnection(conn.Server, c.sshAuthMethods, c.hostKeyCallback)
-		c.connections[i] = conn
+		logger.Debug(conn.Server(), "Reconnecting")
+		c.connections[i] = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback)
 	}
 }
 
-func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod, hostKeyCallback client.HostKeyCallback) *remote.Connection {
-	conn := remote.NewConnection(server, c.UserName, sshAuthMethods, hostKeyCallback)
-	conn.Handler = c.maker.makeHandler(server)
-	conn.Commands = c.maker.makeCommands()
-
-	return conn
+func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod, hostKeyCallback client.HostKeyCallback) connectors.Connector {
+	return connectors.NewServerConnection(server, c.UserName, sshAuthMethods, hostKeyCallback, c.maker.makeHandler(server), c.maker.makeCommands())
 }
 
 func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
diff --git a/internal/clients/connectors/connector.go b/internal/clients/connectors/connector.go
new file mode 100644
index 0000000..3ab6a08
--- /dev/null
+++ b/internal/clients/connectors/connector.go
@@ -0,0 +1,17 @@
+package connectors
+
+import (
+	"context"
+
+	"github.com/mimecast/dtail/internal/clients/handlers"
+)
+
+// Connector interface.
+type Connector interface {
+	// Start the connection.
+	Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{})
+	// Server hostname.
+	Server() string
+	// Handler for the connection.
+	Handler() handlers.Handler
+}
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
new file mode 100644
index 0000000..fab2f87
--- /dev/null
+++ b/internal/clients/connectors/serverconnection.go
@@ -0,0 +1,225 @@
+package connectors
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/ssh/client"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// ServerConnection represents a client connection connection to a single server.
+type ServerConnection struct {
+	// The remote server's hostname connected to.
+	server string
+	// The remote server's port connected to.
+	port int
+	// The SSH client configuration used.
+	config *ssh.ClientConfig
+	// The SSH client handler to use.
+	handler handlers.Handler
+	// DTail commands sent from client to server. When client loses
+	// connection to the server it re-connects automatically and sends the
+	// same commands again.
+	commands []string
+	// Is it a persistent connection or a one-off?
+	isOneOff bool
+	// To deal with SSH server host keys
+	hostKeyCallback client.HostKeyCallback
+	// To determine if connection throttling has finished or not
+	throttlingDone bool
+}
+
+// NewServerConnection returns a new connection.
+func NewServerConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback, handler handlers.Handler, commands []string) *ServerConnection {
+	logger.Debug(server, "Creating new connection")
+
+	c := ServerConnection{
+		hostKeyCallback: hostKeyCallback,
+		server:          server,
+		handler:         handler,
+		commands:        commands,
+		config: &ssh.ClientConfig{
+			User:            userName,
+			Auth:            authMethods,
+			HostKeyCallback: hostKeyCallback.Wrap(),
+			Timeout:         time.Second * 2,
+		},
+	}
+
+	c.initServerPort()
+	return &c
+}
+
+// NewOneOffServerConnection creates new one-off connection (only for sending a series of commands and then quit).
+func NewOneOffServerConnection(server string, userName string, authMethods []ssh.AuthMethod, handler handlers.Handler, commands []string) *ServerConnection {
+	c := ServerConnection{
+		server:   server,
+		handler:  handler,
+		commands: commands,
+		config: &ssh.ClientConfig{
+			User:            userName,
+			Auth:            authMethods,
+			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		},
+		isOneOff: true,
+	}
+
+	c.initServerPort()
+	return &c
+}
+
+// Server hostname
+func (c *ServerConnection) Server() string {
+	return c.server
+}
+
+// Handler for the connection
+func (c *ServerConnection) Handler() handlers.Handler {
+	return c.handler
+}
+
+// Attempt to parse the server port address from the provided server FQDN.
+func (c *ServerConnection) initServerPort() {
+	c.port = config.Common.SSHPort
+	parts := strings.Split(c.server, ":")
+
+	if len(parts) == 2 {
+		logger.Debug("Parsing port from hostname", parts)
+		port, err := strconv.Atoi(parts[1])
+		if err != nil {
+			logger.FatalExit("Unable to parse client port", c.server, parts, err)
+		}
+		c.server = parts[0]
+		c.port = port
+	}
+}
+
+// Start the server connection. Build up SSH session and send some DTail commands.
+func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+	// Throttle how many connections can be established concurrently (based on ch length)
+	logger.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
+
+	select {
+	case throttleCh <- struct{}{}:
+	case <-ctx.Done():
+		logger.Debug(c.server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
+		return
+	}
+
+	logger.Debug(c.server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
+
+	go func() {
+		defer func() {
+			if !c.throttlingDone {
+				logger.Debug(c.server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
+				c.throttlingDone = true
+				<-throttleCh
+			}
+			cancel()
+		}()
+
+		if err := c.dial(ctx, cancel, throttleCh, statsCh); err != nil {
+			logger.Warn(c.server, c.port, err)
+			if c.hostKeyCallback.Untrusted(fmt.Sprintf("%s:%d", c.server, c.port)) {
+				logger.Debug(c.server, "Not trusting host")
+			}
+		}
+	}()
+
+	<-ctx.Done()
+}
+
+// Dail into a new SSH connection. Close connection in case of an error.
+func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) error {
+	logger.Debug(c.server, "Incrementing connection stats")
+	statsCh <- struct{}{}
+	defer func() {
+		logger.Debug(c.server, "Decrementing connection stats")
+		<-statsCh
+	}()
+
+	logger.Debug(c.server, "Dialing into the connection")
+	address := fmt.Sprintf("%s:%d", c.server, c.port)
+
+	client, err := ssh.Dial("tcp", address, c.config)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	return c.session(ctx, cancel, client, throttleCh)
+}
+
+// Create the SSH session. Close the session in case of an error.
+func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
+	logger.Debug(c.server, "session")
+
+	session, err := client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	return c.handle(ctx, cancel, session, throttleCh)
+}
+
+func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
+	logger.Debug(c.server, "handle")
+
+	stdinPipe, err := session.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	stdoutPipe, err := session.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	if err := session.Shell(); err != nil {
+		return err
+	}
+
+	go func() {
+		io.Copy(stdinPipe, c.handler)
+		cancel()
+	}()
+
+	go func() {
+		io.Copy(c.handler, stdoutPipe)
+		cancel()
+	}()
+
+	go func() {
+		select {
+		case <-c.handler.Done():
+		case <-ctx.Done():
+		}
+		cancel()
+	}()
+
+	// Send all commands to client.
+	for _, command := range c.commands {
+		logger.Debug(command)
+		c.handler.SendMessage(command)
+	}
+
+	if !c.throttlingDone {
+		logger.Debug(c.server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
+		c.throttlingDone = true
+		<-throttleCh
+	}
+
+	<-ctx.Done()
+	c.handler.Shutdown()
+	return nil
+}
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index 692464c..47007b6 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -7,8 +7,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mimecast/dtail/internal/clients/connectors"
 	"github.com/mimecast/dtail/internal/clients/handlers"
-	"github.com/mimecast/dtail/internal/clients/remote"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/protocol"
@@ -47,9 +47,13 @@ func (c *HealthClient) Start(ctx context.Context) (status int) {
 	throttleCh := make(chan struct{}, runtime.NumCPU())
 	statsCh := make(chan struct{}, 1)
 
-	conn := remote.NewOneOffConnection(c.server, c.userName, c.sshAuthMethods)
-	conn.Handler = handlers.NewHealthHandler(c.server, receive)
-	conn.Commands = []string{c.mode.String()}
+	conn := connectors.NewOneOffServerConnection(
+		c.server,
+		c.userName,
+		c.sshAuthMethods,
+		handlers.NewHealthHandler(c.server, receive),
+		[]string{c.mode.String()},
+	)
 
 	connCtx, cancel := context.WithCancel(ctx)
 	go conn.Start(connCtx, cancel, throttleCh, statsCh)
diff --git a/internal/clients/remote/connection.go b/internal/clients/remote/connection.go
deleted file mode 100644
index b29ffed..0000000
--- a/internal/clients/remote/connection.go
+++ /dev/null
@@ -1,212 +0,0 @@
-package remote
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/mimecast/dtail/internal/clients/handlers"
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/ssh/client"
-
-	"golang.org/x/crypto/ssh"
-)
-
-// Connection represents a client connection connection to a single server.
-type Connection struct {
-	// The remote server's hostname connected to.
-	Server string
-	// The remote server's port connected to.
-	port int
-	// The SSH client configuration used.
-	config *ssh.ClientConfig
-	// The SSH client handler to use.
-	Handler handlers.Handler
-	// DTail commands sent from client to server. When client loses
-	// connection to the server it re-connects automatically and sends the
-	// same commands again.
-	Commands []string
-	// Is it a persistent connection or a one-off?
-	isOneOff bool
-	// To deal with SSH server host keys
-	hostKeyCallback client.HostKeyCallback
-	// To determine if connection throttling has finished or not
-	throttlingDone bool
-}
-
-// NewConnection returns a new connection.
-func NewConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback) *Connection {
-	logger.Debug(server, "Creating new connection")
-
-	c := Connection{
-		hostKeyCallback: hostKeyCallback,
-		config: &ssh.ClientConfig{
-			User:            userName,
-			Auth:            authMethods,
-			HostKeyCallback: hostKeyCallback.Wrap(),
-			Timeout:         time.Second * 3,
-		},
-	}
-
-	c.initServerPort(server)
-
-	return &c
-}
-
-// NewOneOffConnection creates new one-off connection (only for sending a series of commands and then quit).
-func NewOneOffConnection(server string, userName string, authMethods []ssh.AuthMethod) *Connection {
-	c := Connection{
-		config: &ssh.ClientConfig{
-			User:            userName,
-			Auth:            authMethods,
-			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		},
-		isOneOff: true,
-	}
-
-	c.initServerPort(server)
-
-	return &c
-}
-
-// Attempt to parse the server port address from the provided server FQDN.
-func (c *Connection) initServerPort(server string) {
-	c.Server = server
-	c.port = config.Common.SSHPort
-	parts := strings.Split(server, ":")
-
-	if len(parts) == 2 {
-		logger.Debug("Parsing port from hostname", parts)
-		port, err := strconv.Atoi(parts[1])
-		if err != nil {
-			logger.FatalExit("Unable to parse client port", server, parts, err)
-		}
-		c.Server = parts[0]
-		c.port = port
-	}
-}
-
-// Start the server connection. Build up SSH session and send some DTail commands.
-func (c *Connection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
-	// Throttle how many connections can be established concurrently (based on ch length)
-	logger.Debug(c.Server, "Throttling connection", len(throttleCh), cap(throttleCh))
-
-	select {
-	case throttleCh <- struct{}{}:
-	case <-ctx.Done():
-		logger.Debug(c.Server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
-		return
-	}
-
-	logger.Debug(c.Server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
-
-	go func() {
-		defer func() {
-			if !c.throttlingDone {
-				logger.Debug(c.Server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
-				c.throttlingDone = true
-				<-throttleCh
-			}
-			cancel()
-		}()
-
-		if err := c.dial(ctx, cancel, throttleCh, statsCh); err != nil {
-			logger.Warn(c.Server, c.port, err)
-			if c.hostKeyCallback.Untrusted(fmt.Sprintf("%s:%d", c.Server, c.port)) {
-				logger.Debug(c.Server, "Not trusting host")
-			}
-		}
-	}()
-
-	<-ctx.Done()
-}
-
-// Dail into a new SSH connection. Close connection in case of an error.
-func (c *Connection) dial(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) error {
-	logger.Debug(c.Server, "Incrementing connection stats")
-	statsCh <- struct{}{}
-	defer func() {
-		logger.Debug(c.Server, "Decrementing connection stats")
-		<-statsCh
-	}()
-
-	logger.Debug(c.Server, "Dialing into the connection")
-	address := fmt.Sprintf("%s:%d", c.Server, c.port)
-
-	client, err := ssh.Dial("tcp", address, c.config)
-	if err != nil {
-		return err
-	}
-	defer client.Close()
-
-	return c.session(ctx, cancel, client, throttleCh)
-}
-
-// Create the SSH session. Close the session in case of an error.
-func (c *Connection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
-	logger.Debug(c.Server, "session")
-
-	session, err := client.NewSession()
-	if err != nil {
-		return err
-	}
-	defer session.Close()
-
-	return c.handle(ctx, cancel, session, throttleCh)
-}
-
-func (c *Connection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
-	logger.Debug(c.Server, "handle")
-
-	stdinPipe, err := session.StdinPipe()
-	if err != nil {
-		return err
-	}
-
-	stdoutPipe, err := session.StdoutPipe()
-	if err != nil {
-		return err
-	}
-
-	if err := session.Shell(); err != nil {
-		return err
-	}
-
-	go func() {
-		io.Copy(stdinPipe, c.Handler)
-		cancel()
-	}()
-
-	go func() {
-		io.Copy(c.Handler, stdoutPipe)
-		cancel()
-	}()
-
-	go func() {
-		select {
-		case <-c.Handler.Done():
-		case <-ctx.Done():
-		}
-		cancel()
-	}()
-
-	// Send all commands to client.
-	for _, command := range c.Commands {
-		logger.Debug(command)
-		c.Handler.SendMessage(command)
-	}
-
-	if !c.throttlingDone {
-		logger.Debug(c.Server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
-		c.throttlingDone = true
-		<-throttleCh
-	}
-
-	<-ctx.Done()
-	c.Handler.Shutdown()
-	return nil
-}
-- 
cgit v1.2.3


From abeac87aec44249bf67f1b0eca471a31086265ca Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 18 Sep 2021 19:27:50 +0300
Subject: fix auto reconnect

---
 internal/clients/args.go                        | 43 ++----------
 internal/clients/baseclient.go                  |  9 ++-
 internal/clients/connectors/serverconnection.go | 38 ++++-------
 internal/clients/connectors/serverless.go       | 90 +++++++++++++++++++++++++
 internal/clients/handlers/basehandler.go        | 10 +++
 internal/done.go                                |  9 +++
 internal/server/handlers/serverhandler.go       |  6 +-
 7 files changed, 139 insertions(+), 66 deletions(-)
 create mode 100644 internal/clients/connectors/serverless.go

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
index 081b911..dc71e83 100644
--- a/internal/clients/args.go
+++ b/internal/clients/args.go
@@ -3,12 +3,8 @@ package clients
 import (
 	"flag"
 	"fmt"
-	"os"
-	"path/filepath"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/omode"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -33,6 +29,7 @@ type Args struct {
 	Quiet              bool
 	Spartan            bool
 	NoColor            bool
+	Serverless         bool
 }
 
 // Transform the arguments based on certain conditions.
@@ -50,45 +47,15 @@ func (a *Args) Transform(args []string) {
 		a.Quiet = true
 		a.NoColor = true
 	}
-}
 
-// TransformAfterConfigFile same as Transform, but after the config file has been read.
-func (a *Args) TransformAfterConfigFile() {
 	if a.Discovery == "" && a.ServersStr == "" {
-		a.handleEmptyServer()
+		a.Serverless = true
 	}
 }
 
-func (a *Args) handleEmptyServer() {
-	fqdn, err := os.Hostname()
-	if err != nil {
-		logger.FatalExit(err)
-	}
-	a.ServersStr = fmt.Sprintf("%s:%d", fqdn, config.Common.SSHPort)
-	// I am trusting my own hostname.
-	a.TrustAllHosts = true
-	logger.Debug("Will connect to local server", a.ServersStr)
-
-	cleanPath := func(dirtyPath string) string {
-		cleanPath, err := filepath.EvalSymlinks(dirtyPath)
-		if err != nil {
-			logger.FatalExit("Unable to evaluate symlinks", dirtyPath, err)
-		}
-		cleanPath, err = filepath.Abs(cleanPath)
-		if err != nil {
-			logger.FatalExit("Unable to make file path absolute", dirtyPath, cleanPath, err)
-		}
-		return cleanPath
-	}
-
-	logger.Debug("Dirty file paths", a.What)
-	var filePaths []string
-	for _, dirtyPath := range strings.Split(a.What, ",") {
-		filePaths = append(filePaths, cleanPath(dirtyPath))
-	}
-
-	a.What = strings.Join(filePaths, ",")
-	logger.Debug("Clean file paths", a.What)
+// TransformAfterConfigFile same as Transform, but after the config file has been read.
+func (a *Args) TransformAfterConfigFile() {
+	// TODO: Remove this method. It's not used.
 }
 
 func (a *Args) SerializeOptions() string {
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index d0631fc..5523052 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -113,12 +113,17 @@ func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, con
 
 		time.Sleep(time.Second * 2)
 		logger.Debug(conn.Server(), "Reconnecting")
-		c.connections[i] = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback)
+		conn = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback)
+		c.connections[i] = conn
 	}
 }
 
 func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod, hostKeyCallback client.HostKeyCallback) connectors.Connector {
-	return connectors.NewServerConnection(server, c.UserName, sshAuthMethods, hostKeyCallback, c.maker.makeHandler(server), c.maker.makeCommands())
+	if c.Args.Serverless {
+		return connectors.NewServerless(c.UserName, c.maker.makeHandler(server), c.maker.makeCommands())
+	}
+	return connectors.NewServerConnection(server, c.UserName, sshAuthMethods,
+		hostKeyCallback, c.maker.makeHandler(server), c.maker.makeCommands())
 }
 
 func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index fab2f87..0904ba1 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -16,31 +16,21 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-// ServerConnection represents a client connection connection to a single server.
+// ServerConnection represents a connection to a single remote dtail server via SSH protocol.
 type ServerConnection struct {
-	// The remote server's hostname connected to.
-	server string
-	// The remote server's port connected to.
-	port int
-	// The SSH client configuration used.
-	config *ssh.ClientConfig
-	// The SSH client handler to use.
-	handler handlers.Handler
-	// DTail commands sent from client to server. When client loses
-	// connection to the server it re-connects automatically and sends the
-	// same commands again.
-	commands []string
-	// Is it a persistent connection or a one-off?
-	isOneOff bool
-	// To deal with SSH server host keys
+	server          string
+	port            int
+	config          *ssh.ClientConfig
+	handler         handlers.Handler
+	commands        []string
+	isOneOff        bool
 	hostKeyCallback client.HostKeyCallback
-	// To determine if connection throttling has finished or not
-	throttlingDone bool
+	throttlingDone  bool
 }
 
-// NewServerConnection returns a new connection.
+// NewServerConnection returns a new DTail SSH server connection.
 func NewServerConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback, handler handlers.Handler, commands []string) *ServerConnection {
-	logger.Debug(server, "Creating new connection")
+	logger.Debug(server, "Creating new connection", server, handler, commands)
 
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
@@ -77,12 +67,10 @@ func NewOneOffServerConnection(server string, userName string, authMethods []ssh
 	return &c
 }
 
-// Server hostname
 func (c *ServerConnection) Server() string {
 	return c.server
 }
 
-// Handler for the connection
 func (c *ServerConnection) Handler() handlers.Handler {
 	return c.handler
 }
@@ -103,7 +91,6 @@ func (c *ServerConnection) initServerPort() {
 	}
 }
 
-// Start the server connection. Build up SSH session and send some DTail commands.
 func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
 	// Throttle how many connections can be established concurrently (based on ch length)
 	logger.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
@@ -161,7 +148,7 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 
 // Create the SSH session. Close the session in case of an error.
 func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
-	logger.Debug(c.server, "session")
+	logger.Debug(c.server, "Creating SSH session")
 
 	session, err := client.NewSession()
 	if err != nil {
@@ -173,7 +160,7 @@ func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFun
 }
 
 func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
-	logger.Debug(c.server, "handle")
+	logger.Debug(c.server, "Creating handler for SSH session")
 
 	stdinPipe, err := session.StdinPipe()
 	if err != nil {
@@ -221,5 +208,6 @@ func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc
 
 	<-ctx.Done()
 	c.handler.Shutdown()
+
 	return nil
 }
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
new file mode 100644
index 0000000..0500645
--- /dev/null
+++ b/internal/clients/connectors/serverless.go
@@ -0,0 +1,90 @@
+package connectors
+
+import (
+	"context"
+	"io"
+
+	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/logger"
+	serverHandlers "github.com/mimecast/dtail/internal/server/handlers"
+	user "github.com/mimecast/dtail/internal/user/server"
+)
+
+// Serverless creates a server object directly without TCP.
+type Serverless struct {
+	handler  handlers.Handler
+	commands []string
+	userName string
+}
+
+// NewServerConnection returns a new connection.
+func NewServerless(userName string, handler handlers.Handler, commands []string) *Serverless {
+	s := Serverless{
+		userName: userName,
+		handler:  handler,
+		commands: commands,
+	}
+
+	logger.Debug("Creating new serverless connector", handler, commands)
+	return &s
+}
+
+func (s *Serverless) Server() string {
+	return "local(serverless)"
+}
+
+func (s *Serverless) Handler() handlers.Handler {
+	return s.handler
+}
+
+func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+	go func() {
+		defer cancel()
+
+		if err := s.handle(ctx, cancel); err != nil {
+			logger.Warn(err)
+		}
+	}()
+
+	<-ctx.Done()
+}
+
+func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) error {
+	logger.Debug("Creating server handler for a serverless session")
+
+	serverHandler := serverHandlers.NewServerHandler(
+		user.New(s.userName, s.Server()),
+		make(chan struct{}, config.Server.MaxConcurrentCats),
+		make(chan struct{}, config.Server.MaxConcurrentTails),
+	)
+
+	go func() {
+		io.Copy(serverHandler, s.handler)
+		cancel()
+	}()
+
+	go func() {
+		io.Copy(s.handler, serverHandler)
+		cancel()
+	}()
+
+	go func() {
+		select {
+		case <-s.handler.Done():
+		case <-ctx.Done():
+		}
+		cancel()
+	}()
+
+	// Send all commands to client.
+	for _, command := range s.commands {
+		logger.Debug(command)
+		s.handler.SendMessage(command)
+	}
+
+	<-ctx.Done()
+	s.handler.Shutdown()
+
+	return nil
+}
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 0f2d1b5..af1ad62 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -22,6 +22,16 @@ type baseHandler struct {
 	status       int
 }
 
+func (h *baseHandler) String() string {
+	return fmt.Sprintf("baseHandler(%s,server:%s,shellStarted:%v,status:%d)@%p",
+		h.done,
+		h.server,
+		h.shellStarted,
+		h.status,
+		h,
+	)
+}
+
 func (h *baseHandler) Server() string {
 	return h.server
 }
diff --git a/internal/done.go b/internal/done.go
index 54e5e8e..5ea22a0 100644
--- a/internal/done.go
+++ b/internal/done.go
@@ -17,6 +17,15 @@ func NewDone() *Done {
 	}
 }
 
+func (d *Done) String() string {
+	select {
+	case <-d.Done():
+		return "Done(yes)"
+	default:
+		return "Done(no)"
+	}
+}
+
 // Done returns the done channel (closed when done)
 func (d *Done) Done() <-chan struct{} {
 	return d.ch
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 2f3b73b..4820476 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -331,7 +331,11 @@ func (h *ServerHandler) handleAckCommand(argc int, args []string) {
 		return
 	}
 	if args[1] == "close" && args[2] == "connection" {
-		close(h.ackCloseReceived)
+		select {
+		case <-h.ackCloseReceived:
+		default:
+			close(h.ackCloseReceived)
+		}
 	}
 }
 
-- 
cgit v1.2.3


From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 19 Sep 2021 13:22:59 +0300
Subject: move args to config package

logger package rewrite as dlog
---
 internal/clients/args.go                        |  63 --------
 internal/clients/baseclient.go                  |  28 ++--
 internal/clients/catclient.go                   |  12 +-
 internal/clients/connectors/serverconnection.go |  34 ++--
 internal/clients/connectors/serverless.go       |  21 ++-
 internal/clients/grepclient.go                  |  13 +-
 internal/clients/handlers/basehandler.go        |   6 +-
 internal/clients/handlers/clienthandler.go      |   4 +-
 internal/clients/handlers/maprhandler.go        |   8 +-
 internal/clients/maprclient.go                  |  40 ++---
 internal/clients/stats.go                       |   8 +-
 internal/clients/tailclient.go                  |  16 +-
 internal/color/table.go                         |   8 +-
 internal/config/args.go                         | 105 ++++++++++++
 internal/config/common.go                       |   8 +-
 internal/config/config.go                       |   3 +
 internal/config/read.go                         |  40 -----
 internal/config/server.go                       |   1 +
 internal/config/setup.go                        |  35 ++++
 internal/discovery/comma.go                     |   4 +-
 internal/discovery/discovery.go                 |  18 +--
 internal/discovery/file.go                      |   8 +-
 internal/io/dlog/dlog.go                        | 206 ++++++++++++++++++++++++
 internal/io/dlog/level.go                       |  89 ++++++++++
 internal/io/dlog/loggers/factory.go             |  60 +++++++
 internal/io/dlog/loggers/file.go                | 156 ++++++++++++++++++
 internal/io/dlog/loggers/fout.go                |  46 ++++++
 internal/io/dlog/loggers/logger.go              |  18 +++
 internal/io/dlog/loggers/none.go                |  21 +++
 internal/io/dlog/loggers/stdout.go              |  73 +++++++++
 internal/io/dlog/rotation.go                    |  27 ++++
 internal/io/dlog/source.go                      |  19 +++
 internal/io/dlog/strategy.go                    |  22 +++
 internal/io/fs/permissions/permission.go        |   4 +-
 internal/io/fs/readfile.go                      |  16 +-
 internal/io/logger/logger.go                    |  23 ++-
 internal/io/logger/modes.go                     |   9 +-
 internal/io/prompt/prompt.go                    |   6 +-
 internal/mapr/aggregateset.go                   |   6 +-
 internal/mapr/client/aggregate.go               |   4 +-
 internal/mapr/funcs/function.go                 |   4 +-
 internal/mapr/groupset.go                       |   4 +-
 internal/mapr/logformat/default.go              |   1 +
 internal/mapr/logformat/generickv.go            |   2 +-
 internal/mapr/logformat/parser.go               |   3 -
 internal/mapr/query.go                          |   4 -
 internal/mapr/server/aggregate.go               |  22 +--
 internal/mapr/token.go                          |  10 +-
 internal/mapr/whereclause.go                    |   6 +-
 internal/mapr/wherecondition.go                 |   6 +-
 internal/regex/regex.go                         |  11 +-
 internal/regex/regex_test.go                    |  16 +-
 internal/server/continuous.go                   |  21 ++-
 internal/server/handlers/controlhandler.go      |  12 +-
 internal/server/handlers/readcommand.go         |  26 +--
 internal/server/handlers/serverhandler.go       |  46 +++---
 internal/server/scheduler.go                    |  22 +--
 internal/server/server.go                       |  52 +++---
 internal/server/stats.go                        |   6 +-
 internal/ssh/client/authmethods.go              |  28 ++--
 internal/ssh/client/knownhostscallback.go       |  14 +-
 internal/ssh/server/hostkey.go                  |  17 +-
 internal/ssh/server/publickeycallback.go        |  12 +-
 internal/ssh/ssh.go                             |   6 +-
 internal/user/server/user.go                    |  22 +--
 65 files changed, 1236 insertions(+), 425 deletions(-)
 delete mode 100644 internal/clients/args.go
 create mode 100644 internal/config/args.go
 delete mode 100644 internal/config/read.go
 create mode 100644 internal/config/setup.go
 create mode 100644 internal/io/dlog/dlog.go
 create mode 100644 internal/io/dlog/level.go
 create mode 100644 internal/io/dlog/loggers/factory.go
 create mode 100644 internal/io/dlog/loggers/file.go
 create mode 100644 internal/io/dlog/loggers/fout.go
 create mode 100644 internal/io/dlog/loggers/logger.go
 create mode 100644 internal/io/dlog/loggers/none.go
 create mode 100644 internal/io/dlog/loggers/stdout.go
 create mode 100644 internal/io/dlog/rotation.go
 create mode 100644 internal/io/dlog/source.go
 create mode 100644 internal/io/dlog/strategy.go

(limited to 'internal')

diff --git a/internal/clients/args.go b/internal/clients/args.go
deleted file mode 100644
index dc71e83..0000000
--- a/internal/clients/args.go
+++ /dev/null
@@ -1,63 +0,0 @@
-package clients
-
-import (
-	"flag"
-	"fmt"
-	"strings"
-
-	"github.com/mimecast/dtail/internal/omode"
-
-	gossh "golang.org/x/crypto/ssh"
-)
-
-// Args is a helper struct to summarize common client arguments.
-type Args struct {
-	Mode               omode.Mode
-	ServersStr         string
-	UserName           string
-	What               string
-	Arguments          []string
-	RegexStr           string
-	RegexInvert        bool
-	TrustAllHosts      bool
-	Discovery          string
-	ConnectionsPerCPU  int
-	Timeout            int
-	SSHAuthMethods     []gossh.AuthMethod
-	SSHHostKeyCallback gossh.HostKeyCallback
-	PrivateKeyPathFile string
-	Quiet              bool
-	Spartan            bool
-	NoColor            bool
-	Serverless         bool
-}
-
-// Transform the arguments based on certain conditions.
-func (a *Args) Transform(args []string) {
-	// Interpret additional args as file list.
-	if a.What == "" {
-		var files []string
-		for _, file := range flag.Args() {
-			files = append(files, file)
-		}
-		a.What = strings.Join(files, ",")
-	}
-
-	if a.Spartan {
-		a.Quiet = true
-		a.NoColor = true
-	}
-
-	if a.Discovery == "" && a.ServersStr == "" {
-		a.Serverless = true
-	}
-}
-
-// TransformAfterConfigFile same as Transform, but after the config file has been read.
-func (a *Args) TransformAfterConfigFile() {
-	// TODO: Remove this method. It's not used.
-}
-
-func (a *Args) SerializeOptions() string {
-	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
-}
diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 5523052..fc01955 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -6,8 +6,9 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/clients/connectors"
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/discovery"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 	"github.com/mimecast/dtail/internal/ssh/client"
@@ -17,7 +18,7 @@ import (
 
 // This is the main client data structure.
 type baseClient struct {
-	Args
+	config.Args
 	// To display client side stats
 	stats *stats
 	// List of remote servers to connect to.
@@ -39,7 +40,8 @@ type baseClient struct {
 }
 
 func (c *baseClient) init() {
-	logger.Debug("Initiating base client")
+	dlog.Client.Debug("Initiating base client")
+	dlog.Client.Debug(c.Args.String())
 
 	flag := regex.Default
 	if c.Args.RegexInvert {
@@ -47,11 +49,14 @@ func (c *baseClient) init() {
 	}
 	regex, err := regex.New(c.Args.RegexStr, flag)
 	if err != nil {
-		logger.FatalExit(c.Regex, "invalid regex!", err, regex)
+		dlog.Client.FatalPanic(c.Regex, "invalid regex!", err, regex)
 	}
 	c.Regex = regex
-	logger.Debug("Regex", c.Regex)
+	dlog.Client.Debug("Regex", c.Regex)
 
+	if c.Args.Serverless {
+		return
+	}
 	c.sshAuthMethods, c.hostKeyCallback = client.InitSSHAuthMethods(c.Args.SSHAuthMethods, c.Args.SSHHostKeyCallback, c.Args.TrustAllHosts, c.throttleCh, c.Args.PrivateKeyPathFile)
 }
 
@@ -67,8 +72,11 @@ func (c *baseClient) makeConnections(maker maker) {
 }
 
 func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status int) {
-	// Periodically check for unknown hosts, and ask the user whether to trust them or not.
-	go c.hostKeyCallback.PromptAddHosts(ctx)
+	// Can be nil when serverless.
+	if c.hostKeyCallback != nil {
+		// Periodically check for unknown hosts, and ask the user whether to trust them or not.
+		go c.hostKeyCallback.PromptAddHosts(ctx)
+	}
 	// Print client stats every time something on statsCh is recieved.
 	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet)
 
@@ -112,7 +120,7 @@ func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, con
 		}
 
 		time.Sleep(time.Second * 2)
-		logger.Debug(conn.Server(), "Reconnecting")
+		dlog.Client.Debug(conn.Server(), "Reconnecting")
 		conn = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback)
 		c.connections[i] = conn
 	}
@@ -127,7 +135,7 @@ func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMe
 }
 
 func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
-	defer logger.Debug("Terminated connection")
+	defer dlog.Client.Debug("Terminated connection")
 
 	// We want to have at least one active connection
 	<-active
@@ -143,7 +151,7 @@ func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
 		if numActive == 0 {
 			return
 		}
-		logger.Debug("Active connections", numActive)
+		dlog.Client.Debug("Active connections", numActive)
 		time.Sleep(time.Second)
 	}
 }
diff --git a/internal/clients/catclient.go b/internal/clients/catclient.go
index d14cdcc..2726e7e 100644
--- a/internal/clients/catclient.go
+++ b/internal/clients/catclient.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 )
 
@@ -16,7 +18,7 @@ type CatClient struct {
 }
 
 // NewCatClient returns a new cat client.
-func NewCatClient(args Args) (*CatClient, error) {
+func NewCatClient(args config.Args) (*CatClient, error) {
 	if args.RegexStr != "" {
 		return nil, errors.New("Can't use regex with 'cat' operating mode")
 	}
@@ -42,11 +44,13 @@ func (c CatClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c CatClient) makeCommands() (commands []string) {
+	regex, err := c.Regex.Serialize()
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
-			c.Mode.String(),
-			c.Args.SerializeOptions(),
-			file, c.Regex.Serialize()))
+			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
 	return
 }
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 0904ba1..5bc63ee 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/ssh/client"
 
 	"golang.org/x/crypto/ssh"
@@ -30,7 +30,7 @@ type ServerConnection struct {
 
 // NewServerConnection returns a new DTail SSH server connection.
 func NewServerConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback, handler handlers.Handler, commands []string) *ServerConnection {
-	logger.Debug(server, "Creating new connection", server, handler, commands)
+	dlog.Client.Debug(server, "Creating new connection", server, handler, commands)
 
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
@@ -81,10 +81,10 @@ func (c *ServerConnection) initServerPort() {
 	parts := strings.Split(c.server, ":")
 
 	if len(parts) == 2 {
-		logger.Debug("Parsing port from hostname", parts)
+		dlog.Client.Debug("Parsing port from hostname", parts)
 		port, err := strconv.Atoi(parts[1])
 		if err != nil {
-			logger.FatalExit("Unable to parse client port", c.server, parts, err)
+			dlog.Client.FatalPanic("Unable to parse client port", c.server, parts, err)
 		}
 		c.server = parts[0]
 		c.port = port
@@ -93,21 +93,21 @@ func (c *ServerConnection) initServerPort() {
 
 func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
 	// Throttle how many connections can be established concurrently (based on ch length)
-	logger.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
+	dlog.Client.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
 
 	select {
 	case throttleCh <- struct{}{}:
 	case <-ctx.Done():
-		logger.Debug(c.server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
+		dlog.Client.Debug(c.server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
 		return
 	}
 
-	logger.Debug(c.server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
+	dlog.Client.Debug(c.server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
 
 	go func() {
 		defer func() {
 			if !c.throttlingDone {
-				logger.Debug(c.server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
+				dlog.Client.Debug(c.server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
 				c.throttlingDone = true
 				<-throttleCh
 			}
@@ -115,9 +115,9 @@ func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
 		}()
 
 		if err := c.dial(ctx, cancel, throttleCh, statsCh); err != nil {
-			logger.Warn(c.server, c.port, err)
+			dlog.Client.Warn(c.server, c.port, err)
 			if c.hostKeyCallback.Untrusted(fmt.Sprintf("%s:%d", c.server, c.port)) {
-				logger.Debug(c.server, "Not trusting host")
+				dlog.Client.Debug(c.server, "Not trusting host")
 			}
 		}
 	}()
@@ -127,14 +127,14 @@ func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
 
 // Dail into a new SSH connection. Close connection in case of an error.
 func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) error {
-	logger.Debug(c.server, "Incrementing connection stats")
+	dlog.Client.Debug(c.server, "Incrementing connection stats")
 	statsCh <- struct{}{}
 	defer func() {
-		logger.Debug(c.server, "Decrementing connection stats")
+		dlog.Client.Debug(c.server, "Decrementing connection stats")
 		<-statsCh
 	}()
 
-	logger.Debug(c.server, "Dialing into the connection")
+	dlog.Client.Debug(c.server, "Dialing into the connection")
 	address := fmt.Sprintf("%s:%d", c.server, c.port)
 
 	client, err := ssh.Dial("tcp", address, c.config)
@@ -148,7 +148,7 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 
 // Create the SSH session. Close the session in case of an error.
 func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
-	logger.Debug(c.server, "Creating SSH session")
+	dlog.Client.Debug(c.server, "Creating SSH session")
 
 	session, err := client.NewSession()
 	if err != nil {
@@ -160,7 +160,7 @@ func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFun
 }
 
 func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
-	logger.Debug(c.server, "Creating handler for SSH session")
+	dlog.Client.Debug(c.server, "Creating handler for SSH session")
 
 	stdinPipe, err := session.StdinPipe()
 	if err != nil {
@@ -196,12 +196,12 @@ func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc
 
 	// Send all commands to client.
 	for _, command := range c.commands {
-		logger.Debug(command)
+		dlog.Client.Debug(command)
 		c.handler.SendMessage(command)
 	}
 
 	if !c.throttlingDone {
-		logger.Debug(c.server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
+		dlog.Client.Debug(c.server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
 		c.throttlingDone = true
 		<-throttleCh
 	}
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index 0500645..c7b5f62 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -6,7 +6,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	serverHandlers "github.com/mimecast/dtail/internal/server/handlers"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
@@ -26,7 +26,7 @@ func NewServerless(userName string, handler handlers.Handler, commands []string)
 		commands: commands,
 	}
 
-	logger.Debug("Creating new serverless connector", handler, commands)
+	dlog.Client.Debug("Creating new serverless connector", handler, commands)
 	return &s
 }
 
@@ -43,7 +43,7 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 		defer cancel()
 
 		if err := s.handle(ctx, cancel); err != nil {
-			logger.Warn(err)
+			dlog.Client.Warn(err)
 		}
 	}()
 
@@ -51,7 +51,7 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 }
 
 func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) error {
-	logger.Debug("Creating server handler for a serverless session")
+	dlog.Client.Debug("Creating server handler for a serverless session")
 
 	serverHandler := serverHandlers.NewServerHandler(
 		user.New(s.userName, s.Server()),
@@ -59,14 +59,19 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 		make(chan struct{}, config.Server.MaxConcurrentTails),
 	)
 
+	terminate := func() {
+		serverHandler.Shutdown()
+		cancel()
+	}
+
 	go func() {
 		io.Copy(serverHandler, s.handler)
-		cancel()
+		terminate()
 	}()
 
 	go func() {
 		io.Copy(s.handler, serverHandler)
-		cancel()
+		terminate()
 	}()
 
 	go func() {
@@ -74,12 +79,12 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 		case <-s.handler.Done():
 		case <-ctx.Done():
 		}
-		cancel()
+		terminate()
 	}()
 
 	// Send all commands to client.
 	for _, command := range s.commands {
-		logger.Debug(command)
+		dlog.Client.Debug(command)
 		s.handler.SendMessage(command)
 	}
 
diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index ea5022b..ae21ff2 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 )
 
@@ -16,7 +18,7 @@ type GrepClient struct {
 }
 
 // NewGrepClient creates a new grep client.
-func NewGrepClient(args Args) (*GrepClient, error) {
+func NewGrepClient(args config.Args) (*GrepClient, error) {
 	if args.RegexStr == "" {
 		return nil, errors.New("No regex specified, use '-regex' flag")
 	}
@@ -41,12 +43,13 @@ func (c GrepClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c GrepClient) makeCommands() (commands []string) {
+	regex, err := c.Regex.Serialize()
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
-			c.Mode.String(),
-			c.Args.SerializeOptions(),
-			file,
-			c.Regex.Serialize()))
+			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
 
 	return
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index af1ad62..3291b43 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -51,7 +51,7 @@ func (h *baseHandler) Shutdown() {
 // SendMessage to the server.
 func (h *baseHandler) SendMessage(command string) error {
 	encoded := base64.StdEncoding.EncodeToString([]byte(command))
-	logger.Debug("Sending command", h.server, command, encoded)
+	dlog.Client.Debug("Sending command", h.server, command, encoded)
 
 	select {
 	case h.commands <- fmt.Sprintf("protocol %s base64 %v;", protocol.ProtocolCompat, encoded):
@@ -112,7 +112,7 @@ func (h *baseHandler) handleMessageType(message string) {
 		return
 	}
 
-	logger.Raw(message)
+	dlog.Client.Raw(message)
 }
 
 // Handle messages received from server which are not meant to be displayed
diff --git a/internal/clients/handlers/clienthandler.go b/internal/clients/handlers/clienthandler.go
index 2bcb038..27ac85e 100644
--- a/internal/clients/handlers/clienthandler.go
+++ b/internal/clients/handlers/clienthandler.go
@@ -2,7 +2,7 @@ package handlers
 
 import (
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ClientHandler is the basic client handler interface.
@@ -12,7 +12,7 @@ type ClientHandler struct {
 
 // NewClientHandler creates a new client handler.
 func NewClientHandler(server string) *ClientHandler {
-	logger.Debug(server, "Creating new client handler")
+	dlog.Client.Debug(server, "Creating new client handler")
 
 	return &ClientHandler{
 		baseHandler{
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index 65b1454..848e7f0 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -4,7 +4,7 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/client"
 	"github.com/mimecast/dtail/internal/protocol"
@@ -40,7 +40,7 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 			continue
 		case protocol.MessageDelimiter:
 			message := h.baseHandler.receiveBuf.String()
-			logger.Debug(message)
+			dlog.Client.Debug(message)
 			if message[0] == 'A' {
 				h.handleAggregateMessage(message)
 			} else {
@@ -60,10 +60,10 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 func (h *MaprHandler) handleAggregateMessage(message string) {
 	parts := strings.SplitN(message, protocol.FieldDelimiter, 3)
 	if len(parts) != 3 {
-		logger.Error("Unable to aggregate data", h.server, message, parts, len(parts), "expected 3 parts")
+		dlog.Client.Error("Unable to aggregate data", h.server, message, parts, len(parts), "expected 3 parts")
 		return
 	}
 	if err := h.aggregate.Aggregate(parts[2]); err != nil {
-		logger.Error("Unable to aggregate data", h.server, message, err)
+		dlog.Client.Error("Unable to aggregate data", h.server, message, err)
 	}
 }
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index e6ab96b..f23aa08 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -11,7 +11,7 @@ import (
 	"github.com/mimecast/dtail/internal/clients/handlers"
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/omode"
 )
@@ -44,14 +44,14 @@ type MaprClient struct {
 }
 
 // NewMaprClient returns a new mapreduce client.
-func NewMaprClient(args Args, queryStr string, maprClientMode MaprClientMode) (*MaprClient, error) {
+func NewMaprClient(args config.Args, queryStr string, maprClientMode MaprClientMode) (*MaprClient, error) {
 	if queryStr == "" {
 		return nil, errors.New("No mapreduce query specified, use '-query' flag")
 	}
 
 	query, err := mapr.NewQuery(queryStr)
 	if err != nil {
-		logger.FatalExit(queryStr, "Can't parse mapr query", err)
+		dlog.Client.FatalPanic(queryStr, "Can't parse mapr query", err)
 	}
 
 	// Don't retry connection if in tail mode and no outfile specified.
@@ -68,7 +68,7 @@ func NewMaprClient(args Args, queryStr string, maprClientMode MaprClientMode) (*
 		cumulative = args.Mode == omode.MapClient || query.HasOutfile()
 	}
 
-	logger.Debug("Cumulative mapreduce mode?", cumulative)
+	dlog.Client.Debug("Cumulative mapreduce mode?", cumulative)
 
 	c := MaprClient{
 		baseClient: baseClient{
@@ -103,7 +103,7 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 
 	status = c.baseClient.Start(ctx, statsCh)
 	if c.cumulative {
-		logger.Debug("Received final mapreduce result")
+		dlog.Client.Debug("Received final mapreduce result")
 		c.reportResults()
 	}
 
@@ -123,15 +123,17 @@ func (c MaprClient) makeCommands() (commands []string) {
 	}
 
 	for _, file := range strings.Split(c.What, ",") {
+		regex, err := c.Regex.Serialize()
+		if err != nil {
+			dlog.Client.FatalPanic(err)
+		}
 		if c.Timeout > 0 {
-			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout, modeStr, file, c.Regex.Serialize()))
+			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout,
+				modeStr, file, regex))
 			continue
 		}
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
-			modeStr,
-			c.Args.SerializeOptions(),
-			file,
-			c.Regex.Serialize()))
+			modeStr, c.Args.SerializeOptions(), file, regex))
 	}
 
 	return
@@ -141,7 +143,7 @@ func (c *MaprClient) periodicReportResults(ctx context.Context) {
 	for {
 		select {
 		case <-time.After(c.query.Interval):
-			logger.Debug("Gathering interim mapreduce result")
+			dlog.Client.Debug("Gathering interim mapreduce result")
 			c.reportResults()
 		case <-ctx.Done():
 			return
@@ -177,17 +179,17 @@ func (c *MaprClient) printResults() {
 	}
 
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Client.FatalPanic(err)
 	}
 
 	if result == c.lastResult {
-		logger.Debug("Result hasn't changed compared to last time...")
+		dlog.Client.Debug("Result hasn't changed compared to last time...")
 		return
 	}
 	c.lastResult = result
 
 	if numRows == 0 {
-		logger.Debug("Empty result set this time...")
+		dlog.Client.Debug("Empty result set this time...")
 		return
 	}
 
@@ -198,24 +200,24 @@ func (c *MaprClient) printResults() {
 			config.Client.TermColors.MaprTable.RawQueryBg,
 			config.Client.TermColors.MaprTable.RawQueryAttr)
 	}
-	logger.Raw(rawQuery)
+	dlog.Client.Raw(rawQuery)
 
 	if rowsLimit > 0 && numRows > rowsLimit {
-		logger.Warn(fmt.Sprintf("Got %d results but limited output to %d rows! Use 'limit' clause to override!",
+		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited output to %d rows! Use 'limit' clause to override!",
 			numRows, rowsLimit))
 	}
-	logger.Raw(result)
+	dlog.Client.Raw(result)
 }
 
 func (c *MaprClient) writeResultsToOutfile() {
 	if c.cumulative {
 		if err := c.globalGroup.WriteResult(c.query); err != nil {
-			logger.FatalExit(err)
+			dlog.Client.FatalPanic(err)
 		}
 		return
 	}
 
 	if err := c.globalGroup.SwapOut().WriteResult(c.query); err != nil {
-		logger.FatalExit(err)
+		dlog.Client.FatalPanic(err)
 	}
 }
diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index 6da443c..fbef572 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
@@ -67,7 +67,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 			s.printStatsDueInterrupt(messages)
 		default:
 			data := s.statsData(connected, newConnections, throttle)
-			logger.Mapreduce("STATS", data)
+			dlog.Client.Mapreduce("STATS", data)
 		}
 
 		connectedLast = connected
@@ -78,7 +78,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 }
 
 func (s *stats) printStatsDueInterrupt(messages []string) {
-	logger.Pause()
+	dlog.Client.Pause()
 	for i, message := range messages {
 		if i > 0 && config.Client.TermColorsEnable {
 			fmt.Println(color.PaintStrWithAttr(message,
@@ -91,7 +91,7 @@ func (s *stats) printStatsDueInterrupt(messages []string) {
 		fmt.Println(fmt.Sprintf(" %s", message))
 	}
 	time.Sleep(time.Second * time.Duration(config.InterruptTimeoutS))
-	logger.Resume()
+	dlog.Client.Resume()
 }
 
 func (s *stats) statsData(connected, newConnections int, throttle int) map[string]interface{} {
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index 360354b..d42a0e4 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -6,7 +6,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 )
 
@@ -16,7 +17,7 @@ type TailClient struct {
 }
 
 // NewTailClient returns a new TailClient.
-func NewTailClient(args Args) (*TailClient, error) {
+func NewTailClient(args config.Args) (*TailClient, error) {
 	args.Mode = omode.TailClient
 
 	c := TailClient{
@@ -38,14 +39,15 @@ func (c TailClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c TailClient) makeCommands() (commands []string) {
+	regex, err := c.Regex.Serialize()
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
 	for _, file := range strings.Split(c.What, ",") {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
-			c.Mode.String(),
-			c.Args.SerializeOptions(),
-			file,
-			c.Regex.Serialize()))
+			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
-	logger.Debug(commands)
+	dlog.Client.Debug(commands)
 
 	return
 }
diff --git a/internal/color/table.go b/internal/color/table.go
index 7ecfbca..2115edf 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -7,17 +7,17 @@ import (
 
 const sampleParagraph string = "Mimecast is Making Email Safer for Business. We believe that securely operating a business in the cloud requires new levels of IT preparedness, centered around cyber resilience. This is why we unify the delivery and management of security, continuity and data protection for email via one, simple-to-use cloud platform. Thousands of organizations trust us to increase their cyber resilience preparedness, streamline compliance, reduce IT complexity and keep their business running. We give employees fast and secure access to sensitive business information, and ensure email keeps flowing in the event of an outage. Mimecast will remain committed to protecting your IT assets through constant innovation and focus on your success."
 
-func TablePrintAndExit(useSampleParagraph bool) {
+func TablePrintAndExit(displaySampleParagraph bool) {
 	for _, attr := range AttributeNames {
 		if attr == "Hidden" || attr == "SlowBlink" {
 			continue
 		}
-		printColorTable(attr, useSampleParagraph)
+		printColorTable(attr, displaySampleParagraph)
 	}
 	os.Exit(0)
 }
 
-func printColorTable(attr string, useSampleParagraph bool) {
+func printColorTable(attr string, displaySampleParagraph bool) {
 	for _, fg := range ColorNames {
 		fgColor, _ := ToFgColor(fg)
 		for _, bg := range ColorNames {
@@ -29,7 +29,7 @@ func printColorTable(attr string, useSampleParagraph bool) {
 
 			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
 			fmt.Print(PaintStrWithAttr(text, fgColor, bgColor, attribute))
-			if useSampleParagraph {
+			if displaySampleParagraph {
 				fmt.Print("\n")
 				fmt.Print(PaintStrWithAttr(sampleParagraph, fgColor, bgColor, attribute))
 				fmt.Print("\n")
diff --git a/internal/config/args.go b/internal/config/args.go
new file mode 100644
index 0000000..89e4bc9
--- /dev/null
+++ b/internal/config/args.go
@@ -0,0 +1,105 @@
+package config
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+
+	"github.com/mimecast/dtail/internal/omode"
+
+	gossh "golang.org/x/crypto/ssh"
+)
+
+// Args is a helper struct to summarize common client arguments.
+type Args struct {
+	Arguments          []string
+	ConfigFile         string
+	ConnectionsPerCPU  int
+	Discovery          string
+	LogLevel           string
+	Mode               omode.Mode
+	NoColor            bool
+	PrivateKeyPathFile string
+	Quiet              bool
+	RegexInvert        bool
+	RegexStr           string
+	Serverless         bool
+	ServersStr         string
+	Spartan            bool
+	SSHAuthMethods     []gossh.AuthMethod
+	SSHHostKeyCallback gossh.HostKeyCallback
+	SSHPort            int
+	Timeout            int
+	TrustAllHosts      bool
+	UserName           string
+	What               string
+}
+
+func (a *Args) String() string {
+	var sb strings.Builder
+
+	sb.WriteString("Args(")
+	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "ConnectionsPerCPU", a.ConnectionsPerCPU))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Discovery", a.Discovery))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Spartan", a.Spartan))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHAuthMethods", a.SSHAuthMethods))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHHostKeyCallback", a.SSHHostKeyCallback))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHPort", a.SSHPort))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Timeout", a.Timeout))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "TrustAllHosts", a.TrustAllHosts))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "UserName", a.UserName))
+	sb.WriteString(fmt.Sprintf("%s:%v", "What", a.What))
+	sb.WriteString(")")
+
+	return sb.String()
+}
+
+// Based on the argument list, transform/manipulate some of the arguments.
+func (a *Args) transform(args []string) {
+	if a.LogLevel != "" {
+		Common.LogLevel = a.LogLevel
+	}
+
+	if a.SSHPort != 2222 {
+		Common.SSHPort = a.SSHPort
+	}
+	if a.NoColor {
+		Client.TermColorsEnable = false
+	}
+
+	if a.Spartan {
+		a.Quiet = true
+		a.NoColor = true
+	}
+
+	if a.Discovery == "" && a.ServersStr == "" {
+		a.Serverless = true
+	}
+
+	// Interpret additional args as file list.
+	if a.What == "" {
+		var files []string
+		for _, file := range flag.Args() {
+			files = append(files, file)
+		}
+		a.What = strings.Join(files, ",")
+	}
+}
+
+// SerializeOptions returns a string ready to be sent over the wire to the server.
+func (a *Args) SerializeOptions() string {
+	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
+}
+
+// TODO: Put the DeseializeOptions function here (move it away from the internal/server package)
diff --git a/internal/config/common.go b/internal/config/common.go
index c3e203e..7d45261 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -6,10 +6,8 @@ type CommonConfig struct {
 	SSHPort int
 	// Enable experimental features (mainly for dev purposes)
 	ExperimentalFeaturesEnable bool `json:",omitempty"`
-	// Enable debug logging. Don't enable in production.
-	DebugEnable bool `json:",omitempty"`
-	// Enable trace logging. Don't enable in production.
-	TraceEnable bool `json:",omitempty"`
+	// LogLevel defines how much is logged. TODO: Adjust JSONschema
+	LogLevel string `json:",omitempty"`
 	// The log strategy to use, one of
 	//   stdout: only log to stdout (useful when used with systemd)
 	//   daily: create a log file for every day
@@ -26,8 +24,6 @@ type CommonConfig struct {
 func newDefaultCommonConfig() *CommonConfig {
 	return &CommonConfig{
 		SSHPort:                    2222,
-		DebugEnable:                false,
-		TraceEnable:                false,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
 		CacheDir:                   "cache",
diff --git a/internal/config/config.go b/internal/config/config.go
index 276ddcf..2d77041 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -15,6 +15,9 @@ const ScheduleUser string = "DTAIL-SCHEDULE"
 // ContinuousUser is used for non-interactive continuous mapreduce queries.
 const ContinuousUser string = "DTAIL-CONTINUOUS"
 
+// TestUser is used for unit tests and potentially also for integration tests.
+const TestUser string = "DTAIL-TEST"
+
 // InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
 const InterruptTimeoutS int = 3
 
diff --git a/internal/config/read.go b/internal/config/read.go
deleted file mode 100644
index ea358f8..0000000
--- a/internal/config/read.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package config
-
-import (
-	"os"
-)
-
-// Read the DTail configuration.
-func Read(configFile string, sshPort int, noColor bool) {
-	initializer := configInitializer{
-		Common: newDefaultCommonConfig(),
-		Server: newDefaultServerConfig(),
-		Client: newDefaultClientConfig(),
-	}
-
-	if configFile == "" {
-		configFile = "./cfg/dtail.json"
-	}
-
-	if _, err := os.Stat(configFile); !os.IsNotExist(err) {
-		initializer.parseConfig(configFile)
-	}
-
-	// Assign pointers to global variables, so that we can access the
-	// configuration from any place of the program.
-	Common = initializer.Common
-	Server = initializer.Server
-	Client = initializer.Client
-
-	if Server.MapreduceLogFormat == "" {
-		Server.MapreduceLogFormat = "default"
-	}
-
-	// If non-standard port specified, overwrite config
-	if sshPort != 2222 {
-		Common.SSHPort = sshPort
-	}
-	if noColor {
-		Client.TermColorsEnable = false
-	}
-}
diff --git a/internal/config/server.go b/internal/config/server.go
index dc0d587..8bbb394 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -76,6 +76,7 @@ func newDefaultServerConfig() *ServerConfig {
 		MaxConcurrentTails: 50,
 		HostKeyFile:        "./cache/ssh_host_key",
 		HostKeyBits:        4096,
+		MapreduceLogFormat: "default",
 		Permissions: Permissions{
 			Default: defaultPermissions,
 		},
diff --git a/internal/config/setup.go b/internal/config/setup.go
new file mode 100644
index 0000000..3c4bcc4
--- /dev/null
+++ b/internal/config/setup.go
@@ -0,0 +1,35 @@
+package config
+
+import (
+	"os"
+)
+
+const NoConfigFile string = "Don't read a config file - use defaults only"
+
+// Setup the DTail configuration.
+func Setup(args *Args, additionalArgs []string) {
+	initializer := configInitializer{
+		Common: newDefaultCommonConfig(),
+		Server: newDefaultServerConfig(),
+		Client: newDefaultClientConfig(),
+	}
+
+	if args.ConfigFile == "" {
+		// TODO: Search more paths for config file (e.g. in /etc and in ~/.config/...
+		args.ConfigFile = "./cfg/dtail.json"
+	}
+
+	if args.ConfigFile != NoConfigFile {
+		if _, err := os.Stat(args.ConfigFile); !os.IsNotExist(err) {
+			initializer.parseConfig(args.ConfigFile)
+		}
+	}
+
+	// Assign pointers to global variables, so that we can access the
+	// configuration from any place of the program.
+	Common = initializer.Common
+	Server = initializer.Server
+	Client = initializer.Client
+
+	args.transform(additionalArgs)
+}
diff --git a/internal/discovery/comma.go b/internal/discovery/comma.go
index 4344240..9bea89c 100644
--- a/internal/discovery/comma.go
+++ b/internal/discovery/comma.go
@@ -3,11 +3,11 @@ package discovery
 import (
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ServerListFromCOMMA retrieves a list of servers from comma separated input list.
 func (d *Discovery) ServerListFromCOMMA() []string {
-	logger.Debug("Retrieving server list from comma separated list", d.server)
+	dlog.Common.Debug("Retrieving server list from comma separated list", d.server)
 	return strings.Split(d.server, ",")
 }
diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go
index 3608ce7..83ee95e 100644
--- a/internal/discovery/discovery.go
+++ b/internal/discovery/discovery.go
@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ServerOrder to specify how to sort the server list.
@@ -42,7 +42,7 @@ func New(method, server string, order ServerOrder) *Discovery {
 	if strings.Contains(module, ":") {
 		s := strings.Split(module, ":")
 		if len(s) != 2 {
-			logger.FatalExit("Unable to parse discovery module", module)
+			dlog.Common.FatalPanic("Unable to parse discovery module", module)
 		}
 		module = s[0]
 		options = s[1]
@@ -72,11 +72,11 @@ func (d *Discovery) initRegex() {
 	}
 
 	regexStr := string(runes)
-	logger.Debug("Using filter regex", regexStr)
+	dlog.Common.Debug("Using filter regex", regexStr)
 
 	regex, err := regexp.Compile(regexStr)
 	if err != nil {
-		logger.FatalExit("Could not compile regex", regexStr, err)
+		dlog.Common.FatalPanic("Could not compile regex", regexStr, err)
 	}
 
 	d.regex = regex
@@ -97,7 +97,7 @@ func (d *Discovery) ServerList() []string {
 		servers = d.shuffleList(servers)
 	}
 
-	logger.Debug("Discovered servers", len(servers), servers)
+	dlog.Common.Debug("Discovered servers", len(servers), servers)
 	return servers
 }
 
@@ -124,7 +124,7 @@ func (d *Discovery) serverListFromReflectedModule() []string {
 	rt := reflect.TypeOf(d)
 	reflectedMethod, ok := rt.MethodByName(methodName)
 	if !ok {
-		logger.FatalExit("No such server discovery module", d.module, methodName)
+		dlog.Common.FatalPanic("No such server discovery module", d.module, methodName)
 	}
 
 	inputValues := make([]reflect.Value, 1)
@@ -138,7 +138,7 @@ func (d *Discovery) serverListFromReflectedModule() []string {
 
 // Filter server list based on a regexp.
 func (d *Discovery) filterList(servers []string) (filtered []string) {
-	logger.Debug("Filtering server list")
+	dlog.Common.Debug("Filtering server list")
 
 	for _, server := range servers {
 		if d.regex.MatchString(server) {
@@ -160,13 +160,13 @@ func (d *Discovery) dedupList(servers []string) (deduped []string) {
 		}
 	}
 
-	logger.Debug("Deduped server list", len(servers), len(deduped))
+	dlog.Common.Debug("Deduped server list", len(servers), len(deduped))
 	return
 }
 
 // Randomly shuffle the server list.
 func (d *Discovery) shuffleList(servers []string) []string {
-	logger.Debug("Shuffling server list")
+	dlog.Common.Debug("Shuffling server list")
 
 	r := rand.New(rand.NewSource(time.Now().Unix()))
 	shuffled := make([]string, len(servers))
diff --git a/internal/discovery/file.go b/internal/discovery/file.go
index 1250755..fb46eeb 100644
--- a/internal/discovery/file.go
+++ b/internal/discovery/file.go
@@ -4,16 +4,16 @@ import (
 	"bufio"
 	"os"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ServerListFromFILE retrieves a list of servers from a file.
 func (d *Discovery) ServerListFromFILE() (servers []string) {
-	logger.Debug("Retrieving server list from file", d.server)
+	dlog.Common.Debug("Retrieving server list from file", d.server)
 
 	file, err := os.Open(d.server)
 	if err != nil {
-		logger.FatalExit(d.server, err)
+		dlog.Common.FatalPanic(d.server, err)
 	}
 	defer file.Close()
 
@@ -22,7 +22,7 @@ func (d *Discovery) ServerListFromFILE() (servers []string) {
 		servers = append(servers, scanner.Text())
 	}
 	if err := scanner.Err(); err != nil {
-		logger.FatalExit(d.server, err)
+		dlog.Common.FatalPanic(d.server, err)
 	}
 
 	return
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
new file mode 100644
index 0000000..7282741
--- /dev/null
+++ b/internal/io/dlog/dlog.go
@@ -0,0 +1,206 @@
+package dlog
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mimecast/dtail/internal/color/brush"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog/loggers"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/protocol"
+)
+
+// Client is the log handler for the client packages.
+var Client *DLog
+
+// Server is the log handler for the server packages.
+var Server *DLog
+
+// Common is the log handler for all other packages.
+// TODO: Rename Common to Common
+var Common *DLog
+
+var mutex sync.Mutex
+var started bool
+
+// Start logger(s).
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLevel string) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	if started {
+		Common.FatalPanic("Logger already started")
+	}
+
+	level := newLevel(logLevel)
+	switch sourceProcess {
+	case CLIENT:
+		// This is a DTail client process running.
+		impl := loggers.FOUT
+		Client = New(CLIENT, CLIENT, impl, level)
+		Server = New(CLIENT, SERVER, impl, level)
+		Common = Client
+	case SERVER:
+		// This is a DTail server process running.
+		impl := loggers.FILE
+		Client = New(SERVER, CLIENT, impl, level)
+		Server = New(SERVER, SERVER, impl, level)
+		Common = Server
+	}
+
+	var wg2 sync.WaitGroup
+	wg2.Add(2)
+	Client.start(ctx, &wg2)
+	Server.start(ctx, &wg2)
+	started = true
+
+	go rotation(ctx)
+	go func() {
+		wg2.Wait()
+		wg.Done()
+	}()
+}
+
+// DLog is the DTail logger.
+type DLog struct {
+	logger loggers.Logger
+	// Is this a DTail server or client process logging?
+	sourceProcess source
+	// Is this a DTail server or client package logging? In serverless mode
+	// the client can also execute code from the server package.
+	sourcePackage source
+	// Max log level to log.
+	maxLevel level
+}
+
+// New creates a new DTail logger.
+func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+	return &DLog{
+		logger:        loggers.Factory(sourceProcess.String(), impl),
+		sourceProcess: sourceProcess,
+		sourcePackage: sourcePackage,
+		maxLevel:      maxLevel,
+	}
+}
+
+func (d *DLog) start(ctx context.Context, wg *sync.WaitGroup) {
+	go func() {
+		defer wg.Done()
+		var wg2 sync.WaitGroup
+		wg2.Add(1)
+		d.logger.Start(ctx, &wg2)
+		<-ctx.Done()
+		wg2.Wait()
+	}()
+}
+
+func (d *DLog) log(level level, args []interface{}) string {
+	if d.maxLevel < level {
+		return ""
+	}
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
+	now := time.Now()
+
+	sb.WriteString(d.sourcePackage.String())
+	sb.WriteString(protocol.FieldDelimiter)
+	sb.WriteString(now.Format("20060102-150405"))
+	sb.WriteString(protocol.FieldDelimiter)
+	sb.WriteString(level.String())
+	sb.WriteString(protocol.FieldDelimiter)
+	d.writeArgStrings(sb, args)
+
+	message := sb.String()
+	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
+		d.logger.Log(now, message)
+		return message
+	}
+
+	d.logger.LogWithColors(now, message, brush.Colorfy(message))
+	return message
+}
+
+func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
+	for i, arg := range args {
+		if i > 0 {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+		switch v := arg.(type) {
+		case string:
+			sb.WriteString(v)
+		case error:
+			sb.WriteString(v.Error())
+		default:
+			sb.WriteString(fmt.Sprintf("%v", v))
+		}
+	}
+}
+
+func (d *DLog) FatalPanic(args ...interface{}) {
+	d.log(FATAL, args)
+	d.logger.Flush()
+	panic("Not recovering from this fatal error...")
+}
+
+func (d *DLog) Fatal(args ...interface{}) string {
+	return d.log(FATAL, args)
+}
+
+func (d *DLog) Error(args ...interface{}) string {
+	return d.log(ERROR, args)
+}
+
+func (d *DLog) Warn(args ...interface{}) string {
+	return d.log(WARN, args)
+}
+
+func (d *DLog) Info(args ...interface{}) string {
+	return d.log(INFO, args)
+}
+
+func (d *DLog) Verbose(args ...interface{}) string {
+	return d.log(VERBOSE, args)
+}
+
+func (d *DLog) Debug(args ...interface{}) string {
+	return d.log(DEBUG, args)
+}
+
+func (d *DLog) Trace(args ...interface{}) string {
+	return d.log(TRACE, args)
+}
+
+func (d *DLog) Devel(args ...interface{}) string {
+	return d.log(DEVEL, args)
+}
+
+func (d *DLog) Raw(message string) string {
+	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
+		d.logger.Log(time.Now(), message)
+		return message
+	}
+
+	d.logger.Log(time.Now(), brush.Colorfy(message))
+	return message
+}
+
+func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
+	args := make([]interface{}, len(data)+1)
+	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
+
+	i := 1
+	for k, v := range data {
+		args[i] = fmt.Sprintf("%s=%v", k, v)
+		i++
+	}
+
+	return d.log(INFO, args)
+}
+
+func (d *DLog) Flush()  { d.logger.Flush() }
+func (d *DLog) Pause()  { d.logger.Pause() }
+func (d *DLog) Resume() { d.logger.Resume() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
new file mode 100644
index 0000000..84550f0
--- /dev/null
+++ b/internal/io/dlog/level.go
@@ -0,0 +1,89 @@
+package dlog
+
+import (
+	"fmt"
+	"strings"
+)
+
+type level int
+
+const (
+	FATAL   level = iota
+	ERROR   level = iota
+	WARN    level = iota
+	INFO    level = iota
+	DEFAULT level = iota
+	VERBOSE level = iota
+	DEBUG   level = iota
+	DEVEL   level = iota
+	TRACE   level = iota
+	ALL     level = iota
+)
+
+var allLevels = []level{
+	FATAL,
+	ERROR,
+	WARN,
+	INFO,
+	DEFAULT,
+	VERBOSE,
+	DEBUG,
+	DEVEL,
+	TRACE,
+	ALL,
+}
+
+func newLevel(l string) level {
+	switch strings.ToUpper(l) {
+	case "FATAL":
+		return FATAL
+	case "ERROR":
+		return ERROR
+	case "WARN":
+		return WARN
+	case "INFO":
+		return INFO
+	case "":
+		fallthrough
+	case "DEFAULT":
+		return DEFAULT
+	case "VERBOSE":
+		return VERBOSE
+	case "DEBUG":
+		return DEBUG
+	case "DEVEL":
+		return DEVEL
+	case "TRACE":
+		return TRACE
+	case "ALL":
+		return ALL
+	}
+	panic(fmt.Sprintf("Unknown log level %s, must be one of: %v", l, allLevels))
+}
+
+func (l level) String() string {
+	switch l {
+	case FATAL:
+		return "FATAL"
+	case ERROR:
+		return "ERROR"
+	case WARN:
+		return "WARN"
+	case INFO:
+		return "INFO"
+	case DEFAULT:
+		return "DEFAULT"
+	case VERBOSE:
+		return "VERBOSE"
+	case DEBUG:
+		return "DEBUG"
+	case DEVEL:
+		return "DEVEL"
+	case TRACE:
+		return "TRACE"
+	case ALL:
+		return "ALL"
+	}
+
+	panic("Unknown log level " + fmt.Sprintf("%d", l))
+}
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
new file mode 100644
index 0000000..3eb29c5
--- /dev/null
+++ b/internal/io/dlog/loggers/factory.go
@@ -0,0 +1,60 @@
+package loggers
+
+import (
+	"fmt"
+	"sync"
+)
+
+type Impl int
+
+const (
+	NONE   Impl = iota
+	STDOUT Impl = iota
+	FILE   Impl = iota
+	FOUT   Impl = iota
+)
+
+var factoryMap map[string]Logger
+var factoryMutex sync.Mutex
+
+func Factory(name string, impl Impl) Logger {
+	factoryMutex.Lock()
+	defer factoryMutex.Unlock()
+
+	id := fmt.Sprintf("name:%s,impl:%v", name, impl)
+
+	if factoryMap == nil {
+		factoryMap = make(map[string]Logger)
+	}
+
+	singleton, ok := factoryMap[id]
+	if !ok {
+		switch impl {
+		case NONE:
+			singleton = none{}
+		case STDOUT:
+			singleton = newStdout()
+			factoryMap[id] = singleton
+		case FILE:
+			singleton = newFile()
+			factoryMap[id] = singleton
+		case FOUT:
+			singleton = newFout()
+			factoryMap[id] = singleton
+		}
+	}
+
+	return singleton
+}
+
+func FactoryRotate() {
+	factoryMutex.Lock()
+	defer factoryMutex.Unlock()
+	if factoryMap == nil {
+		return
+	}
+
+	for _, impl := range factoryMap {
+		impl.Rotate()
+	}
+}
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
new file mode 100644
index 0000000..1c525c9
--- /dev/null
+++ b/internal/io/dlog/loggers/file.go
@@ -0,0 +1,156 @@
+package loggers
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/mimecast/dtail/internal/config"
+)
+
+type fileMessageBuf struct {
+	now     time.Time
+	message string
+}
+
+type file struct {
+	bufferCh    chan *fileMessageBuf
+	pauseCh     chan struct{}
+	resumeCh    chan struct{}
+	rotateCh    chan struct{}
+	flushCh     chan struct{}
+	lastDateStr string
+	fd          *os.File
+	writer      *bufio.Writer
+	mutex       sync.Mutex
+	started     bool
+}
+
+func newFile() *file {
+	f := file{
+		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
+		pauseCh:  make(chan struct{}),
+		resumeCh: make(chan struct{}),
+		rotateCh: make(chan struct{}),
+		flushCh:  make(chan struct{}),
+	}
+	f.getWriter(time.Now().Format("20060102"))
+	return &f
+}
+
+func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	// Logger already started from another Goroutine.
+	if s.started {
+		wg.Done()
+		return
+	}
+
+	pause := func(ctx context.Context) {
+		select {
+		case <-s.resumeCh:
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	go func() {
+		defer wg.Done()
+
+		for {
+			select {
+			case m := <-s.bufferCh:
+				s.write(m)
+			case <-s.pauseCh:
+				pause(ctx)
+			case <-s.flushCh:
+				s.flush()
+			case <-ctx.Done():
+				s.flush()
+				s.fd.Close()
+				return
+			}
+		}
+	}()
+
+	s.started = true
+}
+
+func (s *file) Log(now time.Time, message string) {
+	s.bufferCh <- &fileMessageBuf{now, message}
+}
+
+func (s *file) LogWithColors(now time.Time, message, coloredMessage string) {
+	panic("Colors not supported in file logger")
+}
+
+func (s *file) Pause()  { s.pauseCh <- struct{}{} }
+func (s *file) Resume() { s.resumeCh <- struct{}{} }
+func (s *file) Flush()  { s.flushCh <- struct{}{} }
+
+// TODO: Test that Rotate() actually works.
+func (s *file) Rotate()           { s.rotateCh <- struct{}{} }
+func (file) SupportsColors() bool { return false }
+
+func (s *file) write(m *fileMessageBuf) {
+	select {
+	case <-s.rotateCh:
+		// Force re-opening the outfile.
+		s.lastDateStr = ""
+	default:
+	}
+
+	writer := s.getWriter(m.now.Format("20060102"))
+	writer.WriteString(m.message)
+	writer.WriteByte('\n')
+}
+
+func (s *file) getWriter(dateStr string) *bufio.Writer {
+	if s.lastDateStr == dateStr {
+		return s.writer
+	}
+
+	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
+		if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil {
+			panic(err)
+		}
+	}
+
+	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr)
+	newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
+	if err != nil {
+		panic(err)
+	}
+
+	// Close old writer.
+	if s.fd != nil {
+		s.writer.Flush()
+		s.fd.Close()
+	}
+
+	s.fd = newFd
+	s.writer = bufio.NewWriterSize(s.fd, 1)
+	s.lastDateStr = dateStr
+
+	return s.writer
+}
+
+func (s *file) flush() {
+	defer s.writer.Flush()
+
+	for {
+		select {
+		case m := <-s.bufferCh:
+			s.write(m)
+		default:
+			return
+		}
+	}
+}
diff --git a/internal/io/dlog/loggers/fout.go b/internal/io/dlog/loggers/fout.go
new file mode 100644
index 0000000..603dbe9
--- /dev/null
+++ b/internal/io/dlog/loggers/fout.go
@@ -0,0 +1,46 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+type fout struct {
+	file   *file
+	stdout *stdout
+}
+
+// Logs to both, a file and stdout
+func newFout() *fout {
+	return &fout{file: newFile(), stdout: newStdout()}
+}
+
+func (f *fout) Start(ctx context.Context, wg *sync.WaitGroup) {
+	go func() {
+		defer wg.Done()
+
+		var wg2 sync.WaitGroup
+		wg2.Add(2)
+		f.file.Start(ctx, &wg2)
+		f.stdout.Start(ctx, &wg2)
+		wg2.Wait()
+	}()
+}
+
+func (f *fout) Log(now time.Time, message string) {
+	f.stdout.Log(now, message)
+	f.file.Log(now, message)
+}
+
+func (f *fout) LogWithColors(now time.Time, message, coloredMessage string) {
+	f.stdout.LogWithColors(now, "", coloredMessage)
+	f.file.Log(now, message)
+}
+
+func (f *fout) Flush()  { f.stdout.Flush(); f.file.Flush() }
+func (f *fout) Pause()  { f.stdout.Pause(); f.file.Pause() }
+func (f *fout) Resume() { f.stdout.Resume(); f.file.Resume() }
+func (f *fout) Rotate() { f.file.Rotate() }
+
+func (fout) SupportsColors() bool { return true }
diff --git a/internal/io/dlog/loggers/logger.go b/internal/io/dlog/loggers/logger.go
new file mode 100644
index 0000000..c88900d
--- /dev/null
+++ b/internal/io/dlog/loggers/logger.go
@@ -0,0 +1,18 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+type Logger interface {
+	Log(now time.Time, message string)
+	LogWithColors(now time.Time, message, messageWithColors string)
+	Start(ctx context.Context, wg *sync.WaitGroup)
+	Flush()
+	Pause()
+	Resume()
+	Rotate()
+	SupportsColors() bool
+}
diff --git a/internal/io/dlog/loggers/none.go b/internal/io/dlog/loggers/none.go
new file mode 100644
index 0000000..270027f
--- /dev/null
+++ b/internal/io/dlog/loggers/none.go
@@ -0,0 +1,21 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+// don't log anything
+type none struct{}
+
+func (none) Start(ctx context.Context, wg *sync.WaitGroup) { wg.Done() }
+func (none) Log(now time.Time, message string)             {}
+
+func (none) LogWithColors(now time.Time, message, coloredMessage string) {}
+
+func (none) Flush()               {}
+func (none) Pause()               {}
+func (none) Resume()              {}
+func (none) Rotate()              {}
+func (none) SupportsColors() bool { return false }
diff --git a/internal/io/dlog/loggers/stdout.go b/internal/io/dlog/loggers/stdout.go
new file mode 100644
index 0000000..9738323
--- /dev/null
+++ b/internal/io/dlog/loggers/stdout.go
@@ -0,0 +1,73 @@
+package loggers
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type stdout struct {
+	bufferCh chan string
+	pauseCh  chan struct{}
+	resumeCh chan struct{}
+}
+
+func newStdout() *stdout {
+	return &stdout{
+		bufferCh: make(chan string, 100),
+		pauseCh:  make(chan struct{}),
+		resumeCh: make(chan struct{}),
+	}
+}
+
+func (s *stdout) Start(ctx context.Context, wg *sync.WaitGroup) {
+	pause := func(ctx context.Context) {
+		select {
+		case <-s.resumeCh:
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	go func() {
+		defer wg.Done()
+
+		for {
+			select {
+			case message := <-s.bufferCh:
+				fmt.Println(message)
+			case <-s.pauseCh:
+				pause(ctx)
+			case <-ctx.Done():
+				s.Flush()
+				return
+			}
+		}
+	}()
+}
+
+func (s *stdout) Log(now time.Time, message string) {
+	s.bufferCh <- message
+}
+
+func (s *stdout) LogWithColors(now time.Time, message, coloredMessage string) {
+	s.bufferCh <- coloredMessage
+}
+
+func (s *stdout) Flush() {
+	for {
+		select {
+		case message := <-s.bufferCh:
+			fmt.Println(message)
+		default:
+			return
+		}
+	}
+}
+
+func (s *stdout) Pause()            { s.pauseCh <- struct{}{} }
+func (s *stdout) Resume()           { s.resumeCh <- struct{}{} }
+func (s *stdout) Rotate()           {}
+func (stdout) SupportsColors() bool { return true }
diff --git a/internal/io/dlog/rotation.go b/internal/io/dlog/rotation.go
new file mode 100644
index 0000000..15ce1fd
--- /dev/null
+++ b/internal/io/dlog/rotation.go
@@ -0,0 +1,27 @@
+package dlog
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/mimecast/dtail/internal/io/dlog/loggers"
+)
+
+func rotation(ctx context.Context) {
+	rotateCh := make(chan os.Signal, 1)
+	signal.Notify(rotateCh, syscall.SIGHUP)
+	go func() {
+		for {
+			select {
+			case <-rotateCh:
+				Common.Debug("Invoking log rotation")
+				loggers.FactoryRotate()
+				return
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+}
diff --git a/internal/io/dlog/source.go b/internal/io/dlog/source.go
new file mode 100644
index 0000000..265885e
--- /dev/null
+++ b/internal/io/dlog/source.go
@@ -0,0 +1,19 @@
+package dlog
+
+type source int
+
+const (
+	CLIENT source = iota
+	SERVER source = iota
+)
+
+func (s source) String() string {
+	switch s {
+	case CLIENT:
+		return "CLIENT"
+	case SERVER:
+		return "SERVER"
+	}
+
+	panic("Unknown log source type")
+}
diff --git a/internal/io/dlog/strategy.go b/internal/io/dlog/strategy.go
new file mode 100644
index 0000000..32d8298
--- /dev/null
+++ b/internal/io/dlog/strategy.go
@@ -0,0 +1,22 @@
+package dlog
+
+import "github.com/mimecast/dtail/internal/config"
+
+// Strategy allows to specify a log rotation strategy.
+type Strategy int
+
+// Possible log strategies.
+const (
+	NormalStrategy Strategy = iota
+	DailyStrategy  Strategy = iota
+	StdoutStrategy Strategy = iota
+)
+
+func logStrategy() Strategy {
+	switch config.Common.LogStrategy {
+	case "daily":
+		return DailyStrategy
+	default:
+	}
+	return StdoutStrategy
+}
diff --git a/internal/io/fs/permissions/permission.go b/internal/io/fs/permissions/permission.go
index cc5dd9b..bbcb74e 100644
--- a/internal/io/fs/permissions/permission.go
+++ b/internal/io/fs/permissions/permission.go
@@ -3,12 +3,12 @@
 package permissions
 
 import (
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // ToRead is to check whether user has read permissions to a given file.
 func ToRead(user, filePath string) (bool, error) {
 	// Only implemented for Linux, always expect true
-	logger.Warn(user, filePath, "Not performing ACL check, not supported on this platform")
+	dlog.Common.Warn(user, filePath, "Not performing ACL check, not supported on this platform")
 	return true, nil
 }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index ec33c60..07486a1 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/regex"
 
@@ -62,7 +62,7 @@ func (f readFile) Retry() bool {
 
 // Start tailing a log file.
 func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Regex) error {
-	logger.Debug("readFile", f)
+	dlog.Common.Debug("readFile", f)
 	defer func() {
 		select {
 		case <-f.limiter:
@@ -74,7 +74,7 @@ func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Re
 	case f.limiter <- struct{}{}:
 	default:
 		select {
-		case f.serverMessages <- logger.Warn(f.filePath, f.globID, "Server limit reached. Queuing file..."):
+		case f.serverMessages <- dlog.Common.Warn(f.filePath, f.globID, "Server limit reached. Queuing file..."):
 		case <-ctx.Done():
 			return nil
 		}
@@ -126,7 +126,7 @@ func (f readFile) makeReader(fd *os.File) (reader *bufio.Reader, err error) {
 	case strings.HasSuffix(f.FilePath(), ".gz"):
 		fallthrough
 	case strings.HasSuffix(f.FilePath(), ".gzip"):
-		logger.Info(f.FilePath(), "Detected gzip compression format")
+		dlog.Common.Info(f.FilePath(), "Detected gzip compression format")
 		var gzipReader *gzip.Reader
 		gzipReader, err = gzip.NewReader(fd)
 		if err != nil {
@@ -134,7 +134,7 @@ func (f readFile) makeReader(fd *os.File) (reader *bufio.Reader, err error) {
 		}
 		reader = bufio.NewReader(gzipReader)
 	case strings.HasSuffix(f.FilePath(), ".zst"):
-		logger.Info(f.FilePath(), "Detected zstd compression format")
+		dlog.Common.Info(f.FilePath(), "Detected zstd compression format")
 		reader = bufio.NewReader(zstd.NewReader(fd))
 	default:
 		reader = bufio.NewReader(fd)
@@ -172,7 +172,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 			default:
 			}
 			if !f.seekEOF {
-				logger.Info(f.FilePath(), "End of file reached")
+				dlog.Common.Info(f.FilePath(), "End of file reached")
 				return nil
 			}
 			time.Sleep(time.Millisecond * 100)
@@ -201,7 +201,7 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 		default:
 			if message.Len() >= lineLengthThreshold {
 				if !warnedAboutLongLine {
-					f.serverMessages <- logger.Warn(f.filePath, "Long log line, splitting into multiple lines")
+					f.serverMessages <- dlog.Common.Warn(f.filePath, "Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
 				message.WriteString("\n")
@@ -268,7 +268,7 @@ func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int, r
 
 // Check wether log file is truncated. Returns nil if not.
 func (f readFile) truncated(fd *os.File) (bool, error) {
-	logger.Debug(f.filePath, "File truncation check")
+	dlog.Common.Debug(f.filePath, "File truncation check")
 
 	// Can not seek currently open FD.
 	curPos, err := fd.Seek(0, os.SEEK_CUR)
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 6890201..6a6b5ec 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -1,5 +1,7 @@
 package logger
 
+// TODO: Rewrite this logger
+
 import (
 	"bufio"
 	"context"
@@ -64,12 +66,25 @@ var resumeCh chan struct{}
 // Tell the logger about logrotation
 var rotateCh chan os.Signal
 
+// Override the logger with a custom callack (e.g. for the t.Log for unit tests)
+type unitTestCallback func(message string)
+
+var unitTestOkCb unitTestCallback
+var unitTestErrorCb unitTestCallback
+
 // Helper type to make logging non-blocking.
 type buf struct {
 	time    time.Time
 	message string
 }
 
+// StartUnitTests enables to log all messages to the unit tests.
+func StartUnitTests(ctx context.Context, okCb, errCb unitTestCallback) {
+	unitTestOkCb = okCb
+	unitTestErrorCb = errCb
+	Start(ctx, Modes{UnitTest: true})
+}
+
 // Start logging.
 func Start(ctx context.Context, mode Modes) {
 	Mode = mode
@@ -91,12 +106,12 @@ func Start(ctx context.Context, mode Modes) {
 	switch strategy {
 	case DailyStrategy:
 		_, err := os.Stat(config.Common.LogDir)
-		Mode.logToFile = !os.IsNotExist(err)
+		Mode.logToFile = !os.IsNotExist(err) && !Mode.UnitTest
 		Mode.logToStdout = !Mode.Server || Mode.Debug || Mode.Trace || Mode.Quiet
 	case StdoutStrategy:
 		fallthrough
 	default:
-		Mode.logToFile = !Mode.Server
+		Mode.logToFile = !Mode.Server && !Mode.UnitTest
 		Mode.logToStdout = true
 	}
 
@@ -182,8 +197,8 @@ func Fatal(args ...interface{}) string {
 	return log(clientStr, fatalStr, args)
 }
 
-// FatalExit logs an error and exists the process.
-func FatalExit(args ...interface{}) {
+// FatalPanic logs an error and exists the process.
+func FatalPanic(args ...interface{}) {
 	what := clientStr
 	if Mode.Server {
 		what = serverStr
diff --git a/internal/io/logger/modes.go b/internal/io/logger/modes.go
index 8864179..85f90a5 100644
--- a/internal/io/logger/modes.go
+++ b/internal/io/logger/modes.go
@@ -2,11 +2,12 @@ package logger
 
 // Modes specifies the logging mode.
 type Modes struct {
-	Server      bool
-	Trace       bool
 	Debug       bool
+	logToFile   bool
+	logToStdout bool
 	Nothing     bool
 	Quiet       bool
-	logToStdout bool
-	logToFile   bool
+	Server      bool
+	Trace       bool
+	UnitTest    bool
 }
diff --git a/internal/io/prompt/prompt.go b/internal/io/prompt/prompt.go
index 36ebdb5..7c3cdb5 100644
--- a/internal/io/prompt/prompt.go
+++ b/internal/io/prompt/prompt.go
@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // Answer is a user input of a prompt question.
@@ -58,7 +58,7 @@ func (p *Prompt) Add(answer Answer) {
 // Ask a question.
 func (p *Prompt) Ask() {
 	reader := bufio.NewReader(os.Stdin)
-	logger.Pause()
+	dlog.Common.Pause()
 
 	for {
 		fmt.Print(p.askString())
@@ -70,7 +70,7 @@ func (p *Prompt) Ask() {
 			}
 
 			if !a.AskAgain {
-				logger.Resume()
+				dlog.Common.Resume()
 				if a.EndCallback != nil {
 					a.EndCallback()
 				}
diff --git a/internal/mapr/aggregateset.go b/internal/mapr/aggregateset.go
index 47f4925..14e6943 100644
--- a/internal/mapr/aggregateset.go
+++ b/internal/mapr/aggregateset.go
@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 )
@@ -37,7 +37,7 @@ func (s *AggregateSet) String() string {
 // Merge one aggregate set into this one.
 func (s *AggregateSet) Merge(query *Query, set *AggregateSet) error {
 	s.Samples += set.Samples
-	//logger.Trace("Merge", set)
+	//dlog.Common.Trace("Merge", set)
 
 	for _, sc := range query.Select {
 		storage := sc.FieldStorage
@@ -70,7 +70,7 @@ func (s *AggregateSet) Merge(query *Query, set *AggregateSet) error {
 
 // Serialize the aggregate set so it can be sent over the wire.
 func (s *AggregateSet) Serialize(ctx context.Context, groupKey string, ch chan<- string) {
-	logger.Trace("Serialising mapr.AggregateSet", s)
+	dlog.Common.Trace("Serialising mapr.AggregateSet", s)
 	sb := pool.BuilderBuffer.Get().(*strings.Builder)
 	defer pool.RecycleBuilderBuffer(sb)
 
diff --git a/internal/mapr/client/aggregate.go b/internal/mapr/client/aggregate.go
index 5cc09a1..d0c1d70 100644
--- a/internal/mapr/client/aggregate.go
+++ b/internal/mapr/client/aggregate.go
@@ -5,7 +5,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/protocol"
 )
@@ -52,7 +52,7 @@ func (a *Aggregate) Aggregate(message string) error {
 	for _, sc := range a.query.Select {
 		if val, ok := fields[sc.FieldStorage]; ok {
 			if err := set.Aggregate(sc.FieldStorage, sc.Operation, val, true); err != nil {
-				logger.Error(err)
+				dlog.Common.Error(err)
 				continue
 			}
 			addedSamples = true
diff --git a/internal/mapr/funcs/function.go b/internal/mapr/funcs/function.go
index 1a89c3a..0433b9a 100644
--- a/internal/mapr/funcs/function.go
+++ b/internal/mapr/funcs/function.go
@@ -58,9 +58,9 @@ func NewFunctionStack(in string) (FunctionStack, string, error) {
 // Call the function stack.
 func (fs FunctionStack) Call(str string) string {
 	for i := len(fs) - 1; i >= 0; i-- {
-		//logger.Debug("Call", fs[i].Name, str)
+		//dlog.Common.Debug("Call", fs[i].Name, str)
 		str = fs[i].call(str)
-		//logger.Debug("Call.result", fs[i].Name, str)
+		//dlog.Common.Debug("Call.result", fs[i].Name, str)
 	}
 
 	return str
diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index 9bff790..df8c603 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -11,7 +11,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/color"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
 )
@@ -189,7 +189,7 @@ func (g *GroupSet) WriteResult(query *Query) error {
 		return err
 	}
 
-	logger.Info("Writing outfile", query.Outfile)
+	dlog.Common.Info("Writing outfile", query.Outfile)
 	tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile)
 
 	file, err := os.Create(tmpOutfile)
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index c67137e..e0bbc30 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -26,6 +26,7 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	fields["$timeoffset"] = p.timeZoneOffset
 
 	fields["$severity"] = splitted[0]
+	fields["$loglevel"] = splitted[0]
 	// TODO: Parse time like we do at Mimecast
 	fields["$time"] = splitted[1]
 
diff --git a/internal/mapr/logformat/generickv.go b/internal/mapr/logformat/generickv.go
index 23d75cb..3769c22 100644
--- a/internal/mapr/logformat/generickv.go
+++ b/internal/mapr/logformat/generickv.go
@@ -21,7 +21,7 @@ func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error)
 	for _, kv := range splitted[0:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
-			//logger.Debug("Unable to parse key-value token, ignoring it", kv)
+			//dlog.Common.Debug("Unable to parse key-value token, ignoring it", kv)
 			continue
 		}
 		fields[strings.ToLower(keyAndValue[0])] = keyAndValue[1]
diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go
index 6582b5f..a352580 100644
--- a/internal/mapr/logformat/parser.go
+++ b/internal/mapr/logformat/parser.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/logger"
 	"github.com/mimecast/dtail/internal/mapr"
 )
 
@@ -76,12 +75,10 @@ func (p *Parser) MakeFields(maprLine string) (fields map[string]string, err erro
 
 	if errInterface == nil {
 		fields, err = returnValues[0].Interface().(map[string]string), nil
-		logger.Trace("parser.MakeFields", fields, err)
 		return
 	}
 
 	fields, err = returnValues[0].Interface().(map[string]string), errInterface.(error)
-	logger.Trace("parser.MakeFields", fields, err)
 
 	return
 }
diff --git a/internal/mapr/query.go b/internal/mapr/query.go
index 01852da..6c1d849 100644
--- a/internal/mapr/query.go
+++ b/internal/mapr/query.go
@@ -6,8 +6,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/mimecast/dtail/internal/io/logger"
 )
 
 const (
@@ -67,8 +65,6 @@ func NewQuery(queryStr string) (*Query, error) {
 	}
 
 	err := q.parse(tokens)
-
-	logger.Debug(q)
 	return &q, err
 }
 
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index d11ed7d..767aada 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -9,7 +9,7 @@ import (
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/logformat"
@@ -40,7 +40,7 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 
 	fqdn, err := os.Hostname()
 	if err != nil {
-		logger.Error(err)
+		dlog.Common.Error(err)
 	}
 	s := strings.Split(fqdn, ".")
 
@@ -55,12 +55,12 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 		parserName = query.LogFormat
 	}
 
-	logger.Info("Creating log format parser", parserName)
+	dlog.Common.Info("Creating log format parser", parserName)
 	logParser, err := logformat.NewParser(parserName, query)
 	if err != nil {
-		logger.Error("Could not create log format parser. Falling back to 'generic'", err)
+		dlog.Common.Error("Could not create log format parser. Falling back to 'generic'", err)
 		if logParser, err = logformat.NewParser("generic", query); err != nil {
-			logger.FatalExit("Could not create log format parser", err)
+			dlog.Common.FatalPanic("Could not create log format parser", err)
 		}
 	}
 
@@ -153,7 +153,7 @@ func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]strin
 				if err != nil {
 					// Should fields be ignored anyway?
 					if err != logformat.IgnoreFieldsErr {
-						logger.Error(fields, err)
+						dlog.Common.Error(fields, err)
 					}
 					continue
 				}
@@ -187,7 +187,7 @@ func (a *Aggregate) setAdditionalFields(ctx context.Context, fieldsCh <-chan map
 				return
 			}
 			if err := a.query.SetClause(fields); err != nil {
-				logger.Error(err)
+				dlog.Common.Error(err)
 			}
 
 			select {
@@ -204,7 +204,7 @@ func (a *Aggregate) aggregateAndSerialize(ctx context.Context, fieldsCh <-chan m
 	group := mapr.NewGroupSet()
 
 	serialize := func() {
-		logger.Info("Serializing mapreduce result")
+		dlog.Common.Info("Serializing mapreduce result")
 		group.Serialize(ctx, maprMessages)
 		group = mapr.NewGroupSet()
 	}
@@ -243,7 +243,7 @@ func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 	for _, sc := range a.query.Select {
 		if val, ok := fields[sc.Field]; ok {
 			if err := set.Aggregate(sc.FieldStorage, sc.Operation, val, false); err != nil {
-				logger.Error(err)
+				dlog.Common.Error(err)
 				continue
 			}
 			addedSample = true
@@ -255,7 +255,7 @@ func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 		return
 	}
 
-	logger.Trace("Aggregated data locally without adding new samples")
+	dlog.Common.Trace("Aggregated data locally without adding new samples")
 }
 
 // Serialize all the aggregated data.
@@ -263,7 +263,7 @@ func (a *Aggregate) Serialize(ctx context.Context) {
 	select {
 	case a.serialize <- struct{}{}:
 	case <-time.After(time.Minute):
-		logger.Warn("Starting to serialize mapredice data takes over a minute")
+		dlog.Common.Warn("Starting to serialize mapredice data takes over a minute")
 	case <-ctx.Done():
 	}
 }
diff --git a/internal/mapr/token.go b/internal/mapr/token.go
index 8972188..7c6578b 100644
--- a/internal/mapr/token.go
+++ b/internal/mapr/token.go
@@ -58,12 +58,12 @@ func tokenize(queryStr string) []token {
 }
 
 func tokensConsume(tokens []token) ([]token, []token) {
-	//logger.Trace("=====================")
+	//dlog.Common.Trace("=====================")
 	var consumed []token
 
 	for i, t := range tokens {
 		if t.isKeyword() {
-			//logger.Trace("keyword", t)
+			//dlog.Common.Trace("keyword", t)
 			return tokens[i:], consumed
 		}
 		// strip escapes, such as ` from `foo`, this allows to use keywords as field names
@@ -73,7 +73,7 @@ func tokensConsume(tokens []token) ([]token, []token) {
 		}
 		if t.str[0] == '`' && t.str[length-1] == '`' {
 			stripped := t.str[1 : length-1]
-			//logger.Trace("stripped", stripped)
+			//dlog.Common.Trace("stripped", stripped)
 			t := token{
 				str:        stripped,
 				isBareword: t.isBareword,
@@ -81,11 +81,11 @@ func tokensConsume(tokens []token) ([]token, []token) {
 			consumed = append(consumed, t)
 			continue
 		}
-		//logger.Trace("bare", token)
+		//dlog.Common.Trace("bare", token)
 		consumed = append(consumed, t)
 	}
 
-	//logger.Trace("result", consumed)
+	//dlog.Common.Trace("result", consumed)
 	return nil, consumed
 }
 
diff --git a/internal/mapr/whereclause.go b/internal/mapr/whereclause.go
index cc1c164..6356d94 100644
--- a/internal/mapr/whereclause.go
+++ b/internal/mapr/whereclause.go
@@ -3,7 +3,7 @@ package mapr
 import (
 	"strconv"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // WhereClause interprets the where clause of the mapreduce query.
@@ -55,7 +55,7 @@ func whereClauseFloatValue(fields map[string]string, str string, float float64,
 		}
 		return f, true
 	default:
-		logger.Error("Unexpected argument in 'where' clause", str, float, t)
+		dlog.Common.Error("Unexpected argument in 'where' clause", str, float, t)
 		return 0, false
 	}
 }
@@ -71,7 +71,7 @@ func whereClauseStringValue(fields map[string]string, str string, t fieldType) (
 	case String:
 		return str, true
 	default:
-		logger.Error("Unexpected argument in 'where' clause", str, t)
+		dlog.Common.Error("Unexpected argument in 'where' clause", str, t)
 		return str, false
 	}
 }
diff --git a/internal/mapr/wherecondition.go b/internal/mapr/wherecondition.go
index 7a60dba..c60c0a5 100644
--- a/internal/mapr/wherecondition.go
+++ b/internal/mapr/wherecondition.go
@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // QueryOperation determines the mapreduce operation.
@@ -168,7 +168,7 @@ func (wc *whereCondition) floatClause(lValue float64, rValue float64) bool {
 	case FloatGe:
 		return lValue >= rValue
 	default:
-		logger.Error("Unknown float operation", lValue, wc.Operation, rValue)
+		dlog.Common.Error("Unknown float operation", lValue, wc.Operation, rValue)
 	}
 
 	return false
@@ -193,7 +193,7 @@ func (wc *whereCondition) stringClause(lValue string, rValue string) bool {
 	case StringNotHasSuffix:
 		return !strings.HasSuffix(lValue, rValue)
 	default:
-		logger.Error("Unknown string operation", lValue, wc.Operation, rValue)
+		dlog.Common.Error("Unknown string operation", lValue, wc.Operation, rValue)
 	}
 
 	return false
diff --git a/internal/regex/regex.go b/internal/regex/regex.go
index 2561659..352ffd6 100644
--- a/internal/regex/regex.go
+++ b/internal/regex/regex.go
@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
-
-	"github.com/mimecast/dtail/internal/io/logger"
 )
 
 // Regex for filtering lines.
@@ -91,17 +89,17 @@ func (r Regex) MatchString(str string) bool {
 }
 
 // Serialize the regex.
-func (r Regex) Serialize() string {
+func (r Regex) Serialize() (string, error) {
 	var flags []string
 	for _, flag := range r.flags {
 		flags = append(flags, flag.String())
 	}
 
 	if !r.initialized {
-		logger.FatalExit("Unable to serialize regex as not initialized properly", r)
+		return "", fmt.Errorf("Unable to serialize regex as not initialized properly: %v", r)
 	}
 
-	return fmt.Sprintf("regex:%s %s", strings.Join(flags, ","), r.regexStr)
+	return fmt.Sprintf("regex:%s %s", strings.Join(flags, ","), r.regexStr), nil
 }
 
 // Deserialize the regex.
@@ -109,7 +107,6 @@ func Deserialize(str string) (Regex, error) {
 	// Get regex string
 	s := strings.SplitN(str, " ", 2)
 	if len(s) < 2 {
-		logger.Debug("Using noop regex", str)
 		return NewNoop(), nil
 	}
 
@@ -127,10 +124,8 @@ func Deserialize(str string) (Regex, error) {
 		for _, flagStr := range strings.Split(s[1], ",") {
 			flag, err := NewFlag(flagStr)
 			if err != nil {
-				logger.Error("ignoring flag", err)
 				continue
 			}
-			logger.Debug("Adding regex flag", flag)
 			flags = append(flags, flag)
 		}
 	}
diff --git a/internal/regex/regex_test.go b/internal/regex/regex_test.go
index a5e7faf..2ce49ac 100644
--- a/internal/regex/regex_test.go
+++ b/internal/regex/regex_test.go
@@ -20,9 +20,13 @@ func TestRegex(t *testing.T) {
 		t.Errorf("expected to match string '%s' with regex '%v' but didn't\n", input, r)
 	}
 
-	r2, err := Deserialize(r.Serialize())
+	serialized, err := r.Serialize()
 	if err != nil {
-		t.Errorf("unable to serialize deserialized regex: %v: %v\n", r.Serialize(), err)
+		t.Errorf("unable to serialize regex: %v: %v\n", serialized, err)
+	}
+	r2, err := Deserialize(serialized)
+	if err != nil {
+		t.Errorf("unable to serialize deserialized regex: %v: %v\n", serialized, err)
 	}
 	if r.String() != r2.String() {
 		t.Errorf("regex should be the same after deserialize(serialize(..)), got '%s' but expected '%s'.\n",
@@ -37,9 +41,13 @@ func TestRegex(t *testing.T) {
 		t.Errorf("expected to not match string '%s' with regex '%v' but matched\n", input, r)
 	}
 
-	r2, err = Deserialize(r.Serialize())
+	serialized, err = r.Serialize()
+	if err != nil {
+		t.Errorf("unable to serialize regex: %v: %v\n", serialized, err)
+	}
+	r2, err = Deserialize(serialized)
 	if err != nil {
-		t.Errorf("unable to serialize deserialized regex: %v: %v\n", r.Serialize(), err)
+		t.Errorf("unable to serialize deserialized regex: %v: %v\n", serialized, err)
 	}
 	if r.String() != r2.String() {
 		t.Errorf("regex should be the same after deserialize(serialize(..)), got '%s' but expected '%s'.\n",
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index f75c732..5f4c454 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -8,9 +8,8 @@ import (
 
 	"github.com/mimecast/dtail/internal/clients"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
-
 	gossh "golang.org/x/crypto/ssh"
 )
 
@@ -22,7 +21,7 @@ func newContinuous() *continuous {
 }
 
 func (c *continuous) start(ctx context.Context) {
-	logger.Info("Starting continuous job runner after 10s")
+	dlog.Server.Info("Starting continuous job runner after 10s")
 	time.Sleep(time.Second * 10)
 
 	c.runJobs(ctx)
@@ -31,7 +30,7 @@ func (c *continuous) start(ctx context.Context) {
 func (c *continuous) runJobs(ctx context.Context) {
 	for _, job := range config.Server.Continuous {
 		if !job.Enable {
-			logger.Debug(job.Name, "Not running job as not enabled")
+			dlog.Server.Debug(job.Name, "Not running job as not enabled")
 			continue
 		}
 
@@ -51,7 +50,7 @@ func (c *continuous) runJobs(ctx context.Context) {
 }
 
 func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
-	logger.Debug(job.Name, "Processing job")
+	dlog.Server.Debug(job.Name, "Processing job")
 
 	files := fillDates(job.Files)
 	outfile := fillDates(job.Outfile)
@@ -61,7 +60,7 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 		servers = config.Server.SSHBindAddress
 	}
 
-	args := clients.Args{
+	args := config.Args{
 		ConnectionsPerCPU: 10,
 		Discovery:         job.Discovery,
 		ServersStr:        servers,
@@ -75,7 +74,7 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	query := fmt.Sprintf("%s outfile %s", job.Query, outfile)
 	client, err := clients.NewMaprClient(args, query, clients.NonCumulativeMode)
 	if err != nil {
-		logger.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
+		dlog.Server.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
 		return
 	}
 
@@ -85,21 +84,21 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	if job.RestartOnDayChange {
 		go func() {
 			if c.waitForDayChange(ctx) {
-				logger.Info(fmt.Sprintf("Canceling job %s due to day change", job.Name))
+				dlog.Server.Info(fmt.Sprintf("Canceling job %s due to day change", job.Name))
 				cancel()
 			}
 		}()
 	}
 
-	logger.Info(fmt.Sprintf("Starting job %s", job.Name))
+	dlog.Server.Info(fmt.Sprintf("Starting job %s", job.Name))
 	status := client.Start(jobCtx, make(chan string))
 	logMessage := fmt.Sprintf("Job exited with status %d", status)
 
 	if status != 0 {
-		logger.Warn(logMessage)
+		dlog.Server.Warn(logMessage)
 		return
 	}
-	logger.Info(logMessage)
+	dlog.Server.Info(logMessage)
 }
 
 func (c *continuous) waitForDayChange(ctx context.Context) bool {
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
index 1e17c78..ae70675 100644
--- a/internal/server/handlers/controlhandler.go
+++ b/internal/server/handlers/controlhandler.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
@@ -22,7 +22,7 @@ type ControlHandler struct {
 
 // NewControlHandler returns a new control handler.
 func NewControlHandler(user *user.User) *ControlHandler {
-	logger.Debug(user, "Creating control handler")
+	dlog.Server.Debug(user, "Creating control handler")
 
 	h := ControlHandler{
 		done:           internal.NewDone(),
@@ -32,7 +32,7 @@ func NewControlHandler(user *user.User) *ControlHandler {
 
 	fqdn, err := os.Hostname()
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Server.FatalPanic(err)
 	}
 
 	s := strings.Split(fqdn, ".")
@@ -84,15 +84,15 @@ func (h *ControlHandler) Write(p []byte) (n int, err error) {
 }
 
 func (h *ControlHandler) handleCommand(command string) {
-	logger.Info(h.user, command)
+	dlog.Server.Info(h.user, command)
 	s := strings.Split(command, " ")
-	logger.Debug(h.user, "Receiving command", command, s)
+	dlog.Server.Debug(h.user, "Receiving command", command, s)
 
 	switch s[0] {
 	case "health":
 		h.serverMessages <- "OK: DTail SSH Server seems fine"
 		h.serverMessages <- "done;"
 	default:
-		h.serverMessages <- logger.Error(h.user, "Received unknown control command", command, s)
+		h.serverMessages <- dlog.Server.Error(h.user, "Received unknown control command", command, s)
 	}
 }
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 69dd4a5..60ad2a0 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/io/fs"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 )
@@ -32,13 +32,13 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string, retrie
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			r.server.sendServerMessage(logger.Error(r.server.user, commandParseWarning, err))
+			r.server.sendServerMessage(dlog.Server.Error(r.server.user, commandParseWarning, err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.sendServerWarnMessage(logger.Warn(r.server.user, commandParseWarning, args, argc))
+		r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, commandParseWarning, args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re, retries)
@@ -51,14 +51,14 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 	for retryCount := 0; retryCount < retries; retryCount++ {
 		paths, err := filepath.Glob(glob)
 		if err != nil {
-			logger.Warn(r.server.user, glob, err)
+			dlog.Server.Warn(r.server.user, glob, err)
 			time.Sleep(retryInterval)
 			continue
 		}
 
 		if numPaths := len(paths); numPaths == 0 {
-			logger.Error(r.server.user, "No such file(s) to read", glob)
-			r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Unable to read file(s), check server logs"))
+			dlog.Server.Error(r.server.user, "No such file(s) to read", glob)
+			r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
 			select {
 			case <-ctx.Done():
 				return
@@ -72,7 +72,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 		return
 	}
 
-	r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Giving up to read file(s)"))
+	r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Giving up to read file(s)"))
 	return
 }
 
@@ -92,8 +92,8 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGr
 	globID := r.makeGlobID(path, glob)
 
 	if !r.server.user.HasFilePermission(path, "readfiles") {
-		logger.Error(r.server.user, "No permission to read file", path, globID)
-		r.server.sendServerWarnMessage(logger.Warn(r.server.user, "Unable to read file(s), check server logs"))
+		dlog.Server.Error(r.server.user, "No permission to read file", path, globID)
+		r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
 		return
 	}
 
@@ -101,7 +101,7 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGr
 }
 
 func (r *readCommand) readFile(ctx context.Context, path, globID string, re regex.Regex) {
-	logger.Info(r.server.user, "Start reading file", path, globID)
+	dlog.Server.Info(r.server.user, "Start reading file", path, globID)
 
 	var reader fs.FileReader
 	switch r.mode {
@@ -122,7 +122,7 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 			aggregate.NextLinesCh <- lines
 		}
 		if err := reader.Start(ctx, lines, re); err != nil {
-			logger.Error(r.server.user, path, globID, err)
+			dlog.Server.Error(r.server.user, path, globID, err)
 		}
 		if aggregate != nil {
 			// Also makes aggregate to Flush
@@ -139,7 +139,7 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 		}
 
 		time.Sleep(time.Second * 2)
-		logger.Info(path, globID, "Reading file again")
+		dlog.Server.Info(path, globID, "Reading file again")
 	}
 }
 
@@ -161,6 +161,6 @@ func (r *readCommand) makeGlobID(path, glob string) string {
 		return pathParts[len(pathParts)-1]
 	}
 
-	r.server.sendServerWarnMessage(logger.Warn("Empty file path given?", path, glob))
+	r.server.sendServerWarnMessage(dlog.Server.Warn("Empty file path given?", path, glob))
 	return ""
 }
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 4820476..b664566 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -16,7 +16,7 @@ import (
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
@@ -66,7 +66,7 @@ func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *S
 
 	fqdn, err := os.Hostname()
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Server.FatalPanic(err)
 	}
 
 	s := strings.Split(fqdn, ".")
@@ -165,18 +165,18 @@ func (h *ServerHandler) Write(p []byte) (n int, err error) {
 }
 
 func (h *ServerHandler) handleCommand(commandStr string) {
-	logger.Debug(h.user, commandStr)
+	dlog.Server.Debug(h.user, commandStr)
 	ctx := context.Background()
 
 	args, argc, add, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
 	if err != nil {
-		h.send(h.serverMessages, logger.Error(h.user, err)+add)
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err)+add)
 		return
 	}
 
 	args, argc, err = h.handleBase64(args, argc)
 	if err != nil {
-		h.send(h.serverMessages, logger.Error(h.user, err))
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
 	}
 
@@ -239,7 +239,7 @@ func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, er
 
 	args = strings.Split(decodedStr, " ")
 	argc = len(decodedStr)
-	logger.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
+	dlog.Server.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
 
 	return args, argc, nil
 }
@@ -247,14 +247,14 @@ func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, er
 func (h *ServerHandler) handleControlCommand(argc int, args []string) {
 	switch args[0] {
 	case "debug":
-		h.send(h.serverMessages, logger.Debug(h.user, "Receiving debug command", argc, args))
+		h.send(h.serverMessages, dlog.Server.Debug(h.user, "Receiving debug command", argc, args))
 	default:
-		logger.Warn(h.user, "Received unknown control command", argc, args)
+		dlog.Server.Warn(h.user, "Received unknown control command", argc, args)
 	}
 }
 
 func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string) {
-	logger.Debug(h.user, "handleUserCommand", argc, args)
+	dlog.Server.Debug(h.user, "handleUserCommand", argc, args)
 
 	h.incrementActiveCommands()
 	commandFinished := func() {
@@ -268,19 +268,19 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 
 	options, err := readOptions(splitted[1:])
 	if err != nil {
-		h.sendServerMessage(logger.Error(h.user, err))
+		h.sendServerMessage(dlog.Server.Error(h.user, err))
 		commandFinished()
 		return
 	}
 	if quiet, ok := options["quiet"]; ok {
 		if quiet == "true" {
-			logger.Debug(h.user, "Enabling quiet mode")
+			dlog.Server.Debug(h.user, "Enabling quiet mode")
 			h.quiet = true
 		}
 	}
 	if spartan, ok := options["spartan"]; ok {
 		if spartan == "true" {
-			logger.Debug(h.user, "Enabling spartan mode")
+			dlog.Server.Debug(h.user, "Enabling spartan mode")
 			h.spartan = true
 		}
 	}
@@ -304,7 +304,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		command, aggregate, err := newMapCommand(h, argc, args)
 		if err != nil {
 			h.sendServerMessage(err.Error())
-			logger.Error(h.user, err)
+			dlog.Server.Error(h.user, err)
 			commandFinished()
 			return
 		}
@@ -320,14 +320,14 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		commandFinished()
 
 	default:
-		h.sendServerMessage(logger.Error(h.user, "Received unknown user command", commandName, argc, args, options))
+		h.sendServerMessage(dlog.Server.Error(h.user, "Received unknown user command", commandName, argc, args, options))
 		commandFinished()
 	}
 }
 
 func (h *ServerHandler) handleAckCommand(argc int, args []string) {
 	if argc < 3 {
-		h.sendServerWarnMessage(logger.Warn(h.user, commandParseWarning, args, argc))
+		h.sendServerWarnMessage(dlog.Server.Warn(h.user, commandParseWarning, args, argc))
 		return
 	}
 	if args[1] == "close" && args[2] == "connection" {
@@ -362,25 +362,25 @@ func (h *ServerHandler) serverMessageC() chan<- string {
 }
 
 func (h *ServerHandler) flushMessages() {
-	logger.Debug(h.user, "flushMessages()")
+	dlog.Server.Debug(h.user, "flushMessages()")
 
 	unsentMessages := func() int {
 		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
 	}
 	for i := 0; i < 3; i++ {
 		if unsentMessages() == 0 {
-			logger.Debug(h.user, "All lines sent")
+			dlog.Server.Debug(h.user, "All lines sent")
 			return
 		}
-		logger.Debug(h.user, "Still lines to be sent")
+		dlog.Server.Debug(h.user, "Still lines to be sent")
 		time.Sleep(time.Second)
 	}
 
-	logger.Warn(h.user, "Some lines remain unsent", unsentMessages())
+	dlog.Server.Warn(h.user, "Some lines remain unsent", unsentMessages())
 }
 
 func (h *ServerHandler) shutdown() {
-	logger.Debug(h.user, "shutdown()")
+	dlog.Server.Debug(h.user, "shutdown()")
 	h.flushMessages()
 
 	go func() {
@@ -393,7 +393,7 @@ func (h *ServerHandler) shutdown() {
 	select {
 	case <-h.ackCloseReceived:
 	case <-time.After(time.Second * 5):
-		logger.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
+		dlog.Server.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
 	case <-h.done.Done():
 	}
 
@@ -410,7 +410,7 @@ func (h *ServerHandler) decrementActiveCommands() int32 {
 }
 
 func readOptions(opts []string) (map[string]string, error) {
-	logger.Debug("Parsing options", opts)
+	dlog.Server.Debug("Parsing options", opts)
 	options := make(map[string]string, len(opts))
 
 	for _, o := range opts {
@@ -430,7 +430,7 @@ func readOptions(opts []string) (map[string]string, error) {
 			val = string(decoded)
 		}
 
-		logger.Debug("Setting option", key, val)
+		dlog.Server.Debug("Setting option", key, val)
 		options[key] = val
 	}
 
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index a1e9e36..f474cc8 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/clients"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -24,7 +24,7 @@ func newScheduler() *scheduler {
 }
 
 func (s *scheduler) start(ctx context.Context) {
-	logger.Info("Starting scheduled job runner after 10s")
+	dlog.Server.Info("Starting scheduled job runner after 10s")
 	// First run after just 10s!
 	time.Sleep(time.Second * 10)
 	s.runJobs(ctx)
@@ -42,18 +42,18 @@ func (s *scheduler) start(ctx context.Context) {
 func (s *scheduler) runJobs(ctx context.Context) {
 	for _, job := range config.Server.Schedule {
 		if !job.Enable {
-			logger.Debug(job.Name, "Not running job as not enabled")
+			dlog.Server.Debug(job.Name, "Not running job as not enabled")
 			continue
 		}
 
 		hour, err := strconv.Atoi(time.Now().Format("15"))
 		if err != nil {
-			logger.Error(job.Name, "Unable to create job", err)
+			dlog.Server.Error(job.Name, "Unable to create job", err)
 			continue
 		}
 
 		if hour < job.TimeRange[0] || hour >= job.TimeRange[1] {
-			logger.Debug(job.Name, "Not running job out of time range")
+			dlog.Server.Debug(job.Name, "Not running job out of time range")
 			continue
 		}
 
@@ -62,7 +62,7 @@ func (s *scheduler) runJobs(ctx context.Context) {
 
 		_, err = os.Stat(outfile)
 		if !os.IsNotExist(err) {
-			logger.Debug(job.Name, "Not running job as outfile already exists", outfile)
+			dlog.Server.Debug(job.Name, "Not running job as outfile already exists", outfile)
 			continue
 		}
 
@@ -71,7 +71,7 @@ func (s *scheduler) runJobs(ctx context.Context) {
 			servers = config.Server.SSHBindAddress
 		}
 
-		args := clients.Args{
+		args := config.Args{
 			ConnectionsPerCPU: 10,
 			Discovery:         job.Discovery,
 			ServersStr:        servers,
@@ -85,21 +85,21 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		query := fmt.Sprintf("%s outfile %s", job.Query, outfile)
 		client, err := clients.NewMaprClient(args, query, clients.CumulativeMode)
 		if err != nil {
-			logger.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
+			dlog.Server.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
 			continue
 		}
 
 		jobCtx, cancel := context.WithCancel(ctx)
 		defer cancel()
 
-		logger.Info(fmt.Sprintf("Starting job %s", job.Name))
+		dlog.Server.Info(fmt.Sprintf("Starting job %s", job.Name))
 		status := client.Start(jobCtx, make(chan string))
 		logMessage := fmt.Sprintf("Job exited with status %d", status)
 
 		if status != 0 {
-			logger.Warn(logMessage)
+			dlog.Server.Warn(logMessage)
 			continue
 		}
-		logger.Info(logMessage)
+		dlog.Server.Info(logMessage)
 	}
 }
diff --git a/internal/server/server.go b/internal/server/server.go
index a20737e..a8f541b 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/server/handlers"
 	"github.com/mimecast/dtail/internal/ssh/server"
 	user "github.com/mimecast/dtail/internal/user/server"
@@ -36,7 +36,7 @@ type Server struct {
 
 // New returns a new server.
 func New() *Server {
-	logger.Info("Creating server", version.String())
+	dlog.Server.Info("Creating server", version.String())
 
 	s := Server{
 		sshServerConfig: &gossh.ServerConfig{},
@@ -51,7 +51,7 @@ func New() *Server {
 
 	private, err := gossh.ParsePrivateKey(server.PrivateHostKey())
 	if err != nil {
-		logger.FatalExit(err)
+		dlog.Server.FatalPanic(err)
 	}
 	s.sshServerConfig.AddHostKey(private)
 
@@ -60,14 +60,14 @@ func New() *Server {
 
 // Start the server.
 func (s *Server) Start(ctx context.Context) int {
-	logger.Info("Starting server")
+	dlog.Server.Info("Starting server")
 
 	bindAt := fmt.Sprintf("%s:%d", config.Server.SSHBindAddress, config.Common.SSHPort)
-	logger.Info("Binding server", bindAt)
+	dlog.Server.Info("Binding server", bindAt)
 
 	listener, err := net.Listen("tcp", bindAt)
 	if err != nil {
-		logger.FatalExit("Failed to open listening TCP socket", err)
+		dlog.Server.FatalPanic("Failed to open listening TCP socket", err)
 	}
 
 	go s.stats.start(ctx)
@@ -82,7 +82,7 @@ func (s *Server) Start(ctx context.Context) int {
 }
 
 func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
-	logger.Debug("Starting listener loop")
+	dlog.Server.Debug("Starting listener loop")
 
 	for {
 		conn, err := listener.Accept() // Blocking
@@ -92,12 +92,12 @@ func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 				return
 			default:
 			}
-			logger.Error("Failed to accept incoming connection", err)
+			dlog.Server.Error("Failed to accept incoming connection", err)
 			continue
 		}
 
 		if err := s.stats.serverLimitExceeded(); err != nil {
-			logger.Error(err)
+			dlog.Server.Error(err)
 			conn.Close()
 			continue
 		}
@@ -107,11 +107,11 @@ func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 }
 
 func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
-	logger.Info("Handling connection")
+	dlog.Server.Info("Handling connection")
 
 	sshConn, chans, reqs, err := gossh.NewServerConn(conn, s.sshServerConfig)
 	if err != nil {
-		logger.Error("Something just happened", err)
+		dlog.Server.Error("Something just happened", err)
 		return
 	}
 
@@ -125,29 +125,29 @@ func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
 
 func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChannel gossh.NewChannel) {
 	user := user.New(sshConn.User(), sshConn.RemoteAddr().String())
-	logger.Info(user, "Invoking channel handler")
+	dlog.Server.Info(user, "Invoking channel handler")
 
 	if newChannel.ChannelType() != "session" {
 		err := errors.New("Don'w allow other channel types than session")
-		logger.Error(user, err)
+		dlog.Server.Error(user, err)
 		newChannel.Reject(gossh.Prohibited, err.Error())
 		return
 	}
 
 	channel, requests, err := newChannel.Accept()
 	if err != nil {
-		logger.Error(user, "Could not accept channel", err)
+		dlog.Server.Error(user, "Could not accept channel", err)
 		return
 	}
 
 	if err := s.handleRequests(ctx, sshConn, requests, channel, user); err != nil {
-		logger.Error(user, err)
+		dlog.Server.Error(user, err)
 		sshConn.Close()
 	}
 }
 
 func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-chan *gossh.Request, channel gossh.Channel, user *user.User) error {
-	logger.Info(user, "Invoking request handler")
+	dlog.Server.Info(user, "Invoking request handler")
 
 	for req := range in {
 		var payload = struct{ Value string }{}
@@ -190,10 +190,10 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 
 			go func() {
 				if err := sshConn.Wait(); err != nil && err != io.EOF {
-					logger.Error(user, err)
+					dlog.Server.Error(user, err)
 				}
 				s.stats.decrementConnections()
-				logger.Info(user, "Good bye Mister!")
+				dlog.Server.Info(user, "Good bye Mister!")
 				terminate()
 			}()
 
@@ -216,7 +216,7 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	user := user.New(c.User(), c.RemoteAddr().String())
 
 	if config.ServerRelaxedAuthEnable {
-		logger.Fatal(user, "Granting permissions via relaxed-auth")
+		dlog.Server.Fatal(user, "Granting permissions via relaxed-auth")
 		return nil, nil
 	}
 
@@ -228,20 +228,20 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	switch user.Name {
 	case config.ControlUser:
 		if authInfo == config.ControlUser {
-			logger.Debug(user, "Granting permissions to control user")
+			dlog.Server.Debug(user, "Granting permissions to control user")
 			return nil, nil
 		}
 	case config.ScheduleUser:
 		for _, job := range config.Server.Schedule {
 			if s.backgroundCanSSH(user, authInfo, remoteIP, job.Name, job.AllowFrom) {
-				logger.Debug(user, "Granting SSH connection")
+				dlog.Server.Debug(user, "Granting SSH connection")
 				return nil, nil
 			}
 		}
 	case config.ContinuousUser:
 		for _, job := range config.Server.Continuous {
 			if s.backgroundCanSSH(user, authInfo, remoteIP, job.Name, job.AllowFrom) {
-				logger.Debug(user, "Granting SSH connection")
+				dlog.Server.Debug(user, "Granting SSH connection")
 				return nil, nil
 			}
 		}
@@ -252,22 +252,22 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 }
 
 func (s *Server) backgroundCanSSH(user *user.User, jobName, remoteIP, allowedJobName string, allowFrom []string) bool {
-	logger.Debug("backgroundCanSSH", user, jobName, remoteIP, allowedJobName, allowFrom)
+	dlog.Server.Debug("backgroundCanSSH", user, jobName, remoteIP, allowedJobName, allowFrom)
 
 	if jobName != allowedJobName {
-		logger.Debug(user, jobName, "backgroundCanSSH", "Job name does not match, skipping to next one...", allowedJobName)
+		dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Job name does not match, skipping to next one...", allowedJobName)
 		return false
 	}
 
 	for _, myAddr := range allowFrom {
 		ips, err := net.LookupIP(myAddr)
 		if err != nil {
-			logger.Debug(user, jobName, "backgroundCanSSH", "Unable to lookup IP address for allowed hosts lookup, skipping to next one...", myAddr, err)
+			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Unable to lookup IP address for allowed hosts lookup, skipping to next one...", myAddr, err)
 			continue
 		}
 
 		for _, ip := range ips {
-			logger.Debug(user, jobName, "backgroundCanSSH", "Comparing IP addresses", remoteIP, ip.String())
+			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Comparing IP addresses", remoteIP, ip.String())
 			if remoteIP == ip.String() {
 				return true
 			}
diff --git a/internal/server/stats.go b/internal/server/stats.go
index 3e8c71d..8583318 100644
--- a/internal/server/stats.go
+++ b/internal/server/stats.go
@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 // Used to collect and display various server stats.
@@ -41,7 +41,7 @@ func (s *stats) hasConnections() bool {
 	s.mutex.Unlock()
 
 	has := currentConnections > 0
-	logger.Info("stats", "Server with open connections?", has, currentConnections)
+	dlog.Server.Info("stats", "Server with open connections?", has, currentConnections)
 
 	return has
 }
@@ -57,7 +57,7 @@ func (s *stats) logServerStats() {
 	data["cgocalls"] = runtime.NumCgoCall()
 	data["cpu"] = runtime.NumCPU()
 
-	logger.Mapreduce("STATS", data)
+	dlog.Server.Mapreduce("STATS", data)
 }
 
 func (s *stats) serverLimitExceeded() error {
diff --git a/internal/ssh/client/authmethods.go b/internal/ssh/client/authmethods.go
index 2ff80b2..4508319 100644
--- a/internal/ssh/client/authmethods.go
+++ b/internal/ssh/client/authmethods.go
@@ -4,7 +4,7 @@ import (
 	"os"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/ssh"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -15,7 +15,7 @@ func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod, hostKeyCallback gossh
 	if len(sshAuthMethods) > 0 {
 		simpleCallback, err := NewSimpleCallback()
 		if err != nil {
-			logger.FatalExit(err)
+			dlog.Common.FatalPanic(err)
 		}
 		return sshAuthMethods, simpleCallback
 	}
@@ -29,13 +29,13 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, pri
 	knownHostsPath := os.Getenv("HOME") + "/.ssh/known_hosts"
 	knownHostsCallback, err := NewKnownHostsCallback(knownHostsPath, trustAllHosts, throttleCh)
 	if err != nil {
-		logger.FatalExit(knownHostsPath, err)
+		dlog.Common.FatalPanic(knownHostsPath, err)
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
 
 	if config.Common.ExperimentalFeaturesEnable {
 		sshAuthMethods = append(sshAuthMethods, gossh.Password("experimental feature test"))
-		logger.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
+		dlog.Common.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
 	}
 
 	// First try to read custom private key path.
@@ -43,41 +43,41 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, pri
 		authMethod, err := ssh.PrivateKey(privateKeyPath)
 		if err == nil {
 			sshAuthMethods = append(sshAuthMethods, authMethod)
-			logger.Debug("initKnownHostsAuthMethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+			dlog.Common.Debug("initKnownHostsAuthMethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
 			return sshAuthMethods, knownHostsCallback
 		}
-		logger.FatalExit("Unable to use private SSH key", privateKeyPath, err)
+		dlog.Common.FatalPanic("Unable to use private SSH key", privateKeyPath, err)
 	}
 
 	// Second, try SSH Agent
 	authMethod, err := ssh.Agent()
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK) to list of auth methods, not adding further methods")
+		dlog.Common.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK) to list of auth methods, not adding further methods")
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err)
 
 	// Third, try Linux/UNIX default key paths
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_rsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Common.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
 
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_dsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Common.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
 
-	logger.FatalExit("Unable to find private SSH key information")
+	dlog.Common.FatalPanic("Unable to find private SSH key information")
 
 	// Never reach this point.
 	return sshAuthMethods, knownHostsCallback
diff --git a/internal/ssh/client/knownhostscallback.go b/internal/ssh/client/knownhostscallback.go
index 1ccf6c6..a73d612 100644
--- a/internal/ssh/client/knownhostscallback.go
+++ b/internal/ssh/client/knownhostscallback.go
@@ -10,7 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/prompt"
 
 	"golang.org/x/crypto/ssh"
@@ -97,7 +97,7 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 			responseCh: make(chan response),
 		}
 
-		logger.Warn("Encountered unknown host", unknown)
+		dlog.Common.Warn("Encountered unknown host", unknown)
 		// Notify user that there is an unknown host
 		c.unknownCh <- unknown
 
@@ -139,7 +139,7 @@ func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 				hosts = []unknownHost{}
 			}
 		case <-ctx.Done():
-			logger.Debug("Stopping goroutine prompting new hosts...")
+			dlog.Common.Debug("Stopping goroutine prompting new hosts...")
 			return
 		}
 	}
@@ -154,7 +154,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 
 	select {
 	case <-c.trustAllHostsCh:
-		logger.Warn("Trusting host keys of servers", servers)
+		dlog.Common.Warn("Trusting host keys of servers", servers)
 		c.trustHosts(hosts)
 		return
 	default:
@@ -175,7 +175,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.trustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Added hosts to known hosts file", c.knownHostsPath)
+			dlog.Common.Info("Added hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
@@ -188,7 +188,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.trustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Added hosts to known hosts file", c.knownHostsPath)
+			dlog.Common.Info("Added hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
@@ -200,7 +200,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.dontTrustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Didn't add hosts to known hosts file", c.knownHostsPath)
+			dlog.Common.Info("Didn't add hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
diff --git a/internal/ssh/server/hostkey.go b/internal/ssh/server/hostkey.go
index 07790ad..20de1f0 100644
--- a/internal/ssh/server/hostkey.go
+++ b/internal/ssh/server/hostkey.go
@@ -1,11 +1,12 @@
 package server
 
 import (
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
-	"github.com/mimecast/dtail/internal/ssh"
 	"io/ioutil"
 	"os"
+
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/ssh"
 )
 
 // PrivateHostKey retrieves the private server RSA host key.
@@ -14,24 +15,24 @@ func PrivateHostKey() []byte {
 	_, err := os.Stat(hostKeyFile)
 
 	if os.IsNotExist(err) {
-		logger.Info("Generating private server RSA host key")
+		dlog.Common.Info("Generating private server RSA host key")
 		privateKey, err := ssh.GeneratePrivateRSAKey(config.Server.HostKeyBits)
 
 		if err != nil {
-			logger.FatalExit("Failed to generate private server RSA host key", err)
+			dlog.Common.FatalPanic("Failed to generate private server RSA host key", err)
 		}
 
 		pem := ssh.EncodePrivateKeyToPEM(privateKey)
 		if err := ioutil.WriteFile(hostKeyFile, pem, 0600); err != nil {
-			logger.Error("Unable to write private server RSA host key to file", hostKeyFile, err)
+			dlog.Common.Error("Unable to write private server RSA host key to file", hostKeyFile, err)
 		}
 		return pem
 	}
 
-	logger.Info("Reading private server RSA host key from file", hostKeyFile)
+	dlog.Common.Info("Reading private server RSA host key from file", hostKeyFile)
 	pem, err := ioutil.ReadFile(hostKeyFile)
 	if err != nil {
-		logger.FatalExit("Failed to load private server RSA host key", err)
+		dlog.Common.FatalPanic("Failed to load private server RSA host key", err)
 	}
 	return pem
 }
diff --git a/internal/ssh/server/publickeycallback.go b/internal/ssh/server/publickeycallback.go
index e81f019..65ecdd1 100644
--- a/internal/ssh/server/publickeycallback.go
+++ b/internal/ssh/server/publickeycallback.go
@@ -7,7 +7,7 @@ import (
 	osUser "os/user"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	user "github.com/mimecast/dtail/internal/user/server"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -16,7 +16,7 @@ import (
 // PublicKeyCallback is for the server to check whether a public SSH key is authorized ot not.
 func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
 	user := user.New(c.User(), c.RemoteAddr().String())
-	logger.Info(user, "Incoming authorization")
+	dlog.Common.Info(user, "Incoming authorization")
 
 	cwd, err := os.Getwd()
 	if err != nil {
@@ -24,7 +24,7 @@ func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*go
 	}
 
 	if config.ServerRelaxedAuthEnable {
-		logger.Fatal(user, "Granting permissions via relaxed-auth")
+		dlog.Common.Fatal(user, "Granting permissions via relaxed-auth")
 		return nil, nil
 	}
 
@@ -38,7 +38,7 @@ func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*go
 		authorizedKeysFile = user.HomeDir + "/.ssh/authorized_keys"
 	}
 
-	logger.Info(user, "Reading", authorizedKeysFile)
+	dlog.Common.Info(user, "Reading", authorizedKeysFile)
 	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysFile)
 	if err != nil {
 		return nil, fmt.Errorf("Unable to read authorized keys file|%s|%s|%s", authorizedKeysFile, user, err.Error())
@@ -53,10 +53,10 @@ func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*go
 		authorizedKeysMap[string(authorizedPubKey.Marshal())] = true
 		authorizedKeysBytes = restBytes
 
-		logger.Debug(user, "Authorized public key fingerprint", gossh.FingerprintSHA256(authorizedPubKey))
+		dlog.Common.Debug(user, "Authorized public key fingerprint", gossh.FingerprintSHA256(authorizedPubKey))
 	}
 
-	logger.Debug(user, "Offered public key fingerprint", gossh.FingerprintSHA256(offeredPubKey))
+	dlog.Common.Debug(user, "Offered public key fingerprint", gossh.FingerprintSHA256(offeredPubKey))
 
 	if authorizedKeysMap[string(offeredPubKey.Marshal())] {
 		return &gossh.Permissions{
diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go
index 78bf99e..56494a7 100644
--- a/internal/ssh/ssh.go
+++ b/internal/ssh/ssh.go
@@ -11,7 +11,7 @@ import (
 	"os"
 	"syscall"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 
 	gossh "golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
@@ -58,7 +58,7 @@ func Agent() (gossh.AuthMethod, error) {
 		return nil, err
 	}
 	for i, key := range keys {
-		logger.Debug("Public key", i, key)
+		dlog.Common.Debug("Public key", i, key)
 	}
 	return gossh.PublicKeysCallback(agentClient.Signers), nil
 }
@@ -106,7 +106,7 @@ func KeyFile(keyFile string) (gossh.AuthMethod, error) {
 func PrivateKey(keyFile string) (gossh.AuthMethod, error) {
 	signer, err := KeyFile(keyFile)
 	if err != nil {
-		logger.Debug(keyFile, err)
+		dlog.Common.Debug(keyFile, err)
 		return nil, err
 	}
 	return gossh.AuthMethod(signer), nil
diff --git a/internal/user/server/user.go b/internal/user/server/user.go
index 637945c..99cd211 100644
--- a/internal/user/server/user.go
+++ b/internal/user/server/user.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/fs/permissions"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 )
 
 const maxLinkDepth int = 100
@@ -39,9 +39,9 @@ func (u *User) String() string {
 
 // HasFilePermission is used to determine whether user is alowed to read a file.
 func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission bool) {
-	logger.Debug(u, filePath, permissionType, "Checking config permissions")
+	dlog.Common.Debug(u, filePath, permissionType, "Checking config permissions")
 	if config.ServerRelaxedAuthEnable {
-		logger.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
+		dlog.Common.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
 		return true
 	}
 
@@ -52,25 +52,25 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 
 	cleanPath, err := filepath.EvalSymlinks(filePath)
 	if err != nil {
-		logger.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
+		dlog.Common.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
 		hasPermission = false
 		return
 	}
 
 	cleanPath, err = filepath.Abs(cleanPath)
 	if err != nil {
-		logger.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
+		dlog.Common.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
 		hasPermission = false
 		return
 	}
 
 	if cleanPath != filePath {
-		logger.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
+		dlog.Common.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
 	}
 
 	hasPermission, err = u.hasFilePermission(cleanPath, permissionType)
 	if err != nil {
-		logger.Warn(u, cleanPath, err)
+		dlog.Common.Warn(u, cleanPath, err)
 	}
 
 	return
@@ -81,7 +81,7 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 	if _, err := permissions.ToRead(u.Name, cleanPath); err != nil {
 		return false, fmt.Errorf("User without OS file system permissions to read path: '%v'", err)
 	}
-	logger.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
+	dlog.Common.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
 
 	// Only allow to follow regular files or symlinks.
 	info, err := os.Lstat(cleanPath)
@@ -123,7 +123,7 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 			permission = strings.Join(splitted[1:], ":")
 		}
 
-		logger.Debug(u, cleanPath, typeStr, permission)
+		dlog.Common.Debug(u, cleanPath, typeStr, permission)
 
 		if typeStr != permissionType {
 			continue
@@ -141,12 +141,12 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 		}
 
 		if negate && re.MatchString(cleanPath) {
-			logger.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
+			dlog.Common.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
 			hasPermission = false
 		}
 
 		if !negate && re.MatchString(cleanPath) {
-			logger.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
+			dlog.Common.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
 			hasPermission = true
 		}
 	}
-- 
cgit v1.2.3


From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 26 Sep 2021 16:42:47 +0300
Subject: refactor config reader - also looks in additional search paths for
 config file unless NONE is specified

---
 internal/clients/connectors/serverless.go |  7 +++-
 internal/clients/handlers/basehandler.go  |  2 +-
 internal/config/args.go                   | 25 ++++++++++---
 internal/config/common.go                 |  2 +-
 internal/config/config.go                 | 61 +++++++++++++++++++++----------
 internal/config/setup.go                  | 32 ++++------------
 internal/io/dlog/dlog.go                  | 37 ++++++++++++++-----
 internal/io/logger/logger.go              |  2 -
 internal/mapr/logformat/default.go        |  2 +-
 internal/server/continuous.go             |  2 +-
 internal/server/handlers/readcommand.go   |  2 +-
 internal/server/scheduler.go              |  2 +-
 internal/server/server.go                 | 12 +++++-
 internal/ssh/server/publickeycallback.go  |  5 ++-
 internal/user/server/user.go              | 37 +++++++++----------
 15 files changed, 140 insertions(+), 90 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index c7b5f62..7740aab 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -53,8 +53,13 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) error {
 	dlog.Client.Debug("Creating server handler for a serverless session")
 
+	user, err := user.New(s.userName, s.Server())
+	if err != nil {
+		return err
+	}
+
 	serverHandler := serverHandlers.NewServerHandler(
-		user.New(s.userName, s.Server()),
+		user,
 		make(chan struct{}, config.Server.MaxConcurrentCats),
 		make(chan struct{}, config.Server.MaxConcurrentTails),
 	)
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 3291b43..8acb45f 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -69,7 +69,7 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		switch b {
 		/*
-			// TODO: Next DTail version make it so that '\n' gets ignored. For now
+			// NEXT: Next DTail version make it so that '\n' gets ignored. For now
 			// leave it for compatibility with older DTail server + ability to display
 			// the protocol mismatch warn message.
 			case '\n' {
diff --git a/internal/config/args.go b/internal/config/args.go
index 89e4bc9..767cc65 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -16,6 +16,7 @@ type Args struct {
 	ConfigFile         string
 	ConnectionsPerCPU  int
 	Discovery          string
+	LogDir             string
 	LogLevel           string
 	Mode               omode.Mode
 	NoColor            bool
@@ -39,6 +40,8 @@ func (a *Args) String() string {
 	var sb strings.Builder
 
 	sb.WriteString("Args(")
+	// TODO: All commands should make use of this
+	sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir))
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile))
@@ -66,16 +69,24 @@ func (a *Args) String() string {
 }
 
 // Based on the argument list, transform/manipulate some of the arguments.
-func (a *Args) transform(args []string) {
+func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
+	if a.LogDir != "" {
+		common.LogDir = a.LogDir
+		if common.LogStrategy == "" {
+			// TODO: Implement the other (not-daily) log strategy for the server.
+			common.LogStrategy = "daily"
+		}
+	}
+
 	if a.LogLevel != "" {
-		Common.LogLevel = a.LogLevel
+		common.LogLevel = a.LogLevel
 	}
 
-	if a.SSHPort != 2222 {
-		Common.SSHPort = a.SSHPort
+	if a.SSHPort != DefaultSSHPort {
+		common.SSHPort = a.SSHPort
 	}
 	if a.NoColor {
-		Client.TermColorsEnable = false
+		client.TermColorsEnable = false
 	}
 
 	if a.Spartan {
@@ -95,6 +106,8 @@ func (a *Args) transform(args []string) {
 		}
 		a.What = strings.Join(files, ",")
 	}
+
+	return client, server, common
 }
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
@@ -102,4 +115,4 @@ func (a *Args) SerializeOptions() string {
 	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
 }
 
-// TODO: Put the DeseializeOptions function here (move it away from the internal/server package)
+// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package)
diff --git a/internal/config/common.go b/internal/config/common.go
index 7d45261..acc8e6a 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -23,7 +23,7 @@ type CommonConfig struct {
 // Create a new default configuration.
 func newDefaultCommonConfig() *CommonConfig {
 	return &CommonConfig{
-		SSHPort:                    2222,
+		SSHPort:                    DefaultSSHPort,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
 		CacheDir:                   "cache",
diff --git a/internal/config/config.go b/internal/config/config.go
index 2d77041..c9f411c 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -2,24 +2,26 @@ package config
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 )
 
-// ControlUser is used for various DTail specific operations.
-const ControlUser string = "DTAIL-CONTROL"
-
-// ScheduleUser is used for non-interactive scheduled mapreduce queries.
-const ScheduleUser string = "DTAIL-SCHEDULE"
-
-// ContinuousUser is used for non-interactive continuous mapreduce queries.
-const ContinuousUser string = "DTAIL-CONTINUOUS"
-
-// TestUser is used for unit tests and potentially also for integration tests.
-const TestUser string = "DTAIL-TEST"
-
-// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
-const InterruptTimeoutS int = 3
+const (
+	// ControlUser is used for various DTail specific operations.
+	ControlUser string = "DTAIL-CONTROL"
+	// ScheduleUser is used for non-interactive scheduled mapreduce queries.
+	ScheduleUser string = "DTAIL-SCHEDULE"
+	// ContinuousUser is used for non-interactive continuous mapreduce queries.
+	ContinuousUser string = "DTAIL-CONTINUOUS"
+	// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
+	InterruptTimeoutS int = 3
+	// ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start)
+	DefaultConnectionsPerCPU int = 10
+	// DTailSSHServerDefaultPort is the default DServer port.
+	DefaultSSHPort int = 2222
+)
 
 // Client holds a DTail client configuration.
 var Client *ClientConfig
@@ -37,21 +39,42 @@ type configInitializer struct {
 	Client *ClientConfig
 }
 
-// Parse and read a given config file in JSON format.
-func (c *configInitializer) parseConfig(configFile string) {
+func (c *configInitializer) parseConfig(args *Args) {
+	if strings.ToUpper(args.ConfigFile) == "NONE" {
+		return
+	}
+
+	if args.ConfigFile != "" {
+		c.parseSpecificConfig(args.ConfigFile)
+		return
+	}
+
+	if homeDir, err := os.UserHomeDir(); err != nil {
+		var paths []string
+		paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir))
+		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
+		for _, configPath := range paths {
+			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
+				c.parseSpecificConfig(configPath)
+			}
+		}
+	}
+}
+
+func (c *configInitializer) parseSpecificConfig(configFile string) {
 	fd, err := os.Open(configFile)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to read config file: %v", err))
 	}
 	defer fd.Close()
 
 	cfgBytes, err := ioutil.ReadAll(fd)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
 	}
 
 	err = json.Unmarshal([]byte(cfgBytes), c)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
 	}
 }
diff --git a/internal/config/setup.go b/internal/config/setup.go
index 3c4bcc4..be8e867 100644
--- a/internal/config/setup.go
+++ b/internal/config/setup.go
@@ -1,11 +1,5 @@
 package config
 
-import (
-	"os"
-)
-
-const NoConfigFile string = "Don't read a config file - use defaults only"
-
 // Setup the DTail configuration.
 func Setup(args *Args, additionalArgs []string) {
 	initializer := configInitializer{
@@ -13,23 +7,11 @@ func Setup(args *Args, additionalArgs []string) {
 		Server: newDefaultServerConfig(),
 		Client: newDefaultClientConfig(),
 	}
-
-	if args.ConfigFile == "" {
-		// TODO: Search more paths for config file (e.g. in /etc and in ~/.config/...
-		args.ConfigFile = "./cfg/dtail.json"
-	}
-
-	if args.ConfigFile != NoConfigFile {
-		if _, err := os.Stat(args.ConfigFile); !os.IsNotExist(err) {
-			initializer.parseConfig(args.ConfigFile)
-		}
-	}
-
-	// Assign pointers to global variables, so that we can access the
-	// configuration from any place of the program.
-	Common = initializer.Common
-	Server = initializer.Server
-	Client = initializer.Client
-
-	args.transform(additionalArgs)
+	initializer.parseConfig(args)
+	Client, Server, Common = args.transformConfig(
+		additionalArgs,
+		initializer.Client,
+		initializer.Server,
+		initializer.Common,
+	)
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 7282741..49b405d 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -3,6 +3,7 @@ package dlog
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -21,7 +22,6 @@ var Client *DLog
 var Server *DLog
 
 // Common is the log handler for all other packages.
-// TODO: Rename Common to Common
 var Common *DLog
 
 var mutex sync.Mutex
@@ -75,15 +75,22 @@ type DLog struct {
 	sourcePackage source
 	// Max log level to log.
 	maxLevel level
+	// Current hostname.
+	hostname string
 }
 
 // New creates a new DTail logger.
 func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+	hostname, err := os.Hostname()
+	if err != nil {
+		panic(err)
+	}
 	return &DLog{
 		logger:        loggers.Factory(sourceProcess.String(), impl),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      maxLevel,
+		hostname:      hostname,
 	}
 }
 
@@ -106,11 +113,18 @@ func (d *DLog) log(level level, args []interface{}) string {
 	defer pool.RecycleBuilderBuffer(sb)
 	now := time.Now()
 
-	sb.WriteString(d.sourcePackage.String())
-	sb.WriteString(protocol.FieldDelimiter)
-	sb.WriteString(now.Format("20060102-150405"))
-	sb.WriteString(protocol.FieldDelimiter)
-	sb.WriteString(level.String())
+	switch d.sourceProcess {
+	case CLIENT:
+		sb.WriteString(d.sourcePackage.String())
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(d.hostname)
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(level.String())
+	default:
+		sb.WriteString(level.String())
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(now.Format("20060102-150405"))
+	}
 	sb.WriteString(protocol.FieldDelimiter)
 	d.writeArgStrings(sb, args)
 
@@ -159,6 +173,11 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
+	if d.sourcePackage == SERVER && d.sourceProcess != CLIENT {
+		// This can be dtail client in serverless mode. In this case log all
+		// info server messages as verbose.
+		return d.log(VERBOSE, args)
+	}
 	return d.log(INFO, args)
 }
 
@@ -183,13 +202,14 @@ func (d *DLog) Raw(message string) string {
 		d.logger.Log(time.Now(), message)
 		return message
 	}
-
-	d.logger.Log(time.Now(), brush.Colorfy(message))
+	d.logger.LogWithColors(time.Now(), message, brush.Colorfy(message))
 	return message
 }
 
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
+
+	// TODO: mC compatible SERVER mapreduce fields, no MAPREDUCE keyword in CLIENT mode
 	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
 
 	i := 1
@@ -197,7 +217,6 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 		args[i] = fmt.Sprintf("%s=%v", k, v)
 		i++
 	}
-
 	return d.log(INFO, args)
 }
 
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 6a6b5ec..905d1cf 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -1,7 +1,5 @@
 package logger
 
-// TODO: Rewrite this logger
-
 import (
 	"bufio"
 	"context"
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index e0bbc30..7fb1700 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -27,7 +27,7 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 
 	fields["$severity"] = splitted[0]
 	fields["$loglevel"] = splitted[0]
-	// TODO: Parse time like we do at Mimecast
+	// NEXT: Parse time like we do at Mimecast
 	fields["$time"] = splitted[1]
 
 	for _, kv := range splitted[4:] {
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index 5f4c454..87c8889 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -61,7 +61,7 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	}
 
 	args := config.Args{
-		ConnectionsPerCPU: 10,
+		ConnectionsPerCPU: config.DefaultConnectionsPerCPU,
 		Discovery:         job.Discovery,
 		ServersStr:        servers,
 		What:              files,
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 60ad2a0..c76ae2a 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -7,9 +7,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/fs"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 )
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index f474cc8..64e6573 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -72,7 +72,7 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		}
 
 		args := config.Args{
-			ConnectionsPerCPU: 10,
+			ConnectionsPerCPU: config.DefaultConnectionsPerCPU,
 			Discovery:         job.Discovery,
 			ServersStr:        servers,
 			What:              files,
diff --git a/internal/server/server.go b/internal/server/server.go
index a8f541b..d1cd57d 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -124,7 +124,12 @@ func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
 }
 
 func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChannel gossh.NewChannel) {
-	user := user.New(sshConn.User(), sshConn.RemoteAddr().String())
+	user, err := user.New(sshConn.User(), sshConn.RemoteAddr().String())
+	if err != nil {
+		dlog.Server.Error(user, err)
+		newChannel.Reject(gossh.Prohibited, err.Error())
+		return
+	}
 	dlog.Server.Info(user, "Invoking channel handler")
 
 	if newChannel.ChannelType() != "session" {
@@ -213,7 +218,10 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 
 // Callback for SSH authentication.
 func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Permissions, error) {
-	user := user.New(c.User(), c.RemoteAddr().String())
+	user, err := user.New(c.User(), c.RemoteAddr().String())
+	if err != nil {
+		return nil, err
+	}
 
 	if config.ServerRelaxedAuthEnable {
 		dlog.Server.Fatal(user, "Granting permissions via relaxed-auth")
diff --git a/internal/ssh/server/publickeycallback.go b/internal/ssh/server/publickeycallback.go
index 65ecdd1..59d1f31 100644
--- a/internal/ssh/server/publickeycallback.go
+++ b/internal/ssh/server/publickeycallback.go
@@ -15,7 +15,10 @@ import (
 
 // PublicKeyCallback is for the server to check whether a public SSH key is authorized ot not.
 func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
-	user := user.New(c.User(), c.RemoteAddr().String())
+	user, err := user.New(c.User(), c.RemoteAddr().String())
+	if err != nil {
+		return nil, err
+	}
 	dlog.Common.Info(user, "Incoming authorization")
 
 	cwd, err := os.Getwd()
diff --git a/internal/user/server/user.go b/internal/user/server/user.go
index 99cd211..70ead1c 100644
--- a/internal/user/server/user.go
+++ b/internal/user/server/user.go
@@ -8,8 +8,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/fs/permissions"
 	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/fs/permissions"
 )
 
 const maxLinkDepth int = 100
@@ -25,11 +25,16 @@ type User struct {
 }
 
 // New returns a new user.
-func New(name, remoteAddress string) *User {
+func New(name, remoteAddress string) (*User, error) {
+	permissions, err := config.ServerUserPermissions(name)
+	if err != nil {
+		return nil, err
+	}
 	return &User{
 		Name:          name,
 		remoteAddress: remoteAddress,
-	}
+		permissions:   permissions,
+	}, nil
 }
 
 // String representation of the user.
@@ -39,9 +44,9 @@ func (u *User) String() string {
 
 // HasFilePermission is used to determine whether user is alowed to read a file.
 func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission bool) {
-	dlog.Common.Debug(u, filePath, permissionType, "Checking config permissions")
+	dlog.Server.Debug(u, filePath, permissionType, "Checking config permissions")
 	if config.ServerRelaxedAuthEnable {
-		dlog.Common.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
+		dlog.Server.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
 		return true
 	}
 
@@ -52,25 +57,25 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 
 	cleanPath, err := filepath.EvalSymlinks(filePath)
 	if err != nil {
-		dlog.Common.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
+		dlog.Server.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
 		hasPermission = false
 		return
 	}
 
 	cleanPath, err = filepath.Abs(cleanPath)
 	if err != nil {
-		dlog.Common.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
+		dlog.Server.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
 		hasPermission = false
 		return
 	}
 
 	if cleanPath != filePath {
-		dlog.Common.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
+		dlog.Server.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
 	}
 
 	hasPermission, err = u.hasFilePermission(cleanPath, permissionType)
 	if err != nil {
-		dlog.Common.Warn(u, cleanPath, err)
+		dlog.Server.Warn(u, cleanPath, err)
 	}
 
 	return
@@ -81,7 +86,7 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 	if _, err := permissions.ToRead(u.Name, cleanPath); err != nil {
 		return false, fmt.Errorf("User without OS file system permissions to read path: '%v'", err)
 	}
-	dlog.Common.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
+	dlog.Server.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
 
 	// Only allow to follow regular files or symlinks.
 	info, err := os.Lstat(cleanPath)
@@ -93,12 +98,6 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 		return false, fmt.Errorf("Can only open regular files or follow symlinks")
 	}
 
-	permissions, err := config.ServerUserPermissions(u.Name)
-	if err != nil {
-		return false, err
-	}
-	u.permissions = permissions
-
 	hasPermission, err := u.iteratePaths(cleanPath, permissionType)
 	if err != nil {
 		return false, err
@@ -123,7 +122,7 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 			permission = strings.Join(splitted[1:], ":")
 		}
 
-		dlog.Common.Debug(u, cleanPath, typeStr, permission)
+		dlog.Server.Debug(u, cleanPath, typeStr, permission)
 
 		if typeStr != permissionType {
 			continue
@@ -141,12 +140,12 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 		}
 
 		if negate && re.MatchString(cleanPath) {
-			dlog.Common.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
 			hasPermission = false
 		}
 
 		if !negate && re.MatchString(cleanPath) {
-			dlog.Common.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
 			hasPermission = true
 		}
 	}
-- 
cgit v1.2.3


From 609921f9c783941eaa9019a92b78ec45b49d681c Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 28 Sep 2021 21:11:50 +0300
Subject: can have daily and normal file log rotation

---
 internal/config/args.go             |  43 ----
 internal/config/config.go           |  43 ++++
 internal/config/setup.go            |   4 +-
 internal/io/dlog/dlog.go            |  14 +-
 internal/io/dlog/loggers/factory.go |   8 +-
 internal/io/dlog/loggers/file.go    | 117 +++++-----
 internal/io/dlog/loggers/fout.go    |   4 +-
 internal/io/dlog/strategy.go        |  22 --
 internal/io/logger/logger.go        | 443 ------------------------------------
 internal/io/logger/modes.go         |  13 --
 internal/io/logger/strategy.go      |  22 --
 11 files changed, 123 insertions(+), 610 deletions(-)
 delete mode 100644 internal/io/dlog/strategy.go
 delete mode 100644 internal/io/logger/logger.go
 delete mode 100644 internal/io/logger/modes.go
 delete mode 100644 internal/io/logger/strategy.go

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index 767cc65..7f24348 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -1,7 +1,6 @@
 package config
 
 import (
-	"flag"
 	"fmt"
 	"strings"
 
@@ -68,48 +67,6 @@ func (a *Args) String() string {
 	return sb.String()
 }
 
-// Based on the argument list, transform/manipulate some of the arguments.
-func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
-	if a.LogDir != "" {
-		common.LogDir = a.LogDir
-		if common.LogStrategy == "" {
-			// TODO: Implement the other (not-daily) log strategy for the server.
-			common.LogStrategy = "daily"
-		}
-	}
-
-	if a.LogLevel != "" {
-		common.LogLevel = a.LogLevel
-	}
-
-	if a.SSHPort != DefaultSSHPort {
-		common.SSHPort = a.SSHPort
-	}
-	if a.NoColor {
-		client.TermColorsEnable = false
-	}
-
-	if a.Spartan {
-		a.Quiet = true
-		a.NoColor = true
-	}
-
-	if a.Discovery == "" && a.ServersStr == "" {
-		a.Serverless = true
-	}
-
-	// Interpret additional args as file list.
-	if a.What == "" {
-		var files []string
-		for _, file := range flag.Args() {
-			files = append(files, file)
-		}
-		a.What = strings.Join(files, ",")
-	}
-
-	return client, server, common
-}
-
 // SerializeOptions returns a string ready to be sent over the wire to the server.
 func (a *Args) SerializeOptions() string {
 	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
diff --git a/internal/config/config.go b/internal/config/config.go
index c9f411c..76dcc65 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -2,6 +2,7 @@ package config
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -78,3 +79,45 @@ func (c *configInitializer) parseSpecificConfig(configFile string) {
 		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
 	}
 }
+
+func (c *configInitializer) transformConfig(args *Args, additionalArgs []string,
+	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
+	if args.LogDir != "" {
+		common.LogDir = args.LogDir
+		if common.LogStrategy == "" {
+			// TODO: Implement the other (not-daily) log strategy for the server.
+			common.LogStrategy = "daily"
+		}
+	}
+
+	if args.LogLevel != "" {
+		common.LogLevel = args.LogLevel
+	}
+
+	if args.SSHPort != DefaultSSHPort {
+		common.SSHPort = args.SSHPort
+	}
+	if args.NoColor {
+		client.TermColorsEnable = false
+	}
+
+	if args.Spartan {
+		args.Quiet = true
+		args.NoColor = true
+	}
+
+	if args.Discovery == "" && args.ServersStr == "" {
+		args.Serverless = true
+	}
+
+	// Interpret additional args as file list.
+	if args.What == "" {
+		var files []string
+		for _, file := range flag.Args() {
+			files = append(files, file)
+		}
+		args.What = strings.Join(files, ",")
+	}
+
+	return client, server, common
+}
diff --git a/internal/config/setup.go b/internal/config/setup.go
index be8e867..7800914 100644
--- a/internal/config/setup.go
+++ b/internal/config/setup.go
@@ -8,8 +8,8 @@ func Setup(args *Args, additionalArgs []string) {
 		Client: newDefaultClientConfig(),
 	}
 	initializer.parseConfig(args)
-	Client, Server, Common = args.transformConfig(
-		additionalArgs,
+	Client, Server, Common = initializer.transformConfig(
+		args, additionalArgs,
 		initializer.Client,
 		initializer.Server,
 		initializer.Common,
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 49b405d..49533a5 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -36,19 +36,21 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 		Common.FatalPanic("Logger already started")
 	}
 
+	strategy := loggers.GetStrategy(config.Common.LogStrategy)
 	level := newLevel(logLevel)
+
 	switch sourceProcess {
 	case CLIENT:
 		// This is a DTail client process running.
 		impl := loggers.FOUT
-		Client = New(CLIENT, CLIENT, impl, level)
-		Server = New(CLIENT, SERVER, impl, level)
+		Client = New(CLIENT, CLIENT, level, impl, strategy)
+		Server = New(CLIENT, SERVER, level, impl, strategy)
 		Common = Client
 	case SERVER:
 		// This is a DTail server process running.
 		impl := loggers.FILE
-		Client = New(SERVER, CLIENT, impl, level)
-		Server = New(SERVER, SERVER, impl, level)
+		Client = New(SERVER, CLIENT, level, impl, strategy)
+		Server = New(SERVER, SERVER, level, impl, strategy)
 		Common = Server
 	}
 
@@ -80,13 +82,13 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+func New(sourceProcess, sourcePackage source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
 	}
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), impl),
+		logger:        loggers.Factory(sourceProcess.String(), impl, strategy),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      maxLevel,
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 3eb29c5..8697dc4 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -17,11 +17,11 @@ const (
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
-func Factory(name string, impl Impl) Logger {
+func Factory(name string, impl Impl, strategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("name:%s,impl:%v", name, impl)
+	id := fmt.Sprintf("name:%s,fileBase:%s,impl:%v", name, strategy.FileBase, impl)
 
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
@@ -36,10 +36,10 @@ func Factory(name string, impl Impl) Logger {
 			singleton = newStdout()
 			factoryMap[id] = singleton
 		case FILE:
-			singleton = newFile()
+			singleton = newFile(strategy)
 			factoryMap[id] = singleton
 		case FOUT:
-			singleton = newFout()
+			singleton = newFout(strategy)
 			factoryMap[id] = singleton
 		}
 	}
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 1c525c9..dcdd7d0 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -12,49 +12,54 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 )
 
+type fileWriter struct {
+}
+
 type fileMessageBuf struct {
 	now     time.Time
 	message string
 }
 
 type file struct {
-	bufferCh    chan *fileMessageBuf
-	pauseCh     chan struct{}
-	resumeCh    chan struct{}
-	rotateCh    chan struct{}
-	flushCh     chan struct{}
-	lastDateStr string
-	fd          *os.File
-	writer      *bufio.Writer
-	mutex       sync.Mutex
-	started     bool
+	bufferCh     chan *fileMessageBuf
+	pauseCh      chan struct{}
+	resumeCh     chan struct{}
+	rotateCh     chan struct{}
+	flushCh      chan struct{}
+	fd           *os.File
+	writer       *bufio.Writer
+	mutex        sync.Mutex
+	started      bool
+	lastFileName string
+	strategy     Strategy
 }
 
-func newFile() *file {
+func newFile(strategy Strategy) *file {
 	f := file{
 		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
 		rotateCh: make(chan struct{}),
 		flushCh:  make(chan struct{}),
+		strategy: strategy,
 	}
-	f.getWriter(time.Now().Format("20060102"))
+
 	return &f
 }
 
-func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
-	s.mutex.Lock()
-	defer s.mutex.Unlock()
+func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
 
 	// Logger already started from another Goroutine.
-	if s.started {
+	if f.started {
 		wg.Done()
 		return
 	}
 
 	pause := func(ctx context.Context) {
 		select {
-		case <-s.resumeCh:
+		case <-f.resumeCh:
 			return
 		case <-ctx.Done():
 			return
@@ -66,55 +71,61 @@ func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 
 		for {
 			select {
-			case m := <-s.bufferCh:
-				s.write(m)
-			case <-s.pauseCh:
+			case m := <-f.bufferCh:
+				f.write(m)
+			case <-f.pauseCh:
 				pause(ctx)
-			case <-s.flushCh:
-				s.flush()
+			case <-f.flushCh:
+				f.flush()
 			case <-ctx.Done():
-				s.flush()
-				s.fd.Close()
+				f.flush()
+				f.fd.Close()
 				return
 			}
 		}
 	}()
 
-	s.started = true
+	f.started = true
 }
 
-func (s *file) Log(now time.Time, message string) {
-	s.bufferCh <- &fileMessageBuf{now, message}
+func (f *file) Log(now time.Time, message string) {
+	f.bufferCh <- &fileMessageBuf{now, message}
 }
 
-func (s *file) LogWithColors(now time.Time, message, coloredMessage string) {
+func (f *file) LogWithColors(now time.Time, message, coloredMessage string) {
 	panic("Colors not supported in file logger")
 }
 
-func (s *file) Pause()  { s.pauseCh <- struct{}{} }
-func (s *file) Resume() { s.resumeCh <- struct{}{} }
-func (s *file) Flush()  { s.flushCh <- struct{}{} }
+func (f *file) Pause()  { f.pauseCh <- struct{}{} }
+func (f *file) Resume() { f.resumeCh <- struct{}{} }
+func (f *file) Flush()  { f.flushCh <- struct{}{} }
 
 // TODO: Test that Rotate() actually works.
-func (s *file) Rotate()           { s.rotateCh <- struct{}{} }
-func (file) SupportsColors() bool { return false }
+func (f *file) Rotate()            { f.rotateCh <- struct{}{} }
+func (*file) SupportsColors() bool { return false }
 
-func (s *file) write(m *fileMessageBuf) {
+func (f *file) write(m *fileMessageBuf) {
 	select {
-	case <-s.rotateCh:
-		// Force re-opening the outfile.
-		s.lastDateStr = ""
+	case <-f.rotateCh:
+		// Force re-opening the outfile next time in getWriter.
+		f.lastFileName = ""
 	default:
 	}
 
-	writer := s.getWriter(m.now.Format("20060102"))
+	var writer *bufio.Writer
+	if f.strategy.Rotation == DailyRotation {
+		writer = f.getWriter(m.now.Format("20060102"))
+	} else {
+		writer = f.getWriter(f.strategy.FileBase)
+	}
+
 	writer.WriteString(m.message)
 	writer.WriteByte('\n')
 }
 
-func (s *file) getWriter(dateStr string) *bufio.Writer {
-	if s.lastDateStr == dateStr {
-		return s.writer
+func (f *file) getWriter(name string) *bufio.Writer {
+	if f.lastFileName == name {
+		return f.writer
 	}
 
 	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
@@ -123,32 +134,32 @@ func (s *file) getWriter(dateStr string) *bufio.Writer {
 		}
 	}
 
-	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr)
+	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, name)
 	newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
 	if err != nil {
 		panic(err)
 	}
 
 	// Close old writer.
-	if s.fd != nil {
-		s.writer.Flush()
-		s.fd.Close()
+	if f.fd != nil {
+		f.writer.Flush()
+		f.fd.Close()
 	}
 
-	s.fd = newFd
-	s.writer = bufio.NewWriterSize(s.fd, 1)
-	s.lastDateStr = dateStr
+	f.fd = newFd
+	f.writer = bufio.NewWriterSize(f.fd, 1)
+	f.lastFileName = name
 
-	return s.writer
+	return f.writer
 }
 
-func (s *file) flush() {
-	defer s.writer.Flush()
+func (f *file) flush() {
+	defer f.writer.Flush()
 
 	for {
 		select {
-		case m := <-s.bufferCh:
-			s.write(m)
+		case m := <-f.bufferCh:
+			f.write(m)
 		default:
 			return
 		}
diff --git a/internal/io/dlog/loggers/fout.go b/internal/io/dlog/loggers/fout.go
index 603dbe9..60c318d 100644
--- a/internal/io/dlog/loggers/fout.go
+++ b/internal/io/dlog/loggers/fout.go
@@ -12,8 +12,8 @@ type fout struct {
 }
 
 // Logs to both, a file and stdout
-func newFout() *fout {
-	return &fout{file: newFile(), stdout: newStdout()}
+func newFout(strategy Strategy) *fout {
+	return &fout{file: newFile(strategy), stdout: newStdout()}
 }
 
 func (f *fout) Start(ctx context.Context, wg *sync.WaitGroup) {
diff --git a/internal/io/dlog/strategy.go b/internal/io/dlog/strategy.go
deleted file mode 100644
index 32d8298..0000000
--- a/internal/io/dlog/strategy.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package dlog
-
-import "github.com/mimecast/dtail/internal/config"
-
-// Strategy allows to specify a log rotation strategy.
-type Strategy int
-
-// Possible log strategies.
-const (
-	NormalStrategy Strategy = iota
-	DailyStrategy  Strategy = iota
-	StdoutStrategy Strategy = iota
-)
-
-func logStrategy() Strategy {
-	switch config.Common.LogStrategy {
-	case "daily":
-		return DailyStrategy
-	default:
-	}
-	return StdoutStrategy
-}
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
deleted file mode 100644
index 905d1cf..0000000
--- a/internal/io/logger/logger.go
+++ /dev/null
@@ -1,443 +0,0 @@
-package logger
-
-import (
-	"bufio"
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"runtime"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/mimecast/dtail/internal/color/brush"
-	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/pool"
-	"github.com/mimecast/dtail/internal/protocol"
-)
-
-const (
-	clientStr string = "CLIENT"
-	serverStr string = "SERVER"
-	infoStr   string = "INFO"
-	warnStr   string = "WARN"
-	errorStr  string = "ERROR"
-	fatalStr  string = "FATAL"
-	debugStr  string = "DEBUG"
-	traceStr  string = "TRACE"
-)
-
-// Mode specifies the configured logging mode(s)
-var Mode Modes
-
-// Strategy is the current log strattegy used.
-var strategy Strategy
-
-// Synchronise access to logging.
-var mutex sync.Mutex
-
-// File descriptor of log file when Mode.logToFile enabled.
-var fd *os.File
-
-// File write buffer of log file when Mode.logToFile enabled.
-var writer *bufio.Writer
-
-// File write buffer of stdout when Mode.logToStdout enabled.
-var stdoutWriter *bufio.Writer
-
-// Current hostname.
-var hostname string
-
-// Used to detect change of day (create one log file per day0
-var lastDateStr string
-
-// Used to make logging non-blocking.
-var fileLogBufCh chan buf
-var stdoutBufCh chan string
-
-// Stdout channel, required to pause output
-var pauseCh chan struct{}
-var resumeCh chan struct{}
-
-// Tell the logger about logrotation
-var rotateCh chan os.Signal
-
-// Override the logger with a custom callack (e.g. for the t.Log for unit tests)
-type unitTestCallback func(message string)
-
-var unitTestOkCb unitTestCallback
-var unitTestErrorCb unitTestCallback
-
-// Helper type to make logging non-blocking.
-type buf struct {
-	time    time.Time
-	message string
-}
-
-// StartUnitTests enables to log all messages to the unit tests.
-func StartUnitTests(ctx context.Context, okCb, errCb unitTestCallback) {
-	unitTestOkCb = okCb
-	unitTestErrorCb = errCb
-	Start(ctx, Modes{UnitTest: true})
-}
-
-// Start logging.
-func Start(ctx context.Context, mode Modes) {
-	Mode = mode
-
-	switch {
-	case Mode.Nothing:
-		return
-	case Mode.Quiet:
-		Mode.Trace = false
-		Mode.Debug = false
-	case Mode.Trace:
-		Mode.Debug = true
-	default:
-	}
-
-	strategy := logStrategy()
-	stdoutWriter = bufio.NewWriter(os.Stdout)
-
-	switch strategy {
-	case DailyStrategy:
-		_, err := os.Stat(config.Common.LogDir)
-		Mode.logToFile = !os.IsNotExist(err) && !Mode.UnitTest
-		Mode.logToStdout = !Mode.Server || Mode.Debug || Mode.Trace || Mode.Quiet
-	case StdoutStrategy:
-		fallthrough
-	default:
-		Mode.logToFile = !Mode.Server && !Mode.UnitTest
-		Mode.logToStdout = true
-	}
-
-	fqdn, err := os.Hostname()
-	if err != nil {
-		panic(err)
-	}
-	s := strings.Split(fqdn, ".")
-	hostname = s[0]
-
-	pauseCh = make(chan struct{})
-	resumeCh = make(chan struct{})
-
-	// Setup logrotation
-	rotateCh = make(chan os.Signal, 1)
-	signal.Notify(rotateCh, syscall.SIGHUP)
-
-	if Mode.logToStdout {
-		stdoutBufCh = make(chan string, runtime.NumCPU()*100)
-		go writeToStdout(ctx)
-	}
-
-	if Mode.logToFile {
-		fileLogBufCh = make(chan buf, runtime.NumCPU()*100)
-		go writeToFile(ctx)
-	}
-}
-
-// Info message logging.
-func Info(args ...interface{}) string {
-	if Mode.Server {
-		return log(serverStr, infoStr, args)
-	}
-
-	return log(clientStr, infoStr, args)
-}
-
-// Mapreduce message logging.
-func Mapreduce(table string, data map[string]interface{}) string {
-	args := make([]interface{}, len(data)+1)
-
-	args[0] = fmt.Sprintf("MAPREDUCE:%s", strings.ToUpper(table))
-	i := 1
-	for k, v := range data {
-		args[i] = fmt.Sprintf("%s=%v", k, v)
-		i++
-	}
-
-	if Mode.Server {
-		return log(serverStr, infoStr, args)
-	}
-
-	return log(clientStr, infoStr, args)
-}
-
-// Warn message logging.
-func Warn(args ...interface{}) string {
-	if !Mode.Quiet {
-		if Mode.Server {
-			return log(serverStr, warnStr, args)
-		}
-		return log(clientStr, warnStr, args)
-	}
-
-	return ""
-}
-
-// Error message logging.
-func Error(args ...interface{}) string {
-	if Mode.Server {
-		return log(serverStr, errorStr, args)
-	}
-
-	return log(clientStr, errorStr, args)
-}
-
-// Fatal message logging.
-func Fatal(args ...interface{}) string {
-	if Mode.Server {
-		return log(serverStr, fatalStr, args)
-	}
-
-	return log(clientStr, fatalStr, args)
-}
-
-// FatalPanic logs an error and exists the process.
-func FatalPanic(args ...interface{}) {
-	what := clientStr
-	if Mode.Server {
-		what = serverStr
-	}
-	log(what, fatalStr, args)
-
-	time.Sleep(time.Second)
-	mutex.Lock()
-	defer mutex.Unlock()
-
-	closeWriter()
-	os.Exit(3)
-}
-
-// Debug message logging.
-func Debug(args ...interface{}) string {
-	if Mode.Debug {
-		if Mode.Server {
-			return log(serverStr, debugStr, args)
-		}
-		return log(clientStr, debugStr, args)
-	}
-
-	return ""
-}
-
-// Trace message logging.
-func Trace(args ...interface{}) string {
-	if Mode.Trace {
-		if Mode.Server {
-			return log(serverStr, traceStr, args)
-		}
-		return log(clientStr, traceStr, args)
-	}
-
-	return ""
-}
-
-// Write log line to buffer and/or log file.
-func write(what, severity, message string) {
-	if Mode.logToStdout {
-		line := fmt.Sprintf("%s|%s|%s|%s", what, hostname, severity, message)
-
-		if config.Client.TermColorsEnable {
-			line = brush.Colorfy(line)
-		}
-
-		stdoutBufCh <- line
-	}
-
-	if Mode.logToFile {
-		t := time.Now()
-		timeStr := t.Format("20060102-150405")
-		fileLogBufCh <- buf{
-			time:    t,
-			message: fmt.Sprintf("%s|%s|%s|%s", severity, timeStr, what, message),
-		}
-	}
-}
-
-// Generig log message.
-func log(what string, severity string, args []interface{}) string {
-	if Mode.Nothing {
-		return ""
-	}
-
-	sb := pool.BuilderBuffer.Get().(*strings.Builder)
-
-	for i, arg := range args {
-		if i > 0 {
-			sb.WriteString(protocol.FieldDelimiter)
-		}
-
-		switch v := arg.(type) {
-		case string:
-			sb.WriteString(v)
-		case int:
-			sb.WriteString(fmt.Sprintf("%d", v))
-		case error:
-			sb.WriteString(v.Error())
-		default:
-			sb.WriteString(fmt.Sprintf("%v", v))
-		}
-	}
-
-	message := sb.String()
-	pool.RecycleBuilderBuffer(sb)
-	write(what, severity, message)
-	return fmt.Sprintf("%s|%s", severity, message)
-}
-
-// Raw message logging.
-func Raw(message string) {
-	if Mode.Nothing {
-		return
-	}
-
-	if Mode.logToFile {
-		fileLogBufCh <- buf{time.Now(), message}
-	}
-
-	if Mode.logToStdout {
-		if config.Client.TermColorsEnable {
-			message = brush.Colorfy(message)
-		}
-		stdoutBufCh <- message
-	}
-}
-
-// Close log writer (e.g. on change of day).
-func closeWriter() {
-	if writer != nil {
-		writer.Flush()
-		fd.Close()
-	}
-}
-
-// Return the correct log file writer
-func fileWriter(dateStr string) *bufio.Writer {
-	if dateStr != lastDateStr {
-		return updateFileWriter(dateStr)
-	}
-
-	// Check for log rotation signal
-	select {
-	case <-rotateCh:
-		stdoutWriter.WriteString("Received signal for logrotation\n")
-		return updateFileWriter(dateStr)
-	default:
-	}
-
-	return writer
-}
-
-// Update log file writer
-func updateFileWriter(dateStr string) *bufio.Writer {
-	// Detected change of day. Close current writer and create a new one.
-	mutex.Lock()
-	defer mutex.Unlock()
-	closeWriter()
-
-	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
-		if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil {
-			panic(err)
-		}
-	}
-
-	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr)
-	newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
-	if err != nil {
-		panic(err)
-	}
-
-	fd = newFd
-	writer = bufio.NewWriterSize(fd, 1)
-	lastDateStr = dateStr
-
-	return writer
-}
-
-// Flush all outstanding lines.
-func Flush() {
-	for {
-		select {
-		case message := <-stdoutBufCh:
-			stdoutWriter.WriteString(message)
-			stdoutWriter.WriteString("\n")
-		default:
-			stdoutWriter.Flush()
-			return
-		}
-	}
-}
-
-func writeToStdout(ctx context.Context) {
-	for {
-		select {
-		case message := <-stdoutBufCh:
-			stdoutWriter.WriteString(message)
-			stdoutWriter.WriteString("\n")
-		case <-time.After(time.Millisecond * 100):
-			stdoutWriter.Flush()
-		case <-pauseCh:
-		PAUSE:
-			for {
-				select {
-				case <-stdoutBufCh:
-				case <-resumeCh:
-					break PAUSE
-				case <-ctx.Done():
-					return
-				}
-			}
-		case <-ctx.Done():
-			Flush()
-			return
-		}
-	}
-}
-
-func writeToFile(ctx context.Context) {
-	for {
-		select {
-		case buf := <-fileLogBufCh:
-			dateStr := buf.time.Format("20060102")
-			w := fileWriter(dateStr)
-			w.WriteString(buf.message)
-			w.WriteString("\n")
-		case <-pauseCh:
-		PAUSE:
-			for {
-				select {
-				case <-stdoutBufCh:
-				case <-resumeCh:
-					break PAUSE
-				case <-ctx.Done():
-					return
-				}
-			}
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-// Pause logging.
-func Pause() {
-	if Mode.logToStdout {
-		pauseCh <- struct{}{}
-	}
-	if Mode.logToFile {
-		pauseCh <- struct{}{}
-	}
-}
-
-// Resume logging (after pausing).
-func Resume() {
-	if Mode.logToStdout {
-		resumeCh <- struct{}{}
-	}
-	if Mode.logToFile {
-		resumeCh <- struct{}{}
-	}
-}
diff --git a/internal/io/logger/modes.go b/internal/io/logger/modes.go
deleted file mode 100644
index 85f90a5..0000000
--- a/internal/io/logger/modes.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package logger
-
-// Modes specifies the logging mode.
-type Modes struct {
-	Debug       bool
-	logToFile   bool
-	logToStdout bool
-	Nothing     bool
-	Quiet       bool
-	Server      bool
-	Trace       bool
-	UnitTest    bool
-}
diff --git a/internal/io/logger/strategy.go b/internal/io/logger/strategy.go
deleted file mode 100644
index 44bf393..0000000
--- a/internal/io/logger/strategy.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package logger
-
-import "github.com/mimecast/dtail/internal/config"
-
-// Strategy allows to specify a log rotation strategy.
-type Strategy int
-
-// Possible log strategies.
-const (
-	NormalStrategy Strategy = iota
-	DailyStrategy  Strategy = iota
-	StdoutStrategy Strategy = iota
-)
-
-func logStrategy() Strategy {
-	switch config.Common.LogStrategy {
-	case "daily":
-		return DailyStrategy
-	default:
-	}
-	return StdoutStrategy
-}
-- 
cgit v1.2.3


From 764ef99a3d779a0db1fb60679292af52425ba2f6 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 10:46:47 +0300
Subject: add more default fields to MAPREDUCE

---
 internal/config/args.go                 |  2 +-
 internal/config/config.go               |  1 -
 internal/io/dlog/dlog.go                | 38 ++++++++++++++++++--
 internal/io/dlog/loggers/file.go        |  1 -
 internal/mapr/logformat/default.go      | 23 +++++++++---
 internal/mapr/logformat/default_test.go | 62 +++++++++++++++++++++++++++------
 internal/server/stats.go                |  4 ---
 7 files changed, 108 insertions(+), 23 deletions(-)

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index 7f24348..484aa8b 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -39,7 +39,7 @@ func (a *Args) String() string {
 	var sb strings.Builder
 
 	sb.WriteString("Args(")
-	// TODO: All commands should make use of this
+
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir))
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
diff --git a/internal/config/config.go b/internal/config/config.go
index 76dcc65..3d05a11 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -85,7 +85,6 @@ func (c *configInitializer) transformConfig(args *Args, additionalArgs []string,
 	if args.LogDir != "" {
 		common.LogDir = args.LogDir
 		if common.LogStrategy == "" {
-			// TODO: Implement the other (not-daily) log strategy for the server.
 			common.LogStrategy = "daily"
 		}
 	}
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 49533a5..bc9b2f8 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -3,7 +3,11 @@ package dlog
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -211,8 +215,38 @@ func (d *DLog) Raw(message string) string {
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
-	// TODO: mC compatible SERVER mapreduce fields, no MAPREDUCE keyword in CLIENT mode
-	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
+	if d.sourceProcess == SERVER {
+		// level|date-time|process|caller|cpus|goroutines|cgocalls|loadavg|uptime|MAPREDUCE:TABLE|key=value|...
+
+		var loadAvg string
+		if loadAvgBytes, err := ioutil.ReadFile("/proc/loadavg"); err == nil {
+			tmp := string(loadAvgBytes)
+			s := strings.SplitN(tmp, " ", 2)
+			loadAvg = s[0]
+		}
+
+		var uptime string
+		if uptimeBytes, err := ioutil.ReadFile("/proc/uptime"); err == nil {
+			tmp := string(uptimeBytes)
+			s := strings.SplitN(tmp, ".", 2)
+			i, _ := strconv.ParseInt(s[0], 10, 64)
+			t := time.Duration(i) * time.Second
+			uptime = fmt.Sprintf("%v", t)
+		}
+
+		_, file, line, _ := runtime.Caller(1)
+		args[0] = fmt.Sprintf("%d|%s:%d|%d|%d|%d|%s|%s|MAPREDUCE:%s",
+			os.Getpid(),
+			filepath.Base(file), line,
+			runtime.NumCPU(),
+			runtime.NumGoroutine(),
+			runtime.NumCgoCall(),
+			loadAvg,
+			uptime,
+			strings.ToUpper(table))
+	} else {
+		args[0] = fmt.Sprintf("STATS:%s", strings.ToUpper(table))
+	}
 
 	i := 1
 	for k, v := range data {
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index dcdd7d0..6e692a3 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -100,7 +100,6 @@ func (f *file) Pause()  { f.pauseCh <- struct{}{} }
 func (f *file) Resume() { f.resumeCh <- struct{}{} }
 func (f *file) Flush()  { f.flushCh <- struct{}{} }
 
-// TODO: Test that Rotate() actually works.
 func (f *file) Rotate()            { f.rotateCh <- struct{}{} }
 func (*file) SupportsColors() bool { return false }
 
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 7fb1700..8016667 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -11,7 +11,7 @@ import (
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
 
-	if len(splitted) < 3 || !strings.HasPrefix(splitted[3], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") {
+	if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") {
 		// Not a DTail mapreduce log line.
 		return nil, IgnoreFieldsErr
 	}
@@ -27,10 +27,25 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 
 	fields["$severity"] = splitted[0]
 	fields["$loglevel"] = splitted[0]
-	// NEXT: Parse time like we do at Mimecast
-	fields["$time"] = splitted[1]
 
-	for _, kv := range splitted[4:] {
+	time := splitted[1]
+	fields["$time"] = time
+	if len(time) == 15 {
+		// Example: 20211002-071209
+		fields["$date"] = time[0:8]
+		fields["$hour"] = time[9:11]
+		fields["$minute"] = time[11:13]
+		fields["$second"] = time[13:]
+	}
+	fields["$pid"] = splitted[2]
+	fields["$caller"] = splitted[3]
+	fields["$cpus"] = splitted[4]
+	fields["$goroutines"] = splitted[5]
+	fields["$cgocalls"] = splitted[6]
+	fields["$loadavg"] = splitted[7]
+	fields["$uptime"] = splitted[8]
+
+	for _, kv := range splitted[10:] {
 		keyAndValue := strings.SplitN(kv, "=", 2)
 		if len(keyAndValue) != 2 {
 			return fields, fmt.Errorf("Unable to parse key-value token '%s'", kv)
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index 79911d1..02c03a3 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -1,6 +1,7 @@
 package logformat
 
 import (
+	"fmt"
 	"testing"
 )
 
@@ -10,9 +11,15 @@ func TestDefaultLogFormat(t *testing.T) {
 		t.Errorf("Unable to create parser: %s", err.Error())
 	}
 
+	date := "20211002"
+	hour := "07"
+	minute := "23"
+	second := "42"
+	time := fmt.Sprintf("%s-%s%s%s", date, hour, minute, second)
+
 	inputs := []string{
-		"INFO|20210907-065632|SERVER|MAPREDUCE:TEST|foo=bar|baz=bay",
-		"INFO|20210907-065732|CLIENT|MAPREDUCE:YOMAN|baz=bay|foo=bar",
+		fmt.Sprintf("INFO|%s|1|default_test.go:0|8|14|7|0.21|471h0m21s|MAPREDUCE:STATS|foo=bar|bar=foo", time),
+		fmt.Sprintf("INFO|%s|1|default_test.go:0|8|14|7|0.21|471h0m21s|MAPREDUCE:STATS|bar=foo|foo=bar", time),
 	}
 
 	for _, input := range inputs {
@@ -22,18 +29,53 @@ func TestDefaultLogFormat(t *testing.T) {
 			t.Errorf("Parser unable to make fields: %s", err.Error())
 		}
 
-		if bar, ok := fields["foo"]; !ok {
-			t.Errorf("Expected field 'foo', but no such field there in '%s'\n", input)
-		} else if bar != "bar" {
-			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n", bar, input)
+		if val, ok := fields["$severity"]; !ok {
+			t.Errorf("Expected field '$severity', but no such field there in '%s'\n", input)
+		} else if val != "INFO" {
+			t.Errorf("Expected 'INFO' stored in field '$severity', but got '%s' in '%s'\n", val, input)
+		}
+
+		if val, ok := fields["$time"]; !ok {
+			t.Errorf("Expected field '$time', but no such field there in '%s'\n", input)
+		} else if val != time {
+			t.Errorf("Expected '%s' stored in field '$time', but got '%s' in '%s'\n", time, val, input)
+		}
+
+		if val, ok := fields["$date"]; !ok {
+			t.Errorf("Expected field '$date', but no such field there in '%s'\n", input)
+		} else if val != date {
+			t.Errorf("Expected '%s' stored in field '$date', but got '%s' in '%s'\n", date, val, input)
+		}
+
+		if val, ok := fields["$hour"]; !ok {
+			t.Errorf("Expected field '$hour', but no such field there in '%s'\n", input)
+		} else if val != hour {
+			t.Errorf("Expected '%s' stored in field '$hour', but got '%s' in '%s'\n", hour, val, input)
+		}
+
+		if val, ok := fields["$minute"]; !ok {
+			t.Errorf("Expected field '$minute', but no such field there in '%s'\n", input)
+		} else if val != minute {
+			t.Errorf("Expected '%s' stored in field '$minute', but got '%s' in '%s'\n", minute, val, input)
 		}
 
-		if bay, ok := fields["baz"]; !ok {
-			t.Errorf("Expected field 'baz', but no such field there in '%s'\n", input)
-		} else if bay != "bay" {
-			t.Errorf("Expected 'bay' stored in field 'baz', but got '%s' in '%s'\n", bay, input)
+		if val, ok := fields["$second"]; !ok {
+			t.Errorf("Expected field '$second', but no such field there in '%s'\n", input)
+		} else if val != second {
+			t.Errorf("Expected '%s' stored in field '$second', but got '%s' in '%s'\n", second, val, input)
 		}
 
+		if val, ok := fields["foo"]; !ok {
+			t.Errorf("Expected field 'foo', but no such field there in '%s'\n", input)
+		} else if val != "bar" {
+			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n", val, input)
+		}
+
+		if val, ok := fields["bar"]; !ok {
+			t.Errorf("Expected field 'bar', but no such field there in '%s'\n", input)
+		} else if val != "foo" {
+			t.Errorf("Expected 'foo' stored in field 'bar', but got '%s' in '%s'\n", val, input)
+		}
 	}
 
 	fields, err := parser.MakeFields("foozoo=bar|bazbay")
diff --git a/internal/server/stats.go b/internal/server/stats.go
index 8583318..c07634d 100644
--- a/internal/server/stats.go
+++ b/internal/server/stats.go
@@ -3,7 +3,6 @@ package server
 import (
 	"context"
 	"fmt"
-	"runtime"
 	"sync"
 	"time"
 
@@ -53,9 +52,6 @@ func (s *stats) logServerStats() {
 	data := make(map[string]interface{})
 	data["currentConnections"] = s.currentConnections
 	data["lifetimeConnections"] = s.lifetimeConnections
-	data["goroutines"] = runtime.NumGoroutine()
-	data["cgocalls"] = runtime.NumCgoCall()
-	data["cpu"] = runtime.NumCPU()
 
 	dlog.Server.Mapreduce("STATS", data)
 }
-- 
cgit v1.2.3


From 6e1af993924bc7bebe898b403962db5a6b3505d1 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 11:23:08 +0300
Subject: Client default log dir is ~/log

---
 internal/config/config.go            | 11 +++++++++--
 internal/io/dlog/dlog.go             | 27 ++++++++++++++-------------
 internal/io/dlog/loggers/strategy.go | 27 +++++++++++++++++++++++++++
 internal/io/dlog/source.go           | 19 -------------------
 internal/source/source.go            | 19 +++++++++++++++++++
 5 files changed, 69 insertions(+), 34 deletions(-)
 create mode 100644 internal/io/dlog/loggers/strategy.go
 delete mode 100644 internal/io/dlog/source.go
 create mode 100644 internal/source/source.go

(limited to 'internal')

diff --git a/internal/config/config.go b/internal/config/config.go
index 3d05a11..6d4730a 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -84,9 +84,16 @@ func (c *configInitializer) transformConfig(args *Args, additionalArgs []string,
 	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
 	if args.LogDir != "" {
 		common.LogDir = args.LogDir
-		if common.LogStrategy == "" {
-			common.LogStrategy = "daily"
+	}
+	if strings.Contains(common.LogDir, "~/") {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			panic(err)
 		}
+		common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
+	}
+	if common.LogStrategy == "" {
+		common.LogStrategy = "daily"
 	}
 
 	if args.LogLevel != "" {
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index bc9b2f8..3de3120 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -17,6 +17,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog/loggers"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
+	"github.com/mimecast/dtail/internal/source"
 )
 
 // Client is the log handler for the client packages.
@@ -32,7 +33,7 @@ var mutex sync.Mutex
 var started bool
 
 // Start logger(s).
-func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLevel string) {
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source, logLevel string) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
@@ -44,17 +45,17 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 	level := newLevel(logLevel)
 
 	switch sourceProcess {
-	case CLIENT:
+	case source.Client:
 		// This is a DTail client process running.
 		impl := loggers.FOUT
-		Client = New(CLIENT, CLIENT, level, impl, strategy)
-		Server = New(CLIENT, SERVER, level, impl, strategy)
+		Client = New(source.Client, source.Client, level, impl, strategy)
+		Server = New(source.Client, source.Server, level, impl, strategy)
 		Common = Client
-	case SERVER:
+	case source.Server:
 		// This is a DTail server process running.
 		impl := loggers.FILE
-		Client = New(SERVER, CLIENT, level, impl, strategy)
-		Server = New(SERVER, SERVER, level, impl, strategy)
+		Client = New(source.Server, source.Client, level, impl, strategy)
+		Server = New(source.Server, source.Server, level, impl, strategy)
 		Common = Server
 	}
 
@@ -75,10 +76,10 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 type DLog struct {
 	logger loggers.Logger
 	// Is this a DTail server or client process logging?
-	sourceProcess source
+	sourceProcess source.Source
 	// Is this a DTail server or client package logging? In serverless mode
 	// the client can also execute code from the server package.
-	sourcePackage source
+	sourcePackage source.Source
 	// Max log level to log.
 	maxLevel level
 	// Current hostname.
@@ -86,7 +87,7 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
+func New(sourceProcess, sourcePackage source.Source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
@@ -120,7 +121,7 @@ func (d *DLog) log(level level, args []interface{}) string {
 	now := time.Now()
 
 	switch d.sourceProcess {
-	case CLIENT:
+	case source.Client:
 		sb.WriteString(d.sourcePackage.String())
 		sb.WriteString(protocol.FieldDelimiter)
 		sb.WriteString(d.hostname)
@@ -179,7 +180,7 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	if d.sourcePackage == SERVER && d.sourceProcess != CLIENT {
+	if d.sourcePackage == source.Server && d.sourceProcess != source.Client {
 		// This can be dtail client in serverless mode. In this case log all
 		// info server messages as verbose.
 		return d.log(VERBOSE, args)
@@ -215,7 +216,7 @@ func (d *DLog) Raw(message string) string {
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
-	if d.sourceProcess == SERVER {
+	if d.sourceProcess == source.Server {
 		// level|date-time|process|caller|cpus|goroutines|cgocalls|loadavg|uptime|MAPREDUCE:TABLE|key=value|...
 
 		var loadAvg string
diff --git a/internal/io/dlog/loggers/strategy.go b/internal/io/dlog/loggers/strategy.go
new file mode 100644
index 0000000..a1b9355
--- /dev/null
+++ b/internal/io/dlog/loggers/strategy.go
@@ -0,0 +1,27 @@
+package loggers
+
+import (
+	"os"
+	"path/filepath"
+)
+
+type Rotation int
+
+const (
+	DailyRotation  Rotation = iota
+	SignalRotation Rotation = iota
+)
+
+type Strategy struct {
+	Rotation Rotation
+	FileBase string
+}
+
+func GetStrategy(name string) Strategy {
+	switch name {
+	case "daily":
+		return Strategy{DailyRotation, ""}
+	default:
+		return Strategy{SignalRotation, filepath.Base(os.Args[0])}
+	}
+}
diff --git a/internal/io/dlog/source.go b/internal/io/dlog/source.go
deleted file mode 100644
index 265885e..0000000
--- a/internal/io/dlog/source.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package dlog
-
-type source int
-
-const (
-	CLIENT source = iota
-	SERVER source = iota
-)
-
-func (s source) String() string {
-	switch s {
-	case CLIENT:
-		return "CLIENT"
-	case SERVER:
-		return "SERVER"
-	}
-
-	panic("Unknown log source type")
-}
diff --git a/internal/source/source.go b/internal/source/source.go
new file mode 100644
index 0000000..bf1c880
--- /dev/null
+++ b/internal/source/source.go
@@ -0,0 +1,19 @@
+package source
+
+type Source int
+
+const (
+	Client Source = iota
+	Server Source = iota
+)
+
+func (s Source) String() string {
+	switch s {
+	case Client:
+		return "Client"
+	case Server:
+		return "Server"
+	}
+
+	panic("Unknown log source type")
+}
-- 
cgit v1.2.3


From 12c79f68bb5bda6673819d7b754820ecfe6d08ff Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 11:54:07 +0300
Subject: reduce logging in serverless mode

---
 internal/config/args.go                   |  30 +++++++-
 internal/config/config.go                 | 116 ++++--------------------------
 internal/config/initializer.go            | 109 ++++++++++++++++++++++++++++
 internal/config/setup.go                  |  17 -----
 internal/io/dlog/dlog.go                  |   5 --
 internal/server/handlers/readcommand.go   |  12 ++--
 internal/server/handlers/serverhandler.go |  93 ++++++++----------------
 internal/source/source.go                 |   4 +-
 internal/version/version.go               |   2 +-
 9 files changed, 190 insertions(+), 198 deletions(-)
 create mode 100644 internal/config/initializer.go
 delete mode 100644 internal/config/setup.go

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index 484aa8b..3e2eb1f 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -1,6 +1,7 @@
 package config
 
 import (
+	"encoding/base64"
 	"fmt"
 	"strings"
 
@@ -69,7 +70,32 @@ func (a *Args) String() string {
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
 func (a *Args) SerializeOptions() string {
-	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
+	return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, a.Serverless)
 }
 
-// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package)
+// DeserializeOptions deserializes the options, but into a map.
+func DeserializeOptions(opts []string) (map[string]string, error) {
+	options := make(map[string]string, len(opts))
+
+	for _, o := range opts {
+		kv := strings.SplitN(o, "=", 2)
+		if len(kv) != 2 {
+			return options, fmt.Errorf("Unable to parse options: %v", kv)
+		}
+		key := kv[0]
+		val := kv[1]
+
+		if strings.HasPrefix(val, "base64%") {
+			s := strings.SplitN(val, "%", 2)
+			decoded, err := base64.StdEncoding.DecodeString(s[1])
+			if err != nil {
+				return options, err
+			}
+			val = string(decoded)
+		}
+
+		options[key] = val
+	}
+
+	return options, nil
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index 6d4730a..09ae994 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1,14 +1,5 @@
 package config
 
-import (
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-)
-
 const (
 	// ControlUser is used for various DTail specific operations.
 	ControlUser string = "DTAIL-CONTROL"
@@ -33,97 +24,18 @@ var Server *ServerConfig
 // Common holds common configs of both both, client and server.
 var Common *CommonConfig
 
-// Used to initialize the configuration.
-type configInitializer struct {
-	Common *CommonConfig
-	Server *ServerConfig
-	Client *ClientConfig
-}
-
-func (c *configInitializer) parseConfig(args *Args) {
-	if strings.ToUpper(args.ConfigFile) == "NONE" {
-		return
-	}
-
-	if args.ConfigFile != "" {
-		c.parseSpecificConfig(args.ConfigFile)
-		return
-	}
-
-	if homeDir, err := os.UserHomeDir(); err != nil {
-		var paths []string
-		paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir))
-		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
-		for _, configPath := range paths {
-			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
-				c.parseSpecificConfig(configPath)
-			}
-		}
-	}
-}
-
-func (c *configInitializer) parseSpecificConfig(configFile string) {
-	fd, err := os.Open(configFile)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to read config file: %v", err))
-	}
-	defer fd.Close()
-
-	cfgBytes, err := ioutil.ReadAll(fd)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
-	}
-
-	err = json.Unmarshal([]byte(cfgBytes), c)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
-	}
-}
-
-func (c *configInitializer) transformConfig(args *Args, additionalArgs []string,
-	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
-	if args.LogDir != "" {
-		common.LogDir = args.LogDir
-	}
-	if strings.Contains(common.LogDir, "~/") {
-		homeDir, err := os.UserHomeDir()
-		if err != nil {
-			panic(err)
-		}
-		common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
-	}
-	if common.LogStrategy == "" {
-		common.LogStrategy = "daily"
-	}
-
-	if args.LogLevel != "" {
-		common.LogLevel = args.LogLevel
-	}
-
-	if args.SSHPort != DefaultSSHPort {
-		common.SSHPort = args.SSHPort
-	}
-	if args.NoColor {
-		client.TermColorsEnable = false
-	}
-
-	if args.Spartan {
-		args.Quiet = true
-		args.NoColor = true
-	}
-
-	if args.Discovery == "" && args.ServersStr == "" {
-		args.Serverless = true
-	}
-
-	// Interpret additional args as file list.
-	if args.What == "" {
-		var files []string
-		for _, file := range flag.Args() {
-			files = append(files, file)
-		}
-		args.What = strings.Join(files, ",")
-	}
-
-	return client, server, common
+// Setup the DTail configuration.
+func Setup(args *Args, additionalArgs []string) {
+	initializer := initializer{
+		Common: newDefaultCommonConfig(),
+		Server: newDefaultServerConfig(),
+		Client: newDefaultClientConfig(),
+	}
+	initializer.parseConfig(args)
+	Client, Server, Common = initializer.transformConfig(
+		args, additionalArgs,
+		initializer.Client,
+		initializer.Server,
+		initializer.Common,
+	)
 }
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
new file mode 100644
index 0000000..f1a9ec4
--- /dev/null
+++ b/internal/config/initializer.go
@@ -0,0 +1,109 @@
+package config
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+// Used to initialize the configuration.
+type initializer struct {
+	Common *CommonConfig
+	Server *ServerConfig
+	Client *ClientConfig
+}
+
+func (c *initializer) parseConfig(args *Args) {
+	if strings.ToUpper(args.ConfigFile) == "NONE" {
+		return
+	}
+
+	if args.ConfigFile != "" {
+		c.parseSpecificConfig(args.ConfigFile)
+		return
+	}
+
+	if homeDir, err := os.UserHomeDir(); err != nil {
+		var paths []string
+		paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir))
+		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
+		for _, configPath := range paths {
+			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
+				c.parseSpecificConfig(configPath)
+			}
+		}
+	}
+}
+
+func (c *initializer) parseSpecificConfig(configFile string) {
+	fd, err := os.Open(configFile)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to read config file: %v", err))
+	}
+	defer fd.Close()
+
+	cfgBytes, err := ioutil.ReadAll(fd)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
+	}
+
+	err = json.Unmarshal([]byte(cfgBytes), c)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
+	}
+}
+
+func (c *initializer) transformConfig(args *Args, additionalArgs []string,
+	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
+	if args.LogDir != "" {
+		common.LogDir = args.LogDir
+	}
+	if strings.Contains(common.LogDir, "~/") {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			panic(err)
+		}
+		common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
+	}
+	if common.LogStrategy == "" {
+		common.LogStrategy = "daily"
+	}
+
+	if args.LogLevel != "" {
+		common.LogLevel = args.LogLevel
+	} else if args.ServersStr == "" && args.Discovery == "" {
+		// We are in serverless mode. Default log level is WARN.
+		common.LogLevel = "WARN"
+	}
+
+	if args.SSHPort != DefaultSSHPort {
+		common.SSHPort = args.SSHPort
+	}
+	if args.NoColor {
+		client.TermColorsEnable = false
+	}
+
+	if args.Spartan {
+		args.Quiet = true
+		args.NoColor = true
+	}
+
+	if args.Discovery == "" && args.ServersStr == "" {
+		// We are not connecting to any servers.
+		args.Serverless = true
+	}
+
+	// Interpret additional args as file list.
+	if args.What == "" {
+		var files []string
+		for _, file := range flag.Args() {
+			files = append(files, file)
+		}
+		args.What = strings.Join(files, ",")
+	}
+
+	return client, server, common
+}
diff --git a/internal/config/setup.go b/internal/config/setup.go
deleted file mode 100644
index 7800914..0000000
--- a/internal/config/setup.go
+++ /dev/null
@@ -1,17 +0,0 @@
-package config
-
-// Setup the DTail configuration.
-func Setup(args *Args, additionalArgs []string) {
-	initializer := configInitializer{
-		Common: newDefaultCommonConfig(),
-		Server: newDefaultServerConfig(),
-		Client: newDefaultClientConfig(),
-	}
-	initializer.parseConfig(args)
-	Client, Server, Common = initializer.transformConfig(
-		args, additionalArgs,
-		initializer.Client,
-		initializer.Server,
-		initializer.Common,
-	)
-}
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 3de3120..6cacfe2 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -180,11 +180,6 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	if d.sourcePackage == source.Server && d.sourceProcess != source.Client {
-		// This can be dtail client in serverless mode. In this case log all
-		// info server messages as verbose.
-		return d.log(VERBOSE, args)
-	}
 	return d.log(INFO, args)
 }
 
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index c76ae2a..6579018 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -32,13 +32,13 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string, retrie
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			r.server.sendServerMessage(dlog.Server.Error(r.server.user, commandParseWarning, err))
+			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user, commandParseWarning, err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, commandParseWarning, args, argc))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, commandParseWarning, args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re, retries)
@@ -58,7 +58,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 
 		if numPaths := len(paths); numPaths == 0 {
 			dlog.Server.Error(r.server.user, "No such file(s) to read", glob)
-			r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
+			r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
 			select {
 			case <-ctx.Done():
 				return
@@ -72,7 +72,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 		return
 	}
 
-	r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Giving up to read file(s)"))
+	r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Giving up to read file(s)"))
 	return
 }
 
@@ -93,7 +93,7 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGr
 
 	if !r.server.user.HasFilePermission(path, "readfiles") {
 		dlog.Server.Error(r.server.user, "No permission to read file", path, globID)
-		r.server.sendServerWarnMessage(dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
 		return
 	}
 
@@ -161,6 +161,6 @@ func (r *readCommand) makeGlobID(path, glob string) string {
 		return pathParts[len(pathParts)-1]
 	}
 
-	r.server.sendServerWarnMessage(dlog.Server.Warn("Empty file path given?", path, glob))
+	r.server.send(r.server.serverMessages, dlog.Server.Warn("Empty file path given?", path, glob))
 	return ""
 }
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index b664566..ace2626 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -15,8 +15,8 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
@@ -46,6 +46,7 @@ type ServerHandler struct {
 	activeCommands   int32
 	quiet            bool
 	spartan          bool
+	serverless       bool
 	readBuf          bytes.Buffer
 	writeBuf         bytes.Buffer
 }
@@ -99,6 +100,12 @@ func (h *ServerHandler) Read(p []byte) (n int, err error) {
 			return
 		}
 
+		if h.serverless {
+			// In serverless mode we have logged the server message already via the
+			// dlog logger, no need to send the message again to the client part.
+			return
+		}
+
 		// Handle normal server message (display to the user)
 		h.readBuf.WriteString("SERVER")
 		h.readBuf.WriteString(protocol.FieldDelimiter)
@@ -266,23 +273,24 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
 
-	options, err := readOptions(splitted[1:])
+	options, err := config.DeserializeOptions(splitted[1:])
 	if err != nil {
-		h.sendServerMessage(dlog.Server.Error(h.user, err))
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		commandFinished()
 		return
 	}
-	if quiet, ok := options["quiet"]; ok {
-		if quiet == "true" {
-			dlog.Server.Debug(h.user, "Enabling quiet mode")
-			h.quiet = true
-		}
+
+	if quiet, _ := options["quiet"]; quiet == "true" {
+		dlog.Server.Debug(h.user, "Enabling quiet mode")
+		h.quiet = true
 	}
-	if spartan, ok := options["spartan"]; ok {
-		if spartan == "true" {
-			dlog.Server.Debug(h.user, "Enabling spartan mode")
-			h.spartan = true
-		}
+	if spartan, _ := options["spartan"]; spartan == "true" {
+		dlog.Server.Debug(h.user, "Enabling spartan mode")
+		h.spartan = true
+	}
+	if serverless, _ := options["serverless"]; serverless == "true" {
+		dlog.Server.Debug(h.user, "Enabling serverless mode")
+		h.serverless = true
 	}
 
 	switch commandName {
@@ -303,7 +311,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 	case "map":
 		command, aggregate, err := newMapCommand(h, argc, args)
 		if err != nil {
-			h.sendServerMessage(err.Error())
+			h.send(h.serverMessages, err.Error())
 			dlog.Server.Error(h.user, err)
 			commandFinished()
 			return
@@ -320,14 +328,16 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		commandFinished()
 
 	default:
-		h.sendServerMessage(dlog.Server.Error(h.user, "Received unknown user command", commandName, argc, args, options))
+		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown user command", commandName, argc, args, options))
 		commandFinished()
 	}
 }
 
 func (h *ServerHandler) handleAckCommand(argc int, args []string) {
 	if argc < 3 {
-		h.sendServerWarnMessage(dlog.Server.Warn(h.user, commandParseWarning, args, argc))
+		if !h.quiet {
+			h.send(h.serverMessages, dlog.Server.Warn(h.user, commandParseWarning, args, argc))
+		}
 		return
 	}
 	if args[1] == "close" && args[2] == "connection" {
@@ -346,23 +356,8 @@ func (h *ServerHandler) send(ch chan<- string, message string) {
 	}
 }
 
-func (h *ServerHandler) sendServerMessage(message string) {
-	h.send(h.serverMessageC(), message)
-}
-
-func (h *ServerHandler) sendServerWarnMessage(message string) {
-	if h.quiet {
-		return
-	}
-	h.send(h.serverMessageC(), message)
-}
-
-func (h *ServerHandler) serverMessageC() chan<- string {
-	return h.serverMessages
-}
-
-func (h *ServerHandler) flushMessages() {
-	dlog.Server.Debug(h.user, "flushMessages()")
+func (h *ServerHandler) flush() {
+	dlog.Server.Debug(h.user, "flush()")
 
 	unsentMessages := func() int {
 		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
@@ -381,11 +376,11 @@ func (h *ServerHandler) flushMessages() {
 
 func (h *ServerHandler) shutdown() {
 	dlog.Server.Debug(h.user, "shutdown()")
-	h.flushMessages()
+	h.flush()
 
 	go func() {
 		select {
-		case h.serverMessageC() <- ".syn close connection":
+		case h.serverMessages <- ".syn close connection":
 		case <-h.done.Done():
 		}
 	}()
@@ -408,31 +403,3 @@ func (h *ServerHandler) decrementActiveCommands() int32 {
 	atomic.AddInt32(&h.activeCommands, -1)
 	return atomic.LoadInt32(&h.activeCommands)
 }
-
-func readOptions(opts []string) (map[string]string, error) {
-	dlog.Server.Debug("Parsing options", opts)
-	options := make(map[string]string, len(opts))
-
-	for _, o := range opts {
-		kv := strings.SplitN(o, "=", 2)
-		if len(kv) != 2 {
-			return options, fmt.Errorf("Unable to parse options: %v", kv)
-		}
-		key := kv[0]
-		val := kv[1]
-
-		if strings.HasPrefix(val, "base64%") {
-			s := strings.SplitN(val, "%", 2)
-			decoded, err := base64.StdEncoding.DecodeString(s[1])
-			if err != nil {
-				return options, err
-			}
-			val = string(decoded)
-		}
-
-		dlog.Server.Debug("Setting option", key, val)
-		options[key] = val
-	}
-
-	return options, nil
-}
diff --git a/internal/source/source.go b/internal/source/source.go
index bf1c880..73dccb2 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -10,9 +10,9 @@ const (
 func (s Source) String() string {
 	switch s {
 	case Client:
-		return "Client"
+		return "CLIENT"
 	case Server:
-		return "Server"
+		return "SERVER"
 	}
 
 	panic("Unknown log source type")
diff --git a/internal/version/version.go b/internal/version/version.go
index a54b560..4ff6eae 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -39,7 +39,7 @@ func PaintedString() string {
 		color.FgBlack, color.BgGreen)
 
 	additional := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Additional),
-		color.FgWhite, color.BgMagenta, color.AttrBlink)
+		color.FgWhite, color.BgMagenta, color.AttrUnderline)
 
 	return fmt.Sprintf("%s%v%s%s", name, version, protocol, additional)
 }
-- 
cgit v1.2.3


From 86ec83754e0ee7153ad55091f7b6da448bc529c5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 13:44:27 +0300
Subject: add dcat test

---
 internal/config/common.go          |  2 +-
 internal/config/initializer.go     | 19 ++++++------
 internal/io/dlog/loggers/stdout.go | 59 +++++++++++++-------------------------
 internal/io/fs/readfile.go         | 10 +------
 4 files changed, 33 insertions(+), 57 deletions(-)

(limited to 'internal')

diff --git a/internal/config/common.go b/internal/config/common.go
index acc8e6a..255bd28 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -6,7 +6,7 @@ type CommonConfig struct {
 	SSHPort int
 	// Enable experimental features (mainly for dev purposes)
 	ExperimentalFeaturesEnable bool `json:",omitempty"`
-	// LogLevel defines how much is logged. TODO: Adjust JSONschema
+	// LogLevel defines how much is logged.
 	LogLevel string `json:",omitempty"`
 	// The log strategy to use, one of
 	//   stdout: only log to stdout (useful when used with systemd)
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index f1a9ec4..0e725a6 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -72,6 +72,17 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string,
 		common.LogStrategy = "daily"
 	}
 
+	if args.Spartan {
+		args.Quiet = true
+		args.NoColor = true
+		if args.LogLevel == "" {
+			args.LogLevel = "ERROR"
+		}
+	}
+	if args.NoColor {
+		client.TermColorsEnable = false
+	}
+
 	if args.LogLevel != "" {
 		common.LogLevel = args.LogLevel
 	} else if args.ServersStr == "" && args.Discovery == "" {
@@ -82,14 +93,6 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string,
 	if args.SSHPort != DefaultSSHPort {
 		common.SSHPort = args.SSHPort
 	}
-	if args.NoColor {
-		client.TermColorsEnable = false
-	}
-
-	if args.Spartan {
-		args.Quiet = true
-		args.NoColor = true
-	}
 
 	if args.Discovery == "" && args.ServersStr == "" {
 		// We are not connecting to any servers.
diff --git a/internal/io/dlog/loggers/stdout.go b/internal/io/dlog/loggers/stdout.go
index 9738323..05485c6 100644
--- a/internal/io/dlog/loggers/stdout.go
+++ b/internal/io/dlog/loggers/stdout.go
@@ -8,66 +8,47 @@ import (
 )
 
 type stdout struct {
-	bufferCh chan string
 	pauseCh  chan struct{}
 	resumeCh chan struct{}
+	mutex    sync.Mutex
 }
 
 func newStdout() *stdout {
 	return &stdout{
-		bufferCh: make(chan string, 100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
 	}
 }
 
 func (s *stdout) Start(ctx context.Context, wg *sync.WaitGroup) {
-	pause := func(ctx context.Context) {
-		select {
-		case <-s.resumeCh:
-			return
-		case <-ctx.Done():
-			return
-		}
-	}
-
-	go func() {
-		defer wg.Done()
-
-		for {
-			select {
-			case message := <-s.bufferCh:
-				fmt.Println(message)
-			case <-s.pauseCh:
-				pause(ctx)
-			case <-ctx.Done():
-				s.Flush()
-				return
-			}
-		}
-	}()
+	wg.Done()
 }
 
 func (s *stdout) Log(now time.Time, message string) {
-	s.bufferCh <- message
+	s.log(message)
 }
 
 func (s *stdout) LogWithColors(now time.Time, message, coloredMessage string) {
-	s.bufferCh <- coloredMessage
+	s.log(coloredMessage)
 }
 
-func (s *stdout) Flush() {
-	for {
-		select {
-		case message := <-s.bufferCh:
-			fmt.Println(message)
-		default:
-			return
-		}
+func (s *stdout) log(message string) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	select {
+	case <-s.pauseCh:
+		// Pause until resumed.
+		<-s.resumeCh
+	default:
 	}
+
+	fmt.Println(message)
 }
 
-func (s *stdout) Pause()            { s.pauseCh <- struct{}{} }
-func (s *stdout) Resume()           { s.resumeCh <- struct{}{} }
-func (s *stdout) Rotate()           {}
+func (s *stdout) Pause()  { s.pauseCh <- struct{}{} }
+func (s *stdout) Resume() { s.resumeCh <- struct{}{} }
+func (s *stdout) Flush()  {}
+func (s *stdout) Rotate() {}
+
 func (stdout) SupportsColors() bool { return true }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 07486a1..f128c07 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -13,8 +13,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/regex"
 
@@ -182,14 +182,6 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 
 		switch b {
 		case '\n':
-			/*
-				// dcat/dgrep should actually transfer empty lines
-					if message.Len() == 0 {
-						time.Sleep(time.Millisecond * 100)
-						continue
-					}
-			*/
-			//message.WriteString("\n")
 			select {
 			case rawLines <- message:
 				message = pool.BytesBuffer.Get().(*bytes.Buffer)
-- 
cgit v1.2.3


From 07e1470892beacf0722276f94bacbd822b002540 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 3 Oct 2021 13:09:32 +0300
Subject: add dmap tests

---
 internal/clients/maprclient.go | 2 +-
 internal/config/common.go      | 1 +
 internal/config/config.go      | 5 ++++-
 internal/config/initializer.go | 6 ++++--
 internal/io/dlog/dlog.go       | 7 +++++--
 5 files changed, 15 insertions(+), 6 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index f23aa08..19fe119 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -203,7 +203,7 @@ func (c *MaprClient) printResults() {
 	dlog.Client.Raw(rawQuery)
 
 	if rowsLimit > 0 && numRows > rowsLimit {
-		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited output to %d rows! Use 'limit' clause to override!",
+		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited terminal output to %d rows! Use 'limit' clause to override!",
 			numRows, rowsLimit))
 	}
 	dlog.Client.Raw(result)
diff --git a/internal/config/common.go b/internal/config/common.go
index 255bd28..5e81bc9 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -26,6 +26,7 @@ func newDefaultCommonConfig() *CommonConfig {
 		SSHPort:                    DefaultSSHPort,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
+		LogLevel:                   "INFO",
 		CacheDir:                   "cache",
 		TmpDir:                     "/tmp",
 	}
diff --git a/internal/config/config.go b/internal/config/config.go
index 09ae994..d58162f 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1,5 +1,7 @@
 package config
 
+import "github.com/mimecast/dtail/internal/source"
+
 const (
 	// ControlUser is used for various DTail specific operations.
 	ControlUser string = "DTAIL-CONTROL"
@@ -25,7 +27,7 @@ var Server *ServerConfig
 var Common *CommonConfig
 
 // Setup the DTail configuration.
-func Setup(args *Args, additionalArgs []string) {
+func Setup(sourceProcess source.Source, args *Args, additionalArgs []string) {
 	initializer := initializer{
 		Common: newDefaultCommonConfig(),
 		Server: newDefaultServerConfig(),
@@ -33,6 +35,7 @@ func Setup(args *Args, additionalArgs []string) {
 	}
 	initializer.parseConfig(args)
 	Client, Server, Common = initializer.transformConfig(
+		sourceProcess,
 		args, additionalArgs,
 		initializer.Client,
 		initializer.Server,
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 0e725a6..a58f82a 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -7,6 +7,8 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+
+	"github.com/mimecast/dtail/internal/source"
 )
 
 // Used to initialize the configuration.
@@ -56,7 +58,7 @@ func (c *initializer) parseSpecificConfig(configFile string) {
 	}
 }
 
-func (c *initializer) transformConfig(args *Args, additionalArgs []string,
+func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string,
 	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
 	if args.LogDir != "" {
 		common.LogDir = args.LogDir
@@ -85,7 +87,7 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string,
 
 	if args.LogLevel != "" {
 		common.LogLevel = args.LogLevel
-	} else if args.ServersStr == "" && args.Discovery == "" {
+	} else if sourceProcess == source.Client && args.ServersStr == "" && args.Discovery == "" {
 		// We are in serverless mode. Default log level is WARN.
 		common.LogLevel = "WARN"
 	}
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 6cacfe2..2beda75 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -163,8 +163,11 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 
 func (d *DLog) FatalPanic(args ...interface{}) {
 	d.log(FATAL, args)
-	d.logger.Flush()
-	panic("Not recovering from this fatal error...")
+	d.Flush()
+
+	var sb strings.Builder
+	d.writeArgStrings(&sb, args)
+	panic(sb.String())
 }
 
 func (d *DLog) Fatal(args ...interface{}) string {
-- 
cgit v1.2.3


From b2dbe133347ef220ff781ffeb1f8137245f5235f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 3 Oct 2021 13:32:20 +0300
Subject: when a mapreduce outfile is specified also always write a
 outfile.query file

---
 internal/mapr/groupset.go | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

(limited to 'internal')

diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index df8c603..ce7630d 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -178,11 +178,31 @@ func (g *GroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	return sb.String(), len(rows), nil
 }
 
+func (*GroupSet) writeQueryFile(query *Query) error {
+	queryFile := fmt.Sprintf("%s.query", query.Outfile)
+	tmpQueryFile := fmt.Sprintf("%s.tmp", queryFile)
+	dlog.Common.Debug("Writing query file", queryFile)
+
+	fd, err := os.Create(tmpQueryFile)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	fd.WriteString(query.RawQuery)
+	os.Rename(tmpQueryFile, queryFile)
+
+	return nil
+}
+
 // WriteResult writes the result to an CSV outfile.
 func (g *GroupSet) WriteResult(query *Query) error {
 	if !query.HasOutfile() {
 		return errors.New("No outfile specified")
 	}
+	if err := g.writeQueryFile(query); err != nil {
+		return err
+	}
 
 	rows, _, err := g.result(query, false)
 	if err != nil {
@@ -192,22 +212,22 @@ func (g *GroupSet) WriteResult(query *Query) error {
 	dlog.Common.Info("Writing outfile", query.Outfile)
 	tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile)
 
-	file, err := os.Create(tmpOutfile)
+	fd, err := os.Create(tmpOutfile)
 	if err != nil {
 		return err
 	}
-	defer file.Close()
+	defer fd.Close()
 
 	// Generate header now
 	lastIndex := len(query.Select) - 1
 	for i, sc := range query.Select {
-		file.WriteString(sc.FieldStorage)
+		fd.WriteString(sc.FieldStorage)
 		if i == lastIndex {
 			continue
 		}
-		file.WriteString(protocol.CSVDelimiter)
+		fd.WriteString(protocol.CSVDelimiter)
 	}
-	file.WriteString("\n")
+	fd.WriteString("\n")
 
 	// And now write the data
 	for i, r := range rows {
@@ -215,13 +235,13 @@ func (g *GroupSet) WriteResult(query *Query) error {
 			break
 		}
 		for j, value := range r.values {
-			file.WriteString(value)
+			fd.WriteString(value)
 			if j == lastIndex {
 				continue
 			}
-			file.WriteString(protocol.CSVDelimiter)
+			fd.WriteString(protocol.CSVDelimiter)
 		}
-		file.WriteString("\n")
+		fd.WriteString("\n")
 	}
 
 	if err := os.Rename(tmpOutfile, query.Outfile); err != nil {
-- 
cgit v1.2.3


From f70622f307629a2542ea5eb128dea8c1043d3a40 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 5 Oct 2021 10:00:38 +0300
Subject: more on this

---
 internal/clients/baseclient.go                  |   8 +-
 internal/clients/connectors/serverconnection.go |  19 --
 internal/clients/connectors/serverless.go       |  22 +-
 internal/clients/handlers/healthhandler.go      | 114 ++++----
 internal/clients/healthclient.go                |  97 ++-----
 internal/io/dlog/dlog.go                        |   6 +
 internal/server/handlers/basehandler.go         | 283 ++++++++++++++++++++
 internal/server/handlers/readcommand.go         |   4 +-
 internal/server/handlers/serverhandler.go       | 332 +++---------------------
 internal/source/source.go                       |   9 +-
 10 files changed, 417 insertions(+), 477 deletions(-)
 create mode 100644 internal/server/handlers/basehandler.go

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index fc01955..5ac298f 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -86,7 +86,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	var mutex sync.Mutex
 	for i, conn := range c.connections {
 		go func(i int, conn connectors.Connector) {
-			connStatus := c.start(ctx, active, i, conn)
+			connStatus := c.startConnection(ctx, active, i, conn)
 
 			// Update global status.
 			mutex.Lock()
@@ -97,11 +97,12 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 		}(i, conn)
 	}
 
+	time.Sleep(time.Second * 2)
 	c.waitUntilDone(ctx, active)
 	return
 }
 
-func (c *baseClient) start(ctx context.Context, active chan struct{}, i int, conn connectors.Connector) (status int) {
+func (c *baseClient) startConnection(ctx context.Context, active chan struct{}, i int, conn connectors.Connector) (status int) {
 	// Increment connection count
 	active <- struct{}{}
 	// Derement connection count
@@ -146,12 +147,13 @@ func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
 		<-ctx.Done()
 	}
 
+	// TODO: Rewrite this to use a wait group.
 	for {
 		numActive := len(active)
 		if numActive == 0 {
 			return
 		}
 		dlog.Client.Debug("Active connections", numActive)
-		time.Sleep(time.Second)
+		time.Sleep(time.Second * time.Millisecond * 100)
 	}
 }
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 5bc63ee..1666a79 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -23,7 +23,6 @@ type ServerConnection struct {
 	config          *ssh.ClientConfig
 	handler         handlers.Handler
 	commands        []string
-	isOneOff        bool
 	hostKeyCallback client.HostKeyCallback
 	throttlingDone  bool
 }
@@ -49,24 +48,6 @@ func NewServerConnection(server string, userName string, authMethods []ssh.AuthM
 	return &c
 }
 
-// NewOneOffServerConnection creates new one-off connection (only for sending a series of commands and then quit).
-func NewOneOffServerConnection(server string, userName string, authMethods []ssh.AuthMethod, handler handlers.Handler, commands []string) *ServerConnection {
-	c := ServerConnection{
-		server:   server,
-		handler:  handler,
-		commands: commands,
-		config: &ssh.ClientConfig{
-			User:            userName,
-			Auth:            authMethods,
-			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		},
-		isOneOff: true,
-	}
-
-	c.initServerPort()
-	return &c
-}
-
 func (c *ServerConnection) Server() string {
 	return c.server
 }
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index 7740aab..ae72c9b 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -20,14 +20,12 @@ type Serverless struct {
 
 // NewServerConnection returns a new connection.
 func NewServerless(userName string, handler handlers.Handler, commands []string) *Serverless {
-	s := Serverless{
+	dlog.Client.Debug("Creating new serverless connector", handler, commands)
+	return &Serverless{
 		userName: userName,
 		handler:  handler,
 		commands: commands,
 	}
-
-	dlog.Client.Debug("Creating new serverless connector", handler, commands)
-	return &s
 }
 
 func (s *Serverless) Server() string {
@@ -58,11 +56,17 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 		return err
 	}
 
-	serverHandler := serverHandlers.NewServerHandler(
-		user,
-		make(chan struct{}, config.Server.MaxConcurrentCats),
-		make(chan struct{}, config.Server.MaxConcurrentTails),
-	)
+	var serverHandler serverHandlers.Handler
+	switch s.userName {
+	case config.ControlUser:
+		serverHandler = serverHandlers.NewControlHandler(user)
+	default:
+		serverHandler = serverHandlers.NewServerHandler(
+			user,
+			make(chan struct{}, config.Server.MaxConcurrentCats),
+			make(chan struct{}, config.Server.MaxConcurrentTails),
+		)
+	}
 
 	terminate := func() {
 		serverHandler.Shutdown()
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index eca0348..4949985 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -1,90 +1,72 @@
 package handlers
 
 import (
-	"bytes"
-	"errors"
 	"fmt"
-	"time"
+	"strings"
 
 	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// HealthHandler implements the handler required for health checks.
+// HealthHandler is the handler used on the client side for running mapreduce aggregations.
 type HealthHandler struct {
-	done *internal.Done
-	// Buffer of incoming data from server.
-	receiveBuf bytes.Buffer
-	// To send commands to the server.
-	commands chan string
-	// To receive messages from the server.
-	receive chan<- string
-	// The remote server address
-	server string
-	// The return status.
-	status int
+	baseHandler
+	HealthStatusCh chan<- int
 }
 
-// NewHealthHandler returns a new health check handler.
-func NewHealthHandler(server string, receive chan<- string) *HealthHandler {
-	h := HealthHandler{
-		server:   server,
-		receive:  receive,
-		commands: make(chan string),
-		status:   -1,
-		done:     internal.NewDone(),
+// NewHealthHandler returns a new health client handler.
+func NewHealthHandler(server string) *HealthHandler {
+	dlog.Client.Debug(server, "Creating new health handler")
+	return &HealthHandler{
+		baseHandler: baseHandler{
+			server:       server,
+			shellStarted: false,
+			commands:     make(chan string),
+			status:       -1,
+			done:         internal.NewDone(),
+		},
+		HealthStatusCh: make(chan int),
 	}
-
-	return &h
-}
-
-// Server returns the remote server name.
-func (h *HealthHandler) Server() string {
-	return h.server
-}
-
-// Status of the handler.
-func (h *HealthHandler) Status() int {
-	return h.status
-}
-
-// Done returns done channel of the handler.
-func (h *HealthHandler) Done() <-chan struct{} {
-	return h.done.Done()
 }
 
-// Shutdown the handler.
-func (h *HealthHandler) Shutdown() {
-	h.done.Shutdown()
-}
-
-// SendMessage sends a DTail command to the server.
-func (h *HealthHandler) SendMessage(command string) error {
-	select {
-	case h.commands <- fmt.Sprintf("%s;", command):
-	case <-time.NewTimer(time.Second * 10).C:
-		return errors.New("Timed out sending command " + command)
-	case <-h.Done():
-	}
-
-	return nil
-}
-
-// Server writes byte stream to client.
+// Read data from the dtail server via Writer interface.
 func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
-		h.receiveBuf.WriteByte(b)
-		if b == protocol.MessageDelimiter {
-			h.receive <- h.receiveBuf.String()
-			h.receiveBuf.Reset()
+		switch b {
+		case '\n':
+			continue
+		case protocol.MessageDelimiter:
+			message := h.baseHandler.receiveBuf.String()
+			dlog.Client.Debug(message)
+			h.handleHealthMessage(message)
+			h.baseHandler.receiveBuf.Reset()
+		default:
+			h.baseHandler.receiveBuf.WriteByte(b)
 		}
 	}
 
 	return len(p), nil
 }
 
-// Server reads byte stream from client.
-func (h *HealthHandler) Read(p []byte) (n int, err error) {
-	n = copy(p, []byte(<-h.commands))
-	return
+func (h *HealthHandler) handleHealthMessage(message string) {
+	s := strings.Split(message, protocol.FieldDelimiter)
+	message = s[len(s)-1]
+	status := strings.Split(message, ":")
+	fmt.Println(status)
+	/*
+		switch status {
+		case "OK":
+			h.HealthStatusCh <- 0
+		case "WARNING":
+			h.HealthStatusCh <- 1
+		case "CRITICAL":
+			h.HealthStatusCh <- 2
+		case "UNKNOWN":
+			h.HealthStatusCh <- 3
+		default:
+			fmt.Println("CRITICAL: Unexpected server response: '%s'")
+			h.HealthStatusCh <- 2
+		}
+	*/
 }
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index 47007b6..df919ae 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -1,101 +1,44 @@
 package clients
 
 import (
-	"context"
-	"fmt"
 	"runtime"
-	"strings"
-	"time"
 
-	"github.com/mimecast/dtail/internal/clients/connectors"
 	"github.com/mimecast/dtail/internal/clients/handlers"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/omode"
-	"github.com/mimecast/dtail/internal/protocol"
 
 	gossh "golang.org/x/crypto/ssh"
 )
 
-// HealthClient is used for health checking (e.g. via Nagios)
+// HealthClient is used to perform a basic server health check.
 type HealthClient struct {
-	// Client operating mode
-	mode omode.Mode
-	// The remote server address
-	server string
-	// SSH user name
-	userName string
-	// SSH auth methods to use to connect to the remote servers.
-	sshAuthMethods []gossh.AuthMethod
+	baseClient
 }
 
-// NewHealthClient returns a new healh client.
-func NewHealthClient(mode omode.Mode) (*HealthClient, error) {
+// NewHealthClient returns a new health client.
+func NewHealthClient(args config.Args) (*HealthClient, error) {
+	args.Mode = omode.HealthClient
+	args.UserName = config.ControlUser
 	c := HealthClient{
-		mode:     mode,
-		server:   fmt.Sprintf("%s:%d", config.Server.SSHBindAddress, config.Common.SSHPort),
-		userName: config.ControlUser,
+		baseClient: baseClient{
+			Args:       args,
+			throttleCh: make(chan struct{}, args.ConnectionsPerCPU*runtime.NumCPU()),
+			retry:      false,
+		},
 	}
-	c.initSSHAuthMethods()
+
+	c.init()
+	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.ControlUser))
+	c.makeConnections(c)
 
 	return &c, nil
 }
 
-// Start the health client.
-func (c *HealthClient) Start(ctx context.Context) (status int) {
-	receive := make(chan string)
-
-	throttleCh := make(chan struct{}, runtime.NumCPU())
-	statsCh := make(chan struct{}, 1)
-
-	conn := connectors.NewOneOffServerConnection(
-		c.server,
-		c.userName,
-		c.sshAuthMethods,
-		handlers.NewHealthHandler(c.server, receive),
-		[]string{c.mode.String()},
-	)
-
-	connCtx, cancel := context.WithCancel(ctx)
-	go conn.Start(connCtx, cancel, throttleCh, statsCh)
-
-	for {
-		select {
-		case data := <-receive:
-			// Parse recieved data.
-			s := strings.Split(data, protocol.FieldDelimiter)
-			message := s[len(s)-1]
-			if strings.HasPrefix(message, "done;") {
-				return
-			}
-
-			// Set severity.
-			s = strings.Split(message, ":")
-			switch s[0] {
-			case "OK":
-			case "WARNING":
-				if status < 1 {
-					status = 1
-				}
-			case "CRITICAL":
-				status = 2
-			case "UNKNOWN":
-				status = 3
-			default:
-				fmt.Printf("CRITICAL: Unexpected server response: '%s'\n", message)
-				status = 2
-				return
-			}
-			fmt.Print(message)
-
-		case <-time.After(time.Second * 2):
-			status = 2
-			fmt.Println("CRITICAL: Could not communicate with DTail server")
-			return
-		}
-	}
+func (c HealthClient) makeHandler(server string) handlers.Handler {
+	return handlers.NewHealthHandler(server)
 }
 
-// Initialize SSH auth methods.
-func (c *HealthClient) initSSHAuthMethods() {
-	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.ControlUser))
+func (c HealthClient) makeCommands() (commands []string) {
+	commands = append(commands, "health")
+	return
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 2beda75..db99307 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -57,6 +57,12 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Client = New(source.Server, source.Client, level, impl, strategy)
 		Server = New(source.Server, source.Server, level, impl, strategy)
 		Common = Server
+	case source.HealthCheck:
+		// Health check isn't logging anything.
+		impl := loggers.STDOUT
+		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
+		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
+		Common = Client
 	}
 
 	var wg2 sync.WaitGroup
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
new file mode 100644
index 0000000..12fb2b3
--- /dev/null
+++ b/internal/server/handlers/basehandler.go
@@ -0,0 +1,283 @@
+package handlers
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/mapr/server"
+	"github.com/mimecast/dtail/internal/protocol"
+	user "github.com/mimecast/dtail/internal/user/server"
+)
+
+type handleCommandCb func(context.Context, int, []string)
+
+type baseHandler struct {
+	done             *internal.Done
+	handleCommandCb  handleCommandCb
+	lines            chan line.Line
+	aggregate        *server.Aggregate
+	maprMessages     chan string
+	serverMessages   chan string
+	hostname         string
+	user             *user.User
+	ackCloseReceived chan struct{}
+	activeCommands   int32
+	quiet            bool
+	spartan          bool
+	serverless       bool
+	readBuf          bytes.Buffer
+	writeBuf         bytes.Buffer
+}
+
+// Shutdown the handler.
+func (h *baseHandler) Shutdown() {
+	h.done.Shutdown()
+}
+
+// Done channel of the handler.
+func (h *baseHandler) Done() <-chan struct{} {
+	return h.done.Done()
+}
+
+// Read is to send data to the dtail client via Reader interface.
+func (h *baseHandler) Read(p []byte) (n int, err error) {
+	defer h.readBuf.Reset()
+
+	select {
+	case message := <-h.serverMessages:
+		if message[0] == '.' {
+			// Handle hidden message (don't display to the user, interpreted by dtail client)
+			h.readBuf.WriteString(message)
+			h.readBuf.WriteByte(protocol.MessageDelimiter)
+			n = copy(p, h.readBuf.Bytes())
+			return
+		}
+
+		if h.serverless {
+			// In serverless mode we have logged the server message already via the
+			// dlog logger, no need to send the message again to the client part.
+			return
+		}
+
+		// Handle normal server message (display to the user)
+		h.readBuf.WriteString("SERVER")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(message)
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+
+	case message := <-h.maprMessages:
+		// Send mapreduce-aggregated data as a message.
+		h.readBuf.WriteString("AGGREGATE")
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(h.hostname)
+		h.readBuf.WriteString(protocol.FieldDelimiter)
+		h.readBuf.WriteString(message)
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+
+	case line := <-h.lines:
+		if !h.spartan {
+			h.readBuf.WriteString("REMOTE")
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(h.hostname)
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+			h.readBuf.WriteString(line.SourceID)
+			h.readBuf.WriteString(protocol.FieldDelimiter)
+		}
+		h.readBuf.WriteString(line.Content.String())
+		h.readBuf.WriteByte(protocol.MessageDelimiter)
+		n = copy(p, h.readBuf.Bytes())
+		pool.RecycleBytesBuffer(line.Content)
+
+	case <-time.After(time.Second):
+		// Once in a while check whether we are done.
+		select {
+		case <-h.done.Done():
+			err = io.EOF
+			return
+		default:
+		}
+	}
+	return
+}
+
+// Write is to receive data from the dtail client via Writer interface.
+func (h *baseHandler) Write(p []byte) (n int, err error) {
+	for _, b := range p {
+		switch b {
+		case ';':
+			h.handleCommand(string(h.writeBuf.Bytes()))
+			h.writeBuf.Reset()
+		default:
+			h.writeBuf.WriteByte(b)
+		}
+	}
+
+	n = len(p)
+	return
+}
+
+func (h *baseHandler) handleCommand(commandStr string) {
+	dlog.Server.Debug(h.user, commandStr)
+
+	args, argc, add, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
+	if err != nil {
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err)+add)
+		return
+	}
+
+	args, argc, err = h.handleBase64(args, argc)
+	if err != nil {
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		<-h.done.Done()
+		cancel()
+	}()
+
+	h.handleCommandCb(ctx, argc, args)
+}
+
+func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
+	argc := len(args)
+	var add string
+
+	if argc <= 2 || args[0] != "protocol" {
+		return args, argc, add, errors.New("unable to determine protocol version")
+	}
+
+	if args[1] != protocol.ProtocolCompat {
+		clientCompat, _ := strconv.Atoi(args[1])
+		serverCompat, _ := strconv.Atoi(protocol.ProtocolCompat)
+		if clientCompat <= 3 {
+			// Protocol version 3 or lower expect a newline as message separator
+			// One day (after 2 major versions) this exception may be removed!
+			add = "\n"
+		}
+
+		toUpdate := "client"
+		if clientCompat > serverCompat {
+			toUpdate = "server"
+		}
+
+		err := fmt.Errorf("DTail server protocol version '%s' does not match client protocol version '%s', please update DTail %s!",
+			protocol.ProtocolCompat, args[1], toUpdate)
+		return args, argc, add, err
+	}
+
+	return args[2:], argc - 2, add, nil
+}
+
+func (h *baseHandler) handleBase64(args []string, argc int) ([]string, int, error) {
+	err := errors.New("Unable to decode client message, DTail server and client versions may not be compatible")
+
+	if argc != 2 || args[0] != "base64" {
+		return args, argc, err
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(args[1])
+	if err != nil {
+		return args, argc, err
+	}
+	decodedStr := string(decoded)
+
+	args = strings.Split(decodedStr, " ")
+	argc = len(decodedStr)
+	dlog.Server.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
+
+	return args, argc, nil
+}
+
+func (h *baseHandler) handleAckCommand(argc int, args []string) {
+	if argc < 3 {
+		if !h.quiet {
+			h.send(h.serverMessages, dlog.Server.Warn(h.user, "Unable to parse command", args, argc))
+		}
+		return
+	}
+	if args[1] == "close" && args[2] == "connection" {
+		select {
+		case <-h.ackCloseReceived:
+		default:
+			close(h.ackCloseReceived)
+		}
+	}
+}
+
+func (h *baseHandler) send(ch chan<- string, message string) {
+	select {
+	case ch <- message:
+	case <-h.done.Done():
+	}
+}
+
+func (h *baseHandler) flush() {
+	dlog.Server.Debug(h.user, "flush()")
+
+	numUnsentMessages := func() int {
+		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
+	}
+
+	for i := 0; i < 3; i++ {
+		if numUnsentMessages() == 0 {
+			dlog.Server.Debug(h.user, "All lines sent")
+			return
+		}
+		dlog.Server.Debug(h.user, "Still lines to be sent")
+		time.Sleep(time.Second)
+	}
+
+	dlog.Server.Warn(h.user, "Some lines remain unsent", numUnsentMessages())
+}
+
+func (h *baseHandler) shutdown() {
+	dlog.Server.Debug(h.user, "shutdown()")
+	h.flush()
+
+	go func() {
+		select {
+		case h.serverMessages <- ".syn close connection":
+		case <-h.done.Done():
+		}
+	}()
+
+	select {
+	case <-h.ackCloseReceived:
+	case <-time.After(time.Second * 5):
+		dlog.Server.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
+	case <-h.done.Done():
+	}
+
+	h.done.Shutdown()
+}
+
+func (h *baseHandler) incrementActiveCommands() {
+	atomic.AddInt32(&h.activeCommands, 1)
+}
+
+func (h *baseHandler) decrementActiveCommands() int32 {
+	atomic.AddInt32(&h.activeCommands, -1)
+	return atomic.LoadInt32(&h.activeCommands)
+}
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 6579018..abc44c7 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -32,13 +32,13 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string, retrie
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user, commandParseWarning, err))
+			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user, "Unable to parse command", err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, commandParseWarning, args, argc))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to parse command", args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re, retries)
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index ace2626..2ec4fbf 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -1,69 +1,60 @@
 package handlers
 
 import (
-	"bytes"
 	"context"
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"io"
 	"os"
-	"strconv"
 	"strings"
-	"sync/atomic"
-	"time"
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/pool"
-	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/omode"
-	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
-const (
-	commandParseWarning string = "Unable to parse command"
-)
-
 // ServerHandler implements the Reader and Writer interfaces to handle
 // the Bi-directional communication between SSH client and server.
 // This handler implements the handler of the SSH server.
 type ServerHandler struct {
-	done             *internal.Done
-	lines            chan line.Line
-	regex            string
-	aggregate        *server.Aggregate
-	maprMessages     chan string
-	serverMessages   chan string
-	hostname         string
-	user             *user.User
-	catLimiter       chan struct{}
-	tailLimiter      chan struct{}
-	ackCloseReceived chan struct{}
-	activeCommands   int32
-	quiet            bool
-	spartan          bool
-	serverless       bool
-	readBuf          bytes.Buffer
-	writeBuf         bytes.Buffer
+	baseHandler
+	catLimiter  chan struct{}
+	tailLimiter chan struct{}
+	regex       string
+	/*
+		done             *internal.Done
+		lines            chan line.Line
+		aggregate        *server.Aggregate
+		maprMessages     chan string
+		serverMessages   chan string
+		hostname         string
+		user             *user.User
+		ackCloseReceived chan struct{}
+		activeCommands   int32
+		quiet            bool
+		spartan          bool
+		serverless       bool
+		readBuf          bytes.Buffer
+		writeBuf         bytes.Buffer
+	*/
 }
 
 // NewServerHandler returns the server handler.
 func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
 	h := ServerHandler{
-		done:             internal.NewDone(),
-		lines:            make(chan line.Line, 100),
-		serverMessages:   make(chan string, 10),
-		maprMessages:     make(chan string, 10),
-		ackCloseReceived: make(chan struct{}),
-		catLimiter:       catLimiter,
-		tailLimiter:      tailLimiter,
-		regex:            ".",
-		user:             user,
-	}
+		baseHandler: baseHandler{
+			done:             internal.NewDone(),
+			lines:            make(chan line.Line, 100),
+			serverMessages:   make(chan string, 10),
+			maprMessages:     make(chan string, 10),
+			ackCloseReceived: make(chan struct{}),
+			user:             user,
+		},
+		catLimiter:  catLimiter,
+		tailLimiter: tailLimiter,
+		regex:       ".",
+	}
+	h.handleCommandCb = h.handleUserCommand
 
 	fqdn, err := os.Hostname()
 	if err != nil {
@@ -76,192 +67,8 @@ func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *S
 	return &h
 }
 
-// Shutdown the handler.
-func (h *ServerHandler) Shutdown() {
-	h.done.Shutdown()
-}
-
-// Done channel of the handler.
-func (h *ServerHandler) Done() <-chan struct{} {
-	return h.done.Done()
-}
-
-// Read is to send data to the dtail client via Reader interface.
-func (h *ServerHandler) Read(p []byte) (n int, err error) {
-	defer h.readBuf.Reset()
-
-	select {
-	case message := <-h.serverMessages:
-		if message[0] == '.' {
-			// Handle hidden message (don't display to the user, interpreted by dtail client)
-			h.readBuf.WriteString(message)
-			h.readBuf.WriteByte(protocol.MessageDelimiter)
-			n = copy(p, h.readBuf.Bytes())
-			return
-		}
-
-		if h.serverless {
-			// In serverless mode we have logged the server message already via the
-			// dlog logger, no need to send the message again to the client part.
-			return
-		}
-
-		// Handle normal server message (display to the user)
-		h.readBuf.WriteString("SERVER")
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(h.hostname)
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(message)
-		h.readBuf.WriteByte(protocol.MessageDelimiter)
-		n = copy(p, h.readBuf.Bytes())
-
-	case message := <-h.maprMessages:
-		// Send mapreduce-aggregated data as a message.
-		h.readBuf.WriteString("AGGREGATE")
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(h.hostname)
-		h.readBuf.WriteString(protocol.FieldDelimiter)
-		h.readBuf.WriteString(message)
-		h.readBuf.WriteByte(protocol.MessageDelimiter)
-		n = copy(p, h.readBuf.Bytes())
-
-	case line := <-h.lines:
-		if !h.spartan {
-			h.readBuf.WriteString("REMOTE")
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-			h.readBuf.WriteString(h.hostname)
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-			h.readBuf.WriteString(fmt.Sprintf("%3d", line.TransmittedPerc))
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-			h.readBuf.WriteString(fmt.Sprintf("%v", line.Count))
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-			h.readBuf.WriteString(line.SourceID)
-			h.readBuf.WriteString(protocol.FieldDelimiter)
-		}
-		h.readBuf.WriteString(line.Content.String())
-		h.readBuf.WriteByte(protocol.MessageDelimiter)
-		n = copy(p, h.readBuf.Bytes())
-		pool.RecycleBytesBuffer(line.Content)
-
-	case <-time.After(time.Second):
-		// Once in a while check whether we are done.
-		select {
-		case <-h.done.Done():
-			err = io.EOF
-			return
-		default:
-		}
-	}
-	return
-}
-
-// Write is to receive data from the dtail client via Writer interface.
-func (h *ServerHandler) Write(p []byte) (n int, err error) {
-	for _, b := range p {
-		switch b {
-		case ';':
-			h.handleCommand(string(h.writeBuf.Bytes()))
-			h.writeBuf.Reset()
-		default:
-			h.writeBuf.WriteByte(b)
-		}
-	}
-
-	n = len(p)
-	return
-}
-
-func (h *ServerHandler) handleCommand(commandStr string) {
-	dlog.Server.Debug(h.user, commandStr)
-	ctx := context.Background()
-
-	args, argc, add, err := h.handleProtocolVersion(strings.Split(commandStr, " "))
-	if err != nil {
-		h.send(h.serverMessages, dlog.Server.Error(h.user, err)+add)
-		return
-	}
-
-	args, argc, err = h.handleBase64(args, argc)
-	if err != nil {
-		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
-		return
-	}
-
-	if h.user.Name == config.ControlUser {
-		h.handleControlCommand(argc, args)
-		return
-	}
-
-	ctx, cancel := context.WithCancel(ctx)
-	go func() {
-		<-h.done.Done()
-		cancel()
-	}()
-
-	h.handleUserCommand(ctx, argc, args)
-}
-
-func (h *ServerHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
-	argc := len(args)
-	var add string
-
-	if argc <= 2 || args[0] != "protocol" {
-		return args, argc, add, errors.New("unable to determine protocol version")
-	}
-
-	if args[1] != protocol.ProtocolCompat {
-		clientCompat, _ := strconv.Atoi(args[1])
-		serverCompat, _ := strconv.Atoi(protocol.ProtocolCompat)
-		if clientCompat <= 3 {
-			// Protocol version 3 or lower expect a newline as message separator
-			// One day (after 2 major versions) this exception may be removed!
-			add = "\n"
-		}
-
-		toUpdate := "client"
-		if clientCompat > serverCompat {
-			toUpdate = "server"
-		}
-
-		err := fmt.Errorf("DTail server protocol version '%s' does not match client protocol version '%s', please update DTail %s!",
-			protocol.ProtocolCompat, args[1], toUpdate)
-		return args, argc, add, err
-	}
-
-	return args[2:], argc - 2, add, nil
-}
-
-func (h *ServerHandler) handleBase64(args []string, argc int) ([]string, int, error) {
-	err := errors.New("Unable to decode client message, DTail server and client versions may not be compatible")
-
-	if argc != 2 || args[0] != "base64" {
-		return args, argc, err
-	}
-
-	decoded, err := base64.StdEncoding.DecodeString(args[1])
-	if err != nil {
-		return args, argc, err
-	}
-	decodedStr := string(decoded)
-
-	args = strings.Split(decodedStr, " ")
-	argc = len(decodedStr)
-	dlog.Server.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
-
-	return args, argc, nil
-}
-
-func (h *ServerHandler) handleControlCommand(argc int, args []string) {
-	switch args[0] {
-	case "debug":
-		h.send(h.serverMessages, dlog.Server.Debug(h.user, "Receiving debug command", argc, args))
-	default:
-		dlog.Server.Warn(h.user, "Received unknown control command", argc, args)
-	}
-}
-
 func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string) {
-	dlog.Server.Debug(h.user, "handleUserCommand", argc, args)
+	dlog.Server.Debug(h.user, "Handling user command", argc, args)
 
 	h.incrementActiveCommands()
 	commandFinished := func() {
@@ -332,74 +139,3 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		commandFinished()
 	}
 }
-
-func (h *ServerHandler) handleAckCommand(argc int, args []string) {
-	if argc < 3 {
-		if !h.quiet {
-			h.send(h.serverMessages, dlog.Server.Warn(h.user, commandParseWarning, args, argc))
-		}
-		return
-	}
-	if args[1] == "close" && args[2] == "connection" {
-		select {
-		case <-h.ackCloseReceived:
-		default:
-			close(h.ackCloseReceived)
-		}
-	}
-}
-
-func (h *ServerHandler) send(ch chan<- string, message string) {
-	select {
-	case ch <- message:
-	case <-h.done.Done():
-	}
-}
-
-func (h *ServerHandler) flush() {
-	dlog.Server.Debug(h.user, "flush()")
-
-	unsentMessages := func() int {
-		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
-	}
-	for i := 0; i < 3; i++ {
-		if unsentMessages() == 0 {
-			dlog.Server.Debug(h.user, "All lines sent")
-			return
-		}
-		dlog.Server.Debug(h.user, "Still lines to be sent")
-		time.Sleep(time.Second)
-	}
-
-	dlog.Server.Warn(h.user, "Some lines remain unsent", unsentMessages())
-}
-
-func (h *ServerHandler) shutdown() {
-	dlog.Server.Debug(h.user, "shutdown()")
-	h.flush()
-
-	go func() {
-		select {
-		case h.serverMessages <- ".syn close connection":
-		case <-h.done.Done():
-		}
-	}()
-
-	select {
-	case <-h.ackCloseReceived:
-	case <-time.After(time.Second * 5):
-		dlog.Server.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
-	case <-h.done.Done():
-	}
-
-	h.done.Shutdown()
-}
-
-func (h *ServerHandler) incrementActiveCommands() {
-	atomic.AddInt32(&h.activeCommands, 1)
-}
-
-func (h *ServerHandler) decrementActiveCommands() int32 {
-	atomic.AddInt32(&h.activeCommands, -1)
-	return atomic.LoadInt32(&h.activeCommands)
-}
diff --git a/internal/source/source.go b/internal/source/source.go
index 73dccb2..be7aecd 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -3,8 +3,9 @@ package source
 type Source int
 
 const (
-	Client Source = iota
-	Server Source = iota
+	Client      Source = iota
+	Server      Source = iota
+	HealthCheck Source = iota
 )
 
 func (s Source) String() string {
@@ -13,7 +14,9 @@ func (s Source) String() string {
 		return "CLIENT"
 	case Server:
 		return "SERVER"
+	case HealthCheck:
+		return "HEALTHCHECK"
 	}
 
-	panic("Unknown log source type")
+	panic("Unknown source type")
 }
-- 
cgit v1.2.3


From 9f395a03f25941d8ed98ec43035688daa1e8877f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 5 Oct 2021 22:39:58 +0300
Subject: more on this

---
 internal/clients/baseclient.go            | 47 +++++++------------------------
 internal/clients/connectors/serverless.go |  1 +
 internal/clients/handlers/basehandler.go  | 23 ++++++---------
 internal/clients/maprclient.go            |  2 ++
 internal/config/client.go                 |  4 +--
 internal/server/handlers/basehandler.go   |  2 +-
 internal/server/handlers/serverhandler.go | 16 -----------
 7 files changed, 25 insertions(+), 70 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 5ac298f..9574e13 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -9,7 +9,6 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/discovery"
 	"github.com/mimecast/dtail/internal/io/dlog"
-	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 	"github.com/mimecast/dtail/internal/ssh/client"
 
@@ -80,13 +79,14 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	// Print client stats every time something on statsCh is recieved.
 	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet)
 
-	// Keep count of active connections
-	active := make(chan struct{}, len(c.connections))
+	var wg sync.WaitGroup
+	wg.Add(len(c.connections))
 
 	var mutex sync.Mutex
 	for i, conn := range c.connections {
 		go func(i int, conn connectors.Connector) {
-			connStatus := c.startConnection(ctx, active, i, conn)
+			defer wg.Done()
+			connStatus := c.startConnection(ctx, i, conn)
 
 			// Update global status.
 			mutex.Lock()
@@ -97,17 +97,11 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 		}(i, conn)
 	}
 
-	time.Sleep(time.Second * 2)
-	c.waitUntilDone(ctx, active)
+	wg.Wait()
 	return
 }
 
-func (c *baseClient) startConnection(ctx context.Context, active chan struct{}, i int, conn connectors.Connector) (status int) {
-	// Increment connection count
-	active <- struct{}{}
-	// Derement connection count
-	defer func() { <-active }()
-
+func (c *baseClient) startConnection(ctx context.Context, i int, conn connectors.Connector) (status int) {
 	for {
 		connCtx, cancel := context.WithCancel(ctx)
 		defer cancel()
@@ -127,33 +121,12 @@ func (c *baseClient) startConnection(ctx context.Context, active chan struct{},
 	}
 }
 
-func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod, hostKeyCallback client.HostKeyCallback) connectors.Connector {
+func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMethod,
+	hostKeyCallback client.HostKeyCallback) connectors.Connector {
 	if c.Args.Serverless {
-		return connectors.NewServerless(c.UserName, c.maker.makeHandler(server), c.maker.makeCommands())
+		return connectors.NewServerless(c.UserName, c.maker.makeHandler(server),
+			c.maker.makeCommands())
 	}
 	return connectors.NewServerConnection(server, c.UserName, sshAuthMethods,
 		hostKeyCallback, c.maker.makeHandler(server), c.maker.makeCommands())
 }
-
-func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
-	defer dlog.Client.Debug("Terminated connection")
-
-	// We want to have at least one active connection
-	<-active
-	// Put it back on the channel
-	active <- struct{}{}
-
-	if c.Mode == omode.TailClient && c.retry {
-		<-ctx.Done()
-	}
-
-	// TODO: Rewrite this to use a wait group.
-	for {
-		numActive := len(active)
-		if numActive == 0 {
-			return
-		}
-		dlog.Client.Debug("Active connections", numActive)
-		time.Sleep(time.Second * time.Millisecond * 100)
-	}
-}
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index ae72c9b..2a1cec4 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -69,6 +69,7 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 	}
 
 	terminate := func() {
+		dlog.Client.Debug("Terminating serverless connection")
 		serverHandler.Shutdown()
 		cancel()
 	}
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 8acb45f..04124e7 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -40,14 +40,6 @@ func (h *baseHandler) Status() int {
 	return h.status
 }
 
-func (h *baseHandler) Done() <-chan struct{} {
-	return h.done.Done()
-}
-
-func (h *baseHandler) Shutdown() {
-	h.done.Shutdown()
-}
-
 // SendMessage to the server.
 func (h *baseHandler) SendMessage(command string) error {
 	encoded := base64.StdEncoding.EncodeToString([]byte(command))
@@ -77,12 +69,6 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 		*/
 		case '\n', protocol.MessageDelimiter:
 			message := h.receiveBuf.String()
-			/*
-				// dcat/grep should actually display empty lines.
-					if len(message) == 0 {
-						continue
-					}
-			*/
 			h.handleMessageType(message)
 			h.receiveBuf.Reset()
 		default:
@@ -121,5 +107,14 @@ func (h *baseHandler) handleHiddenMessage(message string) {
 	switch {
 	case strings.HasPrefix(message, ".syn close connection"):
 		h.SendMessage(".ack close connection")
+		h.Shutdown()
 	}
 }
+
+func (h *baseHandler) Done() <-chan struct{} {
+	return h.done.Done()
+}
+
+func (h *baseHandler) Shutdown() {
+	h.done.Shutdown()
+}
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 19fe119..92bbe39 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -110,6 +110,8 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	return
 }
 
+// NEXT: Make this a callback function rather trying to use polymorphism to call this.
+// This applies to all clients.
 func (c MaprClient) makeHandler(server string) handlers.Handler {
 	return handlers.NewMaprHandler(server, c.query, c.globalGroup)
 }
diff --git a/internal/config/client.go b/internal/config/client.go
index 152ab15..ecd05c5 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -162,8 +162,8 @@ func newDefaultClientConfig() *ClientConfig {
 				HostnameBg:    color.BgCyan,
 				HostnameFg:    color.FgBlack,
 				TextAttr:      color.AttrNone,
-				TextBg:        color.BgCyan,
-				TextFg:        color.FgBlack,
+				TextBg:        color.BgBlack,
+				TextFg:        color.FgWhite,
 			},
 			Common: commonTermColors{
 				SeverityErrorAttr: color.AttrBold,
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 12fb2b3..b683578 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -242,7 +242,7 @@ func (h *baseHandler) flush() {
 
 	for i := 0; i < 3; i++ {
 		if numUnsentMessages() == 0 {
-			dlog.Server.Debug(h.user, "All lines sent")
+			dlog.Server.Debug(h.user, "All lines sent", fmt.Sprintf("%p", h))
 			return
 		}
 		dlog.Server.Debug(h.user, "Still lines to be sent")
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 2ec4fbf..25cb8ba 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -21,22 +21,6 @@ type ServerHandler struct {
 	catLimiter  chan struct{}
 	tailLimiter chan struct{}
 	regex       string
-	/*
-		done             *internal.Done
-		lines            chan line.Line
-		aggregate        *server.Aggregate
-		maprMessages     chan string
-		serverMessages   chan string
-		hostname         string
-		user             *user.User
-		ackCloseReceived chan struct{}
-		activeCommands   int32
-		quiet            bool
-		spartan          bool
-		serverless       bool
-		readBuf          bytes.Buffer
-		writeBuf         bytes.Buffer
-	*/
 }
 
 // NewServerHandler returns the server handler.
-- 
cgit v1.2.3


From fab5dc3e70434ea0abc7a0976487a1973b662331 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 6 Oct 2021 09:50:41 +0300
Subject: enable faster shutdown - useful for dgrep/dmap and dcat commands

---
 internal/clients/baseclient.go             | 14 ++---
 internal/clients/connectors/serverless.go  | 15 +++--
 internal/clients/handlers/basehandler.go   |  8 +--
 internal/clients/handlers/healthhandler.go | 14 ++---
 internal/clients/handlers/maprhandler.go   |  2 +-
 internal/clients/healthclient.go           |  4 +-
 internal/config/config.go                  |  4 +-
 internal/io/dlog/dlog.go                   |  2 +
 internal/io/fs/permissions/permission.go   |  2 +-
 internal/server/handlers/basehandler.go    | 20 ++++--
 internal/server/handlers/controlhandler.go | 98 ------------------------------
 internal/server/handlers/healthhandler.go  | 58 ++++++++++++++++++
 internal/server/handlers/serverhandler.go  | 16 ++---
 internal/server/server.go                  | 10 +--
 14 files changed, 118 insertions(+), 149 deletions(-)
 delete mode 100644 internal/server/handlers/controlhandler.go
 create mode 100644 internal/server/handlers/healthhandler.go

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index 9574e13..b474208 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -39,8 +39,7 @@ type baseClient struct {
 }
 
 func (c *baseClient) init() {
-	dlog.Client.Debug("Initiating base client")
-	dlog.Client.Debug(c.Args.String())
+	dlog.Client.Debug("Initiating base client", c.Args.String())
 
 	flag := regex.Default
 	if c.Args.RegexInvert {
@@ -48,15 +47,16 @@ func (c *baseClient) init() {
 	}
 	regex, err := regex.New(c.Args.RegexStr, flag)
 	if err != nil {
-		dlog.Client.FatalPanic(c.Regex, "invalid regex!", err, regex)
+		dlog.Client.FatalPanic(c.Regex, "Invalid regex!", err, regex)
 	}
 	c.Regex = regex
-	dlog.Client.Debug("Regex", c.Regex)
 
 	if c.Args.Serverless {
 		return
 	}
-	c.sshAuthMethods, c.hostKeyCallback = client.InitSSHAuthMethods(c.Args.SSHAuthMethods, c.Args.SSHHostKeyCallback, c.Args.TrustAllHosts, c.throttleCh, c.Args.PrivateKeyPathFile)
+	c.sshAuthMethods, c.hostKeyCallback = client.InitSSHAuthMethods(
+		c.Args.SSHAuthMethods, c.Args.SSHHostKeyCallback, c.Args.TrustAllHosts,
+		c.throttleCh, c.Args.PrivateKeyPathFile)
 }
 
 func (c *baseClient) makeConnections(maker maker) {
@@ -71,6 +71,7 @@ func (c *baseClient) makeConnections(maker maker) {
 }
 
 func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status int) {
+	dlog.Client.Trace("Starting base client")
 	// Can be nil when serverless.
 	if c.hostKeyCallback != nil {
 		// Periodically check for unknown hosts, and ask the user whether to trust them or not.
@@ -81,13 +82,12 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 
 	var wg sync.WaitGroup
 	wg.Add(len(c.connections))
-
 	var mutex sync.Mutex
+
 	for i, conn := range c.connections {
 		go func(i int, conn connectors.Connector) {
 			defer wg.Done()
 			connStatus := c.startConnection(ctx, i, conn)
-
 			// Update global status.
 			mutex.Lock()
 			defer mutex.Unlock()
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index 2a1cec4..768a5ce 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -37,6 +37,7 @@ func (s *Serverless) Handler() handlers.Handler {
 }
 
 func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+	dlog.Client.Debug("Starting serverless connector")
 	go func() {
 		defer cancel()
 
@@ -44,7 +45,6 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 			dlog.Client.Warn(err)
 		}
 	}()
-
 	<-ctx.Done()
 }
 
@@ -58,9 +58,11 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 
 	var serverHandler serverHandlers.Handler
 	switch s.userName {
-	case config.ControlUser:
-		serverHandler = serverHandlers.NewControlHandler(user)
+	case config.HealthUser:
+		dlog.Client.Debug("Creating serverless health handler")
+		serverHandler = serverHandlers.NewHealthHandler(user)
 	default:
+		dlog.Client.Debug("Creating serverless server handler")
 		serverHandler = serverHandlers.NewServerHandler(
 			user,
 			make(chan struct{}, config.Server.MaxConcurrentCats),
@@ -76,29 +78,34 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 
 	go func() {
 		io.Copy(serverHandler, s.handler)
+		dlog.Client.Trace("io.Copy(serverHandler, s.handler) => done")
 		terminate()
 	}()
 
 	go func() {
 		io.Copy(s.handler, serverHandler)
+		dlog.Client.Trace("io.Copy(s.handler, serverHandler) => done")
 		terminate()
 	}()
 
 	go func() {
 		select {
 		case <-s.handler.Done():
+			dlog.Client.Trace("<-s.handler.Done()")
 		case <-ctx.Done():
+			dlog.Client.Trace("<-ctx.Done()")
 		}
 		terminate()
 	}()
 
 	// Send all commands to client.
 	for _, command := range s.commands {
-		dlog.Client.Debug(command)
+		dlog.Client.Debug("Sending command to serverless server", command)
 		s.handler.SendMessage(command)
 	}
 
 	<-ctx.Done()
+	dlog.Client.Trace("s.handler.Shutdown()")
 	s.handler.Shutdown()
 
 	return nil
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 04124e7..b520c25 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -69,7 +69,7 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 		*/
 		case '\n', protocol.MessageDelimiter:
 			message := h.receiveBuf.String()
-			h.handleMessageType(message)
+			h.handleMessage(message)
 			h.receiveBuf.Reset()
 		default:
 			h.receiveBuf.WriteByte(b)
@@ -90,9 +90,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	return
 }
 
-// Handle various message types.
-func (h *baseHandler) handleMessageType(message string) {
-	// Hidden server commands starti with a dot "."
+func (h *baseHandler) handleMessage(message string) {
 	if len(message) > 0 && message[0] == '.' {
 		h.handleHiddenMessage(message)
 		return
@@ -106,7 +104,7 @@ func (h *baseHandler) handleMessageType(message string) {
 func (h *baseHandler) handleHiddenMessage(message string) {
 	switch {
 	case strings.HasPrefix(message, ".syn close connection"):
-		h.SendMessage(".ack close connection")
+		go h.SendMessage(".ack close connection")
 		h.Shutdown()
 	}
 }
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 4949985..4b16ce4 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -12,7 +12,6 @@ import (
 // HealthHandler is the handler used on the client side for running mapreduce aggregations.
 type HealthHandler struct {
 	baseHandler
-	HealthStatusCh chan<- int
 }
 
 // NewHealthHandler returns a new health client handler.
@@ -26,7 +25,6 @@ func NewHealthHandler(server string) *HealthHandler {
 			status:       -1,
 			done:         internal.NewDone(),
 		},
-		HealthStatusCh: make(chan int),
 	}
 }
 
@@ -34,12 +32,10 @@ func NewHealthHandler(server string) *HealthHandler {
 func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		switch b {
-		case '\n':
-			continue
-		case protocol.MessageDelimiter:
+		case '\n', protocol.MessageDelimiter:
 			message := h.baseHandler.receiveBuf.String()
 			dlog.Client.Debug(message)
-			h.handleHealthMessage(message)
+			h.handleMessage(message)
 			h.baseHandler.receiveBuf.Reset()
 		default:
 			h.baseHandler.receiveBuf.WriteByte(b)
@@ -49,7 +45,11 @@ func (h *HealthHandler) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-func (h *HealthHandler) handleHealthMessage(message string) {
+func (h *HealthHandler) handleMessage(message string) {
+	if len(message) > 0 && message[0] == '.' {
+		h.baseHandler.handleHiddenMessage(message)
+		return
+	}
 	s := strings.Split(message, protocol.FieldDelimiter)
 	message = s[len(s)-1]
 	status := strings.Split(message, ":")
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index 848e7f0..d1acfbd 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -44,7 +44,7 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 			if message[0] == 'A' {
 				h.handleAggregateMessage(message)
 			} else {
-				h.baseHandler.handleMessageType(message)
+				h.baseHandler.handleMessage(message)
 			}
 			h.baseHandler.receiveBuf.Reset()
 		default:
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index df919ae..e2c8ccb 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -18,7 +18,7 @@ type HealthClient struct {
 // NewHealthClient returns a new health client.
 func NewHealthClient(args config.Args) (*HealthClient, error) {
 	args.Mode = omode.HealthClient
-	args.UserName = config.ControlUser
+	args.UserName = config.HealthUser
 	c := HealthClient{
 		baseClient: baseClient{
 			Args:       args,
@@ -28,7 +28,7 @@ func NewHealthClient(args config.Args) (*HealthClient, error) {
 	}
 
 	c.init()
-	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.ControlUser))
+	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.HealthUser))
 	c.makeConnections(c)
 
 	return &c, nil
diff --git a/internal/config/config.go b/internal/config/config.go
index d58162f..f216688 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -3,8 +3,8 @@ package config
 import "github.com/mimecast/dtail/internal/source"
 
 const (
-	// ControlUser is used for various DTail specific operations.
-	ControlUser string = "DTAIL-CONTROL"
+	// HealthUser is used for the health check
+	HealthUser string = "DTAIL-HEALTH"
 	// ScheduleUser is used for non-interactive scheduled mapreduce queries.
 	ScheduleUser string = "DTAIL-SCHEDULE"
 	// ContinuousUser is used for non-interactive continuous mapreduce queries.
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index db99307..a17d6e9 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -201,6 +201,8 @@ func (d *DLog) Debug(args ...interface{}) string {
 }
 
 func (d *DLog) Trace(args ...interface{}) string {
+	_, file, line, _ := runtime.Caller(1)
+	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(TRACE, args)
 }
 
diff --git a/internal/io/fs/permissions/permission.go b/internal/io/fs/permissions/permission.go
index bbcb74e..e80dbb2 100644
--- a/internal/io/fs/permissions/permission.go
+++ b/internal/io/fs/permissions/permission.go
@@ -9,6 +9,6 @@ import (
 // ToRead is to check whether user has read permissions to a given file.
 func ToRead(user, filePath string) (bool, error) {
 	// Only implemented for Linux, always expect true
-	dlog.Common.Warn(user, filePath, "Not performing ACL check, not supported on this platform")
+	dlog.Common.Warn(user, filePath, "Not performing ACL check as not compiled in")
 	return true, nil
 }
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index b683578..4fa8f00 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
@@ -21,7 +22,7 @@ import (
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
-type handleCommandCb func(context.Context, int, []string)
+type handleCommandCb func(context.Context, int, []string, string, map[string]string)
 
 type baseHandler struct {
 	done             *internal.Done
@@ -157,7 +158,16 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		cancel()
 	}()
 
-	h.handleCommandCb(ctx, argc, args)
+	splitted := strings.Split(args[0], ":")
+	commandName := splitted[0]
+
+	options, err := config.DeserializeOptions(splitted[1:])
+	if err != nil {
+		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
+		return
+	}
+
+	h.handleCommandCb(ctx, argc, args, commandName, options)
 }
 
 func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
@@ -234,19 +244,19 @@ func (h *baseHandler) send(ch chan<- string, message string) {
 }
 
 func (h *baseHandler) flush() {
-	dlog.Server.Debug(h.user, "flush()")
+	dlog.Server.Trace(h.user, "flush()")
 
 	numUnsentMessages := func() int {
 		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
 	}
 
-	for i := 0; i < 3; i++ {
+	for i := 0; i < 10; i++ {
 		if numUnsentMessages() == 0 {
 			dlog.Server.Debug(h.user, "All lines sent", fmt.Sprintf("%p", h))
 			return
 		}
 		dlog.Server.Debug(h.user, "Still lines to be sent")
-		time.Sleep(time.Second)
+		time.Sleep(time.Millisecond * 10)
 	}
 
 	dlog.Server.Warn(h.user, "Some lines remain unsent", numUnsentMessages())
diff --git a/internal/server/handlers/controlhandler.go b/internal/server/handlers/controlhandler.go
deleted file mode 100644
index ae70675..0000000
--- a/internal/server/handlers/controlhandler.go
+++ /dev/null
@@ -1,98 +0,0 @@
-package handlers
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"strings"
-
-	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/io/dlog"
-	user "github.com/mimecast/dtail/internal/user/server"
-)
-
-// ControlHandler is used for control functions and health monitoring.
-type ControlHandler struct {
-	done           *internal.Done
-	hostname       string
-	payload        []byte
-	serverMessages chan string
-	user           *user.User
-}
-
-// NewControlHandler returns a new control handler.
-func NewControlHandler(user *user.User) *ControlHandler {
-	dlog.Server.Debug(user, "Creating control handler")
-
-	h := ControlHandler{
-		done:           internal.NewDone(),
-		serverMessages: make(chan string, 10),
-		user:           user,
-	}
-
-	fqdn, err := os.Hostname()
-	if err != nil {
-		dlog.Server.FatalPanic(err)
-	}
-
-	s := strings.Split(fqdn, ".")
-	h.hostname = s[0]
-
-	return &h
-}
-
-// Shutdown the handler.
-func (h *ControlHandler) Shutdown() {
-	h.done.Shutdown()
-}
-
-// Done channel of the handler.
-func (h *ControlHandler) Done() <-chan struct{} {
-	return h.done.Done()
-}
-
-// Read is to send data to the client via the Reader interface.
-func (h *ControlHandler) Read(p []byte) (n int, err error) {
-	for {
-		select {
-		case message := <-h.serverMessages:
-			wholePayload := []byte(fmt.Sprintf("SERVER|%s|%s\n", h.hostname, message))
-			n = copy(p, wholePayload)
-			return
-		case <-h.done.Done():
-			return 0, io.EOF
-		}
-	}
-}
-
-// Write is to read data to the client via the Writer interface.
-func (h *ControlHandler) Write(p []byte) (n int, err error) {
-	for _, c := range p {
-		switch c {
-		case ';':
-			wholePayload := strings.TrimSpace(string(h.payload))
-			h.handleCommand(wholePayload)
-			h.payload = nil
-
-		default:
-			h.payload = append(h.payload, c)
-		}
-	}
-
-	n = len(p)
-	return
-}
-
-func (h *ControlHandler) handleCommand(command string) {
-	dlog.Server.Info(h.user, command)
-	s := strings.Split(command, " ")
-	dlog.Server.Debug(h.user, "Receiving command", command, s)
-
-	switch s[0] {
-	case "health":
-		h.serverMessages <- "OK: DTail SSH Server seems fine"
-		h.serverMessages <- "done;"
-	default:
-		h.serverMessages <- dlog.Server.Error(h.user, "Received unknown control command", command, s)
-	}
-}
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
new file mode 100644
index 0000000..3f3b932
--- /dev/null
+++ b/internal/server/handlers/healthhandler.go
@@ -0,0 +1,58 @@
+package handlers
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	"github.com/mimecast/dtail/internal"
+	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/line"
+	user "github.com/mimecast/dtail/internal/user/server"
+)
+
+// HealthHandler is for the remote health check.
+type HealthHandler struct {
+	baseHandler
+}
+
+// NewHealthHandler returns the server handler.
+func NewHealthHandler(user *user.User) *HealthHandler {
+	dlog.Server.Debug(user, "Creating new server health handler")
+	h := HealthHandler{
+		baseHandler: baseHandler{
+			done:             internal.NewDone(),
+			lines:            make(chan line.Line, 100),
+			serverMessages:   make(chan string, 10),
+			maprMessages:     make(chan string, 10),
+			ackCloseReceived: make(chan struct{}),
+			user:             user,
+		},
+	}
+	h.handleCommandCb = h.handleHealthCommand
+
+	fqdn, err := os.Hostname()
+	if err != nil {
+		dlog.Server.FatalPanic(err)
+	}
+
+	s := strings.Split(fqdn, ".")
+	h.hostname = s[0]
+
+	return &h
+}
+
+func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int, args []string,
+	commandName string, options map[string]string) {
+	dlog.Server.Debug(h.user, "Handling health command", argc, args)
+
+	switch commandName {
+	case "health":
+		h.send(h.serverMessages, "OK: DTail SSH Server seems fine")
+	case "ack", ".ack":
+		h.handleAckCommand(argc, args)
+	default:
+		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown health command", commandName, argc, args))
+	}
+	h.shutdown()
+}
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 25cb8ba..aaffe14 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal"
-	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/omode"
@@ -25,6 +24,7 @@ type ServerHandler struct {
 
 // NewServerHandler returns the server handler.
 func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
+	dlog.Server.Debug(user, "Creating new server handler")
 	h := ServerHandler{
 		baseHandler: baseHandler{
 			done:             internal.NewDone(),
@@ -51,7 +51,9 @@ func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *S
 	return &h
 }
 
-func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string) {
+func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string,
+	commandName string, options map[string]string) {
+
 	dlog.Server.Debug(h.user, "Handling user command", argc, args)
 
 	h.incrementActiveCommands()
@@ -61,16 +63,6 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 		}
 	}
 
-	splitted := strings.Split(args[0], ":")
-	commandName := splitted[0]
-
-	options, err := config.DeserializeOptions(splitted[1:])
-	if err != nil {
-		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
-		commandFinished()
-		return
-	}
-
 	if quiet, _ := options["quiet"]; quiet == "true" {
 		dlog.Server.Debug(h.user, "Enabling quiet mode")
 		h.quiet = true
diff --git a/internal/server/server.go b/internal/server/server.go
index d1cd57d..b3d4bff 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -162,8 +162,8 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 		case "shell":
 			var handler handlers.Handler
 			switch user.Name {
-			case config.ControlUser:
-				handler = handlers.NewControlHandler(user)
+			case config.HealthUser:
+				handler = handlers.NewHealthHandler(user)
 			default:
 				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter)
 			}
@@ -234,9 +234,9 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	remoteIP := splitted[0]
 
 	switch user.Name {
-	case config.ControlUser:
-		if authInfo == config.ControlUser {
-			dlog.Server.Debug(user, "Granting permissions to control user")
+	case config.HealthUser:
+		if authInfo == config.HealthUser {
+			dlog.Server.Debug(user, "Granting permissions to health user")
 			return nil, nil
 		}
 	case config.ScheduleUser:
-- 
cgit v1.2.3


From 7306afe9ab073c424ddca0ddc57950f237948118 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 6 Oct 2021 10:55:50 +0300
Subject: move health check to separate client binary

---
 internal/clients/baseclient.go             |  1 -
 internal/clients/handlers/healthhandler.go | 25 ++++---------------------
 internal/clients/healthclient.go           | 29 +++++++++++++++++++++++++++++
 internal/config/initializer.go             | 10 +++++++++-
 internal/io/dlog/dlog.go                   |  2 +-
 internal/io/signal/signal.go               |  5 +++++
 internal/server/handlers/healthhandler.go  |  4 ++--
 internal/server/handlers/serverhandler.go  |  2 +-
 8 files changed, 51 insertions(+), 27 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index b474208..d5d7c2c 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -88,7 +88,6 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 		go func(i int, conn connectors.Connector) {
 			defer wg.Done()
 			connStatus := c.startConnection(ctx, i, conn)
-			// Update global status.
 			mutex.Lock()
 			defer mutex.Unlock()
 			if connStatus > status {
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 4b16ce4..10ba1f7 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -1,7 +1,6 @@
 package handlers
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/mimecast/dtail/internal"
@@ -22,7 +21,7 @@ func NewHealthHandler(server string) *HealthHandler {
 			server:       server,
 			shellStarted: false,
 			commands:     make(chan string),
-			status:       -1,
+			status:       2, // Assume CRITICAL status by default.
 			done:         internal.NewDone(),
 		},
 	}
@@ -34,14 +33,12 @@ func (h *HealthHandler) Write(p []byte) (n int, err error) {
 		switch b {
 		case '\n', protocol.MessageDelimiter:
 			message := h.baseHandler.receiveBuf.String()
-			dlog.Client.Debug(message)
 			h.handleMessage(message)
 			h.baseHandler.receiveBuf.Reset()
 		default:
 			h.baseHandler.receiveBuf.WriteByte(b)
 		}
 	}
-
 	return len(p), nil
 }
 
@@ -52,21 +49,7 @@ func (h *HealthHandler) handleMessage(message string) {
 	}
 	s := strings.Split(message, protocol.FieldDelimiter)
 	message = s[len(s)-1]
-	status := strings.Split(message, ":")
-	fmt.Println(status)
-	/*
-		switch status {
-		case "OK":
-			h.HealthStatusCh <- 0
-		case "WARNING":
-			h.HealthStatusCh <- 1
-		case "CRITICAL":
-			h.HealthStatusCh <- 2
-		case "UNKNOWN":
-			h.HealthStatusCh <- 3
-		default:
-			fmt.Println("CRITICAL: Unexpected server response: '%s'")
-			h.HealthStatusCh <- 2
-		}
-	*/
+	if message == "OK" {
+		h.baseHandler.status = 0
+	}
 }
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index e2c8ccb..ac1dc20 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -1,6 +1,8 @@
 package clients
 
 import (
+	"context"
+	"fmt"
 	"runtime"
 
 	"github.com/mimecast/dtail/internal/clients/handlers"
@@ -42,3 +44,30 @@ func (c HealthClient) makeCommands() (commands []string) {
 	commands = append(commands, "health")
 	return
 }
+
+func (c *HealthClient) Start(ctx context.Context, statsCh <-chan string) int {
+	status := c.baseClient.Start(ctx, statsCh)
+
+	switch status {
+	case 0:
+		if c.Serverless {
+			fmt.Printf("WARNING: All seems fine but the check only run in serverless mode, please specify a remote server via --server hostname:port\n")
+			return 1
+		}
+		fmt.Printf("OK: All fine at %s :-)\n", c.ServersStr)
+	case 2:
+		if c.Serverless {
+			fmt.Printf("CRITICAL: DTail server not operating properly (using serverless connction)!\n")
+			return 2
+		}
+		fmt.Printf("CRITICAL: DTail server not operating properly at %s!\n", c.ServersStr)
+	default:
+		if c.Serverless {
+			fmt.Printf("UNKNOWN: Received unknown status code %d (using serverless connection)\n", status)
+			return status
+		}
+		fmt.Printf("UNKNOWN: Received unknown status code %d from %s!\n", status, c.ServersStr)
+	}
+
+	return status
+}
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index a58f82a..ec758c8 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -96,11 +96,19 @@ func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, a
 		common.SSHPort = args.SSHPort
 	}
 
-	if args.Discovery == "" && args.ServersStr == "" {
+	if args.Discovery == "" && (args.ServersStr == "" ||
+		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
 	}
 
+	if sourceProcess == source.HealthCheck {
+		args.TrustAllHosts = true
+		if !args.Serverless && strings.ToLower(args.ServersStr) == "" {
+			args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort)
+		}
+	}
+
 	// Interpret additional args as file list.
 	if args.What == "" {
 		var files []string
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index a17d6e9..2ae3b04 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -59,7 +59,7 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Common = Server
 	case source.HealthCheck:
 		// Health check isn't logging anything.
-		impl := loggers.STDOUT
+		impl := loggers.NONE
 		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
 		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
 		Common = Client
diff --git a/internal/io/signal/signal.go b/internal/io/signal/signal.go
index 500c530..14056c4 100644
--- a/internal/io/signal/signal.go
+++ b/internal/io/signal/signal.go
@@ -44,3 +44,8 @@ func InterruptCh(ctx context.Context) <-chan string {
 
 	return statsCh
 }
+
+// NoCh doesn't listen on a signal.
+func NoCh(ctx context.Context) <-chan string {
+	return make(chan string)
+}
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
index 3f3b932..347ff66 100644
--- a/internal/server/handlers/healthhandler.go
+++ b/internal/server/handlers/healthhandler.go
@@ -48,8 +48,8 @@ func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int, args
 
 	switch commandName {
 	case "health":
-		h.send(h.serverMessages, "OK: DTail SSH Server seems fine")
-	case "ack", ".ack":
+		h.send(h.serverMessages, "OK")
+	case ".ack":
 		h.handleAckCommand(argc, args)
 	default:
 		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown health command", commandName, argc, args))
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index aaffe14..aed8956 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -106,7 +106,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			commandFinished()
 		}()
 
-	case "ack", ".ack":
+	case ".ack":
 		h.handleAckCommand(argc, args)
 		commandFinished()
 
-- 
cgit v1.2.3


From 2d7ddbeae8286373ac19787dc7dde598a7cb0598 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Fri, 8 Oct 2021 11:43:43 +0300
Subject: refactor

---
 internal/clients/maprclient.go          |  11 +--
 internal/config/args.go                 |   4 +
 internal/config/common.go               |  14 ++--
 internal/config/config.go               |  27 +++++--
 internal/config/initializer.go          | 131 ++++++++++++++++++++------------
 internal/io/dlog/dlog.go                |  53 ++++++-------
 internal/io/dlog/level.go               |  95 ++++++++++-------------
 internal/io/dlog/loggers/factory.go     |  36 ++++-----
 internal/mapr/logformat/default_test.go |   2 +-
 internal/server/continuous.go           |   4 +-
 internal/server/handlers/basehandler.go |   2 +-
 internal/server/scheduler.go            |   4 +-
 12 files changed, 204 insertions(+), 179 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 92bbe39..412a219 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -31,8 +31,6 @@ const (
 // MaprClient is used for running mapreduce aggregations on remote files.
 type MaprClient struct {
 	baseClient
-	// Query string for mapr aggregations
-	queryStr string
 	// Global group set for merged mapr aggregation results
 	globalGroup *mapr.GlobalGroupSet
 	// The query object (constructed from queryStr)
@@ -44,14 +42,14 @@ type MaprClient struct {
 }
 
 // NewMaprClient returns a new mapreduce client.
-func NewMaprClient(args config.Args, queryStr string, maprClientMode MaprClientMode) (*MaprClient, error) {
-	if queryStr == "" {
+func NewMaprClient(args config.Args, maprClientMode MaprClientMode) (*MaprClient, error) {
+	if args.QueryStr == "" {
 		return nil, errors.New("No mapreduce query specified, use '-query' flag")
 	}
 
-	query, err := mapr.NewQuery(queryStr)
+	query, err := mapr.NewQuery(args.QueryStr)
 	if err != nil {
-		dlog.Client.FatalPanic(queryStr, "Can't parse mapr query", err)
+		dlog.Client.FatalPanic(args.QueryStr, "Can't parse mapr query", err)
 	}
 
 	// Don't retry connection if in tail mode and no outfile specified.
@@ -77,7 +75,6 @@ func NewMaprClient(args config.Args, queryStr string, maprClientMode MaprClientM
 			retry:      retry,
 		},
 		query:      query,
-		queryStr:   queryStr,
 		cumulative: cumulative,
 	}
 
diff --git a/internal/config/args.go b/internal/config/args.go
index 3e2eb1f..a671ae3 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -17,10 +17,12 @@ type Args struct {
 	ConnectionsPerCPU  int
 	Discovery          string
 	LogDir             string
+	Logger             string
 	LogLevel           string
 	Mode               omode.Mode
 	NoColor            bool
 	PrivateKeyPathFile string
+	QueryStr           string
 	Quiet              bool
 	RegexInvert        bool
 	RegexStr           string
@@ -42,6 +44,7 @@ func (a *Args) String() string {
 	sb.WriteString("Args(")
 
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir))
+	sb.WriteString(fmt.Sprintf("%s:%s,", "Logger", a.Logger))
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile))
@@ -50,6 +53,7 @@ func (a *Args) String() string {
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "QueryStr", a.QueryStr))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr))
diff --git a/internal/config/common.go b/internal/config/common.go
index 5e81bc9..9d22c95 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -6,14 +6,14 @@ type CommonConfig struct {
 	SSHPort int
 	// Enable experimental features (mainly for dev purposes)
 	ExperimentalFeaturesEnable bool `json:",omitempty"`
+	// LogDir defines the log directory.
+	LogDir string
+	// Logger defines the name of the logger implementation.
+	Logger string
 	// LogLevel defines how much is logged.
 	LogLevel string `json:",omitempty"`
-	// The log strategy to use, one of
-	//   stdout: only log to stdout (useful when used with systemd)
-	//   daily: create a log file for every day
+	// LogStrategy defines the log rotation strategy.
 	LogStrategy string
-	// The log directory
-	LogDir string
 	// The cache directory
 	CacheDir string
 	// The temp directory
@@ -26,7 +26,9 @@ func newDefaultCommonConfig() *CommonConfig {
 		SSHPort:                    DefaultSSHPort,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
-		LogLevel:                   "INFO",
+		Logger:                     "stdout",
+		LogLevel:                   DefaultLogLevel,
+		LogStrategy:                "daily",
 		CacheDir:                   "cache",
 		TmpDir:                     "/tmp",
 	}
diff --git a/internal/config/config.go b/internal/config/config.go
index f216688..077a658 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -15,6 +15,14 @@ const (
 	DefaultConnectionsPerCPU int = 10
 	// DTailSSHServerDefaultPort is the default DServer port.
 	DefaultSSHPort int = 2222
+	// DefaultLogLevel specifies the default log level (obviously)
+	DefaultLogLevel string = "INFO"
+	// DefaultClientLogger specifies the default logger for the client commands.
+	DefaultClientLogger string = "fout"
+	// DefaultServerLogger specifies the default logger for dtail server.
+	DefaultServerLogger string = "file"
+	// DefaultHealthCheckLogger specifies the default logger used for health checks.
+	DefaultHealthCheckLogger string = "none"
 )
 
 // Client holds a DTail client configuration.
@@ -33,12 +41,15 @@ func Setup(sourceProcess source.Source, args *Args, additionalArgs []string) {
 		Server: newDefaultServerConfig(),
 		Client: newDefaultClientConfig(),
 	}
-	initializer.parseConfig(args)
-	Client, Server, Common = initializer.transformConfig(
-		sourceProcess,
-		args, additionalArgs,
-		initializer.Client,
-		initializer.Server,
-		initializer.Common,
-	)
+	if err := initializer.parseConfig(args); err != nil {
+		panic(err)
+	}
+	if err := initializer.transformConfig(sourceProcess, args, additionalArgs); err != nil {
+		panic(err)
+	}
+
+	// Make config accessible globally
+	Server = initializer.Server
+	Client = initializer.Client
+	Common = initializer.Common
 }
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index ec758c8..5247699 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -18,14 +18,15 @@ type initializer struct {
 	Client *ClientConfig
 }
 
-func (c *initializer) parseConfig(args *Args) {
+type transformCb func(*initializer, *Args, []string) error
+
+func (c *initializer) parseConfig(args *Args) error {
 	if strings.ToUpper(args.ConfigFile) == "NONE" {
-		return
+		return nil
 	}
 
 	if args.ConfigFile != "" {
-		c.parseSpecificConfig(args.ConfigFile)
-		return
+		return c.parseSpecificConfig(args.ConfigFile)
 	}
 
 	if homeDir, err := os.UserHomeDir(); err != nil {
@@ -38,85 +39,119 @@ func (c *initializer) parseConfig(args *Args) {
 			}
 		}
 	}
+
+	return nil
 }
 
-func (c *initializer) parseSpecificConfig(configFile string) {
+func (c *initializer) parseSpecificConfig(configFile string) error {
 	fd, err := os.Open(configFile)
 	if err != nil {
-		panic(fmt.Sprintf("Unable to read config file: %v", err))
+		return fmt.Errorf("Unable to read config file: %v", err)
 	}
 	defer fd.Close()
 
 	cfgBytes, err := ioutil.ReadAll(fd)
 	if err != nil {
-		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
+		return fmt.Errorf("Unable to read config file %s: %v", configFile, err)
 	}
 
-	err = json.Unmarshal([]byte(cfgBytes), c)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
+	if err := json.Unmarshal([]byte(cfgBytes), c); err != nil {
+		return fmt.Errorf("Unable to parse config file %s: %v", configFile, err)
 	}
+
+	return nil
 }
 
-func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string,
-	client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
-	if args.LogDir != "" {
-		common.LogDir = args.LogDir
-	}
-	if strings.Contains(common.LogDir, "~/") {
-		homeDir, err := os.UserHomeDir()
-		if err != nil {
-			panic(err)
-		}
-		common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
-	}
-	if common.LogStrategy == "" {
-		common.LogStrategy = "daily"
+func (i *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string) error {
+
+	switch sourceProcess {
+	case source.Server:
+		return i.optimusPrime(transformServer, args, additionalArgs)
+	case source.Client:
+		return i.optimusPrime(transformClient, args, additionalArgs)
+	case source.HealthCheck:
+		return i.optimusPrime(transformHealthCheck, args, additionalArgs)
+	default:
+		return fmt.Errorf("Unable to transform config, unknown source '%s'", sourceProcess)
 	}
+}
 
-	if args.Spartan {
-		args.Quiet = true
-		args.NoColor = true
-		if args.LogLevel == "" {
-			args.LogLevel = "ERROR"
-		}
+func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalArgs []string) error {
+	// Copy args to config objects.
+	if args.SSHPort != DefaultSSHPort {
+		i.Common.SSHPort = args.SSHPort
+	}
+	if args.LogLevel != DefaultLogLevel {
+		i.Common.LogLevel = args.LogLevel
 	}
 	if args.NoColor {
-		client.TermColorsEnable = false
+		i.Client.TermColorsEnable = false
 	}
-
-	if args.LogLevel != "" {
-		common.LogLevel = args.LogLevel
-	} else if sourceProcess == source.Client && args.ServersStr == "" && args.Discovery == "" {
-		// We are in serverless mode. Default log level is WARN.
-		common.LogLevel = "WARN"
+	if args.LogDir != "" {
+		i.Common.LogDir = args.LogDir
+	}
+	if args.Logger != "" {
+		i.Common.Logger = args.Logger
 	}
 
-	if args.SSHPort != DefaultSSHPort {
-		common.SSHPort = args.SSHPort
+	// Setup log directory.
+	if strings.Contains(i.Common.LogDir, "~/") {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			panic(err)
+		}
+		i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
 	}
 
+	// Serverless mode.
 	if args.Discovery == "" && (args.ServersStr == "" ||
 		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
+		i.Common.LogLevel = "WARN"
 	}
 
-	if sourceProcess == source.HealthCheck {
-		args.TrustAllHosts = true
-		if !args.Serverless && strings.ToLower(args.ServersStr) == "" {
-			args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort)
+	// Source type specific transormations.
+	sourceCb(i, args, additionalArgs)
+
+	// Spartan mode.
+	if args.Spartan {
+		args.Quiet = true
+		args.NoColor = true
+		i.Client.TermColorsEnable = false
+		if args.LogLevel == "" {
+			args.LogLevel = "ERROR"
+			i.Common.LogLevel = "ERROR"
 		}
 	}
-
-	// Interpret additional args as file list.
+	// Interpret additional args as file list or as query.
 	if args.What == "" {
 		var files []string
-		for _, file := range flag.Args() {
-			files = append(files, file)
+		for _, arg := range flag.Args() {
+			if args.QueryStr == "" && strings.Contains(strings.ToLower(arg), "select ") {
+				args.QueryStr = arg
+				continue
+			}
+			files = append(files, arg)
 		}
 		args.What = strings.Join(files, ",")
 	}
 
-	return client, server, common
+	return nil
+}
+
+func transformClient(i *initializer, args *Args, additionalArgs []string) error {
+	return nil
+}
+
+func transformServer(i *initializer, args *Args, additionalArgs []string) error {
+	return nil
+}
+
+func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error {
+	args.TrustAllHosts = true
+	if !args.Serverless && args.ServersStr == "" {
+		args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort)
+	}
+	return nil
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 2ae3b04..f3774ba 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -33,7 +33,7 @@ var mutex sync.Mutex
 var started bool
 
 // Start logger(s).
-func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source, logLevel string) {
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
@@ -41,27 +41,18 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Common.FatalPanic("Logger already started")
 	}
 
-	strategy := loggers.GetStrategy(config.Common.LogStrategy)
-	level := newLevel(logLevel)
-
 	switch sourceProcess {
 	case source.Client:
-		// This is a DTail client process running.
-		impl := loggers.FOUT
-		Client = New(source.Client, source.Client, level, impl, strategy)
-		Server = New(source.Client, source.Server, level, impl, strategy)
+		Client = New(source.Client, source.Client)
+		Server = New(source.Client, source.Server)
 		Common = Client
 	case source.Server:
-		// This is a DTail server process running.
-		impl := loggers.FILE
-		Client = New(source.Server, source.Client, level, impl, strategy)
-		Server = New(source.Server, source.Server, level, impl, strategy)
+		Client = New(source.Server, source.Client)
+		Server = New(source.Server, source.Server)
 		Common = Server
 	case source.HealthCheck:
-		// Health check isn't logging anything.
-		impl := loggers.NONE
-		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
-		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
+		Client = New(source.HealthCheck, source.Client)
+		Server = New(source.HealthCheck, source.Server)
 		Common = Client
 	}
 
@@ -93,16 +84,20 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source.Source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
+func New(sourceProcess, sourcePackage source.Source) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
 	}
+	strategy := loggers.GetStrategy(config.Common.LogStrategy)
+	loggerName := config.Common.Logger
+	level := newLevel(config.Common.LogLevel)
+
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), impl, strategy),
+		logger:        loggers.Factory(sourceProcess.String(), loggerName, strategy),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
-		maxLevel:      maxLevel,
+		maxLevel:      level,
 		hostname:      hostname,
 	}
 }
@@ -168,7 +163,7 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 }
 
 func (d *DLog) FatalPanic(args ...interface{}) {
-	d.log(FATAL, args)
+	d.log(Fatal, args)
 	d.Flush()
 
 	var sb strings.Builder
@@ -177,37 +172,37 @@ func (d *DLog) FatalPanic(args ...interface{}) {
 }
 
 func (d *DLog) Fatal(args ...interface{}) string {
-	return d.log(FATAL, args)
+	return d.log(Fatal, args)
 }
 
 func (d *DLog) Error(args ...interface{}) string {
-	return d.log(ERROR, args)
+	return d.log(Error, args)
 }
 
 func (d *DLog) Warn(args ...interface{}) string {
-	return d.log(WARN, args)
+	return d.log(Warn, args)
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	return d.log(INFO, args)
+	return d.log(Info, args)
 }
 
 func (d *DLog) Verbose(args ...interface{}) string {
-	return d.log(VERBOSE, args)
+	return d.log(Verbose, args)
 }
 
 func (d *DLog) Debug(args ...interface{}) string {
-	return d.log(DEBUG, args)
+	return d.log(Debug, args)
 }
 
 func (d *DLog) Trace(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
-	return d.log(TRACE, args)
+	return d.log(Trace, args)
 }
 
 func (d *DLog) Devel(args ...interface{}) string {
-	return d.log(DEVEL, args)
+	return d.log(Devel, args)
 }
 
 func (d *DLog) Raw(message string) string {
@@ -260,7 +255,7 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 		args[i] = fmt.Sprintf("%s=%v", k, v)
 		i++
 	}
-	return d.log(INFO, args)
+	return d.log(Info, args)
 }
 
 func (d *DLog) Flush()  { d.logger.Flush() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 84550f0..248ad83 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -8,80 +8,69 @@ import (
 type level int
 
 const (
-	FATAL   level = iota
-	ERROR   level = iota
-	WARN    level = iota
-	INFO    level = iota
-	DEFAULT level = iota
-	VERBOSE level = iota
-	DEBUG   level = iota
-	DEVEL   level = iota
-	TRACE   level = iota
-	ALL     level = iota
+	Fatal   level = iota
+	Error   level = iota
+	Warn    level = iota
+	Info    level = iota
+	Default level = iota
+	Verbose level = iota
+	Debug   level = iota
+	Devel   level = iota
+	Trace   level = iota
+	All     level = iota
 )
 
-var allLevels = []level{
-	FATAL,
-	ERROR,
-	WARN,
-	INFO,
-	DEFAULT,
-	VERBOSE,
-	DEBUG,
-	DEVEL,
-	TRACE,
-	ALL,
-}
+var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug, Devel, Trace, All}
 
 func newLevel(l string) level {
-	switch strings.ToUpper(l) {
-	case "FATAL":
-		return FATAL
-	case "ERROR":
-		return ERROR
-	case "WARN":
-		return WARN
-	case "INFO":
-		return INFO
+	switch strings.ToLower(l) {
+	case "fatal":
+		return Fatal
+	case "error":
+		return Error
+	case "warn":
+		return Warn
+	case "info":
+		return Info
 	case "":
 		fallthrough
-	case "DEFAULT":
-		return DEFAULT
-	case "VERBOSE":
-		return VERBOSE
-	case "DEBUG":
-		return DEBUG
-	case "DEVEL":
-		return DEVEL
-	case "TRACE":
-		return TRACE
-	case "ALL":
-		return ALL
+	case "default":
+		return Default
+	case "verbose":
+		return Verbose
+	case "debug":
+		return Debug
+	case "devel":
+		return Devel
+	case "trace":
+		return Trace
+	case "all":
+		return All
 	}
 	panic(fmt.Sprintf("Unknown log level %s, must be one of: %v", l, allLevels))
 }
 
 func (l level) String() string {
 	switch l {
-	case FATAL:
+	case Fatal:
 		return "FATAL"
-	case ERROR:
+	case Error:
 		return "ERROR"
-	case WARN:
+	case Warn:
 		return "WARN"
-	case INFO:
+	case Info:
 		return "INFO"
-	case DEFAULT:
+	case Default:
 		return "DEFAULT"
-	case VERBOSE:
+	case Verbose:
 		return "VERBOSE"
-	case DEBUG:
+	case Debug:
 		return "DEBUG"
-	case DEVEL:
+	case Devel:
 		return "DEVEL"
-	case TRACE:
+	case Trace:
 		return "TRACE"
-	case ALL:
+	case All:
 		return "ALL"
 	}
 
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 8697dc4..dda3ee6 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -2,45 +2,38 @@ package loggers
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 )
 
-type Impl int
-
-const (
-	NONE   Impl = iota
-	STDOUT Impl = iota
-	FILE   Impl = iota
-	FOUT   Impl = iota
-)
-
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
-func Factory(name string, impl Impl, strategy Strategy) Logger {
+func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("name:%s,fileBase:%s,impl:%v", name, strategy.FileBase, impl)
-
+	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName, rotationStrategy.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
 
 	singleton, ok := factoryMap[id]
 	if !ok {
-		switch impl {
-		case NONE:
+		switch strings.ToLower(loggerName) {
+		case "none":
 			singleton = none{}
-		case STDOUT:
+		case "stdout":
 			singleton = newStdout()
 			factoryMap[id] = singleton
-		case FILE:
-			singleton = newFile(strategy)
+		case "file":
+			singleton = newFile(rotationStrategy)
 			factoryMap[id] = singleton
-		case FOUT:
-			singleton = newFout(strategy)
+		case "fout":
+			singleton = newFout(rotationStrategy)
 			factoryMap[id] = singleton
+		default:
+			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
 		}
 	}
 
@@ -53,8 +46,7 @@ func FactoryRotate() {
 	if factoryMap == nil {
 		return
 	}
-
-	for _, impl := range factoryMap {
-		impl.Rotate()
+	for _, logger := range factoryMap {
+		logger.Rotate()
 	}
 }
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index 02c03a3..a777156 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -32,7 +32,7 @@ func TestDefaultLogFormat(t *testing.T) {
 		if val, ok := fields["$severity"]; !ok {
 			t.Errorf("Expected field '$severity', but no such field there in '%s'\n", input)
 		} else if val != "INFO" {
-			t.Errorf("Expected 'INFO' stored in field '$severity', but got '%s' in '%s'\n", val, input)
+			t.Errorf("Expected 'Info' stored in field '$severity', but got '%s' in '%s'\n", val, input)
 		}
 
 		if val, ok := fields["$time"]; !ok {
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index 87c8889..5f84afc 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -71,8 +71,8 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 
 	args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(job.Name))
 
-	query := fmt.Sprintf("%s outfile %s", job.Query, outfile)
-	client, err := clients.NewMaprClient(args, query, clients.NonCumulativeMode)
+	args.QueryStr = fmt.Sprintf("%s outfile %s", job.Query, outfile)
+	client, err := clients.NewMaprClient(args, clients.NonCumulativeMode)
 	if err != nil {
 		dlog.Server.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
 		return
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 4fa8f00..f73f82e 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -252,7 +252,7 @@ func (h *baseHandler) flush() {
 
 	for i := 0; i < 10; i++ {
 		if numUnsentMessages() == 0 {
-			dlog.Server.Debug(h.user, "All lines sent", fmt.Sprintf("%p", h))
+			dlog.Server.Debug(h.user, "ALL lines sent", fmt.Sprintf("%p", h))
 			return
 		}
 		dlog.Server.Debug(h.user, "Still lines to be sent")
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index 64e6573..ccb2225 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -82,8 +82,8 @@ func (s *scheduler) runJobs(ctx context.Context) {
 
 		args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(job.Name))
 
-		query := fmt.Sprintf("%s outfile %s", job.Query, outfile)
-		client, err := clients.NewMaprClient(args, query, clients.CumulativeMode)
+		args.QueryStr = fmt.Sprintf("%s outfile %s", job.Query, outfile)
+		client, err := clients.NewMaprClient(args, clients.CumulativeMode)
 		if err != nil {
 			dlog.Server.Error(fmt.Sprintf("Unable to create job %s", job.Name), err)
 			continue
-- 
cgit v1.2.3


From 7a7169791a64190e1002e38bc9c04ad0d5c1ce1f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 9 Oct 2021 16:44:28 +0300
Subject: add dtail health check unit test.

---
 internal/config/initializer.go    | 29 ++++++++++++++++----------
 internal/io/dlog/dlog.go          | 43 ++++++++++++++++-----------------------
 internal/io/dlog/loggers/file.go  | 10 +++++----
 internal/mapr/server/aggregate.go |  6 +++---
 4 files changed, 45 insertions(+), 43 deletions(-)

(limited to 'internal')

diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 5247699..e4cbeaf 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -93,6 +93,9 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA
 	if args.Logger != "" {
 		i.Common.Logger = args.Logger
 	}
+	if args.ConnectionsPerCPU == 0 {
+		args.ConnectionsPerCPU = DefaultConnectionsPerCPU
+	}
 
 	// Setup log directory.
 	if strings.Contains(i.Common.LogDir, "~/") {
@@ -103,14 +106,6 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA
 		i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
 	}
 
-	// Serverless mode.
-	if args.Discovery == "" && (args.ServersStr == "" ||
-		strings.ToLower(args.ServersStr) == "serverless") {
-		// We are not connecting to any servers.
-		args.Serverless = true
-		i.Common.LogLevel = "WARN"
-	}
-
 	// Source type specific transormations.
 	sourceCb(i, args, additionalArgs)
 
@@ -141,6 +136,14 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA
 }
 
 func transformClient(i *initializer, args *Args, additionalArgs []string) error {
+	// Serverless mode.
+	if args.Discovery == "" && (args.ServersStr == "" ||
+		strings.ToLower(args.ServersStr) == "serverless") {
+		// We are not connecting to any servers.
+		args.Serverless = true
+		i.Common.LogLevel = "warn"
+	}
+
 	return nil
 }
 
@@ -149,9 +152,13 @@ func transformServer(i *initializer, args *Args, additionalArgs []string) error
 }
 
 func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error {
-	args.TrustAllHosts = true
-	if !args.Serverless && args.ServersStr == "" {
-		args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort)
+	// Serverless mode.
+	if args.Discovery == "" && (args.ServersStr == "" ||
+		strings.ToLower(args.ServersStr) == "serverless") {
+		// We are not connecting to any servers.
+		args.Serverless = true
+		i.Common.LogLevel = "warn"
 	}
+	args.TrustAllHosts = true
 	return nil
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index f3774ba..28e6882 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -41,32 +41,25 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source)
 		Common.FatalPanic("Logger already started")
 	}
 
-	switch sourceProcess {
-	case source.Client:
-		Client = New(source.Client, source.Client)
-		Server = New(source.Client, source.Server)
-		Common = Client
-	case source.Server:
-		Client = New(source.Server, source.Client)
-		Server = New(source.Server, source.Server)
+	Client = new(sourceProcess, source.Client)
+	Server = new(sourceProcess, source.Server)
+	Common = Client
+	if sourceProcess == source.Server {
 		Common = Server
-	case source.HealthCheck:
-		Client = New(source.HealthCheck, source.Client)
-		Server = New(source.HealthCheck, source.Server)
-		Common = Client
 	}
 
 	var wg2 sync.WaitGroup
 	wg2.Add(2)
-	Client.start(ctx, &wg2)
-	Server.start(ctx, &wg2)
-	started = true
+	go Client.start(ctx, &wg2)
+	go Server.start(ctx, &wg2)
 
 	go rotation(ctx)
 	go func() {
 		wg2.Wait()
 		wg.Done()
 	}()
+
+	started = true
 }
 
 // DLog is the DTail logger.
@@ -83,8 +76,8 @@ type DLog struct {
 	hostname string
 }
 
-// New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source.Source) *DLog {
+// new creates a new DTail logger.
+func new(sourceProcess, sourcePackage source.Source) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
@@ -103,14 +96,12 @@ func New(sourceProcess, sourcePackage source.Source) *DLog {
 }
 
 func (d *DLog) start(ctx context.Context, wg *sync.WaitGroup) {
-	go func() {
-		defer wg.Done()
-		var wg2 sync.WaitGroup
-		wg2.Add(1)
-		d.logger.Start(ctx, &wg2)
-		<-ctx.Done()
-		wg2.Wait()
-	}()
+	defer wg.Done()
+	var wg2 sync.WaitGroup
+	wg2.Add(1)
+	d.logger.Start(ctx, &wg2)
+	<-ctx.Done()
+	wg2.Wait()
 }
 
 func (d *DLog) log(level level, args []interface{}) string {
@@ -202,6 +193,8 @@ func (d *DLog) Trace(args ...interface{}) string {
 }
 
 func (d *DLog) Devel(args ...interface{}) string {
+	_, file, line, _ := runtime.Caller(1)
+	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Devel, args)
 }
 
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 6e692a3..87280fd 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -126,7 +126,6 @@ func (f *file) getWriter(name string) *bufio.Writer {
 	if f.lastFileName == name {
 		return f.writer
 	}
-
 	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
 		if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil {
 			panic(err)
@@ -144,7 +143,7 @@ func (f *file) getWriter(name string) *bufio.Writer {
 		f.writer.Flush()
 		f.fd.Close()
 	}
-
+	// Set new writer.
 	f.fd = newFd
 	f.writer = bufio.NewWriterSize(f.fd, 1)
 	f.lastFileName = name
@@ -153,8 +152,11 @@ func (f *file) getWriter(name string) *bufio.Writer {
 }
 
 func (f *file) flush() {
-	defer f.writer.Flush()
-
+	defer func() {
+		if f.writer != nil {
+			f.writer.Flush()
+		}
+	}()
 	for {
 		select {
 		case m := <-f.bufferCh:
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 767aada..1f5d1c3 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -8,9 +8,8 @@ import (
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/dlog"
-	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/mapr"
 	"github.com/mimecast/dtail/internal/mapr/logformat"
 	"github.com/mimecast/dtail/internal/protocol"
@@ -148,7 +147,8 @@ func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]strin
 
 				maprLine := strings.TrimSpace(line.Content.String())
 				fields, err := a.parser.MakeFields(maprLine)
-				pool.RecycleBytesBuffer(line.Content)
+				// Can not recycle here for some rason.
+				//pool.RecycleBytesBuffer(line.Content)
 
 				if err != nil {
 					// Should fields be ignored anyway?
-- 
cgit v1.2.3


From 97747ea0f3178f7f5890512d483fdccaa82846b0 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 9 Oct 2021 21:10:29 +0300
Subject: vetting and linting and some code restyling

---
 internal/clients/baseclient.go                    |   7 +-
 internal/clients/catclient.go                     |   2 -
 internal/clients/connectors/serverconnection.go   |  48 +++++----
 internal/clients/connectors/serverless.go         |  16 +--
 internal/clients/grepclient.go                    |   5 +-
 internal/clients/handlers/healthhandler.go        |   3 +-
 internal/clients/handlers/maprhandler.go          |  13 ++-
 internal/clients/healthclient.go                  |  17 +--
 internal/clients/maprclient.go                    |  13 +--
 internal/clients/stats.go                         |  11 +-
 internal/clients/tailclient.go                    |   3 -
 internal/color/brush/brush.go                     |   8 +-
 internal/color/color.go                           |  11 +-
 internal/color/paint.go                           |  19 ++--
 internal/color/table.go                           |  19 +++-
 internal/config/args.go                           |   6 +-
 internal/config/client.go                         |  12 +--
 internal/config/config.go                         |   6 +-
 internal/config/initializer.go                    |  59 +++++-----
 internal/config/server.go                         |   7 +-
 internal/discovery/discovery.go                   |  13 +--
 internal/done.go                                  |   1 -
 internal/io/dlog/dlog.go                          |  22 +++-
 internal/io/dlog/level.go                         |   5 +-
 internal/io/dlog/loggers/factory.go               |   6 +-
 internal/io/dlog/loggers/file.go                  |  17 ++-
 internal/io/dlog/loggers/logger.go                |   1 +
 internal/io/dlog/loggers/strategy.go              |  11 +-
 internal/io/fs/catfile.go                         |   4 +-
 internal/io/fs/filereader.go                      |   3 +-
 internal/io/fs/permissions/permission_linuxacl.go |   2 +-
 internal/io/fs/readfile.go                        |  29 ++---
 internal/io/fs/tailfile.go                        |   4 +-
 internal/io/pool/builder.go                       |   3 +
 internal/io/pool/bytesbuffer.go                   |   3 +
 internal/io/prompt/prompt.go                      |   7 +-
 internal/io/signal/signal.go                      |   3 -
 internal/mapr/aggregateset.go                     |   4 -
 internal/mapr/client/aggregate.go                 |   9 +-
 internal/mapr/funcs/function.go                   |   7 +-
 internal/mapr/funcs/function_test.go              |  21 ++--
 internal/mapr/funcs/maskdigits.go                 |   2 -
 internal/mapr/globalgroupset.go                   |   8 --
 internal/mapr/groupset.go                         |   9 +-
 internal/mapr/logformat/default.go                |   5 +-
 internal/mapr/logformat/default_test.go           |  24 +++--
 internal/mapr/logformat/generickv.go              |   2 +-
 internal/mapr/logformat/parser.go                 |  12 +--
 internal/mapr/query.go                            |  17 ++-
 internal/mapr/query_test.go                       | 125 ++++++++++++++--------
 internal/mapr/selectcondition.go                  |  12 +--
 internal/mapr/server/aggregate.go                 |  30 +++---
 internal/mapr/setclause.go                        |   2 -
 internal/mapr/setcondition.go                     |  15 ++-
 internal/mapr/token.go                            |  11 +-
 internal/mapr/whereclause.go                      |  10 +-
 internal/mapr/wherecondition.go                   |  24 ++---
 internal/protocol/protocol.go                     |   1 -
 internal/regex/regex.go                           |   9 +-
 internal/regex/regex_test.go                      |  17 +--
 internal/server/continuous.go                     |   9 +-
 internal/server/handlers/basehandler.go           |  30 ++----
 internal/server/handlers/healthhandler.go         |  11 +-
 internal/server/handlers/mapcommand.go            |   7 +-
 internal/server/handlers/readcommand.go           |  41 ++++---
 internal/server/handlers/serverhandler.go         |  20 ++--
 internal/server/scheduler.go                      |   9 +-
 internal/server/server.go                         |  47 ++++----
 internal/server/stats.go                          |  11 +-
 internal/source/source.go                         |  14 ++-
 internal/ssh/client/authmethods.go                |  41 ++++---
 internal/ssh/client/customkeycallback.go          |   3 +-
 internal/ssh/client/knownhostscallback.go         |  19 +---
 internal/ssh/server/hostkey.go                    |   3 +-
 internal/ssh/server/publickeycallback.go          |  27 +++--
 internal/ssh/ssh.go                               |   4 -
 internal/user/name.go                             |   3 -
 internal/user/server/user.go                      |  30 +++---
 internal/version/version.go                       |   6 +-
 79 files changed, 584 insertions(+), 546 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go
index d5d7c2c..4a7bd84 100644
--- a/internal/clients/baseclient.go
+++ b/internal/clients/baseclient.go
@@ -64,7 +64,8 @@ func (c *baseClient) makeConnections(maker maker) {
 
 	discoveryService := discovery.New(c.Discovery, c.ServersStr, discovery.Shuffle)
 	for _, server := range discoveryService.ServerList() {
-		c.connections = append(c.connections, c.makeConnection(server, c.sshAuthMethods, c.hostKeyCallback))
+		c.connections = append(c.connections, c.makeConnection(server,
+			c.sshAuthMethods, c.hostKeyCallback))
 	}
 
 	c.stats = newTailStats(len(c.connections))
@@ -100,7 +101,9 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	return
 }
 
-func (c *baseClient) startConnection(ctx context.Context, i int, conn connectors.Connector) (status int) {
+func (c *baseClient) startConnection(ctx context.Context, i int,
+	conn connectors.Connector) (status int) {
+
 	for {
 		connCtx, cancel := context.WithCancel(ctx)
 		defer cancel()
diff --git a/internal/clients/catclient.go b/internal/clients/catclient.go
index 2726e7e..bd65560 100644
--- a/internal/clients/catclient.go
+++ b/internal/clients/catclient.go
@@ -22,7 +22,6 @@ func NewCatClient(args config.Args) (*CatClient, error) {
 	if args.RegexStr != "" {
 		return nil, errors.New("Can't use regex with 'cat' operating mode")
 	}
-
 	args.Mode = omode.CatClient
 
 	c := CatClient{
@@ -35,7 +34,6 @@ func NewCatClient(args config.Args) (*CatClient, error) {
 
 	c.init()
 	c.makeConnections(c)
-
 	return &c, nil
 }
 
diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 1666a79..2d7b45a 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -16,7 +16,8 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-// ServerConnection represents a connection to a single remote dtail server via SSH protocol.
+// ServerConnection represents a connection to a single remote dtail server via
+// SSH protocol.
 type ServerConnection struct {
 	server          string
 	port            int
@@ -28,9 +29,11 @@ type ServerConnection struct {
 }
 
 // NewServerConnection returns a new DTail SSH server connection.
-func NewServerConnection(server string, userName string, authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback, handler handlers.Handler, commands []string) *ServerConnection {
-	dlog.Client.Debug(server, "Creating new connection", server, handler, commands)
+func NewServerConnection(server string, userName string,
+	authMethods []ssh.AuthMethod, hostKeyCallback client.HostKeyCallback,
+	handler handlers.Handler, commands []string) *ServerConnection {
 
+	dlog.Client.Debug(server, "Creating new connection", server, handler, commands)
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
 		server:          server,
@@ -48,10 +51,12 @@ func NewServerConnection(server string, userName string, authMethods []ssh.AuthM
 	return &c
 }
 
+// Server returns the server hostname connected to.
 func (c *ServerConnection) Server() string {
 	return c.server
 }
 
+// Handler returns the handler used for the connection.
 func (c *ServerConnection) Handler() handlers.Handler {
 	return c.handler
 }
@@ -72,23 +77,29 @@ func (c *ServerConnection) initServerPort() {
 	}
 }
 
-func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+// Start the connection to the server.
+func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
+	throttleCh, statsCh chan struct{}) {
+
 	// Throttle how many connections can be established concurrently (based on ch length)
 	dlog.Client.Debug(c.server, "Throttling connection", len(throttleCh), cap(throttleCh))
 
 	select {
 	case throttleCh <- struct{}{}:
 	case <-ctx.Done():
-		dlog.Client.Debug(c.server, "Not establishing connection as context is done", len(throttleCh), cap(throttleCh))
+		dlog.Client.Debug(c.server, "Not establishing connection as context is done",
+			len(throttleCh), cap(throttleCh))
 		return
 	}
 
-	dlog.Client.Debug(c.server, "Throttling says that the connection can be established", len(throttleCh), cap(throttleCh))
+	dlog.Client.Debug(c.server, "Throttling says that the connection can be established",
+		len(throttleCh), cap(throttleCh))
 
 	go func() {
 		defer func() {
 			if !c.throttlingDone {
-				dlog.Client.Debug(c.server, "Unthrottling connection (1)", len(throttleCh), cap(throttleCh))
+				dlog.Client.Debug(c.server, "Unthrottling connection (1)",
+					len(throttleCh), cap(throttleCh))
 				c.throttlingDone = true
 				<-throttleCh
 			}
@@ -107,7 +118,9 @@ func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
 }
 
 // Dail into a new SSH connection. Close connection in case of an error.
-func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) error {
+func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
+	throttleCh, statsCh chan struct{}) error {
+
 	dlog.Client.Debug(c.server, "Incrementing connection stats")
 	statsCh <- struct{}{}
 	defer func() {
@@ -128,31 +141,30 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 }
 
 // Create the SSH session. Close the session in case of an error.
-func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc, client *ssh.Client, throttleCh chan struct{}) error {
-	dlog.Client.Debug(c.server, "Creating SSH session")
+func (c *ServerConnection) session(ctx context.Context, cancel context.CancelFunc,
+	client *ssh.Client, throttleCh chan struct{}) error {
 
+	dlog.Client.Debug(c.server, "Creating SSH session")
 	session, err := client.NewSession()
 	if err != nil {
 		return err
 	}
 	defer session.Close()
-
 	return c.handle(ctx, cancel, session, throttleCh)
 }
 
-func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc, session *ssh.Session, throttleCh chan struct{}) error {
-	dlog.Client.Debug(c.server, "Creating handler for SSH session")
+func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc,
+	session *ssh.Session, throttleCh chan struct{}) error {
 
+	dlog.Client.Debug(c.server, "Creating handler for SSH session")
 	stdinPipe, err := session.StdinPipe()
 	if err != nil {
 		return err
 	}
-
 	stdoutPipe, err := session.StdoutPipe()
 	if err != nil {
 		return err
 	}
-
 	if err := session.Shell(); err != nil {
 		return err
 	}
@@ -161,12 +173,10 @@ func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc
 		io.Copy(stdinPipe, c.handler)
 		cancel()
 	}()
-
 	go func() {
 		io.Copy(c.handler, stdoutPipe)
 		cancel()
 	}()
-
 	go func() {
 		select {
 		case <-c.handler.Done():
@@ -182,13 +192,13 @@ func (c *ServerConnection) handle(ctx context.Context, cancel context.CancelFunc
 	}
 
 	if !c.throttlingDone {
-		dlog.Client.Debug(c.server, "Unthrottling connection (2)", len(throttleCh), cap(throttleCh))
+		dlog.Client.Debug(c.server, "Unthrottling connection (2)",
+			len(throttleCh), cap(throttleCh))
 		c.throttlingDone = true
 		<-throttleCh
 	}
 
 	<-ctx.Done()
 	c.handler.Shutdown()
-
 	return nil
 }
diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index 768a5ce..2ff490a 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -18,8 +18,10 @@ type Serverless struct {
 	userName string
 }
 
-// NewServerConnection returns a new connection.
-func NewServerless(userName string, handler handlers.Handler, commands []string) *Serverless {
+// NewServerless starts a new serverless session.
+func NewServerless(userName string, handler handlers.Handler,
+	commands []string) *Serverless {
+
 	dlog.Client.Debug("Creating new serverless connector", handler, commands)
 	return &Serverless{
 		userName: userName,
@@ -28,15 +30,20 @@ func NewServerless(userName string, handler handlers.Handler, commands []string)
 	}
 }
 
+// Server returns serverless server indicator.
 func (s *Serverless) Server() string {
 	return "local(serverless)"
 }
 
+// Handler returns the handler used for the serverless connection.
 func (s *Serverless) Handler() handlers.Handler {
 	return s.handler
 }
 
-func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throttleCh, statsCh chan struct{}) {
+// Start the serverless connection.
+func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc,
+	throttleCh, statsCh chan struct{}) {
+
 	dlog.Client.Debug("Starting serverless connector")
 	go func() {
 		defer cancel()
@@ -81,13 +88,11 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 		dlog.Client.Trace("io.Copy(serverHandler, s.handler) => done")
 		terminate()
 	}()
-
 	go func() {
 		io.Copy(s.handler, serverHandler)
 		dlog.Client.Trace("io.Copy(s.handler, serverHandler) => done")
 		terminate()
 	}()
-
 	go func() {
 		select {
 		case <-s.handler.Done():
@@ -107,6 +112,5 @@ func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) erro
 	<-ctx.Done()
 	dlog.Client.Trace("s.handler.Shutdown()")
 	s.handler.Shutdown()
-
 	return nil
 }
diff --git a/internal/clients/grepclient.go b/internal/clients/grepclient.go
index ae21ff2..7521c67 100644
--- a/internal/clients/grepclient.go
+++ b/internal/clients/grepclient.go
@@ -12,7 +12,8 @@ import (
 	"github.com/mimecast/dtail/internal/omode"
 )
 
-// GrepClient searches a remote file for all lines matching a regular expression. Only the matching lines are displayed.
+// GrepClient searches a remote file for all lines matching a regular
+// expression. Only the matching lines are displayed.
 type GrepClient struct {
 	baseClient
 }
@@ -34,7 +35,6 @@ func NewGrepClient(args config.Args) (*GrepClient, error) {
 
 	c.init()
 	c.makeConnections(c)
-
 	return &c, nil
 }
 
@@ -51,6 +51,5 @@ func (c GrepClient) makeCommands() (commands []string) {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
 			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
-
 	return
 }
diff --git a/internal/clients/handlers/healthhandler.go b/internal/clients/handlers/healthhandler.go
index 10ba1f7..47b594e 100644
--- a/internal/clients/handlers/healthhandler.go
+++ b/internal/clients/handlers/healthhandler.go
@@ -8,7 +8,8 @@ import (
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// HealthHandler is the handler used on the client side for running mapreduce aggregations.
+// HealthHandler is the handler used on the client side for running mapreduce
+// aggregations.
 type HealthHandler struct {
 	baseHandler
 }
diff --git a/internal/clients/handlers/maprhandler.go b/internal/clients/handlers/maprhandler.go
index d1acfbd..8718b35 100644
--- a/internal/clients/handlers/maprhandler.go
+++ b/internal/clients/handlers/maprhandler.go
@@ -10,7 +10,8 @@ import (
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// MaprHandler is the handler used on the client side for running mapreduce aggregations.
+// MaprHandler is the handler used on the client side for running mapreduce
+// aggregations.
 type MaprHandler struct {
 	baseHandler
 	aggregate *client.Aggregate
@@ -18,7 +19,9 @@ type MaprHandler struct {
 }
 
 // NewMaprHandler returns a new mapreduce client handler.
-func NewMaprHandler(server string, query *mapr.Query, globalGroup *mapr.GlobalGroupSet) *MaprHandler {
+func NewMaprHandler(server string, query *mapr.Query,
+	globalGroup *mapr.GlobalGroupSet) *MaprHandler {
+
 	return &MaprHandler{
 		baseHandler: baseHandler{
 			server:       server,
@@ -55,12 +58,12 @@ func (h *MaprHandler) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-// Handle a message received from server including mapr aggregation
-// related data.
+// Handle a message received from server including mapr aggregation related data.
 func (h *MaprHandler) handleAggregateMessage(message string) {
 	parts := strings.SplitN(message, protocol.FieldDelimiter, 3)
 	if len(parts) != 3 {
-		dlog.Client.Error("Unable to aggregate data", h.server, message, parts, len(parts), "expected 3 parts")
+		dlog.Client.Error("Unable to aggregate data", h.server, message, parts,
+			len(parts), "expected 3 parts")
 		return
 	}
 	if err := h.aggregate.Aggregate(parts[2]); err != nil {
diff --git a/internal/clients/healthclient.go b/internal/clients/healthclient.go
index ac1dc20..1a02827 100644
--- a/internal/clients/healthclient.go
+++ b/internal/clients/healthclient.go
@@ -32,7 +32,6 @@ func NewHealthClient(args config.Args) (*HealthClient, error) {
 	c.init()
 	c.sshAuthMethods = append(c.sshAuthMethods, gossh.Password(config.HealthUser))
 	c.makeConnections(c)
-
 	return &c, nil
 }
 
@@ -45,28 +44,34 @@ func (c HealthClient) makeCommands() (commands []string) {
 	return
 }
 
+// Start the health client.
 func (c *HealthClient) Start(ctx context.Context, statsCh <-chan string) int {
 	status := c.baseClient.Start(ctx, statsCh)
 
 	switch status {
 	case 0:
 		if c.Serverless {
-			fmt.Printf("WARNING: All seems fine but the check only run in serverless mode, please specify a remote server via --server hostname:port\n")
+			fmt.Printf("WARNING: All seems fine but the check only run in serverless mode" +
+				", please specify a remote server via --server hostname:port\n")
 			return 1
 		}
 		fmt.Printf("OK: All fine at %s :-)\n", c.ServersStr)
 	case 2:
 		if c.Serverless {
-			fmt.Printf("CRITICAL: DTail server not operating properly (using serverless connction)!\n")
+			fmt.Printf("CRITICAL: DTail server not operating properly (using " +
+				"serverless connction)!\n")
 			return 2
 		}
-		fmt.Printf("CRITICAL: DTail server not operating properly at %s!\n", c.ServersStr)
+		fmt.Printf("CRITICAL: DTail server not operating properly at %s!\n",
+			c.ServersStr)
 	default:
 		if c.Serverless {
-			fmt.Printf("UNKNOWN: Received unknown status code %d (using serverless connection)\n", status)
+			fmt.Printf("UNKNOWN: Received unknown status code %d (using serverless "+
+				"connection)\n", status)
 			return status
 		}
-		fmt.Printf("UNKNOWN: Received unknown status code %d from %s!\n", status, c.ServersStr)
+		fmt.Printf("UNKNOWN: Received unknown status code %d from %s!\n",
+			status, c.ServersStr)
 	}
 
 	return status
diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 412a219..04f258d 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -107,15 +107,14 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	return
 }
 
-// NEXT: Make this a callback function rather trying to use polymorphism to call this.
-// This applies to all clients.
+// NEXT: Make this a callback function rather trying to use polymorphism to call
+// this. This applies to all clients.
 func (c MaprClient) makeHandler(server string) handlers.Handler {
 	return handlers.NewMaprHandler(server, c.query, c.globalGroup)
 }
 
 func (c MaprClient) makeCommands() (commands []string) {
 	commands = append(commands, fmt.Sprintf("map %s", c.query.RawQuery))
-
 	modeStr := "cat"
 	if c.Mode == omode.TailClient {
 		modeStr = "tail"
@@ -134,7 +133,6 @@ func (c MaprClient) makeCommands() (commands []string) {
 		commands = append(commands, fmt.Sprintf("%s:%s %s %s",
 			modeStr, c.Args.SerializeOptions(), file, regex))
 	}
-
 	return
 }
 
@@ -155,7 +153,6 @@ func (c *MaprClient) reportResults() {
 		c.writeResultsToOutfile()
 		return
 	}
-
 	c.printResults()
 }
 
@@ -176,7 +173,6 @@ func (c *MaprClient) printResults() {
 	} else {
 		result, numRows, err = c.globalGroup.SwapOut().Result(c.query, rowsLimit)
 	}
-
 	if err != nil {
 		dlog.Client.FatalPanic(err)
 	}
@@ -202,8 +198,8 @@ func (c *MaprClient) printResults() {
 	dlog.Client.Raw(rawQuery)
 
 	if rowsLimit > 0 && numRows > rowsLimit {
-		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited terminal output to %d rows! Use 'limit' clause to override!",
-			numRows, rowsLimit))
+		dlog.Client.Warn(fmt.Sprintf("Got %d results but limited terminal output "+
+			"to %d rows! Use 'limit' clause to override!", numRows, rowsLimit))
 	}
 	dlog.Client.Raw(result)
 }
@@ -215,7 +211,6 @@ func (c *MaprClient) writeResultsToOutfile() {
 		}
 		return
 	}
-
 	if err := c.globalGroup.SwapOut().WriteResult(c.query); err != nil {
 		dlog.Client.FatalPanic(err)
 	}
diff --git a/internal/clients/stats.go b/internal/clients/stats.go
index fbef572..1315aea 100644
--- a/internal/clients/stats.go
+++ b/internal/clients/stats.go
@@ -36,9 +36,10 @@ func newTailStats(servers int) *stats {
 
 // Start starts printing client connection stats every time a signal is recieved or
 // connection count has changed.
-func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string, quiet bool) {
-	var connectedLast int
+func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{},
+	statsCh <-chan string, quiet bool) {
 
+	var connectedLast int
 	for {
 		var force bool
 		var messages []string
@@ -94,7 +95,9 @@ func (s *stats) printStatsDueInterrupt(messages []string) {
 	dlog.Client.Resume()
 }
 
-func (s *stats) statsData(connected, newConnections int, throttle int) map[string]interface{} {
+func (s *stats) statsData(connected, newConnections int,
+	throttle int) map[string]interface{} {
+
 	percConnected := percentOf(float64(s.servers), float64(connected))
 
 	data := make(map[string]interface{})
@@ -112,7 +115,6 @@ func (s *stats) statsData(connected, newConnections int, throttle int) map[strin
 
 func (s *stats) statsLine(connected, newConnections int, throttle int) string {
 	sb := strings.Builder{}
-
 	i := 0
 	for k, v := range s.statsData(connected, newConnections, throttle) {
 		if i > 0 {
@@ -123,7 +125,6 @@ func (s *stats) statsLine(connected, newConnections int, throttle int) string {
 		sb.WriteString(fmt.Sprintf("%v", v))
 		i++
 	}
-
 	return sb.String()
 }
 
diff --git a/internal/clients/tailclient.go b/internal/clients/tailclient.go
index d42a0e4..35c01d4 100644
--- a/internal/clients/tailclient.go
+++ b/internal/clients/tailclient.go
@@ -19,7 +19,6 @@ type TailClient struct {
 // NewTailClient returns a new TailClient.
 func NewTailClient(args config.Args) (*TailClient, error) {
 	args.Mode = omode.TailClient
-
 	c := TailClient{
 		baseClient: baseClient{
 			Args:       args,
@@ -30,7 +29,6 @@ func NewTailClient(args config.Args) (*TailClient, error) {
 
 	c.init()
 	c.makeConnections(c)
-
 	return &c, nil
 }
 
@@ -48,6 +46,5 @@ func (c TailClient) makeCommands() (commands []string) {
 			c.Mode.String(), c.Args.SerializeOptions(), file, regex))
 	}
 	dlog.Client.Debug(commands)
-
 	return
 }
diff --git a/internal/color/brush/brush.go b/internal/color/brush/brush.go
index 82fa410..63d63d8 100644
--- a/internal/color/brush/brush.go
+++ b/internal/color/brush/brush.go
@@ -32,7 +32,6 @@ func paintSeverity(sb *strings.Builder, text string) bool {
 	default:
 		return false
 	}
-
 	return true
 }
 
@@ -87,9 +86,9 @@ func paintRemote(sb *strings.Builder, line string) {
 		config.Client.TermColors.Remote.DelimiterAttr)
 
 	color.PaintWithAttr(sb, splitted[4],
-		config.Client.TermColors.Remote.IdFg,
-		config.Client.TermColors.Remote.IdBg,
-		config.Client.TermColors.Remote.IdAttr)
+		config.Client.TermColors.Remote.IDFg,
+		config.Client.TermColors.Remote.IDBg,
+		config.Client.TermColors.Remote.IDAttr)
 	color.PaintWithAttr(sb, protocol.FieldDelimiter,
 		config.Client.TermColors.Remote.DelimiterFg,
 		config.Client.TermColors.Remote.DelimiterBg,
@@ -191,6 +190,5 @@ func Colorfy(line string) string {
 			color.BgDefault,
 			color.AttrNone)
 	}
-
 	return sb.String()
 }
diff --git a/internal/color/color.go b/internal/color/color.go
index 5c25855..9d0bc2e 100644
--- a/internal/color/color.go
+++ b/internal/color/color.go
@@ -52,15 +52,19 @@ const (
 	AttrHidden     Attribute = escape + "[8m"
 )
 
+// ColorNames is the list of all supported terminal colors.
 var ColorNames = []string{
 	"Black", "Red", "Green", "Yellow", "Blue", "Magenta", "Cyan", "White", "Default",
 }
 
+// AttributeNames is the list of all supported terminal text attributes.
 var AttributeNames = []string{
-	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink", "Reverse", "Hidden", "None",
+	"Bold", "Dim", "Italic", "Underline", "Blink", "SlowBlink", "RapidBlink",
+	"Reverse", "Hidden", "None",
 }
 
-// ToFgColor converts a given string (e.g. from a config file) into a foreground color code.
+// ToFgColor converts a given string (e.g. from a config file) into a foreground
+// color code.
 func ToFgColor(s string) (FgColor, error) {
 	switch strings.ToLower(s) {
 	case "black":
@@ -86,7 +90,8 @@ func ToFgColor(s string) (FgColor, error) {
 	}
 }
 
-// ToBgColor converts a given string (e.g. from a config file) into a background color code.
+// ToBgColor converts a given string (e.g. from a config file) into a background
+// color code.
 func ToBgColor(s string) (BgColor, error) {
 	switch strings.ToLower(s) {
 	case "black":
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 53c9abb..7735d87 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -10,12 +10,14 @@ func PaintStr(text string, fg FgColor, bg BgColor) string {
 	return fmt.Sprintf("%s%s%s%s%s", fg, bg, text, BgDefault, FgDefault)
 }
 
-// PaintStrWithAttr paints a given text in a given foreground/background/attribute combination
+// PaintStrWithAttr paints a given text in a given foreground/background/attribute
+// combination
 func PaintStrWithAttr(text string, fg FgColor, bg BgColor, attr Attribute) string {
 	if attr == AttrNone {
 		return PaintStr(text, fg, bg)
 	}
-	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset, BgDefault, FgDefault)
+	return fmt.Sprintf("%s%s%s%s%s%s%s", fg, bg, attr, text, AttrReset,
+		BgDefault, FgDefault)
 }
 
 // PaintStrFg paints a given text in a given foreground color.
@@ -48,8 +50,11 @@ func Reset(sb *strings.Builder) {
 	sb.WriteString(string(FgDefault))
 }
 
-// PaintWithAttr starts painting a given text in a given foreground/background/attribute combination.
-func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, attr Attribute) {
+// PaintWithAttr starts painting a given text in a given foreground/background/
+// attribute combination.
+func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor,
+	attr Attribute) {
+
 	if attr == AttrNone {
 		Paint(sb, text, fg, bg)
 		return
@@ -63,8 +68,10 @@ func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor, att
 	sb.WriteString(string(FgDefault))
 }
 
-// PaintWithAttrs is similar to PaintWithAttr, but it takes multiple text attributes.
-func PaintWithAttrs(sb *strings.Builder, text string, fg FgColor, bg BgColor, attrs []Attribute) {
+// PaintWithAttrs is similar to PaintWithAttr, but it takes multiple attributes.
+func PaintWithAttrs(sb *strings.Builder, text string, fg FgColor, bg BgColor,
+	attrs []Attribute) {
+
 	sb.WriteString(string(fg))
 	sb.WriteString(string(bg))
 	for _, attr := range attrs {
diff --git a/internal/color/table.go b/internal/color/table.go
index 2115edf..e0e4946 100644
--- a/internal/color/table.go
+++ b/internal/color/table.go
@@ -5,8 +5,19 @@ import (
 	"os"
 )
 
-const sampleParagraph string = "Mimecast is Making Email Safer for Business. We believe that securely operating a business in the cloud requires new levels of IT preparedness, centered around cyber resilience. This is why we unify the delivery and management of security, continuity and data protection for email via one, simple-to-use cloud platform. Thousands of organizations trust us to increase their cyber resilience preparedness, streamline compliance, reduce IT complexity and keep their business running. We give employees fast and secure access to sensitive business information, and ensure email keeps flowing in the event of an outage. Mimecast will remain committed to protecting your IT assets through constant innovation and focus on your success."
+const sampleParagraph string = "Mimecast is Making Email Safer for Business. " +
+	"We believe that securely operating a business in the cloud requires new " +
+	"levels of IT preparedness, centered around cyber resilience. This is why " +
+	"we unify the delivery and management of security, continuity and data " +
+	"protection for email via one, simple-to-use cloud platform. Thousands of " +
+	"organizations trust us to increase their cyber resilience preparedness, " +
+	"streamline compliance, reduce IT complexity and keep their business running. " +
+	"We give employees fast and secure access to sensitive business information, " +
+	"and ensure email keeps flowing in the event of an outage. Mimecast will " +
+	"remain committed to protecting your IT assets through constant innovation " +
+	"and focus on your success."
 
+// TablePrintAndExit prints the color table and then exits the process.
 func TablePrintAndExit(displaySampleParagraph bool) {
 	for _, attr := range AttributeNames {
 		if attr == "Hidden" || attr == "SlowBlink" {
@@ -24,11 +35,13 @@ func printColorTable(attr string, displaySampleParagraph bool) {
 			if fg == bg {
 				continue
 			}
+
 			bgColor, _ := ToBgColor(bg)
 			attribute, _ := ToAttribute(attr)
-
-			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ", fg, bg, attr)
+			text := fmt.Sprintf(" Foreground:%10s  |  Background:%10s  |  Attribute:%10s ",
+				fg, bg, attr)
 			fmt.Print(PaintStrWithAttr(text, fgColor, bgColor, attribute))
+
 			if displaySampleParagraph {
 				fmt.Print("\n")
 				fmt.Print(PaintStrWithAttr(sampleParagraph, fgColor, bgColor, attribute))
diff --git a/internal/config/args.go b/internal/config/args.go
index a671ae3..f721390 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -74,13 +74,13 @@ func (a *Args) String() string {
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
 func (a *Args) SerializeOptions() string {
-	return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, a.Serverless)
+	return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan,
+		a.Serverless)
 }
 
 // DeserializeOptions deserializes the options, but into a map.
 func DeserializeOptions(opts []string) (map[string]string, error) {
 	options := make(map[string]string, len(opts))
-
 	for _, o := range opts {
 		kv := strings.SplitN(o, "=", 2)
 		if len(kv) != 2 {
@@ -97,9 +97,7 @@ func DeserializeOptions(opts []string) (map[string]string, error) {
 			}
 			val = string(decoded)
 		}
-
 		options[key] = val
 	}
-
 	return options, nil
 }
diff --git a/internal/config/client.go b/internal/config/client.go
index ecd05c5..8227c68 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -15,9 +15,9 @@ type remoteTermColors struct {
 	HostnameAttr  color.Attribute
 	HostnameBg    color.BgColor
 	HostnameFg    color.FgColor
-	IdAttr        color.Attribute
-	IdBg          color.BgColor
-	IdFg          color.FgColor
+	IDAttr        color.Attribute
+	IDBg          color.BgColor
+	IDFg          color.FgColor
 	StatsOkAttr   color.Attribute
 	StatsOkBg     color.BgColor
 	StatsOkFg     color.FgColor
@@ -124,9 +124,9 @@ func newDefaultClientConfig() *ClientConfig {
 				HostnameAttr:  color.AttrBold,
 				HostnameBg:    color.BgBlue,
 				HostnameFg:    color.FgWhite,
-				IdAttr:        color.AttrDim,
-				IdBg:          color.BgBlue,
-				IdFg:          color.FgWhite,
+				IDAttr:        color.AttrDim,
+				IDBg:          color.BgBlue,
+				IDFg:          color.FgWhite,
 				StatsOkAttr:   color.AttrNone,
 				StatsOkBg:     color.BgGreen,
 				StatsOkFg:     color.FgBlack,
diff --git a/internal/config/config.go b/internal/config/config.go
index 077a658..b99b22b 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -9,11 +9,11 @@ const (
 	ScheduleUser string = "DTAIL-SCHEDULE"
 	// ContinuousUser is used for non-interactive continuous mapreduce queries.
 	ContinuousUser string = "DTAIL-CONTINUOUS"
-	// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
+	// InterruptTimeoutS specifies the Ctrl+C log pause interval.
 	InterruptTimeoutS int = 3
-	// ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start)
+	// DefaultConnectionsPerCPU controls how many connections are established concurrently.
 	DefaultConnectionsPerCPU int = 10
-	// DTailSSHServerDefaultPort is the default DServer port.
+	// DefaultSSHPort is the default DServer port.
 	DefaultSSHPort int = 2222
 	// DefaultLogLevel specifies the default log level (obviously)
 	DefaultLogLevel string = "INFO"
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index e4cbeaf..0a913db 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -20,13 +20,13 @@ type initializer struct {
 
 type transformCb func(*initializer, *Args, []string) error
 
-func (c *initializer) parseConfig(args *Args) error {
+func (in *initializer) parseConfig(args *Args) error {
 	if strings.ToUpper(args.ConfigFile) == "NONE" {
 		return nil
 	}
 
 	if args.ConfigFile != "" {
-		return c.parseSpecificConfig(args.ConfigFile)
+		return in.parseSpecificConfig(args.ConfigFile)
 	}
 
 	if homeDir, err := os.UserHomeDir(); err != nil {
@@ -35,7 +35,7 @@ func (c *initializer) parseConfig(args *Args) error {
 		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
 		for _, configPath := range paths {
 			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
-				c.parseSpecificConfig(configPath)
+				in.parseSpecificConfig(configPath)
 			}
 		}
 	}
@@ -43,7 +43,7 @@ func (c *initializer) parseConfig(args *Args) error {
 	return nil
 }
 
-func (c *initializer) parseSpecificConfig(configFile string) error {
+func (in *initializer) parseSpecificConfig(configFile string) error {
 	fd, err := os.Open(configFile)
 	if err != nil {
 		return fmt.Errorf("Unable to read config file: %v", err)
@@ -55,68 +55,74 @@ func (c *initializer) parseSpecificConfig(configFile string) error {
 		return fmt.Errorf("Unable to read config file %s: %v", configFile, err)
 	}
 
-	if err := json.Unmarshal([]byte(cfgBytes), c); err != nil {
+	if err := json.Unmarshal([]byte(cfgBytes), in); err != nil {
 		return fmt.Errorf("Unable to parse config file %s: %v", configFile, err)
 	}
 
 	return nil
 }
 
-func (i *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string) error {
+func (in *initializer) transformConfig(sourceProcess source.Source, args *Args,
+	additionalArgs []string) error {
 
 	switch sourceProcess {
 	case source.Server:
-		return i.optimusPrime(transformServer, args, additionalArgs)
+		return in.optimusPrime(transformServer, args, additionalArgs)
 	case source.Client:
-		return i.optimusPrime(transformClient, args, additionalArgs)
+		return in.optimusPrime(transformClient, args, additionalArgs)
 	case source.HealthCheck:
-		return i.optimusPrime(transformHealthCheck, args, additionalArgs)
+		return in.optimusPrime(transformHealthCheck, args, additionalArgs)
 	default:
-		return fmt.Errorf("Unable to transform config, unknown source '%s'", sourceProcess)
+		return fmt.Errorf("Unable to transform config, unknown source '%s'",
+			sourceProcess)
 	}
 }
 
-func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalArgs []string) error {
+func (in *initializer) optimusPrime(sourceCb transformCb, args *Args,
+	additionalArgs []string) error {
+
 	// Copy args to config objects.
+	// NEXT: Maybe unify args and config structs?
 	if args.SSHPort != DefaultSSHPort {
-		i.Common.SSHPort = args.SSHPort
+		in.Common.SSHPort = args.SSHPort
 	}
 	if args.LogLevel != DefaultLogLevel {
-		i.Common.LogLevel = args.LogLevel
+		in.Common.LogLevel = args.LogLevel
 	}
 	if args.NoColor {
-		i.Client.TermColorsEnable = false
+		in.Client.TermColorsEnable = false
 	}
 	if args.LogDir != "" {
-		i.Common.LogDir = args.LogDir
+		in.Common.LogDir = args.LogDir
 	}
 	if args.Logger != "" {
-		i.Common.Logger = args.Logger
+		in.Common.Logger = args.Logger
 	}
 	if args.ConnectionsPerCPU == 0 {
 		args.ConnectionsPerCPU = DefaultConnectionsPerCPU
 	}
 
 	// Setup log directory.
-	if strings.Contains(i.Common.LogDir, "~/") {
+	if strings.Contains(in.Common.LogDir, "~/") {
 		homeDir, err := os.UserHomeDir()
 		if err != nil {
 			panic(err)
 		}
-		i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir))
+		in.Common.LogDir = strings.ReplaceAll(in.Common.LogDir, "~/",
+			fmt.Sprintf("%s/", homeDir))
 	}
 
 	// Source type specific transormations.
-	sourceCb(i, args, additionalArgs)
+	sourceCb(in, args, additionalArgs)
 
 	// Spartan mode.
 	if args.Spartan {
 		args.Quiet = true
 		args.NoColor = true
-		i.Client.TermColorsEnable = false
+		in.Client.TermColorsEnable = false
 		if args.LogLevel == "" {
 			args.LogLevel = "ERROR"
-			i.Common.LogLevel = "ERROR"
+			in.Common.LogLevel = "ERROR"
 		}
 	}
 	// Interpret additional args as file list or as query.
@@ -135,29 +141,28 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA
 	return nil
 }
 
-func transformClient(i *initializer, args *Args, additionalArgs []string) error {
+func transformClient(in *initializer, args *Args, additionalArgs []string) error {
 	// Serverless mode.
 	if args.Discovery == "" && (args.ServersStr == "" ||
 		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
-		i.Common.LogLevel = "warn"
+		in.Common.LogLevel = "warn"
 	}
-
 	return nil
 }
 
-func transformServer(i *initializer, args *Args, additionalArgs []string) error {
+func transformServer(in *initializer, args *Args, additionalArgs []string) error {
 	return nil
 }
 
-func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error {
+func transformHealthCheck(in *initializer, args *Args, additionalArgs []string) error {
 	// Serverless mode.
 	if args.Discovery == "" && (args.ServersStr == "" ||
 		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
-		i.Common.LogLevel = "warn"
+		in.Common.LogLevel = "warn"
 	}
 	args.TrustAllHosts = true
 	return nil
diff --git a/internal/config/server.go b/internal/config/server.go
index 8bbb394..677f5ac 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -4,8 +4,8 @@ import (
 	"errors"
 )
 
-// Permissions map. Each SSH user has a list of permissions which
-// log files it is allowed to follow and which ones not.
+// Permissions map. Each SSH user has a list of permissions which log files it
+// is allowed to follow and which ones not.
 type Permissions struct {
 	// The default user permissions.
 	Default []string
@@ -68,7 +68,6 @@ var ServerRelaxedAuthEnable bool
 func newDefaultServerConfig() *ServerConfig {
 	defaultPermissions := []string{"^/.*"}
 	defaultBindAddress := "0.0.0.0"
-
 	return &ServerConfig{
 		SSHBindAddress:     defaultBindAddress,
 		MaxConnections:     10,
@@ -89,10 +88,8 @@ func ServerUserPermissions(userName string) (permissions []string, err error) {
 	if p, ok := Server.Permissions.Users[userName]; ok {
 		permissions = p
 	}
-
 	if len(permissions) == 0 {
 		err = errors.New("Empty set of permission, user won't be able to open any files")
 	}
-
 	return
 }
diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go
index 83ee95e..8bb1e85 100644
--- a/internal/discovery/discovery.go
+++ b/internal/discovery/discovery.go
@@ -73,7 +73,6 @@ func (d *Discovery) initRegex() {
 
 	regexStr := string(runes)
 	dlog.Common.Debug("Using filter regex", regexStr)
-
 	regex, err := regexp.Compile(regexStr)
 	if err != nil {
 		dlog.Common.FatalPanic("Could not compile regex", regexStr, err)
@@ -90,9 +89,7 @@ func (d *Discovery) ServerList() []string {
 	if d.regex != nil {
 		servers = d.filterList(servers)
 	}
-
 	servers = d.dedupList(servers)
-
 	if d.order == Shuffle {
 		servers = d.shuffleList(servers)
 	}
@@ -105,12 +102,10 @@ func (d *Discovery) serverListFromModule() []string {
 	if d.module != "" {
 		return d.serverListFromReflectedModule()
 	}
-
 	if _, err := os.Stat(d.server); err == nil {
 		// Appears to be a file name, now try to read from that file.
 		return d.ServerListFromFILE()
 	}
-
 	// Appears to be a list of FQDNs (or a single FQDN)
 	return d.ServerListFromCOMMA()
 }
@@ -120,18 +115,16 @@ func (d *Discovery) serverListFromModule() []string {
 // Discovery. Whereas MODULENAME must be a upeprcase string.
 func (d *Discovery) serverListFromReflectedModule() []string {
 	methodName := fmt.Sprintf("ServerListFrom%s", d.module)
-
+	// Now we are reflecting the serve discovery function by it's name.
 	rt := reflect.TypeOf(d)
 	reflectedMethod, ok := rt.MethodByName(methodName)
 	if !ok {
 		dlog.Common.FatalPanic("No such server discovery module", d.module, methodName)
 	}
-
 	inputValues := make([]reflect.Value, 1)
 	// Thist input value is method receiver.
 	inputValues[0] = reflect.ValueOf(d)
 	returnValues := reflectedMethod.Func.Call(inputValues)
-
 	// First return value is server list.
 	return returnValues[0].Interface().([]string)
 }
@@ -139,27 +132,23 @@ func (d *Discovery) serverListFromReflectedModule() []string {
 // Filter server list based on a regexp.
 func (d *Discovery) filterList(servers []string) (filtered []string) {
 	dlog.Common.Debug("Filtering server list")
-
 	for _, server := range servers {
 		if d.regex.MatchString(server) {
 			filtered = append(filtered, server)
 		}
 	}
-
 	return
 }
 
 // Deduplicate the server list.
 func (d *Discovery) dedupList(servers []string) (deduped []string) {
 	serverMap := make(map[string]struct{}, len(servers))
-
 	for _, server := range servers {
 		if _, ok := serverMap[server]; !ok {
 			serverMap[server] = struct{}{}
 			deduped = append(deduped, server)
 		}
 	}
-
 	dlog.Common.Debug("Deduped server list", len(servers), len(deduped))
 	return
 }
diff --git a/internal/done.go b/internal/done.go
index 5ea22a0..94f9289 100644
--- a/internal/done.go
+++ b/internal/done.go
@@ -35,7 +35,6 @@ func (d *Done) Done() <-chan struct{} {
 func (d *Done) Shutdown() {
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
-
 	select {
 	case <-d.ch:
 		return
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 28e6882..da67585 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -82,7 +82,7 @@ func new(sourceProcess, sourcePackage source.Source) *DLog {
 	if err != nil {
 		panic(err)
 	}
-	strategy := loggers.GetStrategy(config.Common.LogStrategy)
+	strategy := loggers.NewStrategy(config.Common.LogStrategy)
 	loggerName := config.Common.Logger
 	level := newLevel(config.Common.LogLevel)
 
@@ -153,6 +153,7 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 	}
 }
 
+// FatalPanic terminates the process with a fatal error.
 func (d *DLog) FatalPanic(args ...interface{}) {
 	d.log(Fatal, args)
 	d.Flush()
@@ -162,42 +163,51 @@ func (d *DLog) FatalPanic(args ...interface{}) {
 	panic(sb.String())
 }
 
+// Fatal logs a fatal error.
 func (d *DLog) Fatal(args ...interface{}) string {
 	return d.log(Fatal, args)
 }
 
+// Error logging.
 func (d *DLog) Error(args ...interface{}) string {
 	return d.log(Error, args)
 }
 
+// Warn logs a warning message.
 func (d *DLog) Warn(args ...interface{}) string {
 	return d.log(Warn, args)
 }
 
+// Info logging.
 func (d *DLog) Info(args ...interface{}) string {
 	return d.log(Info, args)
 }
 
+// Verbose logging.
 func (d *DLog) Verbose(args ...interface{}) string {
 	return d.log(Verbose, args)
 }
 
+// Debug logging.
 func (d *DLog) Debug(args ...interface{}) string {
 	return d.log(Debug, args)
 }
 
+// Trace logging.
 func (d *DLog) Trace(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Trace, args)
 }
 
+// Devel used for development purpose only logging (e.g. "print" debugging).
 func (d *DLog) Devel(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Devel, args)
 }
 
+// Raw message logging.
 func (d *DLog) Raw(message string) string {
 	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
 		d.logger.Log(time.Now(), message)
@@ -207,6 +217,7 @@ func (d *DLog) Raw(message string) string {
 	return message
 }
 
+// Mapreduce logging.
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
@@ -251,6 +262,11 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	return d.log(Info, args)
 }
 
-func (d *DLog) Flush()  { d.logger.Flush() }
-func (d *DLog) Pause()  { d.logger.Pause() }
+// Flush the log buffers.
+func (d *DLog) Flush() { d.logger.Flush() }
+
+// Pause the logging.
+func (d *DLog) Pause() { d.logger.Pause() }
+
+// Resume the logging.
 func (d *DLog) Resume() { d.logger.Resume() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 248ad83..0971094 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -7,6 +7,7 @@ import (
 
 type level int
 
+// Available log levels.
 const (
 	Fatal   level = iota
 	Error   level = iota
@@ -20,7 +21,8 @@ const (
 	All     level = iota
 )
 
-var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug, Devel, Trace, All}
+var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug,
+	Devel, Trace, All}
 
 func newLevel(l string) level {
 	switch strings.ToLower(l) {
@@ -73,6 +75,5 @@ func (l level) String() string {
 	case All:
 		return "ALL"
 	}
-
 	panic("Unknown log level " + fmt.Sprintf("%d", l))
 }
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index dda3ee6..415d7fb 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -9,11 +9,13 @@ import (
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
+// Factory is there to retrieve a logger based on various settings.
 func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName, rotationStrategy.FileBase, loggerName)
+	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName,
+		rotationStrategy.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
@@ -36,10 +38,10 @@ func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
 		}
 	}
-
 	return singleton
 }
 
+// FactoryRotate invokes a log rotation of all loggers.
 func FactoryRotate() {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 87280fd..94824fe 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -12,8 +12,7 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 )
 
-type fileWriter struct {
-}
+type fileWriter struct{}
 
 type fileMessageBuf struct {
 	now     time.Time
@@ -35,7 +34,7 @@ type file struct {
 }
 
 func newFile(strategy Strategy) *file {
-	f := file{
+	return &file{
 		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
@@ -43,16 +42,17 @@ func newFile(strategy Strategy) *file {
 		flushCh:  make(chan struct{}),
 		strategy: strategy,
 	}
-
-	return &f
 }
 
 func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 	f.mutex.Lock()
-	defer f.mutex.Unlock()
+	defer func() {
+		f.started = true
+		f.mutex.Unlock()
+	}()
 
-	// Logger already started from another Goroutine.
 	if f.started {
+		// Logger already started from another Goroutine.
 		wg.Done()
 		return
 	}
@@ -68,7 +68,6 @@ func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 
 	go func() {
 		defer wg.Done()
-
 		for {
 			select {
 			case m := <-f.bufferCh:
@@ -84,8 +83,6 @@ func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 			}
 		}
 	}()
-
-	f.started = true
 }
 
 func (f *file) Log(now time.Time, message string) {
diff --git a/internal/io/dlog/loggers/logger.go b/internal/io/dlog/loggers/logger.go
index c88900d..d4e85de 100644
--- a/internal/io/dlog/loggers/logger.go
+++ b/internal/io/dlog/loggers/logger.go
@@ -6,6 +6,7 @@ import (
 	"time"
 )
 
+// Logger is there to plug in your own log implementation.
 type Logger interface {
 	Log(now time.Time, message string)
 	LogWithColors(now time.Time, message, messageWithColors string)
diff --git a/internal/io/dlog/loggers/strategy.go b/internal/io/dlog/loggers/strategy.go
index a1b9355..25d10f0 100644
--- a/internal/io/dlog/loggers/strategy.go
+++ b/internal/io/dlog/loggers/strategy.go
@@ -5,19 +5,26 @@ import (
 	"path/filepath"
 )
 
+// Rotation is the actual strategy used for log rotation..
 type Rotation int
 
 const (
-	DailyRotation  Rotation = iota
+	// DailyRotation tells DTail to rotate its logs on a daily basis or on SIGHUP.
+	DailyRotation Rotation = iota
+	// SignalRotation tells DTail to rotate its logs only on SIGHUP.
 	SignalRotation Rotation = iota
 )
 
+// Strategy is a pair of the rotation and the file base.
 type Strategy struct {
+	// Rotation is the actual rotation strategy used.
 	Rotation Rotation
+	// FileBase can be a name (e.g. "dserver", "dmap") when signal rotation is used.
 	FileBase string
 }
 
-func GetStrategy(name string) Strategy {
+// NewStrategy returns the stratey based on its name.
+func NewStrategy(name string) Strategy {
 	switch name {
 	case "daily":
 		return Strategy{DailyRotation, ""}
diff --git a/internal/io/fs/catfile.go b/internal/io/fs/catfile.go
index 7f387bc..01c15ba 100644
--- a/internal/io/fs/catfile.go
+++ b/internal/io/fs/catfile.go
@@ -6,7 +6,9 @@ type CatFile struct {
 }
 
 // NewCatFile returns a new file catter.
-func NewCatFile(filePath string, globID string, serverMessages chan<- string, limiter chan struct{}) CatFile {
+func NewCatFile(filePath string, globID string, serverMessages chan<- string,
+	limiter chan struct{}) CatFile {
+
 	return CatFile{
 		readFile: readFile{
 			filePath:       filePath,
diff --git a/internal/io/fs/filereader.go b/internal/io/fs/filereader.go
index 0774837..7773142 100644
--- a/internal/io/fs/filereader.go
+++ b/internal/io/fs/filereader.go
@@ -7,7 +7,8 @@ import (
 	"github.com/mimecast/dtail/internal/regex"
 )
 
-// FileReader is the interface used on the dtail server to read/cat/grep/mapr... a file.
+// FileReader is the interface used on the dtail server to read/cat/grep/mapr...
+// a file.
 type FileReader interface {
 	Start(ctx context.Context, lines chan<- line.Line, re regex.Regex) error
 	FilePath() string
diff --git a/internal/io/fs/permissions/permission_linuxacl.go b/internal/io/fs/permissions/permission_linuxacl.go
index 7d2d7ca..904b90f 100644
--- a/internal/io/fs/permissions/permission_linuxacl.go
+++ b/internal/io/fs/permissions/permission_linuxacl.go
@@ -13,7 +13,7 @@ import (
 	"unsafe"
 )
 
-// ToRead checks whether user has Linux file system permissions to read a given file.
+// ToRead checks whether user has Linux file system permissions to read a file.
 func ToRead(user, filePath string) (bool, error) {
 	cUser := C.CString(user)
 	cFilePath := C.CString(filePath)
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index f128c07..92f85b6 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -42,7 +42,8 @@ type readFile struct {
 
 // String returns the string representation of the readFile
 func (f readFile) String() string {
-	return fmt.Sprintf("readFile(filePath:%s,globID:%s,retry:%v,canSkipLines:%v,seekEOF:%v)",
+	return fmt.Sprintf(
+		"readFile(filePath:%s,globID:%s,retry:%v,canSkipLines:%v,seekEOF:%v)",
 		f.filePath,
 		f.globID,
 		f.retry,
@@ -61,7 +62,9 @@ func (f readFile) Retry() bool {
 }
 
 // Start tailing a log file.
-func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Regex) error {
+func (f readFile) Start(ctx context.Context, lines chan<- line.Line,
+	re regex.Regex) error {
+
 	dlog.Common.Debug("readFile", f)
 	defer func() {
 		select {
@@ -74,7 +77,8 @@ func (f readFile) Start(ctx context.Context, lines chan<- line.Line, re regex.Re
 	case f.limiter <- struct{}{}:
 	default:
 		select {
-		case f.serverMessages <- dlog.Common.Warn(f.filePath, f.globID, "Server limit reached. Queuing file..."):
+		case f.serverMessages <- dlog.Common.Warn(f.filePath, f.globID,
+			"Server limit reached. Queuing file..."):
 		case <-ctx.Done():
 			return nil
 		}
@@ -139,13 +143,11 @@ func (f readFile) makeReader(fd *os.File) (reader *bufio.Reader, err error) {
 	default:
 		reader = bufio.NewReader(fd)
 	}
-
 	return
 }
 
 func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Buffer, truncate <-chan struct{}) error {
 	var offset uint64
-
 	reader, err := f.makeReader(fd)
 	if err != nil {
 		return err
@@ -193,7 +195,8 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 		default:
 			if message.Len() >= lineLengthThreshold {
 				if !warnedAboutLongLine {
-					f.serverMessages <- dlog.Common.Warn(f.filePath, "Long log line, splitting into multiple lines")
+					f.serverMessages <- dlog.Common.Warn(f.filePath,
+						"Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
 				message.WriteString("\n")
@@ -210,9 +213,10 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 }
 
 // Filter log lines matching a given regular expression.
-func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
-	defer wg.Done()
+func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup,
+	rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
 
+	defer wg.Done()
 	for {
 		select {
 		case line, ok := <-rawLines:
@@ -231,9 +235,10 @@ func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup, rawLines <-cha
 	}
 }
 
-func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int, re regex.Regex) (line.Line, bool) {
-	var read line.Line
+func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int,
+	re regex.Regex) (line.Line, bool) {
 
+	var read line.Line
 	if !re.Match(lineBytes.Bytes()) {
 		f.updateLineNotMatched()
 		f.updateLineNotTransmitted()
@@ -254,7 +259,6 @@ func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int, r
 		Count:           f.totalLineCount(),
 		TransmittedPerc: f.transmittedPerc(),
 	}
-
 	return read, true
 }
 
@@ -267,7 +271,6 @@ func (f readFile) truncated(fd *os.File) (bool, error) {
 	if err != nil {
 		return true, err
 	}
-
 	// Can not open file at original path.
 	pathFd, err := os.Open(f.filePath)
 	if err != nil {
@@ -280,10 +283,8 @@ func (f readFile) truncated(fd *os.File) (bool, error) {
 	if err != nil {
 		return true, err
 	}
-
 	if curPos > pathPos {
 		return true, errors.New("File got truncated")
 	}
-
 	return false, nil
 }
diff --git a/internal/io/fs/tailfile.go b/internal/io/fs/tailfile.go
index 14994e5..b03b45d 100644
--- a/internal/io/fs/tailfile.go
+++ b/internal/io/fs/tailfile.go
@@ -6,7 +6,9 @@ type TailFile struct {
 }
 
 // NewTailFile returns a new file tailer.
-func NewTailFile(filePath string, globID string, serverMessages chan<- string, limiter chan struct{}) TailFile {
+func NewTailFile(filePath string, globID string, serverMessages chan<- string,
+	limiter chan struct{}) TailFile {
+
 	return TailFile{
 		readFile: readFile{
 			filePath:       filePath,
diff --git a/internal/io/pool/builder.go b/internal/io/pool/builder.go
index c9dc221..89fcf81 100644
--- a/internal/io/pool/builder.go
+++ b/internal/io/pool/builder.go
@@ -5,6 +5,8 @@ import (
 	"sync"
 )
 
+// BuilderBuffer is there to optimize memory allocations (DTail allocates a lot
+// of memory while reading log data otherwise)
 var BuilderBuffer = sync.Pool{
 	New: func() interface{} {
 		sb := strings.Builder{}
@@ -12,6 +14,7 @@ var BuilderBuffer = sync.Pool{
 	},
 }
 
+// RecycleBuilderBuffer recycles the buffer again.
 func RecycleBuilderBuffer(sb *strings.Builder) {
 	sb.Reset()
 	BuilderBuffer.Put(sb)
diff --git a/internal/io/pool/bytesbuffer.go b/internal/io/pool/bytesbuffer.go
index 0a159f5..3d48f2c 100644
--- a/internal/io/pool/bytesbuffer.go
+++ b/internal/io/pool/bytesbuffer.go
@@ -5,6 +5,8 @@ import (
 	"sync"
 )
 
+// BytesBuffer is there to optimize memory allocations. DTail otherwise allocates
+// a lot of memory while reading logs.
 var BytesBuffer = sync.Pool{
 	New: func() interface{} {
 		b := bytes.Buffer{}
@@ -13,6 +15,7 @@ var BytesBuffer = sync.Pool{
 	},
 }
 
+// RecycleBytesBuffer recycles the buffer again.
 func RecycleBytesBuffer(b *bytes.Buffer) {
 	b.Reset()
 	BytesBuffer.Put(b)
diff --git a/internal/io/prompt/prompt.go b/internal/io/prompt/prompt.go
index 7c3cdb5..e82132d 100644
--- a/internal/io/prompt/prompt.go
+++ b/internal/io/prompt/prompt.go
@@ -19,7 +19,8 @@ type Answer struct {
 	Callback func()
 	// Runs after Callback and after logging resumes
 	EndCallback func()
-	AskAgain    bool
+	// AskAgain can be used to not to ask again about the question.
+	AskAgain bool
 }
 
 // Prompt used for interactive user input.
@@ -30,7 +31,6 @@ type Prompt struct {
 
 func (p *Prompt) askString() string {
 	var sb strings.Builder
-
 	sb.WriteString(p.question)
 	sb.WriteString("? (")
 
@@ -41,7 +41,6 @@ func (p *Prompt) askString() string {
 
 	sb.WriteString(strings.Join(ax, ","))
 	sb.WriteString("): ")
-
 	return sb.String()
 }
 
@@ -68,7 +67,6 @@ func (p *Prompt) Ask() {
 			if a.Callback != nil {
 				a.Callback()
 			}
-
 			if !a.AskAgain {
 				dlog.Common.Resume()
 				if a.EndCallback != nil {
@@ -90,6 +88,5 @@ func (p *Prompt) answer(answerStr string) (*Answer, bool) {
 		default:
 		}
 	}
-
 	return nil, false
 }
diff --git a/internal/io/signal/signal.go b/internal/io/signal/signal.go
index 14056c4..584b59c 100644
--- a/internal/io/signal/signal.go
+++ b/internal/io/signal/signal.go
@@ -14,10 +14,8 @@ import (
 func InterruptCh(ctx context.Context) <-chan string {
 	sigIntCh := make(chan os.Signal)
 	gosignal.Notify(sigIntCh, os.Interrupt)
-
 	sigOtherCh := make(chan os.Signal)
 	gosignal.Notify(sigOtherCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT)
-
 	statsCh := make(chan string)
 
 	go func() {
@@ -41,7 +39,6 @@ func InterruptCh(ctx context.Context) <-chan string {
 			}
 		}
 	}()
-
 	return statsCh
 }
 
diff --git a/internal/mapr/aggregateset.go b/internal/mapr/aggregateset.go
index 14e6943..c50c7a1 100644
--- a/internal/mapr/aggregateset.go
+++ b/internal/mapr/aggregateset.go
@@ -38,7 +38,6 @@ func (s *AggregateSet) String() string {
 func (s *AggregateSet) Merge(query *Query, set *AggregateSet) error {
 	s.Samples += set.Samples
 	//dlog.Common.Trace("Merge", set)
-
 	for _, sc := range query.Select {
 		storage := sc.FieldStorage
 		switch sc.Operation {
@@ -115,7 +114,6 @@ func (s *AggregateSet) addFloatMin(key string, value float64) {
 		s.FValues[key] = value
 		return
 	}
-
 	if f > value {
 		s.FValues[key] = value
 	}
@@ -128,7 +126,6 @@ func (s *AggregateSet) addFloatMax(key string, value float64) {
 		s.FValues[key] = value
 		return
 	}
-
 	if f < value {
 		s.FValues[key] = value
 	}
@@ -147,7 +144,6 @@ func (s *AggregateSet) setFloat(key string, value float64) {
 // Aggregate data to the aggregate set.
 func (s *AggregateSet) Aggregate(key string, agg AggregateOperation, value string, clientAggregation bool) (err error) {
 	var f float64
-
 	// First check if we can aggregate anything without converting value to float.
 	switch agg {
 	case Count:
diff --git a/internal/mapr/client/aggregate.go b/internal/mapr/client/aggregate.go
index d0c1d70..02a6a5a 100644
--- a/internal/mapr/client/aggregate.go
+++ b/internal/mapr/client/aggregate.go
@@ -23,7 +23,9 @@ type Aggregate struct {
 }
 
 // NewAggregate create new client aggregator.
-func NewAggregate(server string, query *mapr.Query, globalGroup *mapr.GlobalGroupSet) *Aggregate {
+func NewAggregate(server string, query *mapr.Query,
+	globalGroup *mapr.GlobalGroupSet) *Aggregate {
+
 	return &Aggregate{
 		query:       query,
 		group:       mapr.NewGroupSet(),
@@ -47,8 +49,8 @@ func (a *Aggregate) Aggregate(message string) error {
 
 	fields := a.makeFields(parts[2:])
 	set := a.group.GetSet(groupKey)
-
 	var addedSamples bool
+
 	for _, sc := range a.query.Select {
 		if val, ok := fields[sc.FieldStorage]; ok {
 			if err := set.Aggregate(sc.FieldStorage, sc.Operation, val, true); err != nil {
@@ -71,14 +73,12 @@ func (a *Aggregate) Aggregate(message string) error {
 		// Re-init local group (make it empty again).
 		a.group.InitSet()
 	}
-
 	return nil
 }
 
 // Create a map of key-value pairs from a part list such as ["foo=bar",  "bar=baz"].
 func (a *Aggregate) makeFields(parts []string) map[string]string {
 	fields := make(map[string]string, len(parts))
-
 	for _, part := range parts {
 		kv := strings.SplitN(part, protocol.AggregateKVDelimiter, 2)
 		if len(kv) != 2 {
@@ -86,6 +86,5 @@ func (a *Aggregate) makeFields(parts []string) map[string]string {
 		}
 		fields[kv[0]] = kv[1]
 	}
-
 	return fields
 }
diff --git a/internal/mapr/funcs/function.go b/internal/mapr/funcs/function.go
index 0433b9a..418d86f 100644
--- a/internal/mapr/funcs/function.go
+++ b/internal/mapr/funcs/function.go
@@ -19,13 +19,12 @@ type Function struct {
 // FunctionStack is a list of functions stacked each other
 type FunctionStack []Function
 
-// NewFunctionStack parses the input string, e.g. foo(bar("arg")) and returns a corresponding function stack.
+// NewFunctionStack parses the input string, e.g. foo(bar("arg")) and returns
+// a corresponding function stack.
 func NewFunctionStack(in string) (FunctionStack, string, error) {
 	var fs FunctionStack
-
 	getCallback := func(name string) (CallbackFunc, error) {
 		var cb CallbackFunc
-
 		switch name {
 		case "md5sum":
 			return Md5Sum, nil
@@ -51,7 +50,6 @@ func NewFunctionStack(in string) (FunctionStack, string, error) {
 		fs = append(fs, Function{name, call})
 		aux = aux[index+1 : len(aux)-1]
 	}
-
 	return fs, aux, nil
 }
 
@@ -62,6 +60,5 @@ func (fs FunctionStack) Call(str string) string {
 		str = fs[i].call(str)
 		//dlog.Common.Debug("Call.result", fs[i].Name, str)
 	}
-
 	return str
 }
diff --git a/internal/mapr/funcs/function_test.go b/internal/mapr/funcs/function_test.go
index 415683c..8b5d8b7 100644
--- a/internal/mapr/funcs/function_test.go
+++ b/internal/mapr/funcs/function_test.go
@@ -6,16 +6,19 @@ func TestFunction(t *testing.T) {
 	input := "md5sum($line)"
 	fs, arg, err := NewFunctionStack(input)
 	if err != nil {
-		t.Errorf("error parsing function input '%s': %s (%v)\n", input, err.Error(), fs)
+		t.Errorf("error parsing function input '%s': %s (%v)\n",
+			input, err.Error(), fs)
 	}
 	if arg != "$line" {
-		t.Errorf("error parsing function input '%s': expected argument '$line' but got '%s' (%v)\n", input, arg, fs)
+		t.Errorf("error parsing function input '%s': expected argument '$line' but "+
+			"got '%s' (%v)\n", input, arg, fs)
 	}
 	t.Log(input, fs, arg)
 
 	result := fs.Call(input)
 	if result != "b38699013d79e50d9d122433753959c1" {
-		t.Errorf("error executing function stack '%s': expected result 'b38699013d79e50d9d122433753959c1' but got '%s' (%v)\n", input, result, fs)
+		t.Errorf("error executing function stack '%s': expected result "+
+			"'b38699013d79e50d9d122433753959c1' but got '%s' (%v)\n", input, result, fs)
 	}
 
 	input = "maskdigits(md5sum(maskdigits($line)))"
@@ -24,22 +27,26 @@ func TestFunction(t *testing.T) {
 		t.Errorf("error parsing function input '%s': %s (%v)\n", input, err.Error(), fs)
 	}
 	if arg != "$line" {
-		t.Errorf("error parsing function input '%s': expected argument '$line' but got '%s' (%v)\n", input, arg, fs)
+		t.Errorf("error parsing function input '%s': expected argument '$line' but "+
+			"got '%s' (%v)\n", input, arg, fs)
 	}
 	t.Log(input, fs, arg)
 
 	result = fs.Call(input)
 	if result != ".fac.bbe..bb.........d...a.c..b." {
-		t.Errorf("error executing function stack '%s': expected result '.fac.bbe..bb.........d...a.c..b.' but got '%s' (%v)\n", input, result, fs)
+		t.Errorf("error executing function stack '%s': expected result "+
+			"'.fac.bbe..bb.........d...a.c..b.' but got '%s' (%v)\n", input, result, fs)
 	}
 
 	input = "md5sum$line)"
 	if fs, _, err := NewFunctionStack(input); err == nil {
-		t.Errorf("Expected error parsing function input '%s' (%v) but got no error\n", input, fs)
+		t.Errorf("Expected error parsing function input '%s' (%v) but got no error\n",
+			input, fs)
 	}
 
 	input = "md5sum(makedigits$line))"
 	if fs, _, err := NewFunctionStack(input); err == nil {
-		t.Errorf("Expected error parsing function input '%s' (%v) but got no error\n", input, fs)
+		t.Errorf("Expected error parsing function input '%s' (%v) but got no error\n",
+			input, fs)
 	}
 }
diff --git a/internal/mapr/funcs/maskdigits.go b/internal/mapr/funcs/maskdigits.go
index d51f3d8..925ec4d 100644
--- a/internal/mapr/funcs/maskdigits.go
+++ b/internal/mapr/funcs/maskdigits.go
@@ -3,12 +3,10 @@ package funcs
 // MaskDigits masks all digits (replaces them with .)
 func MaskDigits(input string) string {
 	s := []byte(input)
-
 	for i, b := range s {
 		if '0' <= b && b <= '9' {
 			s[i] = '.'
 		}
 	}
-
 	return string(s)
 }
diff --git a/internal/mapr/globalgroupset.go b/internal/mapr/globalgroupset.go
index 50bac37..2d7f10b 100644
--- a/internal/mapr/globalgroupset.go
+++ b/internal/mapr/globalgroupset.go
@@ -17,7 +17,6 @@ func NewGlobalGroupSet() *GlobalGroupSet {
 		semaphore: make(chan struct{}, 1),
 	}
 	g.InitSet()
-
 	return &g
 }
 
@@ -30,7 +29,6 @@ func (g *GlobalGroupSet) String() string {
 func (g *GlobalGroupSet) Merge(query *Query, group *GroupSet) error {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
-
 	return g.merge(query, group)
 }
 
@@ -48,14 +46,12 @@ func (g *GlobalGroupSet) MergeNoblock(query *Query, group *GroupSet) (bool, erro
 
 // Merge a group set into the global group set.
 func (g *GlobalGroupSet) merge(query *Query, group *GroupSet) error {
-
 	for groupKey, set := range group.sets {
 		s := g.GetSet(groupKey)
 		if err := s.Merge(query, set); err != nil {
 			return err
 		}
 	}
-
 	return nil
 }
 
@@ -68,7 +64,6 @@ func (g *GlobalGroupSet) IsEmpty() bool {
 func (g *GlobalGroupSet) NumSets() int {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
-
 	return len(g.sets)
 }
 
@@ -80,7 +75,6 @@ func (g *GlobalGroupSet) SwapOut() *GroupSet {
 
 	set := &GroupSet{sets: g.sets}
 	g.InitSet()
-
 	return set
 }
 
@@ -88,7 +82,6 @@ func (g *GlobalGroupSet) SwapOut() *GroupSet {
 func (g *GlobalGroupSet) WriteResult(query *Query) error {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
-
 	return g.GroupSet.WriteResult(query)
 }
 
@@ -96,6 +89,5 @@ func (g *GlobalGroupSet) WriteResult(query *Query) error {
 func (g *GlobalGroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	g.semaphore <- struct{}{}
 	defer func() { <-g.semaphore }()
-
 	return g.GroupSet.Result(query, rowsLimit)
 }
diff --git a/internal/mapr/groupset.go b/internal/mapr/groupset.go
index ce7630d..6ffc8b9 100644
--- a/internal/mapr/groupset.go
+++ b/internal/mapr/groupset.go
@@ -73,7 +73,6 @@ func (g *GroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 	if err != nil {
 		return "", 0, err
 	}
-
 	if query.Limit != -1 {
 		rowsLimit = query.Limit
 	}
@@ -91,12 +90,14 @@ func (g *GroupSet) Result(query *Query, rowsLimit int) (string, int, error) {
 			if sc.FieldStorage == query.OrderBy {
 				attrs = append(attrs, config.Client.TermColors.MaprTable.HeaderSortKeyAttr)
 			}
+
 			for _, groupBy := range query.GroupBy {
 				if sc.FieldStorage == groupBy {
 					attrs = append(attrs, config.Client.TermColors.MaprTable.HeaderGroupKeyAttr)
 					break
 				}
 			}
+
 			color.PaintWithAttrs(sb, str,
 				config.Client.TermColors.MaprTable.HeaderFg,
 				config.Client.TermColors.MaprTable.HeaderBg,
@@ -191,7 +192,6 @@ func (*GroupSet) writeQueryFile(query *Query) error {
 
 	fd.WriteString(query.RawQuery)
 	os.Rename(tmpQueryFile, queryFile)
-
 	return nil
 }
 
@@ -256,7 +256,6 @@ func (g *GroupSet) WriteResult(query *Query) error {
 func (g *GroupSet) result(query *Query, gatherWidths bool) ([]result, []int, error) {
 	var rows []result
 	widths := make([]int, len(query.Select))
-
 	var valueStr string
 	var value float64
 
@@ -284,7 +283,8 @@ func (g *GroupSet) result(query *Query, gatherWidths bool) ([]result, []int, err
 				value = set.FValues[sc.FieldStorage] / float64(set.Samples)
 				valueStr = fmt.Sprintf("%f", value)
 			default:
-				return rows, widths, fmt.Errorf("Unknown aggregation method '%v'", sc.Operation)
+				return rows, widths, fmt.Errorf("Unknown aggregation method '%v'",
+					sc.Operation)
 			}
 
 			if sc.FieldStorage == query.OrderBy {
@@ -302,7 +302,6 @@ func (g *GroupSet) result(query *Query, gatherWidths bool) ([]result, []int, err
 				widths[i] = len(valueStr)
 			}
 		}
-
 		rows = append(rows, r)
 	}
 
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index 8016667..9b6c855 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -11,9 +11,10 @@ import (
 func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
 
-	if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") || !strings.HasPrefix(splitted[0], "INFO") {
+	if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") ||
+		!strings.HasPrefix(splitted[0], "INFO") {
 		// Not a DTail mapreduce log line.
-		return nil, IgnoreFieldsErr
+		return nil, ErrIgnoreFields
 	}
 
 	fields := make(map[string]string, len(splitted)+8)
diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go
index a777156..28e1acc 100644
--- a/internal/mapr/logformat/default_test.go
+++ b/internal/mapr/logformat/default_test.go
@@ -32,49 +32,57 @@ func TestDefaultLogFormat(t *testing.T) {
 		if val, ok := fields["$severity"]; !ok {
 			t.Errorf("Expected field '$severity', but no such field there in '%s'\n", input)
 		} else if val != "INFO" {
-			t.Errorf("Expected 'Info' stored in field '$severity', but got '%s' in '%s'\n", val, input)
+			t.Errorf("Expected 'Info' stored in field '$severity', but got '%s' in '%s'\n",
+				val, input)
 		}
 
 		if val, ok := fields["$time"]; !ok {
 			t.Errorf("Expected field '$time', but no such field there in '%s'\n", input)
 		} else if val != time {
-			t.Errorf("Expected '%s' stored in field '$time', but got '%s' in '%s'\n", time, val, input)
+			t.Errorf("Expected '%s' stored in field '$time', but got '%s' in '%s'\n",
+				time, val, input)
 		}
 
 		if val, ok := fields["$date"]; !ok {
 			t.Errorf("Expected field '$date', but no such field there in '%s'\n", input)
 		} else if val != date {
-			t.Errorf("Expected '%s' stored in field '$date', but got '%s' in '%s'\n", date, val, input)
+			t.Errorf("Expected '%s' stored in field '$date', but got '%s' in '%s'\n",
+				date, val, input)
 		}
 
 		if val, ok := fields["$hour"]; !ok {
 			t.Errorf("Expected field '$hour', but no such field there in '%s'\n", input)
 		} else if val != hour {
-			t.Errorf("Expected '%s' stored in field '$hour', but got '%s' in '%s'\n", hour, val, input)
+			t.Errorf("Expected '%s' stored in field '$hour', but got '%s' in '%s'\n",
+				hour, val, input)
 		}
 
 		if val, ok := fields["$minute"]; !ok {
 			t.Errorf("Expected field '$minute', but no such field there in '%s'\n", input)
 		} else if val != minute {
-			t.Errorf("Expected '%s' stored in field '$minute', but got '%s' in '%s'\n", minute, val, input)
+			t.Errorf("Expected '%s' stored in field '$minute', but got '%s' in '%s'\n",
+				minute, val, input)
 		}
 
 		if val, ok := fields["$second"]; !ok {
 			t.Errorf("Expected field '$second', but no such field there in '%s'\n", input)
 		} else if val != second {
-			t.Errorf("Expected '%s' stored in field '$second', but got '%s' in '%s'\n", second, val, input)
+			t.Errorf("Expected '%s' stored in field '$second', but got '%s' in '%s'\n",
+				second, val, input)
 		}
 
 		if val, ok := fields["foo"]; !ok {
 			t.Errorf("Expected field 'foo', but no such field there in '%s'\n", input)
 		} else if val != "bar" {
-			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n", val, input)
+			t.Errorf("Expected 'bar' stored in field 'foo', but got '%s' in '%s'\n",
+				val, input)
 		}
 
 		if val, ok := fields["bar"]; !ok {
 			t.Errorf("Expected field 'bar', but no such field there in '%s'\n", input)
 		} else if val != "foo" {
-			t.Errorf("Expected 'foo' stored in field 'bar', but got '%s' in '%s'\n", val, input)
+			t.Errorf("Expected 'foo' stored in field 'bar', but got '%s' in '%s'\n",
+				val, input)
 		}
 	}
 
diff --git a/internal/mapr/logformat/generickv.go b/internal/mapr/logformat/generickv.go
index 3769c22..433eb5f 100644
--- a/internal/mapr/logformat/generickv.go
+++ b/internal/mapr/logformat/generickv.go
@@ -6,7 +6,7 @@ import (
 	"github.com/mimecast/dtail/internal/protocol"
 )
 
-// MakeFieldsGENERICKV is the generic key-value logfile parser.
+// MakeFieldsGENERIGKV is the generic key-value logfile parser.
 func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error) {
 	splitted := strings.Split(maprLine, protocol.FieldDelimiter)
 	fields := make(map[string]string, len(splitted))
diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go
index a352580..129081d 100644
--- a/internal/mapr/logformat/parser.go
+++ b/internal/mapr/logformat/parser.go
@@ -11,7 +11,8 @@ import (
 	"github.com/mimecast/dtail/internal/mapr"
 )
 
-var IgnoreFieldsErr error = errors.New("Ignore this field set")
+// ErrIgnoreFields indicates that the fields should be ignored.
+var ErrIgnoreFields error = errors.New("Ignore this field set")
 
 // Parser is used to parse the mapreduce information from the server log files.
 type Parser struct {
@@ -26,11 +27,9 @@ type Parser struct {
 // NewParser returns a new log parser.
 func NewParser(logFormatName string, query *mapr.Query) (*Parser, error) {
 	hostname, err := os.Hostname()
-
 	if err != nil {
 		return nil, err
 	}
-
 	now := time.Now()
 	zone, offset := now.Zone()
 
@@ -44,7 +43,6 @@ func NewParser(logFormatName string, query *mapr.Query) (*Parser, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	return &p, nil
 }
 
@@ -53,7 +51,6 @@ func NewParser(logFormatName string, query *mapr.Query) (*Parser, error) {
 // Parser. Whereas MODULENAME must be a upeprcase string.
 func (p *Parser) reflectLogFormat(logFormatName string) error {
 	methodName := fmt.Sprintf("MakeFields%s", strings.ToUpper(logFormatName))
-
 	rt := reflect.TypeOf(p)
 	method, ok := rt.MethodByName(methodName)
 	if !ok {
@@ -62,7 +59,6 @@ func (p *Parser) reflectLogFormat(logFormatName string) error {
 
 	p.makeFieldsFunc = method.Func
 	p.makeFieldsReceiver = reflect.ValueOf(p)
-
 	return nil
 }
 
@@ -70,15 +66,11 @@ func (p *Parser) reflectLogFormat(logFormatName string) error {
 func (p *Parser) MakeFields(maprLine string) (fields map[string]string, err error) {
 	inputValues := []reflect.Value{p.makeFieldsReceiver, reflect.ValueOf(maprLine)}
 	returnValues := p.makeFieldsFunc.Call(inputValues)
-
 	errInterface := returnValues[1].Interface()
-
 	if errInterface == nil {
 		fields, err = returnValues[0].Interface().(map[string]string), nil
 		return
 	}
-
 	fields, err = returnValues[0].Interface().(map[string]string), errInterface.(error)
-
 	return
 }
diff --git a/internal/mapr/query.go b/internal/mapr/query.go
index 6c1d849..d7c32bd 100644
--- a/internal/mapr/query.go
+++ b/internal/mapr/query.go
@@ -32,7 +32,9 @@ type Query struct {
 }
 
 func (q Query) String() string {
-	return fmt.Sprintf("Query(Select:%v,Table:%s,Where:%v,Set:%vGroupBy:%v,GroupKey:%s,OrderBy:%v,ReverseOrder:%v,Interval:%v,Limit:%d,Outfile:%s,RawQuery:%s,tokens:%v,LogFormat:%s)",
+	return fmt.Sprintf("Query(Select:%v,Table:%s,Where:%v,Set:%vGroupBy:%v,"+
+		"GroupKey:%s,OrderBy:%v,ReverseOrder:%v,Interval:%v,Limit:%d,Outfile:%s,"+
+		"RawQuery:%s,tokens:%v,LogFormat:%s)",
 		q.Select,
 		q.Table,
 		q.Where,
@@ -54,18 +56,14 @@ func NewQuery(queryStr string) (*Query, error) {
 	if queryStr == "" {
 		return nil, nil
 	}
-
 	tokens := tokenize(queryStr)
-
 	q := Query{
 		RawQuery: queryStr,
 		tokens:   tokens,
 		Interval: time.Second * 5,
 		Limit:    -1,
 	}
-
-	err := q.parse(tokens)
-	return &q, err
+	return &q, q.parse(tokens)
 }
 
 // HasOutfile returns true if query result will be written to a CVS output file.
@@ -174,13 +172,13 @@ func (q *Query) parse(tokens []token) error {
 	}
 
 	if len(q.Select) < 1 {
-		return errors.New(invalidQuery + "Expected at least one field in 'select' clause but got none")
+		return errors.New(invalidQuery + "Expected at least one field in 'select' " +
+			"clause but got none")
 	}
 	if len(q.GroupBy) == 0 {
 		field := q.Select[0].Field
 		q.GroupBy = append(q.GroupBy, field)
 	}
-
 	if q.OrderBy != "" {
 		var orderFieldIsValid bool
 		for _, sc := range q.Select {
@@ -190,7 +188,8 @@ func (q *Query) parse(tokens []token) error {
 			}
 		}
 		if !orderFieldIsValid {
-			return errors.New(invalidQuery + fmt.Sprintf("Can not '(r)order by' '%s', must be present in 'select' clause", q.OrderBy))
+			return errors.New(invalidQuery + fmt.Sprintf("Can not '(r)order by' '%s',"+
+				"must be present in 'select' clause", q.OrderBy))
 		}
 	}
 
diff --git a/internal/mapr/query_test.go b/internal/mapr/query_test.go
index b0b6c3a..88f7387 100644
--- a/internal/mapr/query_test.go
+++ b/internal/mapr/query_test.go
@@ -13,18 +13,25 @@ func TestParseQuerySimple(t *testing.T) {
 		"select foo from bar where baz <",
 		"select foo from bar where baz < 100 bay eq 12 group",
 		"select foo from bar where baz < 100 bay eq 12 group by foo order by",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit set foo = bar;",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit set foo = bar;",
 	}
 	okQueries := []string{"select foo from bar",
 		"select foo from bar where",
 		"select foo from bar where baz < 100 bay eq 12",
 		"select foo from bar where baz < 100, bay eq 12",
 		"select foo from bar where baz < 100 and bay eq 12",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit 23",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit 23 outfile \"result.csv\"",
-		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz order by foo limit 23 outfile \"result.csv\" set $foo = maskdigits(bar), $baz = 12, $bay = $foo;",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit 23",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit 23 outfile \"result.csv\"",
+		"select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " +
+			"order by foo limit 23 outfile \"result.csv\" " +
+			"set $foo = maskdigits(bar), $baz = 12, $bay = $foo;",
 	}
 
 	for _, queryStr := range errorQueries {
@@ -46,8 +53,13 @@ func TestParseQuerySimple(t *testing.T) {
 
 func TestParseQueryDeep(t *testing.T) {
 	dialects := []string{
-		"select s1, `from`, count(s3) from table where w1 == 2 and w2 eq \"free beer\" group by g1, g2 order by count(s3) interval 10 limit 23 set $foo = maskdigits(bar), $baz = 12, $bay = $foo logformat generic",
-		"SELECT s1, `from`, COUNT(s3) FROM table WHERE w1 == 2 AND w2 eq \"free beer\" GROUP g1, g2 ORDER count(s3) INTERVAL 10 LIMIT 23 SET $foo = maskdigits(bar), $baz = 12, $bay = $foo logformat generic",
+		"select s1, `from`, count(s3) from table where w1 == 2 and w2 eq " +
+			"\"free beer\" group by g1, g2 order by count(s3) interval 10 limit 23 " +
+			"set $foo = maskdigits(bar), $baz = 12, $bay = $foo logformat generic",
+
+		"SELECT s1, `from`, COUNT(s3) FROM table WHERE w1 == 2 AND w2 eq " +
+			"\"free beer\" GROUP g1, g2 ORDER count(s3) INTERVAL 10 LIMIT 23 " +
+			"SET $foo = maskdigits(bar), $baz = 12, $bay = $foo logformat generic",
 	}
 
 	for _, queryStr := range dialects {
@@ -55,119 +67,144 @@ func TestParseQueryDeep(t *testing.T) {
 		if err != nil {
 			t.Errorf("%s: %s", err.Error(), queryStr)
 		}
-
 		t.Log(q)
 
 		// 'select' clause
 		if len(q.Select) != 3 {
-			t.Errorf("Expected three elements in 'select' clause but got '%v': %s\n%v", q.Select, queryStr, q)
+			t.Errorf("Expected three elements in 'select' clause but got '%v': %s\n%v",
+				q.Select, queryStr, q)
 		}
-
 		if q.Select[0].Field != "s1" {
-			t.Errorf("Expected 's1' as first element in 'select' clause but got '%v': %s\n%v", q.Select[0].Field, queryStr, q)
+			t.Errorf("Expected 's1' as first element in 'select' clause but got '%v': %s\n%v",
+				q.Select[0].Field, queryStr, q)
 		}
 		if q.Select[0].Operation != Last {
-			t.Errorf("Expected 'last' as aggregation function of first element in 'select' clause but got '%v': %s\n%v", q.Select[0].Operation, queryStr, q)
+			t.Errorf("Expected 'last' as aggregation function of first element in "+
+				"'select' clause but got '%v': %s\n%v", q.Select[0].Operation, queryStr, q)
 		}
-
 		if q.Select[1].Field != "from" {
-			t.Errorf("Expected 'from' as second element in 'select' clause but got '%v': %s\n%v", q.Select[1].Field, queryStr, q)
+			t.Errorf("Expected 'from' as second element in 'select' clause but got "+
+				"'%v': %s\n%v", q.Select[1].Field, queryStr, q)
 		}
 		if q.Select[1].Operation != Last {
-			t.Errorf("Expected 'last' as aggregation function of second element in 'select' clause but got '%v': %s\n%v", q.Select[1].Operation, queryStr, q)
+			t.Errorf("Expected 'last' as aggregation function of second element in "+
+				"'select' clause but got '%v': %s\n%v", q.Select[1].Operation, queryStr, q)
 		}
-
 		if q.Select[2].Field != "s3" {
-			t.Errorf("Expected 's3' as third element in 'select' clause but got '%v': %s\n%v", q.Select[2].Field, queryStr, q)
+			t.Errorf("Expected 's3' as third element in 'select' clause but got "+
+				"'%v': %s\n%v", q.Select[2].Field, queryStr, q)
 		}
 		if q.Select[2].Operation != Count {
-			t.Errorf("Expected 'count' as aggregation function of third  element in 'select' clause but got '%v': %s\n%v", q.Select[2].Operation, queryStr, q)
+			t.Errorf("Expected 'count' as aggregation function of third  element in "+
+				"'select' clause but got '%v': %s\n%v", q.Select[2].Operation, queryStr, q)
 		}
 		if q.Select[2].FieldStorage != "count(s3)" {
-			t.Errorf("Expected 'count(s3)' as third element's storage in 'select' clause but got '%v': %s\n%v", q.Select[2].FieldStorage, queryStr, q)
+			t.Errorf("Expected 'count(s3)' as third element's storage in 'select' "+
+				"clause but got '%v': %s\n%v", q.Select[2].FieldStorage, queryStr, q)
 		}
 
 		// 'from' clause
 		if q.Table != "TABLE" {
-			t.Errorf("Expected 'TABLE' in 'from' clause but got '%v': %s\n%v", q.Table, queryStr, q)
+			t.Errorf("Expected 'TABLE' in 'from' clause but got '%v': %s\n%v",
+				q.Table, queryStr, q)
 		}
 
 		// 'where' clause
 		if len(q.Where) != 2 {
-			t.Errorf("Expected two elements in 'where' clause but got '%v': %s\n%v", q.Where, queryStr, q)
+			t.Errorf("Expected two elements in 'where' clause but got '%v': %s\n%v",
+				q.Where, queryStr, q)
 		}
 		if q.Where[0].lString != "w1" {
-			t.Errorf("Expected w1 as first element in 'where' clause but got '%v': %s\n%v", q.Where[0].lString, queryStr, q)
+			t.Errorf("Expected w1 as first element in 'where' clause but got '%v': %s\n%v",
+				q.Where[0].lString, queryStr, q)
 		}
 		if q.Where[0].Operation != FloatEq {
-			t.Errorf("Expected FloatEq operation in first 'where' condition but got '%v': %s\n%v", q.Where[0].Operation, queryStr, q)
+			t.Errorf("Expected FloatEq operation in first 'where' condition but got "+
+				"'%v': %s\n%v", q.Where[0].Operation, queryStr, q)
 		}
 		if q.Where[0].rFloat != 2 {
-			t.Errorf("Expected '2' as float argument in first 'where' condition but got '%v': %s\n%v", q.Where[0].rFloat, queryStr, q)
+			t.Errorf("Expected '2' as float argument in first 'where' condition but "+
+				"got '%v': %s\n%v", q.Where[0].rFloat, queryStr, q)
 		}
 		if q.Where[1].lString != "w2" {
-			t.Errorf("Expected w2 as second element in 'where' clause but got '%v': %s\n%v", q.Where[1].lString, queryStr, q)
+			t.Errorf("Expected w2 as second element in 'where' clause but got '%v': "+
+				"%s\n%v", q.Where[1].lString, queryStr, q)
 		}
 		if q.Where[1].Operation != StringEq {
-			t.Errorf("Expected StringEq operation in second 'where' condition but got '%v': %s\n%v", q.Where[0].Operation, queryStr, q)
+			t.Errorf("Expected StringEq operation in second 'where' condition but got "+
+				"'%v': %s\n%v", q.Where[0].Operation, queryStr, q)
 		}
 		if q.Where[1].rString != "free beer" {
-			t.Errorf("Expected 'free beer' as string argument in second 'where' condition but got '%v': %s\n%v", q.Where[0].rString, queryStr, q)
+			t.Errorf("Expected 'free beer' as string argument in second 'where' "+
+				"condition but got '%v': %s\n%v", q.Where[0].rString, queryStr, q)
 		}
 
 		// 'group by' clause
 		if len(q.GroupBy) != 2 {
-			t.Errorf("Expected two elements in 'group by' clause but got '%v': %s\n%v", q.GroupBy, queryStr, q)
+			t.Errorf("Expected two elements in 'group by' clause but got '%v': %s\n%v",
+				q.GroupBy, queryStr, q)
 		}
 		if q.GroupBy[0] != "g1" {
-			t.Errorf("Expected 'g1' as first element in 'group by' clause but got '%v': %s\n%v", q.GroupBy[0], queryStr, q)
+			t.Errorf("Expected 'g1' as first element in 'group by' clause but got "+
+				"'%v': %s\n%v", q.GroupBy[0], queryStr, q)
 		}
 		if q.GroupBy[1] != "g2" {
-			t.Errorf("Expected 'g2' as second element in 'group by' clause but got '%v': %s\n%v", q.GroupBy[1], queryStr, q)
+			t.Errorf("Expected 'g2' as second element in 'group by' clause but got "+
+				"'%v': %s\n%v", q.GroupBy[1], queryStr, q)
 		}
 		if q.GroupKey != "g1,g2" {
-			t.Errorf("Expected 'g1,g2' as group key in 'group by' clause but got '%v': %s\n%v", q.GroupKey, queryStr, q)
+			t.Errorf("Expected 'g1,g2' as group key in 'group by' clause but got "+
+				"'%v': %s\n%v", q.GroupKey, queryStr, q)
 		}
 
 		// 'order by' clause
 		if q.OrderBy != "count(s3)" {
-			t.Errorf("Expected 'count(s3)' as element in 'order by' clause but got '%v': %s\n%v", q.OrderBy, queryStr, q)
+			t.Errorf("Expected 'count(s3)' as element in 'order by' clause but got "+
+				"'%v': %s\n%v", q.OrderBy, queryStr, q)
 		}
 
 		// 'interval' clause
 		if q.Interval != time.Second*time.Duration(10) {
-			t.Errorf("Expected '10s' as duration 'interval' clause but got '%v': %s\n%v", q.Interval, queryStr, q)
+			t.Errorf("Expected '10s' as duration 'interval' clause but got '%v': %s\n%v",
+				q.Interval, queryStr, q)
 		}
 
 		// 'limit' clause
 		if q.Limit != 23 {
-			t.Errorf("Expected '23' as limit in 'limit' clause but got '%v': %s\n%v", q.Limit, queryStr, q)
+			t.Errorf("Expected '23' as limit in 'limit' clause but got '%v': %s\n%v",
+				q.Limit, queryStr, q)
 		}
 
 		// 'set' clause
 		if q.Set[0].lString != "$foo" {
-			t.Errorf("Expected '$foo' lvalue in first 'set' condition clause but got '%v': %s\n%v", q.Set[0].lString, queryStr, q)
+			t.Errorf("Expected '$foo' lvalue in first 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[0].lString, queryStr, q)
 		}
 		if q.Set[0].rString != "bar" {
-			t.Errorf("Expected 'bar' rvalue in first 'set' condition clause but got '%v': %s\n%v", q.Set[0].rString, queryStr, q)
+			t.Errorf("Expected 'bar' rvalue in first 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[0].rString, queryStr, q)
 		}
-
 		if q.Set[1].lString != "$baz" {
-			t.Errorf("Expected '$baz' lvalue in second 'set' condition clause but got '%v': %s\n%v", q.Set[1].lString, queryStr, q)
+			t.Errorf("Expected '$baz' lvalue in second 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[1].lString, queryStr, q)
 		}
 		if q.Set[1].rString != "12" {
-			t.Errorf("Expected '12' rvalue in second 'set' condition clause but got '%v': %s\n%v", q.Set[1].rString, queryStr, q)
+			t.Errorf("Expected '12' rvalue in second 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[1].rString, queryStr, q)
 		}
-
 		if q.Set[2].lString != "$bay" {
-			t.Errorf("Expected '$bay' lvalue in third 'set' condition clause but got '%v': %s\n%v", q.Set[2].lString, queryStr, q)
+			t.Errorf("Expected '$bay' lvalue in third 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[2].lString, queryStr, q)
 		}
 		if q.Set[2].rString != "$foo" {
-			t.Errorf("Expected '$foo' rvalue in third 'set' condition clause but got '%v': %s\n%v", q.Set[2].rString, queryStr, q)
+			t.Errorf("Expected '$foo' rvalue in third 'set' condition clause but got "+
+				"'%v': %s\n%v", q.Set[2].rString, queryStr, q)
 		}
 
+		// 'logformat' clause
 		if q.LogFormat != "generic" {
-			t.Errorf("Expected 'generic' logformat got '%v': %s\n%v", q.LogFormat, queryStr, q)
+			t.Errorf("Expected 'generic' logformat got '%v': %s\n%v",
+				q.LogFormat, queryStr, q)
 		}
 	}
 }
diff --git a/internal/mapr/selectcondition.go b/internal/mapr/selectcondition.go
index d6aa0d4..5cfb8c7 100644
--- a/internal/mapr/selectcondition.go
+++ b/internal/mapr/selectcondition.go
@@ -37,7 +37,6 @@ func (sc selectCondition) String() string {
 
 func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 	var sel []selectCondition
-
 	// Parse select aggregation, e.g. sum(foo)
 	parse := func(token token) (selectCondition, error) {
 		var sc selectCondition
@@ -52,13 +51,15 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 
 		a := strings.Split(tokenStr, "(")
 		if len(a) != 2 {
-			return sc, errors.New(invalidQuery + "Can't parse 'select' aggregation: " + token.str)
+			return sc, errors.New(invalidQuery + "Can't parse 'select' aggregation: " +
+				token.str)
 		}
 		agg := a[0] // Aggregation, e.g. 'sum'
 
 		b := strings.Split(a[1], ")")
 		if len(b) != 2 {
-			return sc, errors.New(invalidQuery + "Can't parse 'select' field name from aggregation: " + token.str)
+			return sc, errors.New(invalidQuery + "Can't parse 'select' field name " +
+				"from aggregation: " + token.str)
 		}
 		sc.Field = b[0]            // Field name, e.g. 'foo'
 		sc.FieldStorage = tokenStr // e.g. 'sum(foo)'
@@ -79,9 +80,9 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 		case "len":
 			sc.Operation = Len
 		default:
-			return sc, errors.New(invalidQuery + "Unknown aggregation in 'select' clause: " + agg)
+			return sc, errors.New(invalidQuery +
+				"Unknown aggregation in 'select' clause: " + agg)
 		}
-
 		return sc, nil
 	}
 
@@ -92,6 +93,5 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) {
 		}
 		sel = append(sel, sc)
 	}
-
 	return sel, nil
 }
diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go
index 1f5d1c3..97fee11 100644
--- a/internal/mapr/server/aggregate.go
+++ b/internal/mapr/server/aggregate.go
@@ -63,16 +63,14 @@ func NewAggregate(queryStr string) (*Aggregate, error) {
 		}
 	}
 
-	a := Aggregate{
+	return &Aggregate{
 		done:        internal.NewDone(),
 		NextLinesCh: make(chan chan line.Line, 10),
 		serialize:   make(chan struct{}),
 		hostname:    s[0],
 		query:       query,
 		parser:      logParser,
-	}
-
-	return &a, nil
+	}, nil
 }
 
 // Shutdown the aggregation engine.
@@ -95,12 +93,10 @@ func (a *Aggregate) Start(ctx context.Context, maprMessages chan<- string) {
 	}()
 
 	fieldsCh := a.fieldsFromLines(myCtx)
-
 	// Add fields (e.g. via 'set' clause)
 	if len(a.query.Set) > 0 {
 		fieldsCh = a.setAdditionalFields(myCtx, fieldsCh)
 	}
-
 	// Periodically pre-aggregate data every a.query.Interval seconds.
 	go a.aggregateTimer(myCtx)
 	a.aggregateAndSerialize(myCtx, fieldsCh, maprMessages)
@@ -147,17 +143,18 @@ func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]strin
 
 				maprLine := strings.TrimSpace(line.Content.String())
 				fields, err := a.parser.MakeFields(maprLine)
-				// Can not recycle here for some rason.
+				// Can't recycle it here yet, as field slices are still
+				// TODO: Add unit test reading from multiple mapreduce files lines.
+				// TODO: Add capability to recycle this bytes buffer.
 				//pool.RecycleBytesBuffer(line.Content)
 
 				if err != nil {
 					// Should fields be ignored anyway?
-					if err != logformat.IgnoreFieldsErr {
+					if err != logformat.ErrIgnoreFields {
 						dlog.Common.Error(fields, err)
 					}
 					continue
 				}
-
 				if !a.query.WhereClause(fields) {
 					continue
 				}
@@ -175,12 +172,12 @@ func (a *Aggregate) fieldsFromLines(ctx context.Context) <-chan map[string]strin
 	return fieldsCh
 }
 
-func (a *Aggregate) setAdditionalFields(ctx context.Context, fieldsCh <-chan map[string]string) <-chan map[string]string {
-	newFieldsCh := make(chan map[string]string)
+func (a *Aggregate) setAdditionalFields(ctx context.Context,
+	fieldsCh <-chan map[string]string) <-chan map[string]string {
 
+	newFieldsCh := make(chan map[string]string)
 	go func() {
 		defer close(newFieldsCh)
-
 		for {
 			fields, ok := <-fieldsCh
 			if !ok {
@@ -196,19 +193,18 @@ func (a *Aggregate) setAdditionalFields(ctx context.Context, fieldsCh <-chan map
 			}
 		}
 	}()
-
 	return newFieldsCh
 }
 
-func (a *Aggregate) aggregateAndSerialize(ctx context.Context, fieldsCh <-chan map[string]string, maprMessages chan<- string) {
-	group := mapr.NewGroupSet()
+func (a *Aggregate) aggregateAndSerialize(ctx context.Context,
+	fieldsCh <-chan map[string]string, maprMessages chan<- string) {
 
+	group := mapr.NewGroupSet()
 	serialize := func() {
 		dlog.Common.Info("Serializing mapreduce result")
 		group.Serialize(ctx, maprMessages)
 		group = mapr.NewGroupSet()
 	}
-
 	for {
 		select {
 		case fields, ok := <-fieldsCh:
@@ -227,7 +223,6 @@ func (a *Aggregate) aggregateAndSerialize(ctx context.Context, fieldsCh <-chan m
 
 func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 	var sb strings.Builder
-
 	for i, field := range a.query.GroupBy {
 		if i > 0 {
 			sb.WriteString(protocol.AggregateGroupKeyCombinator)
@@ -254,7 +249,6 @@ func (a *Aggregate) aggregate(group *mapr.GroupSet, fields map[string]string) {
 		set.Samples++
 		return
 	}
-
 	dlog.Common.Trace("Aggregated data locally without adding new samples")
 }
 
diff --git a/internal/mapr/setclause.go b/internal/mapr/setclause.go
index b4c2f73..1843d31 100644
--- a/internal/mapr/setclause.go
+++ b/internal/mapr/setclause.go
@@ -7,7 +7,6 @@ func (q *Query) SetClause(fields map[string]string) error {
 		if !ok {
 			continue
 		}
-
 		switch sc.rType {
 		case FunctionStack:
 			fields[sc.lString] = sc.functionStack.Call(value)
@@ -15,6 +14,5 @@ func (q *Query) SetClause(fields map[string]string) error {
 			fields[sc.lString] = value
 		}
 	}
-
 	return nil
 }
diff --git a/internal/mapr/setcondition.go b/internal/mapr/setcondition.go
index 8c5cfc9..92b21f4 100644
--- a/internal/mapr/setcondition.go
+++ b/internal/mapr/setcondition.go
@@ -39,20 +39,22 @@ func makeSetConditions(tokens []token) (set []setCondition, err error) {
 		switch setOp {
 		case "=":
 		default:
-			return sc, nil, errors.New(invalidQuery + "Unknown operation in 'set' clause: " + setOp)
+			return sc, nil, errors.New(invalidQuery + "Unknown operation in 'set' " +
+				"clause: " + setOp)
 		}
 
 		if !tokens[0].isBareword {
-			return sc, nil, errors.New(invalidQuery + "Expected bareword at 'set' clause's lValue: " + tokens[0].str)
+			return sc, nil, errors.New(invalidQuery + "Expected bareword at 'set' " +
+				"clause's lValue: " + tokens[0].str)
 		}
-
 		sc.lString = tokens[0].str
 		if !strings.HasPrefix(sc.lString, "$") {
-			return sc, nil, errors.New(invalidQuery + "Expected field variable name (starting with $) at 'set' clause's lValue: " + tokens[0].str)
+			return sc, nil, errors.New(invalidQuery + "Expected field variable name " +
+				"(starting with $) at 'set' clause's lValue: " + tokens[0].str)
 		}
 		sc.rType = Field
-
 		rString := tokens[2].str
+
 		// Seems like a function call?
 		if strings.HasSuffix(rString, ")") {
 			functionStack, functionArg, err := funcs.NewFunctionStack(tokens[2].str)
@@ -72,7 +74,6 @@ func makeSetConditions(tokens []token) (set []setCondition, err error) {
 		} else {
 			sc.rType = Field
 		}
-
 		return sc, tokens[3:], nil
 	}
 
@@ -84,10 +85,8 @@ func makeSetConditions(tokens []token) (set []setCondition, err error) {
 		if err != nil {
 			return nil, err
 		}
-
 		set = append(set, sc)
 		tokens = tokensConsumeOptional(tokens, ",")
 	}
-
 	return
 }
diff --git a/internal/mapr/token.go b/internal/mapr/token.go
index 7c6578b..6ac7631 100644
--- a/internal/mapr/token.go
+++ b/internal/mapr/token.go
@@ -4,7 +4,8 @@ import (
 	"strings"
 )
 
-var keywords = [...]string{"select", "from", "where", "set", "group", "rorder", "order", "interval", "limit", "outfile", "logformat"}
+var keywords = [...]string{"select", "from", "where", "set", "group", "rorder",
+	"order", "interval", "limit", "outfile", "logformat"}
 
 // Represents a parsed token, used to parse the mapr query.
 type token struct {
@@ -16,13 +17,11 @@ func (t token) isKeyword() bool {
 	if !t.isBareword {
 		return false
 	}
-
 	for _, keyword := range keywords {
 		if strings.ToLower(t.str) == keyword {
 			return true
 		}
 	}
-
 	return false
 }
 
@@ -32,7 +31,6 @@ func (t token) String() string {
 
 func tokenize(queryStr string) []token {
 	var tokens []token
-
 	for i, part := range strings.Split(queryStr, "\"") {
 		// Even i, means that it is not a quoted string
 		if i%2 == 0 {
@@ -53,14 +51,12 @@ func tokenize(queryStr string) []token {
 		}
 		tokens = append(tokens, token)
 	}
-
 	return tokens
 }
 
 func tokensConsume(tokens []token) ([]token, []token) {
 	//dlog.Common.Trace("=====================")
 	var consumed []token
-
 	for i, t := range tokens {
 		if t.isKeyword() {
 			//dlog.Common.Trace("keyword", t)
@@ -84,7 +80,6 @@ func tokensConsume(tokens []token) ([]token, []token) {
 		//dlog.Common.Trace("bare", token)
 		consumed = append(consumed, t)
 	}
-
 	//dlog.Common.Trace("result", consumed)
 	return nil, consumed
 }
@@ -95,7 +90,6 @@ func tokensConsumeStr(tokens []token) ([]token, []string) {
 	for _, token := range found {
 		strings = append(strings, token.str)
 	}
-
 	return tokens, strings
 }
 
@@ -106,6 +100,5 @@ func tokensConsumeOptional(tokens []token, optional string) []token {
 	if strings.ToLower(tokens[0].str) == strings.ToLower(optional) {
 		return tokens[1:]
 	}
-
 	return tokens
 }
diff --git a/internal/mapr/whereclause.go b/internal/mapr/whereclause.go
index 6356d94..d9f32eb 100644
--- a/internal/mapr/whereclause.go
+++ b/internal/mapr/whereclause.go
@@ -10,7 +10,6 @@ import (
 func (q *Query) WhereClause(fields map[string]string) bool {
 	for _, wc := range q.Where {
 		var ok bool
-
 		if wc.Operation > FloatOperation {
 			var lValue, rValue float64
 			if lValue, ok = whereClauseFloatValue(fields, wc.lString, wc.lFloat, wc.lType); !ok {
@@ -36,11 +35,12 @@ func (q *Query) WhereClause(fields map[string]string) bool {
 			return false
 		}
 	}
-
 	return true
 }
 
-func whereClauseFloatValue(fields map[string]string, str string, float float64, t fieldType) (float64, bool) {
+func whereClauseFloatValue(fields map[string]string, str string, float float64,
+	t fieldType) (float64, bool) {
+
 	switch t {
 	case Float:
 		return float, true
@@ -60,7 +60,9 @@ func whereClauseFloatValue(fields map[string]string, str string, float float64,
 	}
 }
 
-func whereClauseStringValue(fields map[string]string, str string, t fieldType) (string, bool) {
+func whereClauseStringValue(fields map[string]string, str string,
+	t fieldType) (string, bool) {
+
 	switch t {
 	case Field:
 		value, ok := fields[str]
diff --git a/internal/mapr/wherecondition.go b/internal/mapr/wherecondition.go
index c60c0a5..280dcfb 100644
--- a/internal/mapr/wherecondition.go
+++ b/internal/mapr/wherecondition.go
@@ -46,15 +46,18 @@ type whereCondition struct {
 }
 
 func (wc *whereCondition) String() string {
-	return fmt.Sprintf("whereCondition(Operation:%v,lString:%s,lFloat:%v,lType:%s,rString:%s,rFloat:%v,rType:%s)",
-		wc.Operation, wc.lString, wc.lFloat, wc.lType.String(), wc.rString, wc.rFloat, wc.rType.String())
+	return fmt.Sprintf("whereCondition(Operation:%v,lString:%s,lFloat:%v,"+
+		"lType:%s,rString:%s,rFloat:%v,rType:%s)",
+		wc.Operation, wc.lString, wc.lFloat, wc.lType.String(), wc.rString,
+		wc.rFloat, wc.rType.String())
 }
 
 func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 	parse := func(tokens []token) (whereCondition, []token, error) {
 		var wc whereCondition
 		if len(tokens) < 3 {
-			return wc, nil, errors.New(invalidQuery + "Not enough arguments in 'where' clause")
+			err := errors.New(invalidQuery + "Not enough arguments in 'where' clause")
+			return wc, nil, err
 		}
 
 		whereOp := strings.ToLower(tokens[1].str)
@@ -94,7 +97,8 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 		case "nhassuffix":
 			wc.Operation = StringNotHasSuffix
 		default:
-			return wc, nil, errors.New(invalidQuery + "Unknown operation in 'where' clause: " + whereOp)
+			return wc, nil, errors.New(invalidQuery +
+				"Unknown operation in 'where' clause: " + whereOp)
 		}
 
 		wc.lString = tokens[0].str
@@ -102,7 +106,8 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 
 		if wc.Operation > FloatOperation {
 			if !tokens[0].isBareword {
-				return wc, nil, errors.New(invalidQuery + "Expected bareword at 'where' clause's lValue: " + tokens[0].str)
+				return wc, nil, errors.New(invalidQuery +
+					"Expected bareword at 'where' clause's lValue: " + tokens[0].str)
 			}
 			if f, err := strconv.ParseFloat(wc.lString, 64); err == nil {
 				wc.lFloat = f
@@ -112,7 +117,8 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 			}
 
 			if !tokens[2].isBareword {
-				return wc, nil, errors.New(invalidQuery + "Expected bareword at 'where' clause's rValue: " + tokens[2].str)
+				return wc, nil, errors.New(invalidQuery +
+					"Expected bareword at 'where' clause's rValue: " + tokens[2].str)
 			}
 			if f, err := strconv.ParseFloat(wc.rString, 64); err == nil {
 				wc.rFloat = f
@@ -133,23 +139,19 @@ func makeWhereConditions(tokens []token) (where []whereCondition, err error) {
 		} else {
 			wc.rType = String
 		}
-
 		return wc, tokens[3:], nil
 	}
 
 	for len(tokens) > 0 {
 		var wc whereCondition
 		var err error
-
 		wc, tokens, err = parse(tokens)
 		if err != nil {
 			return nil, err
 		}
-
 		where = append(where, wc)
 		tokens = tokensConsumeOptional(tokens, "and")
 	}
-
 	return
 }
 
@@ -170,7 +172,6 @@ func (wc *whereCondition) floatClause(lValue float64, rValue float64) bool {
 	default:
 		dlog.Common.Error("Unknown float operation", lValue, wc.Operation, rValue)
 	}
-
 	return false
 }
 
@@ -195,6 +196,5 @@ func (wc *whereCondition) stringClause(lValue string, rValue string) bool {
 	default:
 		dlog.Common.Error("Unknown string operation", lValue, wc.Operation, rValue)
 	}
-
 	return false
 }
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index 8c9e861..d29706c 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -13,7 +13,6 @@ const (
 	AggregateKVDelimiter string = "≔"
 	// AggregateDelimiter delimits parts of an aggregation message.
 	AggregateDelimiter string = "∥"
-	// AggregateDelimiter string = "⦀"
 	// AggregateGroupKeyCombinator combines the group set keys.
 	AggregateGroupKeyCombinator string = ","
 )
diff --git a/internal/regex/regex.go b/internal/regex/regex.go
index 352ffd6..eb6e1b3 100644
--- a/internal/regex/regex.go
+++ b/internal/regex/regex.go
@@ -48,9 +48,7 @@ func new(regexStr string, flags []Flag) (Regex, error) {
 		regexStr: regexStr,
 		flags:    flags,
 	}
-
 	re, err := regexp.Compile(regexStr)
-
 	if err != nil {
 		return r, err
 	}
@@ -94,11 +92,9 @@ func (r Regex) Serialize() (string, error) {
 	for _, flag := range r.flags {
 		flags = append(flags, flag.String())
 	}
-
 	if !r.initialized {
 		return "", fmt.Errorf("Unable to serialize regex as not initialized properly: %v", r)
 	}
-
 	return fmt.Sprintf("regex:%s %s", strings.Join(flags, ","), r.regexStr), nil
 }
 
@@ -109,12 +105,12 @@ func Deserialize(str string) (Regex, error) {
 	if len(s) < 2 {
 		return NewNoop(), nil
 	}
-
 	flagsStr := s[0]
 	regexStr := s[1]
 
 	if !strings.HasPrefix(flagsStr, "regex") {
-		return Regex{}, fmt.Errorf("unable to deserialize regex '%s': should start with string 'regex'", str)
+		return Regex{}, fmt.Errorf("unable to deserialize regex '%s': should start "+
+			"with string 'regex'", str)
 	}
 
 	// Parse regex flags, e.g. "regex:flag1,flag2,flag3..."
@@ -129,6 +125,5 @@ func Deserialize(str string) (Regex, error) {
 			flags = append(flags, flag)
 		}
 	}
-
 	return new(regexStr, flags)
 }
diff --git a/internal/regex/regex_test.go b/internal/regex/regex_test.go
index 2ce49ac..033a286 100644
--- a/internal/regex/regex_test.go
+++ b/internal/regex/regex_test.go
@@ -9,7 +9,8 @@ func TestRegex(t *testing.T) {
 
 	r := NewNoop()
 	if !r.MatchString(input) {
-		t.Errorf("expected to match string '%s' with noop regex '%v' but didn't\n", input, r)
+		t.Errorf("expected to match string '%s' with noop regex '%v' but didn't\n",
+			input, r)
 	}
 
 	r, err := New(".hello", Default)
@@ -17,7 +18,8 @@ func TestRegex(t *testing.T) {
 		t.Errorf("unable to create regex: %v\n", err)
 	}
 	if r.MatchString(input) {
-		t.Errorf("expected to match string '%s' with regex '%v' but didn't\n", input, r)
+		t.Errorf("expected to match string '%s' with regex '%v' but didn't\n",
+			input, r)
 	}
 
 	serialized, err := r.Serialize()
@@ -29,8 +31,8 @@ func TestRegex(t *testing.T) {
 		t.Errorf("unable to serialize deserialized regex: %v: %v\n", serialized, err)
 	}
 	if r.String() != r2.String() {
-		t.Errorf("regex should be the same after deserialize(serialize(..)), got '%s' but expected '%s'.\n",
-			r2.String(), r.String())
+		t.Errorf("regex should be the same after deserialize(serialize(..)), got "+
+			"'%s' but expected '%s'.\n", r2.String(), r.String())
 	}
 
 	r, err = New(".hello", Invert)
@@ -38,7 +40,8 @@ func TestRegex(t *testing.T) {
 		t.Errorf("unable to create regex: %v\n", err)
 	}
 	if !r.MatchString(input) {
-		t.Errorf("expected to not match string '%s' with regex '%v' but matched\n", input, r)
+		t.Errorf("expected to not match string '%s' with regex '%v' but matched\n",
+			input, r)
 	}
 
 	serialized, err = r.Serialize()
@@ -50,7 +53,7 @@ func TestRegex(t *testing.T) {
 		t.Errorf("unable to serialize deserialized regex: %v: %v\n", serialized, err)
 	}
 	if r.String() != r2.String() {
-		t.Errorf("regex should be the same after deserialize(serialize(..)), got '%s' but expected '%s'.\n",
-			r2.String(), r.String())
+		t.Errorf("regex should be the same after deserialize(serialize(..)), got "+
+			"'%s' but expected '%s'.\n", r2.String(), r.String())
 	}
 }
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index 5f84afc..93b3fcb 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -13,8 +13,7 @@ import (
 	gossh "golang.org/x/crypto/ssh"
 )
 
-type continuous struct {
-}
+type continuous struct{}
 
 func newContinuous() *continuous {
 	return &continuous{}
@@ -23,7 +22,6 @@ func newContinuous() *continuous {
 func (c *continuous) start(ctx context.Context) {
 	dlog.Server.Info("Starting continuous job runner after 10s")
 	time.Sleep(time.Second * 10)
-
 	c.runJobs(ctx)
 }
 
@@ -33,7 +31,6 @@ func (c *continuous) runJobs(ctx context.Context) {
 			dlog.Server.Debug(job.Name, "Not running job as not enabled")
 			continue
 		}
-
 		go func(job config.Continuous) {
 			c.runJob(ctx, job)
 			for {
@@ -54,7 +51,6 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 
 	files := fillDates(job.Files)
 	outfile := fillDates(job.Outfile)
-
 	servers := strings.Join(job.Servers, ",")
 	if servers == "" {
 		servers = config.Server.SSHBindAddress
@@ -70,7 +66,6 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	}
 
 	args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(job.Name))
-
 	args.QueryStr = fmt.Sprintf("%s outfile %s", job.Query, outfile)
 	client, err := clients.NewMaprClient(args, clients.NonCumulativeMode)
 	if err != nil {
@@ -80,7 +75,6 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 
 	jobCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
-
 	if job.RestartOnDayChange {
 		go func() {
 			if c.waitForDayChange(ctx) {
@@ -93,7 +87,6 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	dlog.Server.Info(fmt.Sprintf("Starting job %s", job.Name))
 	status := client.Start(jobCtx, make(chan string))
 	logMessage := fmt.Sprintf("Job exited with status %d", status)
-
 	if status != 0 {
 		dlog.Server.Warn(logMessage)
 		return
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index f73f82e..847e8f9 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -37,7 +37,7 @@ type baseHandler struct {
 	activeCommands   int32
 	quiet            bool
 	spartan          bool
-	serverless       bool
+	serverless       int32
 	readBuf          bytes.Buffer
 	writeBuf         bytes.Buffer
 }
@@ -59,16 +59,14 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	select {
 	case message := <-h.serverMessages:
 		if message[0] == '.' {
-			// Handle hidden message (don't display to the user, interpreted by dtail client)
+			// Handle hidden message (don't display to the user)
 			h.readBuf.WriteString(message)
 			h.readBuf.WriteByte(protocol.MessageDelimiter)
 			n = copy(p, h.readBuf.Bytes())
 			return
 		}
 
-		if h.serverless {
-			// In serverless mode we have logged the server message already via the
-			// dlog logger, no need to send the message again to the client part.
+		if h.serverless > 0 {
 			return
 		}
 
@@ -132,7 +130,6 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 			h.writeBuf.WriteByte(b)
 		}
 	}
-
 	n = len(p)
 	return
 }
@@ -145,13 +142,11 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err)+add)
 		return
 	}
-
 	args, argc, err = h.handleBase64(args, argc)
 	if err != nil {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
 	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
 		<-h.done.Done()
@@ -160,7 +155,6 @@ func (h *baseHandler) handleCommand(commandStr string) {
 
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
-
 	options, err := config.DeserializeOptions(splitted[1:])
 	if err != nil {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
@@ -191,8 +185,8 @@ func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, strin
 		if clientCompat > serverCompat {
 			toUpdate = "server"
 		}
-
-		err := fmt.Errorf("DTail server protocol version '%s' does not match client protocol version '%s', please update DTail %s!",
+		err := fmt.Errorf("the DTail server protocol version '%s' does not match "+
+			"client protocol version '%s', please update DTail %s",
 			protocol.ProtocolCompat, args[1], toUpdate)
 		return args, argc, add, err
 	}
@@ -201,8 +195,8 @@ func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, strin
 }
 
 func (h *baseHandler) handleBase64(args []string, argc int) ([]string, int, error) {
-	err := errors.New("Unable to decode client message, DTail server and client versions may not be compatible")
-
+	err := errors.New("unable to decode client message, DTail server and client " +
+		"versions may not be compatible")
 	if argc != 2 || args[0] != "base64" {
 		return args, argc, err
 	}
@@ -215,7 +209,8 @@ func (h *baseHandler) handleBase64(args []string, argc int) ([]string, int, erro
 
 	args = strings.Split(decodedStr, " ")
 	argc = len(decodedStr)
-	dlog.Server.Trace(h.user, "Base64 decoded received command", decodedStr, argc, args)
+	dlog.Server.Trace(h.user, "Base64 decoded received command",
+		decodedStr, argc, args)
 
 	return args, argc, nil
 }
@@ -223,7 +218,8 @@ func (h *baseHandler) handleBase64(args []string, argc int) ([]string, int, erro
 func (h *baseHandler) handleAckCommand(argc int, args []string) {
 	if argc < 3 {
 		if !h.quiet {
-			h.send(h.serverMessages, dlog.Server.Warn(h.user, "Unable to parse command", args, argc))
+			h.send(h.serverMessages, dlog.Server.Warn(h.user,
+				"Unable to parse command", args, argc))
 		}
 		return
 	}
@@ -245,11 +241,9 @@ func (h *baseHandler) send(ch chan<- string, message string) {
 
 func (h *baseHandler) flush() {
 	dlog.Server.Trace(h.user, "flush()")
-
 	numUnsentMessages := func() int {
 		return len(h.lines) + len(h.serverMessages) + len(h.maprMessages)
 	}
-
 	for i := 0; i < 10; i++ {
 		if numUnsentMessages() == 0 {
 			dlog.Server.Debug(h.user, "ALL lines sent", fmt.Sprintf("%p", h))
@@ -258,7 +252,6 @@ func (h *baseHandler) flush() {
 		dlog.Server.Debug(h.user, "Still lines to be sent")
 		time.Sleep(time.Millisecond * 10)
 	}
-
 	dlog.Server.Warn(h.user, "Some lines remain unsent", numUnsentMessages())
 }
 
@@ -279,7 +272,6 @@ func (h *baseHandler) shutdown() {
 		dlog.Server.Debug(h.user, "Shutdown timeout reached, enforcing shutdown")
 	case <-h.done.Done():
 	}
-
 	h.done.Shutdown()
 }
 
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
index 347ff66..8d6c400 100644
--- a/internal/server/handlers/healthhandler.go
+++ b/internal/server/handlers/healthhandler.go
@@ -35,24 +35,23 @@ func NewHealthHandler(user *user.User) *HealthHandler {
 	if err != nil {
 		dlog.Server.FatalPanic(err)
 	}
-
 	s := strings.Split(fqdn, ".")
 	h.hostname = s[0]
-
 	return &h
 }
 
-func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int, args []string,
-	commandName string, options map[string]string) {
-	dlog.Server.Debug(h.user, "Handling health command", argc, args)
+func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int,
+	args []string, commandName string, options map[string]string) {
 
+	dlog.Server.Debug(h.user, "Handling health command", argc, args)
 	switch commandName {
 	case "health":
 		h.send(h.serverMessages, "OK")
 	case ".ack":
 		h.handleAckCommand(argc, args)
 	default:
-		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown health command", commandName, argc, args))
+		h.send(h.serverMessages, dlog.Server.Error(h.user,
+			"Received unknown health command", commandName, argc, args))
 	}
 	h.shutdown()
 }
diff --git a/internal/server/handlers/mapcommand.go b/internal/server/handlers/mapcommand.go
index c3e600e..65e0ed8 100644
--- a/internal/server/handlers/mapcommand.go
+++ b/internal/server/handlers/mapcommand.go
@@ -14,18 +14,17 @@ type mapCommand struct {
 }
 
 // NewMapCommand returns a new server side mapreduce command.
-func newMapCommand(serverHandler *ServerHandler, argc int, args []string) (mapCommand, *server.Aggregate, error) {
-	m := mapCommand{server: serverHandler}
+func newMapCommand(serverHandler *ServerHandler, argc int,
+	args []string) (mapCommand, *server.Aggregate, error) {
 
+	m := mapCommand{server: serverHandler}
 	queryStr := strings.Join(args[1:], " ")
 	aggregate, err := server.NewAggregate(queryStr)
 	if err != nil {
 		return m, nil, err
 	}
-
 	m.aggregate = aggregate
 	return m, aggregate, nil
-
 }
 
 func (m mapCommand) Start(ctx context.Context, aggregatedMessages chan<- string) {
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index abc44c7..384e966 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -26,25 +26,30 @@ func newReadCommand(server *ServerHandler, mode omode.Mode) *readCommand {
 	}
 }
 
-func (r *readCommand) Start(ctx context.Context, argc int, args []string, retries int) {
-	re := regex.NewNoop()
+func (r *readCommand) Start(ctx context.Context, argc int, args []string,
+	retries int) {
 
+	re := regex.NewNoop()
 	if argc >= 4 {
 		deserializedRegex, err := regex.Deserialize(strings.Join(args[2:], " "))
 		if err != nil {
-			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user, "Unable to parse command", err))
+			r.server.send(r.server.serverMessages, dlog.Server.Error(r.server.user,
+				"Unable to parse command", err))
 			return
 		}
 		re = deserializedRegex
 	}
 	if argc < 3 {
-		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to parse command", args, argc))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user,
+			"Unable to parse command", args, argc))
 		return
 	}
 	r.readGlob(ctx, args[1], re, retries)
 }
 
-func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex, retries int) {
+func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
+	retries int) {
+
 	retryInterval := time.Second * 5
 	glob = filepath.Clean(glob)
 
@@ -58,7 +63,8 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 
 		if numPaths := len(paths); numPaths == 0 {
 			dlog.Server.Error(r.server.user, "No such file(s) to read", glob)
-			r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
+			r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user,
+				"Unable to read file(s), check server logs"))
 			select {
 			case <-ctx.Done():
 				return
@@ -72,31 +78,33 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 		return
 	}
 
-	r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Giving up to read file(s)"))
+	r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user,
+		"Giving up to read file(s)"))
 	return
 }
 
-func (r *readCommand) readFiles(ctx context.Context, paths []string, glob string, re regex.Regex, retryInterval time.Duration) {
+func (r *readCommand) readFiles(ctx context.Context, paths []string, glob string,
+	re regex.Regex, retryInterval time.Duration) {
+
 	var wg sync.WaitGroup
 	wg.Add(len(paths))
-
 	for _, path := range paths {
 		go r.readFileIfPermissions(ctx, &wg, path, glob, re)
 	}
-
 	wg.Wait()
 }
 
-func (r *readCommand) readFileIfPermissions(ctx context.Context, wg *sync.WaitGroup, path, glob string, re regex.Regex) {
+func (r *readCommand) readFileIfPermissions(ctx context.Context,
+	wg *sync.WaitGroup, path, glob string, re regex.Regex) {
+
 	defer wg.Done()
 	globID := r.makeGlobID(path, glob)
-
 	if !r.server.user.HasFilePermission(path, "readfiles") {
 		dlog.Server.Error(r.server.user, "No permission to read file", path, globID)
-		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user, "Unable to read file(s), check server logs"))
+		r.server.send(r.server.serverMessages, dlog.Server.Warn(r.server.user,
+			"Unable to read file(s), check server logs"))
 		return
 	}
-
 	r.readFile(ctx, path, globID, re)
 }
 
@@ -137,7 +145,6 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 				return
 			}
 		}
-
 		time.Sleep(time.Second * 2)
 		dlog.Server.Info(path, globID, "Reading file again")
 	}
@@ -156,11 +163,11 @@ func (r *readCommand) makeGlobID(path, glob string) string {
 	if len(idParts) > 0 {
 		return strings.Join(idParts, "/")
 	}
-
 	if len(pathParts) > 0 {
 		return pathParts[len(pathParts)-1]
 	}
 
-	r.server.send(r.server.serverMessages, dlog.Server.Warn("Empty file path given?", path, glob))
+	r.server.send(r.server.serverMessages,
+		dlog.Server.Warn("Empty file path given?", path, glob))
 	return ""
 }
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index aed8956..f12d590 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"strings"
+	"sync/atomic"
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/dlog"
@@ -23,7 +24,9 @@ type ServerHandler struct {
 }
 
 // NewServerHandler returns the server handler.
-func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *ServerHandler {
+func NewServerHandler(user *user.User, catLimiter,
+	tailLimiter chan struct{}) *ServerHandler {
+
 	dlog.Server.Debug(user, "Creating new server handler")
 	h := ServerHandler{
 		baseHandler: baseHandler{
@@ -51,11 +54,10 @@ func NewServerHandler(user *user.User, catLimiter, tailLimiter chan struct{}) *S
 	return &h
 }
 
-func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []string,
-	commandName string, options map[string]string) {
+func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
+	args []string, commandName string, options map[string]string) {
 
 	dlog.Server.Debug(h.user, "Handling user command", argc, args)
-
 	h.incrementActiveCommands()
 	commandFinished := func() {
 		if h.decrementActiveCommands() == 0 {
@@ -73,7 +75,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 	}
 	if serverless, _ := options["serverless"]; serverless == "true" {
 		dlog.Server.Debug(h.user, "Enabling serverless mode")
-		h.serverless = true
+		atomic.AddInt32(&h.serverless, 1)
 	}
 
 	switch commandName {
@@ -83,14 +85,12 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			command.Start(ctx, argc, args, 1)
 			commandFinished()
 		}()
-
 	case "tail":
 		command := newReadCommand(h, omode.TailClient)
 		go func() {
 			command.Start(ctx, argc, args, 10)
 			commandFinished()
 		}()
-
 	case "map":
 		command, aggregate, err := newMapCommand(h, argc, args)
 		if err != nil {
@@ -99,19 +99,17 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args []
 			commandFinished()
 			return
 		}
-
 		h.aggregate = aggregate
 		go func() {
 			command.Start(ctx, h.maprMessages)
 			commandFinished()
 		}()
-
 	case ".ack":
 		h.handleAckCommand(argc, args)
 		commandFinished()
-
 	default:
-		h.send(h.serverMessages, dlog.Server.Error(h.user, "Received unknown user command", commandName, argc, args, options))
+		h.send(h.serverMessages, dlog.Server.Error(h.user,
+			"Received unknown user command", commandName, argc, args, options))
 		commandFinished()
 	}
 }
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index ccb2225..0ba65f7 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -16,8 +16,7 @@ import (
 	gossh "golang.org/x/crypto/ssh"
 )
 
-type scheduler struct {
-}
+type scheduler struct{}
 
 func newScheduler() *scheduler {
 	return &scheduler{}
@@ -28,7 +27,6 @@ func (s *scheduler) start(ctx context.Context) {
 	// First run after just 10s!
 	time.Sleep(time.Second * 10)
 	s.runJobs(ctx)
-
 	for {
 		select {
 		case <-time.After(time.Minute):
@@ -45,13 +43,11 @@ func (s *scheduler) runJobs(ctx context.Context) {
 			dlog.Server.Debug(job.Name, "Not running job as not enabled")
 			continue
 		}
-
 		hour, err := strconv.Atoi(time.Now().Format("15"))
 		if err != nil {
 			dlog.Server.Error(job.Name, "Unable to create job", err)
 			continue
 		}
-
 		if hour < job.TimeRange[0] || hour >= job.TimeRange[1] {
 			dlog.Server.Debug(job.Name, "Not running job out of time range")
 			continue
@@ -59,7 +55,6 @@ func (s *scheduler) runJobs(ctx context.Context) {
 
 		files := fillDates(job.Files)
 		outfile := fillDates(job.Outfile)
-
 		_, err = os.Stat(outfile)
 		if !os.IsNotExist(err) {
 			dlog.Server.Debug(job.Name, "Not running job as outfile already exists", outfile)
@@ -70,7 +65,6 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		if servers == "" {
 			servers = config.Server.SSHBindAddress
 		}
-
 		args := config.Args{
 			ConnectionsPerCPU: config.DefaultConnectionsPerCPU,
 			Discovery:         job.Discovery,
@@ -81,7 +75,6 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		}
 
 		args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(job.Name))
-
 		args.QueryStr = fmt.Sprintf("%s outfile %s", job.Query, outfile)
 		client, err := clients.NewMaprClient(args, clients.CumulativeMode)
 		if err != nil {
diff --git a/internal/server/server.go b/internal/server/server.go
index b3d4bff..0cb5e27 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -24,9 +24,9 @@ type Server struct {
 	stats stats
 	// SSH server configuration.
 	sshServerConfig *gossh.ServerConfig
-	// To control the max amount of concurrent cats (which can cause a lot of I/O on the server)
+	// To control the max amount of concurrent cats.
 	catLimiter chan struct{}
-	// To control the max amount of concurrent tails
+	// To control the max amount of concurrent tails.
 	tailLimiter chan struct{}
 	// To run scheduled tasks (if configured)
 	sched *scheduler
@@ -61,7 +61,6 @@ func New() *Server {
 // Start the server.
 func (s *Server) Start(ctx context.Context) int {
 	dlog.Server.Info("Starting server")
-
 	bindAt := fmt.Sprintf("%s:%d", config.Server.SSHBindAddress, config.Common.SSHPort)
 	dlog.Server.Info("Binding server", bindAt)
 
@@ -76,14 +75,12 @@ func (s *Server) Start(ctx context.Context) int {
 	go s.listenerLoop(ctx, listener)
 
 	<-ctx.Done()
-
 	// For future use.
 	return 0
 }
 
 func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 	dlog.Server.Debug("Starting listener loop")
-
 	for {
 		conn, err := listener.Accept() // Blocking
 		if err != nil {
@@ -101,7 +98,6 @@ func (s *Server) listenerLoop(ctx context.Context, listener net.Listener) {
 			conn.Close()
 			continue
 		}
-
 		go s.handleConnection(ctx, conn)
 	}
 }
@@ -116,22 +112,23 @@ func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
 	}
 
 	s.stats.incrementConnections()
-
 	go gossh.DiscardRequests(reqs)
 	for newChannel := range chans {
 		go s.handleChannel(ctx, sshConn, newChannel)
 	}
 }
 
-func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChannel gossh.NewChannel) {
+func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn,
+	newChannel gossh.NewChannel) {
+
 	user, err := user.New(sshConn.User(), sshConn.RemoteAddr().String())
 	if err != nil {
 		dlog.Server.Error(user, err)
 		newChannel.Reject(gossh.Prohibited, err.Error())
 		return
 	}
-	dlog.Server.Info(user, "Invoking channel handler")
 
+	dlog.Server.Info(user, "Invoking channel handler")
 	if newChannel.ChannelType() != "session" {
 		err := errors.New("Don'w allow other channel types than session")
 		dlog.Server.Error(user, err)
@@ -151,9 +148,10 @@ func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChann
 	}
 }
 
-func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-chan *gossh.Request, channel gossh.Channel, user *user.User) error {
-	dlog.Server.Info(user, "Invoking request handler")
+func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn,
+	in <-chan *gossh.Request, channel gossh.Channel, user *user.User) error {
 
+	dlog.Server.Info(user, "Invoking request handler")
 	for req := range in {
 		var payload = struct{ Value string }{}
 		gossh.Unmarshal(req.Payload, &payload)
@@ -167,7 +165,6 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 			default:
 				handler = handlers.NewServerHandler(user, s.catLimiter, s.tailLimiter)
 			}
-
 			terminate := func() {
 				handler.Shutdown()
 				sshConn.Close()
@@ -178,13 +175,11 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 				io.Copy(channel, handler)
 				terminate()
 			}()
-
 			go func() {
 				// Broken pipe, cancel
 				io.Copy(handler, channel)
 				terminate()
 			}()
-
 			go func() {
 				select {
 				case <-ctx.Done():
@@ -192,7 +187,6 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 				}
 				terminate()
 			}()
-
 			go func() {
 				if err := sshConn.Wait(); err != nil && err != io.EOF {
 					dlog.Server.Error(user, err)
@@ -204,20 +198,19 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 
 			// Only serving shell type
 			req.Reply(true, nil)
-
 		default:
 			req.Reply(false, nil)
-
 			return fmt.Errorf("Closing SSH connection as unknown request recieved|%s|%v",
 				req.Type, payload.Value)
 		}
 	}
-
 	return nil
 }
 
 // Callback for SSH authentication.
-func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Permissions, error) {
+func (s *Server) Callback(c gossh.ConnMetadata,
+	authPayload []byte) (*gossh.Permissions, error) {
+
 	user, err := user.New(c.User(), c.RemoteAddr().String())
 	if err != nil {
 		return nil, err
@@ -229,7 +222,6 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	}
 
 	authInfo := string(authPayload)
-
 	splitted := strings.Split(c.RemoteAddr().String(), ":")
 	remoteIP := splitted[0]
 
@@ -259,23 +251,26 @@ func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Perm
 	return nil, fmt.Errorf("user %s not authorized", user)
 }
 
-func (s *Server) backgroundCanSSH(user *user.User, jobName, remoteIP, allowedJobName string, allowFrom []string) bool {
-	dlog.Server.Debug("backgroundCanSSH", user, jobName, remoteIP, allowedJobName, allowFrom)
+func (s *Server) backgroundCanSSH(user *user.User, jobName, remoteIP,
+	allowedJobName string, allowFrom []string) bool {
 
+	dlog.Server.Debug("backgroundCanSSH", user, jobName, remoteIP, allowedJobName, allowFrom)
 	if jobName != allowedJobName {
-		dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Job name does not match, skipping to next one...", allowedJobName)
+		dlog.Server.Debug(user, jobName, "backgroundCanSSH",
+			"Job name does not match, skipping to next one...", allowedJobName)
 		return false
 	}
 
 	for _, myAddr := range allowFrom {
 		ips, err := net.LookupIP(myAddr)
 		if err != nil {
-			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Unable to lookup IP address for allowed hosts lookup, skipping to next one...", myAddr, err)
+			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Unable to lookup IP "+
+				"address for allowed hosts lookup, skipping to next one...", myAddr, err)
 			continue
 		}
-
 		for _, ip := range ips {
-			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Comparing IP addresses", remoteIP, ip.String())
+			dlog.Server.Debug(user, jobName, "backgroundCanSSH", "Comparing IP addresses",
+				remoteIP, ip.String())
 			if remoteIP == ip.String() {
 				return true
 			}
diff --git a/internal/server/stats.go b/internal/server/stats.go
index c07634d..99a644a 100644
--- a/internal/server/stats.go
+++ b/internal/server/stats.go
@@ -19,7 +19,6 @@ type stats struct {
 
 func (s *stats) incrementConnections() {
 	defer s.logServerStats()
-
 	s.mutex.Lock()
 	s.currentConnections++
 	s.lifetimeConnections++
@@ -28,7 +27,6 @@ func (s *stats) incrementConnections() {
 
 func (s *stats) decrementConnections() {
 	defer s.logServerStats()
-
 	s.mutex.Lock()
 	s.currentConnections--
 	s.mutex.Unlock()
@@ -40,8 +38,8 @@ func (s *stats) hasConnections() bool {
 	s.mutex.Unlock()
 
 	has := currentConnections > 0
-	dlog.Server.Info("stats", "Server with open connections?", has, currentConnections)
-
+	dlog.Server.Info("stats", "Server with open connections?",
+		has, currentConnections)
 	return has
 }
 
@@ -52,7 +50,6 @@ func (s *stats) logServerStats() {
 	data := make(map[string]interface{})
 	data["currentConnections"] = s.currentConnections
 	data["lifetimeConnections"] = s.lifetimeConnections
-
 	dlog.Server.Mapreduce("STATS", data)
 }
 
@@ -61,9 +58,9 @@ func (s *stats) serverLimitExceeded() error {
 	defer s.mutex.Unlock()
 
 	if s.currentConnections >= config.Server.MaxConnections {
-		return fmt.Errorf("Exceeded max allowed concurrent connections of %d", config.Server.MaxConnections)
+		return fmt.Errorf("Exceeded max allowed concurrent connections of %d",
+			config.Server.MaxConnections)
 	}
-
 	return nil
 }
 
diff --git a/internal/source/source.go b/internal/source/source.go
index be7aecd..4bb0784 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -1,10 +1,19 @@
 package source
 
+// Source specifies the origin of either the current process (dtail is a client
+// process, dserver is a server process) or the source code package (e.g.
+// dserver server side code or dtail client side code). Notice that dtail client
+// may also executes server code directly (e.g. via serverless mode) and that
+// the dserver may also executes client code (e.g. via scheduled server side
+// mapreduce queries).
 type Source int
 
 const (
-	Client      Source = iota
-	Server      Source = iota
+	// Client process or source code package.
+	Client Source = iota
+	// Server process or source code package.
+	Server Source = iota
+	// HealthCheck process or client source code package.
 	HealthCheck Source = iota
 )
 
@@ -17,6 +26,5 @@ func (s Source) String() string {
 	case HealthCheck:
 		return "HEALTHCHECK"
 	}
-
 	panic("Unknown source type")
 }
diff --git a/internal/ssh/client/authmethods.go b/internal/ssh/client/authmethods.go
index 4508319..ced1fb9 100644
--- a/internal/ssh/client/authmethods.go
+++ b/internal/ssh/client/authmethods.go
@@ -11,7 +11,10 @@ import (
 )
 
 // InitSSHAuthMethods initialises all known SSH auth methods on the client side.
-func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod, hostKeyCallback gossh.HostKeyCallback, trustAllHosts bool, throttleCh chan struct{}, privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
+func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod,
+	hostKeyCallback gossh.HostKeyCallback, trustAllHosts bool, throttleCh chan struct{},
+	privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
+
 	if len(sshAuthMethods) > 0 {
 		simpleCallback, err := NewSimpleCallback()
 		if err != nil {
@@ -19,20 +22,21 @@ func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod, hostKeyCallback gossh
 		}
 		return sshAuthMethods, simpleCallback
 	}
-
 	return initKnownHostsAuthMethods(trustAllHosts, throttleCh, privateKeyPath)
 }
 
-func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
-	var sshAuthMethods []gossh.AuthMethod
+func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{},
+	privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
 
+	var sshAuthMethods []gossh.AuthMethod
 	knownHostsPath := os.Getenv("HOME") + "/.ssh/known_hosts"
-	knownHostsCallback, err := NewKnownHostsCallback(knownHostsPath, trustAllHosts, throttleCh)
+	knownHostsCallback, err := NewKnownHostsCallback(knownHostsPath, trustAllHosts,
+		throttleCh)
 	if err != nil {
 		dlog.Common.FatalPanic(knownHostsPath, err)
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
-
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path",
+		knownHostsPath)
 	if config.Common.ExperimentalFeaturesEnable {
 		sshAuthMethods = append(sshAuthMethods, gossh.Password("experimental feature test"))
 		dlog.Common.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
@@ -43,7 +47,9 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, pri
 		authMethod, err := ssh.PrivateKey(privateKeyPath)
 		if err == nil {
 			sshAuthMethods = append(sshAuthMethods, authMethod)
-			dlog.Common.Debug("initKnownHostsAuthMethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+			dlog.Common.Debug("initKnownHostsAuthMethods",
+				"Added path to list of auth methods, not adding further methods",
+				privateKeyPath)
 			return sshAuthMethods, knownHostsCallback
 		}
 		dlog.Common.FatalPanic("Unable to use private SSH key", privateKeyPath, err)
@@ -53,30 +59,35 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, pri
 	authMethod, err := ssh.Agent()
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK) to list of auth methods, not adding further methods")
+		dlog.Common.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK)"+
+			"to list of auth methods, not adding further methods")
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err)
+	dlog.Common.Debug("initKnownHostsAuthMethods",
+		"Unable to init SSH Agent auth method", err)
 
 	// Third, try Linux/UNIX default key paths
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_rsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Common.Debug("initKnownHostsAuthmethods",
+			"Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key",
+		privateKeyPath, err)
 
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_dsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Common.Debug("initKnownHostsAuthmethods",
+			"Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
-
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key",
+		privateKeyPath, err)
 	dlog.Common.FatalPanic("Unable to find private SSH key information")
 
 	// Never reach this point.
diff --git a/internal/ssh/client/customkeycallback.go b/internal/ssh/client/customkeycallback.go
index 73e5289..53b8e3c 100644
--- a/internal/ssh/client/customkeycallback.go
+++ b/internal/ssh/client/customkeycallback.go
@@ -7,8 +7,7 @@ import (
 )
 
 // CustomCallback is a custom host key callback wrapper.
-type CustomCallback struct {
-}
+type CustomCallback struct{}
 
 // NewCustomCallback returns a new wrapper.
 func NewCustomCallback() (*CustomCallback, error) {
diff --git a/internal/ssh/client/knownhostscallback.go b/internal/ssh/client/knownhostscallback.go
index a73d612..65a590a 100644
--- a/internal/ssh/client/knownhostscallback.go
+++ b/internal/ssh/client/knownhostscallback.go
@@ -46,8 +46,9 @@ type KnownHostsCallback struct {
 }
 
 // NewKnownHostsCallback returns a new wrapper.
-func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool, throttleCh chan struct{}) (HostKeyCallback, error) {
-	// Ensure file exists
+func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool,
+	throttleCh chan struct{}) (HostKeyCallback, error) {
+
 	os.OpenFile(knownHostsPath, os.O_RDONLY|os.O_CREATE, 0666)
 	untrustedHosts := make(map[string]bool)
 
@@ -59,11 +60,9 @@ func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool, throttleCh
 		untrustedHosts:  untrustedHosts,
 		mutex:           &sync.Mutex{},
 	}
-
 	if trustAllHosts {
 		close(c.trustAllHostsCh)
 	}
-
 	return c, nil
 }
 
@@ -75,14 +74,12 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 		if err != nil {
 			return err
 		}
-
 		// Check for valid entry in known_hosts file
 		err = knownHostsCb(server, remote, key)
 		if err == nil {
 			// OK
 			return nil
 		}
-
 		// Make sure that interactive user callback does not interfere with
 		// SSH connection throttler.
 		<-c.throttleCh
@@ -96,11 +93,9 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 			ipLine:     knownhosts.Line([]string{remote.String()}, key),
 			responseCh: make(chan response),
 		}
-
 		dlog.Common.Warn("Encountered unknown host", unknown)
 		// Notify user that there is an unknown host
 		c.unknownCh <- unknown
-
 		// Wait for user input.
 		switch <-unknown.responseCh {
 		case trustHost:
@@ -112,7 +107,6 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 		c.mutex.Lock()
 		defer c.mutex.Unlock()
 		c.untrustedHosts[server] = true
-
 		return err
 	}
 }
@@ -121,7 +115,6 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 // be added to the known hosts or not.
 func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 	var hosts []unknownHost
-
 	for {
 		// Check whether there is a unknown host
 		select {
@@ -147,7 +140,6 @@ func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 
 func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 	var servers []string
-
 	for _, host := range hosts {
 		servers = append(servers, host.server)
 	}
@@ -165,7 +157,6 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 		strings.Join(servers, ","),
 		"Do you want to trust these hosts?",
 	)
-
 	p := prompt.New(question)
 
 	a := prompt.Answer{
@@ -223,7 +214,6 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 
 func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 	tmpKnownHostsPath := fmt.Sprintf("%s.tmp", c.knownHostsPath)
-
 	newFd, err := os.OpenFile(tmpKnownHostsPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 	if err != nil {
 		panic(fmt.Sprintf("%s: %s", tmpKnownHostsPath, err.Error()))
@@ -232,7 +222,6 @@ func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 
 	// Newly trusted hosts in normalized form
 	addresses := make(map[string]struct{})
-
 	// First write to new known hosts file, and keep track of addresses
 	for _, unknown := range hosts {
 		unknown.responseCh <- trustHost
@@ -255,7 +244,6 @@ func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 	defer oldFd.Close()
 
 	scanner := bufio.NewScanner(oldFd)
-
 	// Now, append all still valid old entries to the new host file
 	for scanner.Scan() {
 		line := scanner.Text()
@@ -283,6 +271,5 @@ func (c KnownHostsCallback) Untrusted(server string) bool {
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
 	_, ok := c.untrustedHosts[server]
-
 	return ok
 }
diff --git a/internal/ssh/server/hostkey.go b/internal/ssh/server/hostkey.go
index 20de1f0..33bd4e8 100644
--- a/internal/ssh/server/hostkey.go
+++ b/internal/ssh/server/hostkey.go
@@ -24,7 +24,8 @@ func PrivateHostKey() []byte {
 
 		pem := ssh.EncodePrivateKeyToPEM(privateKey)
 		if err := ioutil.WriteFile(hostKeyFile, pem, 0600); err != nil {
-			dlog.Common.Error("Unable to write private server RSA host key to file", hostKeyFile, err)
+			dlog.Common.Error("Unable to write private server RSA host key to file",
+				hostKeyFile, err)
 		}
 		return pem
 	}
diff --git a/internal/ssh/server/publickeycallback.go b/internal/ssh/server/publickeycallback.go
index 59d1f31..ebc428a 100644
--- a/internal/ssh/server/publickeycallback.go
+++ b/internal/ssh/server/publickeycallback.go
@@ -13,25 +13,28 @@ import (
 	gossh "golang.org/x/crypto/ssh"
 )
 
-// PublicKeyCallback is for the server to check whether a public SSH key is authorized ot not.
-func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
+// PublicKeyCallback is for the server to check whether a public SSH key is
+// authorized ot not.
+func PublicKeyCallback(c gossh.ConnMetadata,
+	offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
+
 	user, err := user.New(c.User(), c.RemoteAddr().String())
 	if err != nil {
 		return nil, err
 	}
-	dlog.Common.Info(user, "Incoming authorization")
 
+	dlog.Common.Info(user, "Incoming authorization")
 	cwd, err := os.Getwd()
 	if err != nil {
 		return nil, fmt.Errorf("Unable to get current working directory|%s|", err.Error())
 	}
-
 	if config.ServerRelaxedAuthEnable {
 		dlog.Common.Fatal(user, "Granting permissions via relaxed-auth")
 		return nil, nil
 	}
 
-	authorizedKeysFile := fmt.Sprintf("%s/%s/%s.authorized_keys", cwd, config.Common.CacheDir, user.Name)
+	authorizedKeysFile := fmt.Sprintf("%s/%s/%s.authorized_keys", cwd,
+		config.Common.CacheDir, user.Name)
 	if _, err := os.Stat(authorizedKeysFile); os.IsNotExist(err) {
 		user, err := osUser.Lookup(user.Name)
 		if err != nil {
@@ -44,23 +47,25 @@ func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*go
 	dlog.Common.Info(user, "Reading", authorizedKeysFile)
 	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysFile)
 	if err != nil {
-		return nil, fmt.Errorf("Unable to read authorized keys file|%s|%s|%s", authorizedKeysFile, user, err.Error())
+		return nil, fmt.Errorf("Unable to read authorized keys file|%s|%s|%s",
+			authorizedKeysFile, user, err.Error())
 	}
 
 	authorizedKeysMap := map[string]bool{}
 	for len(authorizedKeysBytes) > 0 {
 		authorizedPubKey, _, _, restBytes, err := gossh.ParseAuthorizedKey(authorizedKeysBytes)
 		if err != nil {
-			return nil, fmt.Errorf("Unable to parse authorized keys bytes|%s|%s", user, err.Error())
+			return nil, fmt.Errorf("Unable to parse authorized keys bytes|%s|%s",
+				user, err.Error())
 		}
 		authorizedKeysMap[string(authorizedPubKey.Marshal())] = true
 		authorizedKeysBytes = restBytes
-
-		dlog.Common.Debug(user, "Authorized public key fingerprint", gossh.FingerprintSHA256(authorizedPubKey))
+		dlog.Common.Debug(user, "Authorized public key fingerprint",
+			gossh.FingerprintSHA256(authorizedPubKey))
 	}
 
-	dlog.Common.Debug(user, "Offered public key fingerprint", gossh.FingerprintSHA256(offeredPubKey))
-
+	dlog.Common.Debug(user, "Offered public key fingerprint",
+		gossh.FingerprintSHA256(offeredPubKey))
 	if authorizedKeysMap[string(offeredPubKey.Marshal())] {
 		return &gossh.Permissions{
 			Extensions: map[string]string{
diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go
index 56494a7..db5aaf1 100644
--- a/internal/ssh/ssh.go
+++ b/internal/ssh/ssh.go
@@ -24,12 +24,10 @@ func GeneratePrivateRSAKey(size int) (*rsa.PrivateKey, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	err = privateKey.Validate()
 	if err != nil {
 		return nil, err
 	}
-
 	return privateKey, nil
 }
 
@@ -42,7 +40,6 @@ func EncodePrivateKeyToPEM(privateKey *rsa.PrivateKey) []byte {
 		Headers: nil,
 		Bytes:   derFormat,
 	}
-
 	return pem.EncodeToMemory(&block)
 }
 
@@ -80,7 +77,6 @@ func KeyFile(keyFile string) (gossh.AuthMethod, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	key, err := gossh.ParsePrivateKey(buffer)
 	if err != nil {
 		return nil, err
diff --git a/internal/user/name.go b/internal/user/name.go
index 28ab0a4..cd11907 100644
--- a/internal/user/name.go
+++ b/internal/user/name.go
@@ -10,11 +10,9 @@ func NoRootCheck() {
 	if err != nil {
 		panic(err)
 	}
-
 	if user.Uid == "0" {
 		panic("Not allowed to run as UID 0")
 	}
-
 	if user.Gid == "0" {
 		panic("Not allowed to run as GID 0")
 	}
@@ -26,6 +24,5 @@ func Name() string {
 	if err != nil {
 		panic(err)
 	}
-
 	return user.Username
 }
diff --git a/internal/user/server/user.go b/internal/user/server/user.go
index 70ead1c..aa7f8b1 100644
--- a/internal/user/server/user.go
+++ b/internal/user/server/user.go
@@ -49,7 +49,6 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 		dlog.Server.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
 		return true
 	}
-
 	if u.Name == config.ScheduleUser || u.Name == config.ContinuousUser {
 		// Background user has same permissions as dtail process itself.
 		return true
@@ -57,27 +56,29 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 
 	cleanPath, err := filepath.EvalSymlinks(filePath)
 	if err != nil {
-		dlog.Server.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
+		dlog.Server.Error(u, filePath, permissionType,
+			"Unable to evaluate symlinks", err)
 		hasPermission = false
 		return
 	}
 
 	cleanPath, err = filepath.Abs(cleanPath)
 	if err != nil {
-		dlog.Server.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
+		dlog.Server.Error(u, cleanPath, permissionType,
+			"Unable to make file path absolute", err)
 		hasPermission = false
 		return
 	}
 
 	if cleanPath != filePath {
-		dlog.Server.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
+		dlog.Server.Info(u, filePath, cleanPath, permissionType,
+			"Calculated new clean path from original file path (possibly symlink)")
 	}
 
 	hasPermission, err = u.hasFilePermission(cleanPath, permissionType)
 	if err != nil {
 		dlog.Server.Warn(u, cleanPath, err)
 	}
-
 	return
 }
 
@@ -86,18 +87,17 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 	if _, err := permissions.ToRead(u.Name, cleanPath); err != nil {
 		return false, fmt.Errorf("User without OS file system permissions to read path: '%v'", err)
 	}
-	dlog.Server.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
+	dlog.Server.Info(u, cleanPath, permissionType,
+		"User with OS file system permissions to path")
 
 	// Only allow to follow regular files or symlinks.
 	info, err := os.Lstat(cleanPath)
 	if err != nil {
 		return false, fmt.Errorf("Unable to determine file type: '%v'", err)
 	}
-
 	if !info.Mode().IsRegular() {
 		return false, fmt.Errorf("Can only open regular files or follow symlinks")
 	}
-
 	hasPermission, err := u.iteratePaths(cleanPath, permissionType)
 	if err != nil {
 		return false, err
@@ -109,10 +109,8 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 	// By default assume no permissions
 	hasPermission := false
-
 	for _, permission := range u.permissions {
 		typeStr := "readfiles" // Assume ReadFiles by default.
-
 		var regexStr string
 		var negate bool
 
@@ -123,7 +121,6 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 		}
 
 		dlog.Server.Debug(u, cleanPath, typeStr, permission)
-
 		if typeStr != permissionType {
 			continue
 		}
@@ -136,16 +133,17 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 
 		re, err := regexp.Compile(regexStr)
 		if err != nil {
-			return false, fmt.Errorf("Permission test failed, can't compile regex '%s': '%v'", regexStr, err)
+			return false, fmt.Errorf("Permission test failed, can't compile regex "+
+				"'%s': '%v'", regexStr, err)
 		}
-
 		if negate && re.MatchString(cleanPath) {
-			dlog.Server.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test failed partially, "+
+				"matching negative pattern '%s'", permission)
 			hasPermission = false
 		}
-
 		if !negate && re.MatchString(cleanPath) {
-			dlog.Server.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test passed partially, "+
+				"matching positive pattern", permission)
 			hasPermission = true
 		}
 	}
diff --git a/internal/version/version.go b/internal/version/version.go
index 4ff6eae..68b9e6e 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -20,7 +20,8 @@ const (
 
 // String representation of the DTail version.
 func String() string {
-	return fmt.Sprintf("%s %v Protocol %s %s", Name, Version, protocol.ProtocolCompat, Additional)
+	return fmt.Sprintf("%s %v Protocol %s %s", Name, Version,
+		protocol.ProtocolCompat, Additional)
 }
 
 // PaintedString is a prettier string representation of the DTail version.
@@ -31,13 +32,10 @@ func PaintedString() string {
 
 	name := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Name),
 		color.FgYellow, color.BgBlue, color.AttrBold)
-
 	version := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Version),
 		color.FgBlue, color.BgYellow, color.AttrBold)
-
 	protocol := color.PaintStr(fmt.Sprintf(" Protocol %s ", protocol.ProtocolCompat),
 		color.FgBlack, color.BgGreen)
-
 	additional := color.PaintStrWithAttr(fmt.Sprintf(" %s ", Additional),
 		color.FgWhite, color.BgMagenta, color.AttrUnderline)
 
-- 
cgit v1.2.3


From f44792c9102488774c9993b080f35c65287a64b1 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 10 Oct 2021 14:02:12 +0300
Subject: add another dmap test - reading 100 source files at once

fix a data race when reading multiple files on one server from the same session at once
---
 internal/clients/maprclient.go            |  2 +-
 internal/config/config.go                 |  2 +-
 internal/config/initializer.go            |  4 +++-
 internal/io/dlog/level.go                 |  5 ++++
 internal/io/fs/permissions/permission.go  |  2 +-
 internal/server/handlers/basehandler.go   | 40 ++++++++++++++++++++++++++-----
 internal/server/handlers/healthhandler.go |  2 +-
 internal/server/handlers/serverhandler.go | 18 ++------------
 8 files changed, 48 insertions(+), 27 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 04f258d..074494c 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -108,7 +108,7 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 }
 
 // NEXT: Make this a callback function rather trying to use polymorphism to call
-// this. This applies to all clients.
+// this. This applies to all clients. It will make the code easier to read.
 func (c MaprClient) makeHandler(server string) handlers.Handler {
 	return handlers.NewMaprHandler(server, c.query, c.globalGroup)
 }
diff --git a/internal/config/config.go b/internal/config/config.go
index b99b22b..ee23829 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -16,7 +16,7 @@ const (
 	// DefaultSSHPort is the default DServer port.
 	DefaultSSHPort int = 2222
 	// DefaultLogLevel specifies the default log level (obviously)
-	DefaultLogLevel string = "INFO"
+	DefaultLogLevel string = "info"
 	// DefaultClientLogger specifies the default logger for the client commands.
 	DefaultClientLogger string = "fout"
 	// DefaultServerLogger specifies the default logger for dtail server.
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 0a913db..0c6dfdf 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -147,7 +147,9 @@ func transformClient(in *initializer, args *Args, additionalArgs []string) error
 		strings.ToLower(args.ServersStr) == "serverless") {
 		// We are not connecting to any servers.
 		args.Serverless = true
-		in.Common.LogLevel = "warn"
+		if args.LogLevel == DefaultLogLevel {
+			in.Common.LogLevel = "warn"
+		}
 	}
 	return nil
 }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 0971094..05d9ed9 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -9,6 +9,7 @@ type level int
 
 // Available log levels.
 const (
+	None    level = iota
 	Fatal   level = iota
 	Error   level = iota
 	Warn    level = iota
@@ -26,6 +27,8 @@ var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug,
 
 func newLevel(l string) level {
 	switch strings.ToLower(l) {
+	case "none":
+		return None
 	case "fatal":
 		return Fatal
 	case "error":
@@ -54,6 +57,8 @@ func newLevel(l string) level {
 
 func (l level) String() string {
 	switch l {
+	case None:
+		return "NONE"
 	case Fatal:
 		return "FATAL"
 	case Error:
diff --git a/internal/io/fs/permissions/permission.go b/internal/io/fs/permissions/permission.go
index e80dbb2..d621c09 100644
--- a/internal/io/fs/permissions/permission.go
+++ b/internal/io/fs/permissions/permission.go
@@ -9,6 +9,6 @@ import (
 // ToRead is to check whether user has read permissions to a given file.
 func ToRead(user, filePath string) (bool, error) {
 	// Only implemented for Linux, always expect true
-	dlog.Common.Warn(user, filePath, "Not performing ACL check as not compiled in")
+	dlog.Common.Debug(user, filePath, "Not performing ACL check as not compiled in")
 	return true, nil
 }
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 847e8f9..d814cc9 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -9,6 +9,7 @@ import (
 	"io"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -22,7 +23,7 @@ import (
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
-type handleCommandCb func(context.Context, int, []string, string, map[string]string)
+type handleCommandCb func(context.Context, int, []string, string)
 
 type baseHandler struct {
 	done             *internal.Done
@@ -35,11 +36,15 @@ type baseHandler struct {
 	user             *user.User
 	ackCloseReceived chan struct{}
 	activeCommands   int32
-	quiet            bool
-	spartan          bool
-	serverless       int32
 	readBuf          bytes.Buffer
 	writeBuf         bytes.Buffer
+
+	// Some global options + sync primitives required.
+	once       sync.Once
+	mutex      sync.Mutex
+	quiet      bool
+	spartan    bool
+	serverless bool
 }
 
 // Shutdown the handler.
@@ -66,7 +71,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 			return
 		}
 
-		if h.serverless > 0 {
+		if h.serverless {
 			return
 		}
 
@@ -160,8 +165,9 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
 	}
+	h.setOptions(options)
 
-	h.handleCommandCb(ctx, argc, args, commandName, options)
+	h.handleCommandCb(ctx, argc, args, commandName)
 }
 
 func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
@@ -232,6 +238,28 @@ func (h *baseHandler) handleAckCommand(argc int, args []string) {
 	}
 }
 
+func (h *baseHandler) setOptions(options map[string]string) {
+	// We have to make sure that this block is executed only once.
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+	// We can read the options only once, will cause a data race otherwise if
+	// changed multiple times for multiple incoming commands.
+	h.once.Do(func() {
+		if quiet, _ := options["quiet"]; quiet == "true" {
+			dlog.Server.Debug(h.user, "Enabling quiet mode")
+			h.quiet = true
+		}
+		if spartan, _ := options["spartan"]; spartan == "true" {
+			dlog.Server.Debug(h.user, "Enabling spartan mode")
+			h.spartan = true
+		}
+		if serverless, _ := options["serverless"]; serverless == "true" {
+			dlog.Server.Debug(h.user, "Enabling serverless mode")
+			h.serverless = true
+		}
+	})
+}
+
 func (h *baseHandler) send(ch chan<- string, message string) {
 	select {
 	case ch <- message:
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
index 8d6c400..0425696 100644
--- a/internal/server/handlers/healthhandler.go
+++ b/internal/server/handlers/healthhandler.go
@@ -41,7 +41,7 @@ func NewHealthHandler(user *user.User) *HealthHandler {
 }
 
 func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int,
-	args []string, commandName string, options map[string]string) {
+	args []string, commandName string) {
 
 	dlog.Server.Debug(h.user, "Handling health command", argc, args)
 	switch commandName {
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index f12d590..52c4570 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"os"
 	"strings"
-	"sync/atomic"
 
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/dlog"
@@ -55,7 +54,7 @@ func NewServerHandler(user *user.User, catLimiter,
 }
 
 func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
-	args []string, commandName string, options map[string]string) {
+	args []string, commandName string) {
 
 	dlog.Server.Debug(h.user, "Handling user command", argc, args)
 	h.incrementActiveCommands()
@@ -65,19 +64,6 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
 		}
 	}
 
-	if quiet, _ := options["quiet"]; quiet == "true" {
-		dlog.Server.Debug(h.user, "Enabling quiet mode")
-		h.quiet = true
-	}
-	if spartan, _ := options["spartan"]; spartan == "true" {
-		dlog.Server.Debug(h.user, "Enabling spartan mode")
-		h.spartan = true
-	}
-	if serverless, _ := options["serverless"]; serverless == "true" {
-		dlog.Server.Debug(h.user, "Enabling serverless mode")
-		atomic.AddInt32(&h.serverless, 1)
-	}
-
 	switch commandName {
 	case "grep", "cat":
 		command := newReadCommand(h, omode.CatClient)
@@ -109,7 +95,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
 		commandFinished()
 	default:
 		h.send(h.serverMessages, dlog.Server.Error(h.user,
-			"Received unknown user command", commandName, argc, args, options))
+			"Received unknown user command", commandName, argc, args))
 		commandFinished()
 	}
 }
-- 
cgit v1.2.3


From 71f89dc7ec7cf993d1eca98771212afe6310e9c8 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 10 Oct 2021 19:42:48 +0300
Subject: refactor

---
 internal/clients/connectors/serverconnection.go | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 2d7b45a..2737ede 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -37,6 +37,7 @@ func NewServerConnection(server string, userName string,
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
 		server:          server,
+		port:            config.Common.SSHPort,
 		handler:         handler,
 		commands:        commands,
 		config: &ssh.ClientConfig{
@@ -47,25 +48,20 @@ func NewServerConnection(server string, userName string,
 		},
 	}
 
+	// TODO: After reconnecting the port is wrong! Due to string slicing?
 	c.initServerPort()
 	return &c
 }
 
 // Server returns the server hostname connected to.
-func (c *ServerConnection) Server() string {
-	return c.server
-}
+func (c *ServerConnection) Server() string { return c.server }
 
 // Handler returns the handler used for the connection.
-func (c *ServerConnection) Handler() handlers.Handler {
-	return c.handler
-}
+func (c *ServerConnection) Handler() handlers.Handler { return c.handler }
 
 // Attempt to parse the server port address from the provided server FQDN.
 func (c *ServerConnection) initServerPort() {
-	c.port = config.Common.SSHPort
 	parts := strings.Split(c.server, ":")
-
 	if len(parts) == 2 {
 		dlog.Client.Debug("Parsing port from hostname", parts)
 		port, err := strconv.Atoi(parts[1])
-- 
cgit v1.2.3


From c0f4ebc9b3773c37e22c686024c8cc7ce4e71f9f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 11 Oct 2021 11:51:30 +0300
Subject: add dtail integration test

---
 internal/clients/connectors/serverconnection.go | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 2737ede..1df4d73 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -19,7 +19,11 @@ import (
 // ServerConnection represents a connection to a single remote dtail server via
 // SSH protocol.
 type ServerConnection struct {
-	server          string
+	// The full server string as received from the server discovery (can be with port number)
+	server string
+	// Only the hostname or FQDN (without the port number)
+	hostname string
+	// Only the port number.
 	port            int
 	config          *ssh.ClientConfig
 	handler         handlers.Handler
@@ -37,7 +41,6 @@ func NewServerConnection(server string, userName string,
 	c := ServerConnection{
 		hostKeyCallback: hostKeyCallback,
 		server:          server,
-		port:            config.Common.SSHPort,
 		handler:         handler,
 		commands:        commands,
 		config: &ssh.ClientConfig{
@@ -48,7 +51,6 @@ func NewServerConnection(server string, userName string,
 		},
 	}
 
-	// TODO: After reconnecting the port is wrong! Due to string slicing?
 	c.initServerPort()
 	return &c
 }
@@ -61,6 +63,7 @@ func (c *ServerConnection) Handler() handlers.Handler { return c.handler }
 
 // Attempt to parse the server port address from the provided server FQDN.
 func (c *ServerConnection) initServerPort() {
+	c.port = config.Common.SSHPort
 	parts := strings.Split(c.server, ":")
 	if len(parts) == 2 {
 		dlog.Client.Debug("Parsing port from hostname", parts)
@@ -68,7 +71,7 @@ func (c *ServerConnection) initServerPort() {
 		if err != nil {
 			dlog.Client.FatalPanic("Unable to parse client port", c.server, parts, err)
 		}
-		c.server = parts[0]
+		c.hostname = parts[0]
 		c.port = port
 	}
 }
@@ -103,8 +106,8 @@ func (c *ServerConnection) Start(ctx context.Context, cancel context.CancelFunc,
 		}()
 
 		if err := c.dial(ctx, cancel, throttleCh, statsCh); err != nil {
-			dlog.Client.Warn(c.server, c.port, err)
-			if c.hostKeyCallback.Untrusted(fmt.Sprintf("%s:%d", c.server, c.port)) {
+			dlog.Client.Warn(c.server, err)
+			if c.hostKeyCallback.Untrusted(c.server) {
 				dlog.Client.Debug(c.server, "Not trusting host")
 			}
 		}
@@ -125,7 +128,7 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 	}()
 
 	dlog.Client.Debug(c.server, "Dialing into the connection")
-	address := fmt.Sprintf("%s:%d", c.server, c.port)
+	address := fmt.Sprintf("%s:%d", c.hostname, c.port)
 
 	client, err := ssh.Dial("tcp", address, c.config)
 	if err != nil {
-- 
cgit v1.2.3


From a6098084f7150df34edecf1519386bd28a527361 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 11 Oct 2021 17:42:37 +0300
Subject: Update JSON-schema to reflect all recent config file changes.

---
 internal/config/client.go           | 2 +-
 internal/config/common.go           | 9 +++------
 internal/config/server.go           | 2 +-
 internal/io/dlog/dlog.go            | 4 ++--
 internal/io/dlog/loggers/factory.go | 8 ++++----
 5 files changed, 11 insertions(+), 14 deletions(-)

(limited to 'internal')

diff --git a/internal/config/client.go b/internal/config/client.go
index 8227c68..9f4df97 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -102,7 +102,7 @@ type termColors struct {
 // ClientConfig represents a DTail client configuration (empty as of now as there
 // are no available config options yet, but that may changes in the future).
 type ClientConfig struct {
-	TermColorsEnable bool
+	TermColorsEnable bool       `json:",omitempty"`
 	TermColors       termColors `json:",omitempty"`
 }
 
diff --git a/internal/config/common.go b/internal/config/common.go
index 9d22c95..7a72cfe 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -12,12 +12,10 @@ type CommonConfig struct {
 	Logger string
 	// LogLevel defines how much is logged.
 	LogLevel string `json:",omitempty"`
-	// LogStrategy defines the log rotation strategy.
-	LogStrategy string
+	// LogRotation strategy to be used.
+	LogRotation string
 	// The cache directory
 	CacheDir string
-	// The temp directory
-	TmpDir string `json:",omitempty"`
 }
 
 // Create a new default configuration.
@@ -28,8 +26,7 @@ func newDefaultCommonConfig() *CommonConfig {
 		LogDir:                     "log",
 		Logger:                     "stdout",
 		LogLevel:                   DefaultLogLevel,
-		LogStrategy:                "daily",
+		LogRotation:                "daily",
 		CacheDir:                   "cache",
-		TmpDir:                     "/tmp",
 	}
 }
diff --git a/internal/config/server.go b/internal/config/server.go
index 677f5ac..254ea0c 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -47,7 +47,7 @@ type ServerConfig struct {
 	MaxConcurrentCats int
 	// The max amount of concurrent tails per server.
 	MaxConcurrentTails int
-	// The user permissions.
+	// The user permissions. TODO: Add to JSON schema
 	Permissions Permissions `json:",omitempty"`
 	// The mapr log format
 	MapreduceLogFormat string `json:",omitempty"`
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index da67585..5e0c3a1 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -82,12 +82,12 @@ func new(sourceProcess, sourcePackage source.Source) *DLog {
 	if err != nil {
 		panic(err)
 	}
-	strategy := loggers.NewStrategy(config.Common.LogStrategy)
+	logRotation := loggers.NewStrategy(config.Common.LogRotation)
 	loggerName := config.Common.Logger
 	level := newLevel(config.Common.LogLevel)
 
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), loggerName, strategy),
+		logger:        loggers.Factory(sourceProcess.String(), loggerName, logRotation),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      level,
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 415d7fb..a5cc7cf 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -10,12 +10,12 @@ var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
 // Factory is there to retrieve a logger based on various settings.
-func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
+func Factory(sourceName, loggerName string, logRotation Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
 	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName,
-		rotationStrategy.FileBase, loggerName)
+		logRotation.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
@@ -29,10 +29,10 @@ func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 			singleton = newStdout()
 			factoryMap[id] = singleton
 		case "file":
-			singleton = newFile(rotationStrategy)
+			singleton = newFile(logRotation)
 			factoryMap[id] = singleton
 		case "fout":
-			singleton = newFout(rotationStrategy)
+			singleton = newFout(logRotation)
 			factoryMap[id] = singleton
 		default:
 			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
-- 
cgit v1.2.3


From 7b873100d94ddc3c698a620cb83b61dcb2074303 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 13 Oct 2021 09:00:03 +0300
Subject: add another dcat integration test - catting 100 files at once

---
 internal/server/handlers/basehandler.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'internal')

diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index d814cc9..6bc8268 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -63,7 +63,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 
 	select {
 	case message := <-h.serverMessages:
-		if message[0] == '.' {
+		if len(message) > 0 && message[0] == '.' {
 			// Handle hidden message (don't display to the user)
 			h.readBuf.WriteString(message)
 			h.readBuf.WriteByte(protocol.MessageDelimiter)
-- 
cgit v1.2.3


From 1dead22129a26e4f532e68c2c63fe4122b519506 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 13 Oct 2021 21:10:28 +0300
Subject: Merging grep context from master

---
 internal/config/args.go                   |  73 +++++++++++++--
 internal/io/fs/filereader.go              |   4 +-
 internal/io/fs/readfile.go                | 149 ++++++++++++++++++++++++++++--
 internal/lcontext/lcontext.go             |  22 +++++
 internal/server/handlers/basehandler.go   |  12 +--
 internal/server/handlers/healthhandler.go |   5 +-
 internal/server/handlers/readcommand.go   |  30 +++---
 internal/server/handlers/serverhandler.go |   9 +-
 8 files changed, 262 insertions(+), 42 deletions(-)
 create mode 100644 internal/lcontext/lcontext.go

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index f721390..f94e0a9 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -3,8 +3,10 @@ package config
 import (
 	"encoding/base64"
 	"fmt"
+	"strconv"
 	"strings"
 
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/omode"
 
 	gossh "golang.org/x/crypto/ssh"
@@ -12,6 +14,7 @@ import (
 
 // Args is a helper struct to summarize common client arguments.
 type Args struct {
+	lcontext.LContext
 	Arguments          []string
 	ConfigFile         string
 	ConnectionsPerCPU  int
@@ -74,17 +77,50 @@ func (a *Args) String() string {
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
 func (a *Args) SerializeOptions() string {
-	return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan,
-		a.Serverless)
+	options := make(map[string]string)
+
+	if a.Quiet {
+		options["quiet"] = fmt.Sprintf("%v", a.Quiet)
+	}
+	if a.Spartan {
+		options["spartan"] = fmt.Sprintf("%v", a.Spartan)
+	}
+	if a.Serverless {
+		options["serverless"] = fmt.Sprintf("%v", a.Serverless)
+	}
+	if a.LContext.MaxCount != 0 {
+		options["max"] = fmt.Sprintf("%d", a.LContext.MaxCount)
+	}
+	if a.LContext.BeforeContext != 0 {
+		options["before"] = fmt.Sprintf("%d", a.LContext.BeforeContext)
+	}
+	if a.LContext.AfterContext != 0 {
+		options["after"] = fmt.Sprintf("%d", a.LContext.AfterContext)
+	}
+
+	var sb strings.Builder
+	var i int
+	for k, v := range options {
+		if i > 0 {
+			sb.WriteString(":")
+		}
+		sb.WriteString(k)
+		sb.WriteString("=")
+		sb.WriteString(v)
+		i++
+	}
+	return sb.String()
 }
 
 // DeserializeOptions deserializes the options, but into a map.
-func DeserializeOptions(opts []string) (map[string]string, error) {
+func DeserializeOptions(opts []string) (map[string]string, lcontext.LContext, error) {
 	options := make(map[string]string, len(opts))
+	var ltx lcontext.LContext
+
 	for _, o := range opts {
 		kv := strings.SplitN(o, "=", 2)
 		if len(kv) != 2 {
-			return options, fmt.Errorf("Unable to parse options: %v", kv)
+			return options, ltx, fmt.Errorf("Unable to parse options: %v", kv)
 		}
 		key := kv[0]
 		val := kv[1]
@@ -93,11 +129,34 @@ func DeserializeOptions(opts []string) (map[string]string, error) {
 			s := strings.SplitN(val, "%", 2)
 			decoded, err := base64.StdEncoding.DecodeString(s[1])
 			if err != nil {
-				return options, err
+				return options, ltx, err
 			}
 			val = string(decoded)
 		}
-		options[key] = val
+
+		switch key {
+		case "before":
+			iVal, err := strconv.Atoi(val)
+			if err != nil {
+				return options, ltx, err
+			}
+			ltx.BeforeContext = iVal
+		case "after":
+			iVal, err := strconv.Atoi(val)
+			if err != nil {
+				return options, ltx, err
+			}
+			ltx.AfterContext = iVal
+		case "max":
+			iVal, err := strconv.Atoi(val)
+			if err != nil {
+				return options, ltx, err
+			}
+			ltx.MaxCount = iVal
+		default:
+			options[key] = val
+		}
 	}
-	return options, nil
+
+	return options, ltx, nil
 }
diff --git a/internal/io/fs/filereader.go b/internal/io/fs/filereader.go
index 7773142..b05fd39 100644
--- a/internal/io/fs/filereader.go
+++ b/internal/io/fs/filereader.go
@@ -4,13 +4,15 @@ import (
 	"context"
 
 	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/regex"
 )
 
 // FileReader is the interface used on the dtail server to read/cat/grep/mapr...
 // a file.
 type FileReader interface {
-	Start(ctx context.Context, lines chan<- line.Line, re regex.Regex) error
+	Start(ctx context.Context, ltx lcontext.LContext, lines chan<- line.Line,
+		re regex.Regex) error
 	FilePath() string
 	Retry() bool
 }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 92f85b6..88d467e 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -16,6 +16,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/regex"
 
 	"github.com/DataDog/zstd"
@@ -62,8 +63,8 @@ func (f readFile) Retry() bool {
 }
 
 // Start tailing a log file.
-func (f readFile) Start(ctx context.Context, lines chan<- line.Line,
-	re regex.Regex) error {
+func (f readFile) Start(ctx context.Context, ltx lcontext.LContext,
+	lines chan<- line.Line, re regex.Regex) error {
 
 	dlog.Common.Debug("readFile", f)
 	defer func() {
@@ -102,7 +103,7 @@ func (f readFile) Start(ctx context.Context, lines chan<- line.Line,
 	wg.Add(1)
 
 	go f.periodicTruncateCheck(ctx, truncate)
-	go f.filter(ctx, &wg, rawLines, lines, re)
+	go f.filter(ctx, ltx, &wg, rawLines, lines, re)
 
 	err = f.read(ctx, fd, rawLines, truncate)
 	close(rawLines)
@@ -213,10 +214,27 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 }
 
 // Filter log lines matching a given regular expression.
-func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup,
-	rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
+func (f readFile) filter(ctx context.Context, ltx lcontext.LContext,
+	wg *sync.WaitGroup, rawLines <-chan *bytes.Buffer, lines chan<- line.Line,
+	re regex.Regex) {
 
 	defer wg.Done()
+	// Do we have any kind of local context settings? If so then run the more complex
+	// filterWithLContext method.
+	if ltx.Has() {
+		// We can not skip transmitting any lines to the client with a local
+		// grep context specified.
+		f.canSkipLines = false
+		f.filterWithLContext(ctx, ltx, rawLines, lines, re)
+		return
+	}
+
+	f.filterWithoutLContext(ctx, rawLines, lines, re)
+}
+
+func (f readFile) filterWithoutLContext(ctx context.Context, rawLines <-chan *bytes.Buffer,
+	lines chan<- line.Line, re regex.Regex) {
+
 	for {
 		select {
 		case line, ok := <-rawLines:
@@ -235,11 +253,126 @@ func (f readFile) filter(ctx context.Context, wg *sync.WaitGroup,
 	}
 }
 
-func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int,
+// Filter log lines matching a given regular expression, however with local grep context.
+func (f readFile) filterWithLContext(ctx context.Context, ltx lcontext.LContext,
+	rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
+
+	// Scenario 1: Finish once maxCount hits found
+	maxCount := ltx.MaxCount
+	processMaxCount := maxCount > 0
+	maxReached := false
+
+	// Scenario 2: Print prev. N lines when current line matches.
+	before := ltx.BeforeContext
+	processBefore := before > 0
+	var beforeBuf chan *bytes.Buffer
+	if processBefore {
+		beforeBuf = make(chan *bytes.Buffer, before)
+	}
+
+	// Screnario 3: Print next N lines when current line matches.
+	after := 0
+	processAfter := ltx.AfterContext > 0
+
+	for lineBytesBuffer := range rawLines {
+		f.updatePosition()
+
+		if !re.Match(lineBytesBuffer.Bytes()) {
+			f.updateLineNotMatched()
+
+			if processAfter && after > 0 {
+				after--
+				myLine := line.Line{
+					Content:         lineBytesBuffer,
+					SourceID:        f.globID,
+					Count:           f.totalLineCount(),
+					TransmittedPerc: 100,
+				}
+
+				select {
+				case lines <- myLine:
+				case <-ctx.Done():
+					return
+				}
+
+			} else if processBefore {
+				// Keep last num BeforeContext raw messages.
+				select {
+				case beforeBuf <- lineBytesBuffer:
+				default:
+					pool.RecycleBytesBuffer(<-beforeBuf)
+					beforeBuf <- lineBytesBuffer
+				}
+			}
+			continue
+		}
+
+		f.updateLineMatched()
+
+		if processAfter {
+			if maxReached {
+				return
+			}
+			after = ltx.AfterContext
+		}
+
+		if processBefore {
+			i := uint64(len(beforeBuf))
+			for {
+				select {
+				case lineBytesBuffer := <-beforeBuf:
+					myLine := line.Line{
+						Content:         lineBytesBuffer,
+						SourceID:        f.globID,
+						Count:           f.totalLineCount() - i,
+						TransmittedPerc: 100,
+					}
+					i--
+
+					select {
+					case lines <- myLine:
+					case <-ctx.Done():
+						return
+					}
+				default:
+					// beforeBuf is now empty.
+				}
+				if len(beforeBuf) == 0 {
+					break
+				}
+			}
+		}
+
+		line := line.Line{
+			Content:         lineBytesBuffer,
+			SourceID:        f.globID,
+			Count:           f.totalLineCount(),
+			TransmittedPerc: 100,
+		}
+
+		select {
+		case lines <- line:
+			if processMaxCount {
+				maxCount--
+				if maxCount == 0 {
+					if !processAfter || after == 0 {
+						return
+					}
+					// Unfortunatley we have to continue filter, as there might be more lines to print
+					maxReached = true
+				}
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (f readFile) transmittable(lineBytesBuffer *bytes.Buffer, length, capacity int,
 	re regex.Regex) (line.Line, bool) {
 
 	var read line.Line
-	if !re.Match(lineBytes.Bytes()) {
+	if !re.Match(lineBytesBuffer.Bytes()) {
 		f.updateLineNotMatched()
 		f.updateLineNotTransmitted()
 		return read, false
@@ -254,7 +387,7 @@ func (f readFile) transmittable(lineBytes *bytes.Buffer, length, capacity int,
 	f.updateLineTransmitted()
 
 	read = line.Line{
-		Content:         lineBytes,
+		Content:         lineBytesBuffer,
 		SourceID:        f.globID,
 		Count:           f.totalLineCount(),
 		TransmittedPerc: f.transmittedPerc(),
diff --git a/internal/lcontext/lcontext.go b/internal/lcontext/lcontext.go
new file mode 100644
index 0000000..183ceb5
--- /dev/null
+++ b/internal/lcontext/lcontext.go
@@ -0,0 +1,22 @@
+package lcontext
+
+// LContext stands for line context (used by context aware grep queries e.g.)
+type LContext struct {
+	AfterContext  int
+	BeforeContext int
+	MaxCount      int
+}
+
+// Has returns true if it has any parameter set.
+func (c LContext) Has() bool {
+	if c.AfterContext > 0 {
+		return true
+	}
+	if c.BeforeContext > 0 {
+		return true
+	}
+	if c.MaxCount > 0 {
+		return true
+	}
+	return false
+}
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 6bc8268..c25f85a 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -18,12 +18,13 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/mapr/server"
 	"github.com/mimecast/dtail/internal/protocol"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
-type handleCommandCb func(context.Context, int, []string, string)
+type handleCommandCb func(context.Context, lcontext.LContext, int, []string, string)
 
 type baseHandler struct {
 	done             *internal.Done
@@ -160,14 +161,13 @@ func (h *baseHandler) handleCommand(commandStr string) {
 
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
-	options, err := config.DeserializeOptions(splitted[1:])
+	options, ltx, err := config.DeserializeOptions(splitted[1:])
 	if err != nil {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
 	}
-	h.setOptions(options)
-
-	h.handleCommandCb(ctx, argc, args, commandName)
+	h.handleOptions(options)
+	h.handleCommandCb(ctx, ltx, argc, args, commandName)
 }
 
 func (h *baseHandler) handleProtocolVersion(args []string) ([]string, int, string, error) {
@@ -238,7 +238,7 @@ func (h *baseHandler) handleAckCommand(argc int, args []string) {
 	}
 }
 
-func (h *baseHandler) setOptions(options map[string]string) {
+func (h *baseHandler) handleOptions(options map[string]string) {
 	// We have to make sure that this block is executed only once.
 	h.mutex.Lock()
 	defer h.mutex.Unlock()
diff --git a/internal/server/handlers/healthhandler.go b/internal/server/handlers/healthhandler.go
index 0425696..6dd9872 100644
--- a/internal/server/handlers/healthhandler.go
+++ b/internal/server/handlers/healthhandler.go
@@ -8,6 +8,7 @@ import (
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/lcontext"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
 
@@ -40,8 +41,8 @@ func NewHealthHandler(user *user.User) *HealthHandler {
 	return &h
 }
 
-func (h *HealthHandler) handleHealthCommand(ctx context.Context, argc int,
-	args []string, commandName string) {
+func (h *HealthHandler) handleHealthCommand(ctx context.Context,
+	ltx lcontext.LContext, argc int, args []string, commandName string) {
 
 	dlog.Server.Debug(h.user, "Handling health command", argc, args)
 	switch commandName {
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 384e966..4728a55 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -10,6 +10,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/fs"
 	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 )
@@ -26,8 +27,8 @@ func newReadCommand(server *ServerHandler, mode omode.Mode) *readCommand {
 	}
 }
 
-func (r *readCommand) Start(ctx context.Context, argc int, args []string,
-	retries int) {
+func (r *readCommand) Start(ctx context.Context, ltx lcontext.LContext,
+	argc int, args []string, retries int) {
 
 	re := regex.NewNoop()
 	if argc >= 4 {
@@ -44,11 +45,11 @@ func (r *readCommand) Start(ctx context.Context, argc int, args []string,
 			"Unable to parse command", args, argc))
 		return
 	}
-	r.readGlob(ctx, args[1], re, retries)
+	r.readGlob(ctx, ltx, args[1], re, retries)
 }
 
-func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
-	retries int) {
+func (r *readCommand) readGlob(ctx context.Context, ltx lcontext.LContext,
+	glob string, re regex.Regex, retries int) {
 
 	retryInterval := time.Second * 5
 	glob = filepath.Clean(glob)
@@ -74,7 +75,7 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 			continue
 		}
 
-		r.readFiles(ctx, paths, glob, re, retryInterval)
+		r.readFiles(ctx, ltx, paths, glob, re, retryInterval)
 		return
 	}
 
@@ -83,18 +84,18 @@ func (r *readCommand) readGlob(ctx context.Context, glob string, re regex.Regex,
 	return
 }
 
-func (r *readCommand) readFiles(ctx context.Context, paths []string, glob string,
-	re regex.Regex, retryInterval time.Duration) {
+func (r *readCommand) readFiles(ctx context.Context, ltx lcontext.LContext,
+	paths []string, glob string, re regex.Regex, retryInterval time.Duration) {
 
 	var wg sync.WaitGroup
 	wg.Add(len(paths))
 	for _, path := range paths {
-		go r.readFileIfPermissions(ctx, &wg, path, glob, re)
+		go r.readFileIfPermissions(ctx, ltx, &wg, path, glob, re)
 	}
 	wg.Wait()
 }
 
-func (r *readCommand) readFileIfPermissions(ctx context.Context,
+func (r *readCommand) readFileIfPermissions(ctx context.Context, ltx lcontext.LContext,
 	wg *sync.WaitGroup, path, glob string, re regex.Regex) {
 
 	defer wg.Done()
@@ -105,12 +106,13 @@ func (r *readCommand) readFileIfPermissions(ctx context.Context,
 			"Unable to read file(s), check server logs"))
 		return
 	}
-	r.readFile(ctx, path, globID, re)
+	r.readFile(ctx, ltx, path, globID, re)
 }
 
-func (r *readCommand) readFile(ctx context.Context, path, globID string, re regex.Regex) {
-	dlog.Server.Info(r.server.user, "Start reading file", path, globID)
+func (r *readCommand) readFile(ctx context.Context, ltx lcontext.LContext,
+	path, globID string, re regex.Regex) {
 
+	dlog.Server.Info(r.server.user, "Start reading file", path, globID)
 	var reader fs.FileReader
 	switch r.mode {
 	case omode.TailClient:
@@ -129,7 +131,7 @@ func (r *readCommand) readFile(ctx context.Context, path, globID string, re rege
 			lines = make(chan line.Line, 100)
 			aggregate.NextLinesCh <- lines
 		}
-		if err := reader.Start(ctx, lines, re); err != nil {
+		if err := reader.Start(ctx, ltx, lines, re); err != nil {
 			dlog.Server.Error(r.server.user, path, globID, err)
 		}
 		if aggregate != nil {
diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go
index 52c4570..36574a9 100644
--- a/internal/server/handlers/serverhandler.go
+++ b/internal/server/handlers/serverhandler.go
@@ -8,6 +8,7 @@ import (
 	"github.com/mimecast/dtail/internal"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
+	"github.com/mimecast/dtail/internal/lcontext"
 	"github.com/mimecast/dtail/internal/omode"
 	user "github.com/mimecast/dtail/internal/user/server"
 )
@@ -53,8 +54,8 @@ func NewServerHandler(user *user.User, catLimiter,
 	return &h
 }
 
-func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
-	args []string, commandName string) {
+func (h *ServerHandler) handleUserCommand(ctx context.Context, ltx lcontext.LContext,
+	argc int, args []string, commandName string) {
 
 	dlog.Server.Debug(h.user, "Handling user command", argc, args)
 	h.incrementActiveCommands()
@@ -68,13 +69,13 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int,
 	case "grep", "cat":
 		command := newReadCommand(h, omode.CatClient)
 		go func() {
-			command.Start(ctx, argc, args, 1)
+			command.Start(ctx, ltx, argc, args, 1)
 			commandFinished()
 		}()
 	case "tail":
 		command := newReadCommand(h, omode.TailClient)
 		go func() {
-			command.Start(ctx, argc, args, 10)
+			command.Start(ctx, ltx, argc, args, 10)
 			commandFinished()
 		}()
 	case "map":
-- 
cgit v1.2.3


From 06ece112c0dd20c0c211c538216fe64ebe4045c9 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Thu, 14 Oct 2021 20:10:55 +0300
Subject: add dgrep context integration tests

---
 internal/io/fs/readfile.go              | 23 +++++++++++++++--------
 internal/server/handlers/basehandler.go |  2 ++
 2 files changed, 17 insertions(+), 8 deletions(-)

(limited to 'internal')

diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 88d467e..28cbe58 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -99,15 +99,24 @@ func (f readFile) Start(ctx context.Context, ltx lcontext.LContext,
 	rawLines := make(chan *bytes.Buffer, 100)
 	truncate := make(chan struct{})
 
-	var wg sync.WaitGroup
-	wg.Add(1)
+	readCtx, readCancel := context.WithCancel(ctx)
+	var filterWg sync.WaitGroup
+	filterWg.Add(1)
 
 	go f.periodicTruncateCheck(ctx, truncate)
-	go f.filter(ctx, ltx, &wg, rawLines, lines, re)
+	go func() {
+		f.filter(ctx, ltx, rawLines, lines, re)
+		filterWg.Done()
+		// If the filter stopped, make the reader stop too, no need to read
+		// more data if there is nothing more the filter wants to filter for!
+		// E.g. it could be that we only want to filter N matches but not more.
+		readCancel()
+	}()
 
-	err = f.read(ctx, fd, rawLines, truncate)
+	err = f.read(readCtx, fd, rawLines, truncate)
 	close(rawLines)
-	wg.Wait()
+	// Filter may sends some data still. So wait until it is done here.
+	filterWg.Wait()
 
 	return err
 }
@@ -215,10 +224,8 @@ func (f readFile) read(ctx context.Context, fd *os.File, rawLines chan *bytes.Bu
 
 // Filter log lines matching a given regular expression.
 func (f readFile) filter(ctx context.Context, ltx lcontext.LContext,
-	wg *sync.WaitGroup, rawLines <-chan *bytes.Buffer, lines chan<- line.Line,
-	re regex.Regex) {
+	rawLines <-chan *bytes.Buffer, lines chan<- line.Line, re regex.Regex) {
 
-	defer wg.Done()
 	// Do we have any kind of local context settings? If so then run the more complex
 	// filterWithLContext method.
 	if ltx.Has() {
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index c25f85a..934f2bc 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -159,6 +159,8 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		cancel()
 	}()
 
+	dlog.Server.Trace(args)
+	dlog.Server.Trace(args[0])
 	splitted := strings.Split(args[0], ":")
 	commandName := splitted[0]
 	options, ltx, err := config.DeserializeOptions(splitted[1:])
-- 
cgit v1.2.3


From 698fb76b98c46c677abe13fdc93afc6c4f38c39e Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Thu, 14 Oct 2021 20:55:35 +0300
Subject: refactor

---
 internal/server/handlers/basehandler.go | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'internal')

diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 934f2bc..6d10d17 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -159,11 +159,16 @@ func (h *baseHandler) handleCommand(commandStr string) {
 		cancel()
 	}()
 
-	dlog.Server.Trace(args)
-	dlog.Server.Trace(args[0])
-	splitted := strings.Split(args[0], ":")
-	commandName := splitted[0]
-	options, ltx, err := config.DeserializeOptions(splitted[1:])
+	parts := strings.Split(args[0], ":")
+	commandName := parts[0]
+
+	// Either no options or empty options provided.
+	if len(parts) == 1 || len(parts[1]) == 0 {
+		h.handleCommandCb(ctx, lcontext.LContext{}, argc, args, commandName)
+		return
+	}
+
+	options, ltx, err := config.DeserializeOptions(parts[1:])
 	if err != nil {
 		h.send(h.serverMessages, dlog.Server.Error(h.user, err))
 		return
-- 
cgit v1.2.3


From b27fc108ecd6eead5c97cf6e894bf8d639fff75c Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Fri, 15 Oct 2021 13:06:18 +0300
Subject: Execute test directories individually

---
 internal/config/args.go        | 14 ++++++++------
 internal/config/initializer.go |  3 +++
 2 files changed, 11 insertions(+), 6 deletions(-)

(limited to 'internal')

diff --git a/internal/config/args.go b/internal/config/args.go
index f94e0a9..3d7ac7d 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -33,6 +33,7 @@ type Args struct {
 	ServersStr         string
 	Spartan            bool
 	SSHAuthMethods     []gossh.AuthMethod
+	SSHBindAddress     string
 	SSHHostKeyCallback gossh.HostKeyCallback
 	SSHPort            int
 	Timeout            int
@@ -46,13 +47,13 @@ func (a *Args) String() string {
 
 	sb.WriteString("Args(")
 
-	sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir))
-	sb.WriteString(fmt.Sprintf("%s:%s,", "Logger", a.Logger))
-	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "ConnectionsPerCPU", a.ConnectionsPerCPU))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Discovery", a.Discovery))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "LogDir", a.LogDir))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "LogLevel", a.LogLevel))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Logger", a.Logger))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile))
@@ -60,12 +61,13 @@ func (a *Args) String() string {
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr))
-	sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless))
-	sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr))
-	sb.WriteString(fmt.Sprintf("%s:%v,", "Spartan", a.Spartan))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHAuthMethods", a.SSHAuthMethods))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHBindAddress", a.SSHBindAddress))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHHostKeyCallback", a.SSHHostKeyCallback))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "SSHPort", a.SSHPort))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr))
+	sb.WriteString(fmt.Sprintf("%s:%v,", "Spartan", a.Spartan))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Timeout", a.Timeout))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "TrustAllHosts", a.TrustAllHosts))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "UserName", a.UserName))
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 0c6dfdf..8215891 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -155,6 +155,9 @@ func transformClient(in *initializer, args *Args, additionalArgs []string) error
 }
 
 func transformServer(in *initializer, args *Args, additionalArgs []string) error {
+	if args.SSHBindAddress != "" {
+		in.Server.SSHBindAddress = args.SSHBindAddress
+	}
 	return nil
 }
 
-- 
cgit v1.2.3


From 10314cef906fd9b73e003be69c2f6b7b3d66570c Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Fri, 15 Oct 2021 13:20:48 +0300
Subject: Can configure DTail client not to mess with ~/.ssh/known_hosts via
 env var - this is useful for running unit and integration tests in jenkins

---
 internal/config/client.go                 |  3 +++
 internal/config/initializer.go            | 10 ++++++++++
 internal/ssh/client/authmethods.go        |  9 ++++++---
 internal/ssh/client/knownhostscallback.go |  7 +++++++
 4 files changed, 26 insertions(+), 3 deletions(-)

(limited to 'internal')

diff --git a/internal/config/client.go b/internal/config/client.go
index 9f4df97..86f97f0 100644
--- a/internal/config/client.go
+++ b/internal/config/client.go
@@ -104,6 +104,9 @@ type termColors struct {
 type ClientConfig struct {
 	TermColorsEnable bool       `json:",omitempty"`
 	TermColors       termColors `json:",omitempty"`
+	// When unit testing in Jenkins you don't want to touch files in ~jenkins
+	// during integration tests really.
+	SSHDontAddHostsToKnownHostsFile bool `json:",omitempty"`
 }
 
 // Create a new default client configuration.
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 8215891..35105bf 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -65,6 +65,8 @@ func (in *initializer) parseSpecificConfig(configFile string) error {
 func (in *initializer) transformConfig(sourceProcess source.Source, args *Args,
 	additionalArgs []string) error {
 
+	in.readEnvironmentVars()
+
 	switch sourceProcess {
 	case source.Server:
 		return in.optimusPrime(transformServer, args, additionalArgs)
@@ -78,6 +80,14 @@ func (in *initializer) transformConfig(sourceProcess source.Source, args *Args,
 	}
 }
 
+// There are some special options which can be set by environment variable.
+func (in *initializer) readEnvironmentVars() {
+	if len(os.Getenv("DTAIL_SSH_DONT_ADD_HOSTS_TO_KNOWNHOSTS_FILE")) != 0 ||
+		len(os.Getenv("DTAIL_JENKINS")) != 0 {
+		in.Client.SSHDontAddHostsToKnownHostsFile = true
+	}
+}
+
 func (in *initializer) optimusPrime(sourceCb transformCb, args *Args,
 	additionalArgs []string) error {
 
diff --git a/internal/ssh/client/authmethods.go b/internal/ssh/client/authmethods.go
index ced1fb9..089a66a 100644
--- a/internal/ssh/client/authmethods.go
+++ b/internal/ssh/client/authmethods.go
@@ -35,8 +35,7 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{},
 	if err != nil {
 		dlog.Common.FatalPanic(knownHostsPath, err)
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path",
-		knownHostsPath)
+	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
 	if config.Common.ExperimentalFeaturesEnable {
 		sshAuthMethods = append(sshAuthMethods, gossh.Password("experimental feature test"))
 		dlog.Common.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
@@ -88,7 +87,11 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{},
 	}
 	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key",
 		privateKeyPath, err)
-	dlog.Common.FatalPanic("Unable to find private SSH key information")
+
+	// This is only a panic when we expect to do something about it.
+	if !config.Client.SSHDontAddHostsToKnownHostsFile {
+		dlog.Common.FatalPanic("Unable to find private SSH key information")
+	}
 
 	// Never reach this point.
 	return sshAuthMethods, knownHostsCallback
diff --git a/internal/ssh/client/knownhostscallback.go b/internal/ssh/client/knownhostscallback.go
index 65a590a..2aa0168 100644
--- a/internal/ssh/client/knownhostscallback.go
+++ b/internal/ssh/client/knownhostscallback.go
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/prompt"
 
@@ -214,6 +215,12 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 
 func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 	tmpKnownHostsPath := fmt.Sprintf("%s.tmp", c.knownHostsPath)
+
+	if config.Client.SSHDontAddHostsToKnownHostsFile {
+		dlog.Common.Verbose("Not adding hosts to known hosts file, as disabled by config")
+		return
+	}
+
 	newFd, err := os.OpenFile(tmpKnownHostsPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 	if err != nil {
 		panic(fmt.Sprintf("%s: %s", tmpKnownHostsPath, err.Error()))
-- 
cgit v1.2.3


From 860c41441b3bcf542b5701ae2257812879ce47b4 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 19 Oct 2021 19:01:12 +0300
Subject: Set DTAIL_RUN_INTEGRATIONT_TEST to yes for integration tests

---
 internal/config/env.go         | 7 +++++++
 internal/config/initializer.go | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)
 create mode 100644 internal/config/env.go

(limited to 'internal')

diff --git a/internal/config/env.go b/internal/config/env.go
new file mode 100644
index 0000000..88b831d
--- /dev/null
+++ b/internal/config/env.go
@@ -0,0 +1,7 @@
+package config
+
+import "os"
+
+func Env(env string) bool {
+	return "yes" == os.Getenv(env)
+}
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index 35105bf..4d6a73b 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -82,8 +82,8 @@ func (in *initializer) transformConfig(sourceProcess source.Source, args *Args,
 
 // There are some special options which can be set by environment variable.
 func (in *initializer) readEnvironmentVars() {
-	if len(os.Getenv("DTAIL_SSH_DONT_ADD_HOSTS_TO_KNOWNHOSTS_FILE")) != 0 ||
-		len(os.Getenv("DTAIL_JENKINS")) != 0 {
+	if Env("DTAIL_SSH_DONT_ADD_HOSTS_TO_KNOWNHOSTS_FILE") ||
+		Env("DTAIL_JENKINS") {
 		in.Client.SSHDontAddHostsToKnownHostsFile = true
 	}
 }
-- 
cgit v1.2.3


From ddfdb8e01aea4f8b6127834a14b74c5c534820d1 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 19 Oct 2021 19:50:38 +0300
Subject: lowercase log rotation comparison

---
 internal/io/dlog/loggers/strategy.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'internal')

diff --git a/internal/io/dlog/loggers/strategy.go b/internal/io/dlog/loggers/strategy.go
index 25d10f0..48e7d44 100644
--- a/internal/io/dlog/loggers/strategy.go
+++ b/internal/io/dlog/loggers/strategy.go
@@ -3,6 +3,7 @@ package loggers
 import (
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // Rotation is the actual strategy used for log rotation..
@@ -25,7 +26,7 @@ type Strategy struct {
 
 // NewStrategy returns the stratey based on its name.
 func NewStrategy(name string) Strategy {
-	switch name {
+	switch strings.ToLower(name) {
 	case "daily":
 		return Strategy{DailyRotation, ""}
 	default:
-- 
cgit v1.2.3


From 3c16894e997bdb235f1c0ce1291cc85a7b56118c Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 19 Oct 2021 20:33:59 +0300
Subject: Bugfix: Use correct hostname when no port specified

---
 internal/clients/connectors/serverconnection.go | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

(limited to 'internal')

diff --git a/internal/clients/connectors/serverconnection.go b/internal/clients/connectors/serverconnection.go
index 1df4d73..aeb2a41 100644
--- a/internal/clients/connectors/serverconnection.go
+++ b/internal/clients/connectors/serverconnection.go
@@ -63,17 +63,20 @@ func (c *ServerConnection) Handler() handlers.Handler { return c.handler }
 
 // Attempt to parse the server port address from the provided server FQDN.
 func (c *ServerConnection) initServerPort() {
-	c.port = config.Common.SSHPort
 	parts := strings.Split(c.server, ":")
-	if len(parts) == 2 {
-		dlog.Client.Debug("Parsing port from hostname", parts)
-		port, err := strconv.Atoi(parts[1])
-		if err != nil {
-			dlog.Client.FatalPanic("Unable to parse client port", c.server, parts, err)
-		}
-		c.hostname = parts[0]
-		c.port = port
+	if len(parts) == 1 {
+		c.hostname = c.server
+		c.port = config.Common.SSHPort
+		return
+	}
+
+	dlog.Client.Debug("Parsing port from hostname", parts)
+	port, err := strconv.Atoi(parts[1])
+	if err != nil {
+		dlog.Client.FatalPanic("Unable to parse client port", c.server, parts, err)
 	}
+	c.hostname = parts[0]
+	c.port = port
 }
 
 // Start the connection to the server.
@@ -127,8 +130,8 @@ func (c *ServerConnection) dial(ctx context.Context, cancel context.CancelFunc,
 		<-statsCh
 	}()
 
-	dlog.Client.Debug(c.server, "Dialing into the connection")
 	address := fmt.Sprintf("%s:%d", c.hostname, c.port)
+	dlog.Client.Debug(c.server, "Dialing into the connection", address)
 
 	client, err := ssh.Dial("tcp", address, c.config)
 	if err != nil {
-- 
cgit v1.2.3


From 0908282e8b3c09f603fd4083fd7f075c4535e939 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Thu, 21 Oct 2021 21:00:47 +0300
Subject: backport ECDSA key support form master

---
 internal/ssh/client/authmethods.go | 38 ++++++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 14 deletions(-)

(limited to 'internal')

diff --git a/internal/ssh/client/authmethods.go b/internal/ssh/client/authmethods.go
index 089a66a..b1e514d 100644
--- a/internal/ssh/client/authmethods.go
+++ b/internal/ssh/client/authmethods.go
@@ -18,7 +18,7 @@ func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod,
 	if len(sshAuthMethods) > 0 {
 		simpleCallback, err := NewSimpleCallback()
 		if err != nil {
-			dlog.Common.FatalPanic(err)
+			dlog.Client.FatalPanic(err)
 		}
 		return sshAuthMethods, simpleCallback
 	}
@@ -33,12 +33,12 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{},
 	knownHostsCallback, err := NewKnownHostsCallback(knownHostsPath, trustAllHosts,
 		throttleCh)
 	if err != nil {
-		dlog.Common.FatalPanic(knownHostsPath, err)
+		dlog.Client.FatalPanic(knownHostsPath, err)
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
-	if config.Common.ExperimentalFeaturesEnable {
+	dlog.Client.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
+	if config.Client.ExperimentalFeaturesEnable {
 		sshAuthMethods = append(sshAuthMethods, gossh.Password("experimental feature test"))
-		dlog.Common.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
+		dlog.Client.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
 	}
 
 	// First try to read custom private key path.
@@ -46,23 +46,23 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{},
 		authMethod, err := ssh.PrivateKey(privateKeyPath)
 		if err == nil {
 			sshAuthMethods = append(sshAuthMethods, authMethod)
-			dlog.Common.Debug("initKnownHostsAuthMethods",
+			dlog.Client.Debug("initKnownHostsAuthMethods",
 				"Added path to list of auth methods, not adding further methods",
 				privateKeyPath)
 			return sshAuthMethods, knownHostsCallback
 		}
-		dlog.Common.FatalPanic("Unable to use private SSH key", privateKeyPath, err)
+		dlog.Client.FatalPanic("Unable to use private SSH key", privateKeyPath, err)
 	}
 
 	// Second, try SSH Agent
 	authMethod, err := ssh.Agent()
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK)"+
+		dlog.Client.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK)"+
 			"to list of auth methods, not adding further methods")
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods",
+	dlog.Client.Debug("initKnownHostsAuthMethods",
 		"Unable to init SSH Agent auth method", err)
 
 	// Third, try Linux/UNIX default key paths
@@ -70,27 +70,37 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{},
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthmethods",
+		dlog.Client.Debug("initKnownHostsAuthmethods",
 			"Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key",
+	dlog.Client.Debug("initKnownHostsAuthMethods", "Unable to use private key",
 		privateKeyPath, err)
 
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_dsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		dlog.Common.Debug("initKnownHostsAuthmethods",
+		dlog.Client.Debug("initKnownHostsAuthmethods",
 			"Added path to list of auth methods, not adding further methods", privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	dlog.Common.Debug("initKnownHostsAuthMethods", "Unable to use private key",
+
+	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_ecdsa"
+	authMethod, err = ssh.PrivateKey(privateKeyPath)
+	if err == nil {
+		sshAuthMethods = append(sshAuthMethods, authMethod)
+		dlog.Client.Debug("initKnownHostsAuthmethods",
+			"Added path to list of auth methods, not adding further methods", privateKeyPath)
+		return sshAuthMethods, knownHostsCallback
+	}
+
+	dlog.Client.Debug("initKnownHostsAuthMethods", "Unable to use private key",
 		privateKeyPath, err)
 
 	// This is only a panic when we expect to do something about it.
 	if !config.Client.SSHDontAddHostsToKnownHostsFile {
-		dlog.Common.FatalPanic("Unable to find private SSH key information")
+		dlog.Client.FatalPanic("Unable to find private SSH key information")
 	}
 
 	// Never reach this point.
-- 
cgit v1.2.3


From 739205206d63bf42f4e843b39d04d4c8cd8207c3 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Thu, 21 Oct 2021 21:02:58 +0300
Subject: backport mapreduce reporter rampup from master

---
 internal/clients/maprclient.go | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'internal')

diff --git a/internal/clients/maprclient.go b/internal/clients/maprclient.go
index 074494c..246946f 100644
--- a/internal/clients/maprclient.go
+++ b/internal/clients/maprclient.go
@@ -137,6 +137,10 @@ func (c MaprClient) makeCommands() (commands []string) {
 }
 
 func (c *MaprClient) periodicReportResults(ctx context.Context) {
+	rampUpSleep := c.query.Interval / 2
+	dlog.Client.Debug("Ramp up sleeping before processing mapreduce results", rampUpSleep)
+	time.Sleep(rampUpSleep)
+
 	for {
 		select {
 		case <-time.After(c.query.Interval):
-- 
cgit v1.2.3