From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 19 Sep 2021 13:22:59 +0300
Subject: move args to config package

logger package rewrite as dlog
---
 internal/io/dlog/dlog.go            | 206 ++++++++++++++++++++++++++++++++++++
 internal/io/dlog/level.go           |  89 ++++++++++++++++
 internal/io/dlog/loggers/factory.go |  60 +++++++++++
 internal/io/dlog/loggers/file.go    | 156 +++++++++++++++++++++++++++
 internal/io/dlog/loggers/fout.go    |  46 ++++++++
 internal/io/dlog/loggers/logger.go  |  18 ++++
 internal/io/dlog/loggers/none.go    |  21 ++++
 internal/io/dlog/loggers/stdout.go  |  73 +++++++++++++
 internal/io/dlog/rotation.go        |  27 +++++
 internal/io/dlog/source.go          |  19 ++++
 internal/io/dlog/strategy.go        |  22 ++++
 11 files changed, 737 insertions(+)
 create mode 100644 internal/io/dlog/dlog.go
 create mode 100644 internal/io/dlog/level.go
 create mode 100644 internal/io/dlog/loggers/factory.go
 create mode 100644 internal/io/dlog/loggers/file.go
 create mode 100644 internal/io/dlog/loggers/fout.go
 create mode 100644 internal/io/dlog/loggers/logger.go
 create mode 100644 internal/io/dlog/loggers/none.go
 create mode 100644 internal/io/dlog/loggers/stdout.go
 create mode 100644 internal/io/dlog/rotation.go
 create mode 100644 internal/io/dlog/source.go
 create mode 100644 internal/io/dlog/strategy.go

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
new file mode 100644
index 0000000..7282741
--- /dev/null
+++ b/internal/io/dlog/dlog.go
@@ -0,0 +1,206 @@
+package dlog
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mimecast/dtail/internal/color/brush"
+	"github.com/mimecast/dtail/internal/config"
+	"github.com/mimecast/dtail/internal/io/dlog/loggers"
+	"github.com/mimecast/dtail/internal/io/pool"
+	"github.com/mimecast/dtail/internal/protocol"
+)
+
+// Client is the log handler for the client packages.
+var Client *DLog
+
+// Server is the log handler for the server packages.
+var Server *DLog
+
+// Common is the log handler for all other packages.
+// TODO: Rename Common to Common
+var Common *DLog
+
+var mutex sync.Mutex
+var started bool
+
+// Start logger(s).
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLevel string) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	if started {
+		Common.FatalPanic("Logger already started")
+	}
+
+	level := newLevel(logLevel)
+	switch sourceProcess {
+	case CLIENT:
+		// This is a DTail client process running.
+		impl := loggers.FOUT
+		Client = New(CLIENT, CLIENT, impl, level)
+		Server = New(CLIENT, SERVER, impl, level)
+		Common = Client
+	case SERVER:
+		// This is a DTail server process running.
+		impl := loggers.FILE
+		Client = New(SERVER, CLIENT, impl, level)
+		Server = New(SERVER, SERVER, impl, level)
+		Common = Server
+	}
+
+	var wg2 sync.WaitGroup
+	wg2.Add(2)
+	Client.start(ctx, &wg2)
+	Server.start(ctx, &wg2)
+	started = true
+
+	go rotation(ctx)
+	go func() {
+		wg2.Wait()
+		wg.Done()
+	}()
+}
+
+// DLog is the DTail logger.
+type DLog struct {
+	logger loggers.Logger
+	// Is this a DTail server or client process logging?
+	sourceProcess source
+	// Is this a DTail server or client package logging? In serverless mode
+	// the client can also execute code from the server package.
+	sourcePackage source
+	// Max log level to log.
+	maxLevel level
+}
+
+// New creates a new DTail logger.
+func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+	return &DLog{
+		logger:        loggers.Factory(sourceProcess.String(), impl),
+		sourceProcess: sourceProcess,
+		sourcePackage: sourcePackage,
+		maxLevel:      maxLevel,
+	}
+}
+
+func (d *DLog) start(ctx context.Context, wg *sync.WaitGroup) {
+	go func() {
+		defer wg.Done()
+		var wg2 sync.WaitGroup
+		wg2.Add(1)
+		d.logger.Start(ctx, &wg2)
+		<-ctx.Done()
+		wg2.Wait()
+	}()
+}
+
+func (d *DLog) log(level level, args []interface{}) string {
+	if d.maxLevel < level {
+		return ""
+	}
+	sb := pool.BuilderBuffer.Get().(*strings.Builder)
+	defer pool.RecycleBuilderBuffer(sb)
+	now := time.Now()
+
+	sb.WriteString(d.sourcePackage.String())
+	sb.WriteString(protocol.FieldDelimiter)
+	sb.WriteString(now.Format("20060102-150405"))
+	sb.WriteString(protocol.FieldDelimiter)
+	sb.WriteString(level.String())
+	sb.WriteString(protocol.FieldDelimiter)
+	d.writeArgStrings(sb, args)
+
+	message := sb.String()
+	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
+		d.logger.Log(now, message)
+		return message
+	}
+
+	d.logger.LogWithColors(now, message, brush.Colorfy(message))
+	return message
+}
+
+func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
+	for i, arg := range args {
+		if i > 0 {
+			sb.WriteString(protocol.FieldDelimiter)
+		}
+		switch v := arg.(type) {
+		case string:
+			sb.WriteString(v)
+		case error:
+			sb.WriteString(v.Error())
+		default:
+			sb.WriteString(fmt.Sprintf("%v", v))
+		}
+	}
+}
+
+func (d *DLog) FatalPanic(args ...interface{}) {
+	d.log(FATAL, args)
+	d.logger.Flush()
+	panic("Not recovering from this fatal error...")
+}
+
+func (d *DLog) Fatal(args ...interface{}) string {
+	return d.log(FATAL, args)
+}
+
+func (d *DLog) Error(args ...interface{}) string {
+	return d.log(ERROR, args)
+}
+
+func (d *DLog) Warn(args ...interface{}) string {
+	return d.log(WARN, args)
+}
+
+func (d *DLog) Info(args ...interface{}) string {
+	return d.log(INFO, args)
+}
+
+func (d *DLog) Verbose(args ...interface{}) string {
+	return d.log(VERBOSE, args)
+}
+
+func (d *DLog) Debug(args ...interface{}) string {
+	return d.log(DEBUG, args)
+}
+
+func (d *DLog) Trace(args ...interface{}) string {
+	return d.log(TRACE, args)
+}
+
+func (d *DLog) Devel(args ...interface{}) string {
+	return d.log(DEVEL, args)
+}
+
+func (d *DLog) Raw(message string) string {
+	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
+		d.logger.Log(time.Now(), message)
+		return message
+	}
+
+	d.logger.Log(time.Now(), brush.Colorfy(message))
+	return message
+}
+
+func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
+	args := make([]interface{}, len(data)+1)
+	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
+
+	i := 1
+	for k, v := range data {
+		args[i] = fmt.Sprintf("%s=%v", k, v)
+		i++
+	}
+
+	return d.log(INFO, args)
+}
+
+func (d *DLog) Flush()  { d.logger.Flush() }
+func (d *DLog) Pause()  { d.logger.Pause() }
+func (d *DLog) Resume() { d.logger.Resume() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
new file mode 100644
index 0000000..84550f0
--- /dev/null
+++ b/internal/io/dlog/level.go
@@ -0,0 +1,89 @@
+package dlog
+
+import (
+	"fmt"
+	"strings"
+)
+
+type level int
+
+const (
+	FATAL   level = iota
+	ERROR   level = iota
+	WARN    level = iota
+	INFO    level = iota
+	DEFAULT level = iota
+	VERBOSE level = iota
+	DEBUG   level = iota
+	DEVEL   level = iota
+	TRACE   level = iota
+	ALL     level = iota
+)
+
+var allLevels = []level{
+	FATAL,
+	ERROR,
+	WARN,
+	INFO,
+	DEFAULT,
+	VERBOSE,
+	DEBUG,
+	DEVEL,
+	TRACE,
+	ALL,
+}
+
+func newLevel(l string) level {
+	switch strings.ToUpper(l) {
+	case "FATAL":
+		return FATAL
+	case "ERROR":
+		return ERROR
+	case "WARN":
+		return WARN
+	case "INFO":
+		return INFO
+	case "":
+		fallthrough
+	case "DEFAULT":
+		return DEFAULT
+	case "VERBOSE":
+		return VERBOSE
+	case "DEBUG":
+		return DEBUG
+	case "DEVEL":
+		return DEVEL
+	case "TRACE":
+		return TRACE
+	case "ALL":
+		return ALL
+	}
+	panic(fmt.Sprintf("Unknown log level %s, must be one of: %v", l, allLevels))
+}
+
+func (l level) String() string {
+	switch l {
+	case FATAL:
+		return "FATAL"
+	case ERROR:
+		return "ERROR"
+	case WARN:
+		return "WARN"
+	case INFO:
+		return "INFO"
+	case DEFAULT:
+		return "DEFAULT"
+	case VERBOSE:
+		return "VERBOSE"
+	case DEBUG:
+		return "DEBUG"
+	case DEVEL:
+		return "DEVEL"
+	case TRACE:
+		return "TRACE"
+	case ALL:
+		return "ALL"
+	}
+
+	panic("Unknown log level " + fmt.Sprintf("%d", l))
+}
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
new file mode 100644
index 0000000..3eb29c5
--- /dev/null
+++ b/internal/io/dlog/loggers/factory.go
@@ -0,0 +1,60 @@
+package loggers
+
+import (
+	"fmt"
+	"sync"
+)
+
+type Impl int
+
+const (
+	NONE   Impl = iota
+	STDOUT Impl = iota
+	FILE   Impl = iota
+	FOUT   Impl = iota
+)
+
+var factoryMap map[string]Logger
+var factoryMutex sync.Mutex
+
+func Factory(name string, impl Impl) Logger {
+	factoryMutex.Lock()
+	defer factoryMutex.Unlock()
+
+	id := fmt.Sprintf("name:%s,impl:%v", name, impl)
+
+	if factoryMap == nil {
+		factoryMap = make(map[string]Logger)
+	}
+
+	singleton, ok := factoryMap[id]
+	if !ok {
+		switch impl {
+		case NONE:
+			singleton = none{}
+		case STDOUT:
+			singleton = newStdout()
+			factoryMap[id] = singleton
+		case FILE:
+			singleton = newFile()
+			factoryMap[id] = singleton
+		case FOUT:
+			singleton = newFout()
+			factoryMap[id] = singleton
+		}
+	}
+
+	return singleton
+}
+
+func FactoryRotate() {
+	factoryMutex.Lock()
+	defer factoryMutex.Unlock()
+	if factoryMap == nil {
+		return
+	}
+
+	for _, impl := range factoryMap {
+		impl.Rotate()
+	}
+}
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
new file mode 100644
index 0000000..1c525c9
--- /dev/null
+++ b/internal/io/dlog/loggers/file.go
@@ -0,0 +1,156 @@
+package loggers
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/mimecast/dtail/internal/config"
+)
+
+type fileMessageBuf struct {
+	now     time.Time
+	message string
+}
+
+type file struct {
+	bufferCh    chan *fileMessageBuf
+	pauseCh     chan struct{}
+	resumeCh    chan struct{}
+	rotateCh    chan struct{}
+	flushCh     chan struct{}
+	lastDateStr string
+	fd          *os.File
+	writer      *bufio.Writer
+	mutex       sync.Mutex
+	started     bool
+}
+
+func newFile() *file {
+	f := file{
+		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
+		pauseCh:  make(chan struct{}),
+		resumeCh: make(chan struct{}),
+		rotateCh: make(chan struct{}),
+		flushCh:  make(chan struct{}),
+	}
+	f.getWriter(time.Now().Format("20060102"))
+	return &f
+}
+
+func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	// Logger already started from another Goroutine.
+	if s.started {
+		wg.Done()
+		return
+	}
+
+	pause := func(ctx context.Context) {
+		select {
+		case <-s.resumeCh:
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	go func() {
+		defer wg.Done()
+
+		for {
+			select {
+			case m := <-s.bufferCh:
+				s.write(m)
+			case <-s.pauseCh:
+				pause(ctx)
+			case <-s.flushCh:
+				s.flush()
+			case <-ctx.Done():
+				s.flush()
+				s.fd.Close()
+				return
+			}
+		}
+	}()
+
+	s.started = true
+}
+
+func (s *file) Log(now time.Time, message string) {
+	s.bufferCh <- &fileMessageBuf{now, message}
+}
+
+func (s *file) LogWithColors(now time.Time, message, coloredMessage string) {
+	panic("Colors not supported in file logger")
+}
+
+func (s *file) Pause()  { s.pauseCh <- struct{}{} }
+func (s *file) Resume() { s.resumeCh <- struct{}{} }
+func (s *file) Flush()  { s.flushCh <- struct{}{} }
+
+// TODO: Test that Rotate() actually works.
+func (s *file) Rotate()           { s.rotateCh <- struct{}{} }
+func (file) SupportsColors() bool { return false }
+
+func (s *file) write(m *fileMessageBuf) {
+	select {
+	case <-s.rotateCh:
+		// Force re-opening the outfile.
+		s.lastDateStr = ""
+	default:
+	}
+
+	writer := s.getWriter(m.now.Format("20060102"))
+	writer.WriteString(m.message)
+	writer.WriteByte('\n')
+}
+
+func (s *file) getWriter(dateStr string) *bufio.Writer {
+	if s.lastDateStr == dateStr {
+		return s.writer
+	}
+
+	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
+		if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil {
+			panic(err)
+		}
+	}
+
+	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr)
+	newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
+	if err != nil {
+		panic(err)
+	}
+
+	// Close old writer.
+	if s.fd != nil {
+		s.writer.Flush()
+		s.fd.Close()
+	}
+
+	s.fd = newFd
+	s.writer = bufio.NewWriterSize(s.fd, 1)
+	s.lastDateStr = dateStr
+
+	return s.writer
+}
+
+func (s *file) flush() {
+	defer s.writer.Flush()
+
+	for {
+		select {
+		case m := <-s.bufferCh:
+			s.write(m)
+		default:
+			return
+		}
+	}
+}
diff --git a/internal/io/dlog/loggers/fout.go b/internal/io/dlog/loggers/fout.go
new file mode 100644
index 0000000..603dbe9
--- /dev/null
+++ b/internal/io/dlog/loggers/fout.go
@@ -0,0 +1,46 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+type fout struct {
+	file   *file
+	stdout *stdout
+}
+
+// Logs to both, a file and stdout
+func newFout() *fout {
+	return &fout{file: newFile(), stdout: newStdout()}
+}
+
+func (f *fout) Start(ctx context.Context, wg *sync.WaitGroup) {
+	go func() {
+		defer wg.Done()
+
+		var wg2 sync.WaitGroup
+		wg2.Add(2)
+		f.file.Start(ctx, &wg2)
+		f.stdout.Start(ctx, &wg2)
+		wg2.Wait()
+	}()
+}
+
+func (f *fout) Log(now time.Time, message string) {
+	f.stdout.Log(now, message)
+	f.file.Log(now, message)
+}
+
+func (f *fout) LogWithColors(now time.Time, message, coloredMessage string) {
+	f.stdout.LogWithColors(now, "", coloredMessage)
+	f.file.Log(now, message)
+}
+
+func (f *fout) Flush()  { f.stdout.Flush(); f.file.Flush() }
+func (f *fout) Pause()  { f.stdout.Pause(); f.file.Pause() }
+func (f *fout) Resume() { f.stdout.Resume(); f.file.Resume() }
+func (f *fout) Rotate() { f.file.Rotate() }
+
+func (fout) SupportsColors() bool { return true }
diff --git a/internal/io/dlog/loggers/logger.go b/internal/io/dlog/loggers/logger.go
new file mode 100644
index 0000000..c88900d
--- /dev/null
+++ b/internal/io/dlog/loggers/logger.go
@@ -0,0 +1,18 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+type Logger interface {
+	Log(now time.Time, message string)
+	LogWithColors(now time.Time, message, messageWithColors string)
+	Start(ctx context.Context, wg *sync.WaitGroup)
+	Flush()
+	Pause()
+	Resume()
+	Rotate()
+	SupportsColors() bool
+}
diff --git a/internal/io/dlog/loggers/none.go b/internal/io/dlog/loggers/none.go
new file mode 100644
index 0000000..270027f
--- /dev/null
+++ b/internal/io/dlog/loggers/none.go
@@ -0,0 +1,21 @@
+package loggers
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+// don't log anything
+type none struct{}
+
+func (none) Start(ctx context.Context, wg *sync.WaitGroup) { wg.Done() }
+func (none) Log(now time.Time, message string)             {}
+
+func (none) LogWithColors(now time.Time, message, coloredMessage string) {}
+
+func (none) Flush()               {}
+func (none) Pause()               {}
+func (none) Resume()              {}
+func (none) Rotate()              {}
+func (none) SupportsColors() bool { return false }
diff --git a/internal/io/dlog/loggers/stdout.go b/internal/io/dlog/loggers/stdout.go
new file mode 100644
index 0000000..9738323
--- /dev/null
+++ b/internal/io/dlog/loggers/stdout.go
@@ -0,0 +1,73 @@
+package loggers
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type stdout struct {
+	bufferCh chan string
+	pauseCh  chan struct{}
+	resumeCh chan struct{}
+}
+
+func newStdout() *stdout {
+	return &stdout{
+		bufferCh: make(chan string, 100),
+		pauseCh:  make(chan struct{}),
+		resumeCh: make(chan struct{}),
+	}
+}
+
+func (s *stdout) Start(ctx context.Context, wg *sync.WaitGroup) {
+	pause := func(ctx context.Context) {
+		select {
+		case <-s.resumeCh:
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	go func() {
+		defer wg.Done()
+
+		for {
+			select {
+			case message := <-s.bufferCh:
+				fmt.Println(message)
+			case <-s.pauseCh:
+				pause(ctx)
+			case <-ctx.Done():
+				s.Flush()
+				return
+			}
+		}
+	}()
+}
+
+func (s *stdout) Log(now time.Time, message string) {
+	s.bufferCh <- message
+}
+
+func (s *stdout) LogWithColors(now time.Time, message, coloredMessage string) {
+	s.bufferCh <- coloredMessage
+}
+
+func (s *stdout) Flush() {
+	for {
+		select {
+		case message := <-s.bufferCh:
+			fmt.Println(message)
+		default:
+			return
+		}
+	}
+}
+
+func (s *stdout) Pause()            { s.pauseCh <- struct{}{} }
+func (s *stdout) Resume()           { s.resumeCh <- struct{}{} }
+func (s *stdout) Rotate()           {}
+func (stdout) SupportsColors() bool { return true }
diff --git a/internal/io/dlog/rotation.go b/internal/io/dlog/rotation.go
new file mode 100644
index 0000000..15ce1fd
--- /dev/null
+++ b/internal/io/dlog/rotation.go
@@ -0,0 +1,27 @@
+package dlog
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/mimecast/dtail/internal/io/dlog/loggers"
+)
+
+func rotation(ctx context.Context) {
+	rotateCh := make(chan os.Signal, 1)
+	signal.Notify(rotateCh, syscall.SIGHUP)
+	go func() {
+		for {
+			select {
+			case <-rotateCh:
+				Common.Debug("Invoking log rotation")
+				loggers.FactoryRotate()
+				return
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+}
diff --git a/internal/io/dlog/source.go b/internal/io/dlog/source.go
new file mode 100644
index 0000000..265885e
--- /dev/null
+++ b/internal/io/dlog/source.go
@@ -0,0 +1,19 @@
+package dlog
+
+type source int
+
+const (
+	CLIENT source = iota
+	SERVER source = iota
+)
+
+func (s source) String() string {
+	switch s {
+	case CLIENT:
+		return "CLIENT"
+	case SERVER:
+		return "SERVER"
+	}
+
+	panic("Unknown log source type")
+}
diff --git a/internal/io/dlog/strategy.go b/internal/io/dlog/strategy.go
new file mode 100644
index 0000000..32d8298
--- /dev/null
+++ b/internal/io/dlog/strategy.go
@@ -0,0 +1,22 @@
+package dlog
+
+import "github.com/mimecast/dtail/internal/config"
+
+// Strategy allows to specify a log rotation strategy.
+type Strategy int
+
+// Possible log strategies.
+const (
+	NormalStrategy Strategy = iota
+	DailyStrategy  Strategy = iota
+	StdoutStrategy Strategy = iota
+)
+
+func logStrategy() Strategy {
+	switch config.Common.LogStrategy {
+	case "daily":
+		return DailyStrategy
+	default:
+	}
+	return StdoutStrategy
+}
-- 
cgit v1.2.3


From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 26 Sep 2021 16:42:47 +0300
Subject: refactor config reader - also looks in additional search paths for
 config file unless NONE is specified

---
 internal/io/dlog/dlog.go | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 7282741..49b405d 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -3,6 +3,7 @@ package dlog
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -21,7 +22,6 @@ var Client *DLog
 var Server *DLog
 
 // Common is the log handler for all other packages.
-// TODO: Rename Common to Common
 var Common *DLog
 
 var mutex sync.Mutex
@@ -75,15 +75,22 @@ type DLog struct {
 	sourcePackage source
 	// Max log level to log.
 	maxLevel level
+	// Current hostname.
+	hostname string
 }
 
 // New creates a new DTail logger.
 func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+	hostname, err := os.Hostname()
+	if err != nil {
+		panic(err)
+	}
 	return &DLog{
 		logger:        loggers.Factory(sourceProcess.String(), impl),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      maxLevel,
+		hostname:      hostname,
 	}
 }
 
@@ -106,11 +113,18 @@ func (d *DLog) log(level level, args []interface{}) string {
 	defer pool.RecycleBuilderBuffer(sb)
 	now := time.Now()
 
-	sb.WriteString(d.sourcePackage.String())
-	sb.WriteString(protocol.FieldDelimiter)
-	sb.WriteString(now.Format("20060102-150405"))
-	sb.WriteString(protocol.FieldDelimiter)
-	sb.WriteString(level.String())
+	switch d.sourceProcess {
+	case CLIENT:
+		sb.WriteString(d.sourcePackage.String())
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(d.hostname)
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(level.String())
+	default:
+		sb.WriteString(level.String())
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(now.Format("20060102-150405"))
+	}
 	sb.WriteString(protocol.FieldDelimiter)
 	d.writeArgStrings(sb, args)
 
@@ -159,6 +173,11 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
+	if d.sourcePackage == SERVER && d.sourceProcess != CLIENT {
+		// This can be dtail client in serverless mode. In this case log all
+		// info server messages as verbose.
+		return d.log(VERBOSE, args)
+	}
 	return d.log(INFO, args)
 }
 
@@ -183,13 +202,14 @@ func (d *DLog) Raw(message string) string {
 		d.logger.Log(time.Now(), message)
 		return message
 	}
-
-	d.logger.Log(time.Now(), brush.Colorfy(message))
+	d.logger.LogWithColors(time.Now(), message, brush.Colorfy(message))
 	return message
 }
 
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
+
+	// TODO: mC compatible SERVER mapreduce fields, no MAPREDUCE keyword in CLIENT mode
 	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
 
 	i := 1
@@ -197,7 +217,6 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 		args[i] = fmt.Sprintf("%s=%v", k, v)
 		i++
 	}
-
 	return d.log(INFO, args)
 }
 
-- 
cgit v1.2.3


From 609921f9c783941eaa9019a92b78ec45b49d681c Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 28 Sep 2021 21:11:50 +0300
Subject: can have daily and normal file log rotation

---
 internal/io/dlog/dlog.go            |  14 +++--
 internal/io/dlog/loggers/factory.go |   8 +--
 internal/io/dlog/loggers/file.go    | 117 ++++++++++++++++++++----------------
 internal/io/dlog/loggers/fout.go    |   4 +-
 internal/io/dlog/strategy.go        |  22 -------
 5 files changed, 78 insertions(+), 87 deletions(-)
 delete mode 100644 internal/io/dlog/strategy.go

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 49b405d..49533a5 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -36,19 +36,21 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 		Common.FatalPanic("Logger already started")
 	}
 
+	strategy := loggers.GetStrategy(config.Common.LogStrategy)
 	level := newLevel(logLevel)
+
 	switch sourceProcess {
 	case CLIENT:
 		// This is a DTail client process running.
 		impl := loggers.FOUT
-		Client = New(CLIENT, CLIENT, impl, level)
-		Server = New(CLIENT, SERVER, impl, level)
+		Client = New(CLIENT, CLIENT, level, impl, strategy)
+		Server = New(CLIENT, SERVER, level, impl, strategy)
 		Common = Client
 	case SERVER:
 		// This is a DTail server process running.
 		impl := loggers.FILE
-		Client = New(SERVER, CLIENT, impl, level)
-		Server = New(SERVER, SERVER, impl, level)
+		Client = New(SERVER, CLIENT, level, impl, strategy)
+		Server = New(SERVER, SERVER, level, impl, strategy)
 		Common = Server
 	}
 
@@ -80,13 +82,13 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+func New(sourceProcess, sourcePackage source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
 	}
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), impl),
+		logger:        loggers.Factory(sourceProcess.String(), impl, strategy),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      maxLevel,
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 3eb29c5..8697dc4 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -17,11 +17,11 @@ const (
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
-func Factory(name string, impl Impl) Logger {
+func Factory(name string, impl Impl, strategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("name:%s,impl:%v", name, impl)
+	id := fmt.Sprintf("name:%s,fileBase:%s,impl:%v", name, strategy.FileBase, impl)
 
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
@@ -36,10 +36,10 @@ func Factory(name string, impl Impl) Logger {
 			singleton = newStdout()
 			factoryMap[id] = singleton
 		case FILE:
-			singleton = newFile()
+			singleton = newFile(strategy)
 			factoryMap[id] = singleton
 		case FOUT:
-			singleton = newFout()
+			singleton = newFout(strategy)
 			factoryMap[id] = singleton
 		}
 	}
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 1c525c9..dcdd7d0 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -12,49 +12,54 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 )
 
+type fileWriter struct {
+}
+
 type fileMessageBuf struct {
 	now     time.Time
 	message string
 }
 
 type file struct {
-	bufferCh    chan *fileMessageBuf
-	pauseCh     chan struct{}
-	resumeCh    chan struct{}
-	rotateCh    chan struct{}
-	flushCh     chan struct{}
-	lastDateStr string
-	fd          *os.File
-	writer      *bufio.Writer
-	mutex       sync.Mutex
-	started     bool
+	bufferCh     chan *fileMessageBuf
+	pauseCh      chan struct{}
+	resumeCh     chan struct{}
+	rotateCh     chan struct{}
+	flushCh      chan struct{}
+	fd           *os.File
+	writer       *bufio.Writer
+	mutex        sync.Mutex
+	started      bool
+	lastFileName string
+	strategy     Strategy
 }
 
-func newFile() *file {
+func newFile(strategy Strategy) *file {
 	f := file{
 		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
 		rotateCh: make(chan struct{}),
 		flushCh:  make(chan struct{}),
+		strategy: strategy,
 	}
-	f.getWriter(time.Now().Format("20060102"))
+
 	return &f
 }
 
-func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
-	s.mutex.Lock()
-	defer s.mutex.Unlock()
+func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
 
 	// Logger already started from another Goroutine.
-	if s.started {
+	if f.started {
 		wg.Done()
 		return
 	}
 
 	pause := func(ctx context.Context) {
 		select {
-		case <-s.resumeCh:
+		case <-f.resumeCh:
 			return
 		case <-ctx.Done():
 			return
@@ -66,55 +71,61 @@ func (s *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 
 		for {
 			select {
-			case m := <-s.bufferCh:
-				s.write(m)
-			case <-s.pauseCh:
+			case m := <-f.bufferCh:
+				f.write(m)
+			case <-f.pauseCh:
 				pause(ctx)
-			case <-s.flushCh:
-				s.flush()
+			case <-f.flushCh:
+				f.flush()
 			case <-ctx.Done():
-				s.flush()
-				s.fd.Close()
+				f.flush()
+				f.fd.Close()
 				return
 			}
 		}
 	}()
 
-	s.started = true
+	f.started = true
 }
 
-func (s *file) Log(now time.Time, message string) {
-	s.bufferCh <- &fileMessageBuf{now, message}
+func (f *file) Log(now time.Time, message string) {
+	f.bufferCh <- &fileMessageBuf{now, message}
 }
 
-func (s *file) LogWithColors(now time.Time, message, coloredMessage string) {
+func (f *file) LogWithColors(now time.Time, message, coloredMessage string) {
 	panic("Colors not supported in file logger")
 }
 
-func (s *file) Pause()  { s.pauseCh <- struct{}{} }
-func (s *file) Resume() { s.resumeCh <- struct{}{} }
-func (s *file) Flush()  { s.flushCh <- struct{}{} }
+func (f *file) Pause()  { f.pauseCh <- struct{}{} }
+func (f *file) Resume() { f.resumeCh <- struct{}{} }
+func (f *file) Flush()  { f.flushCh <- struct{}{} }
 
 // TODO: Test that Rotate() actually works.
-func (s *file) Rotate()           { s.rotateCh <- struct{}{} }
-func (file) SupportsColors() bool { return false }
+func (f *file) Rotate()            { f.rotateCh <- struct{}{} }
+func (*file) SupportsColors() bool { return false }
 
-func (s *file) write(m *fileMessageBuf) {
+func (f *file) write(m *fileMessageBuf) {
 	select {
-	case <-s.rotateCh:
-		// Force re-opening the outfile.
-		s.lastDateStr = ""
+	case <-f.rotateCh:
+		// Force re-opening the outfile next time in getWriter.
+		f.lastFileName = ""
 	default:
 	}
 
-	writer := s.getWriter(m.now.Format("20060102"))
+	var writer *bufio.Writer
+	if f.strategy.Rotation == DailyRotation {
+		writer = f.getWriter(m.now.Format("20060102"))
+	} else {
+		writer = f.getWriter(f.strategy.FileBase)
+	}
+
 	writer.WriteString(m.message)
 	writer.WriteByte('\n')
 }
 
-func (s *file) getWriter(dateStr string) *bufio.Writer {
-	if s.lastDateStr == dateStr {
-		return s.writer
+func (f *file) getWriter(name string) *bufio.Writer {
+	if f.lastFileName == name {
+		return f.writer
 	}
 
 	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
@@ -123,32 +134,32 @@ func (s *file) getWriter(dateStr string) *bufio.Writer {
 		}
 	}
 
-	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr)
+	logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, name)
 	newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
 	if err != nil {
 		panic(err)
 	}
 
 	// Close old writer.
-	if s.fd != nil {
-		s.writer.Flush()
-		s.fd.Close()
+	if f.fd != nil {
+		f.writer.Flush()
+		f.fd.Close()
 	}
 
-	s.fd = newFd
-	s.writer = bufio.NewWriterSize(s.fd, 1)
-	s.lastDateStr = dateStr
+	f.fd = newFd
+	f.writer = bufio.NewWriterSize(f.fd, 1)
+	f.lastFileName = name
 
-	return s.writer
+	return f.writer
 }
 
-func (s *file) flush() {
-	defer s.writer.Flush()
+func (f *file) flush() {
+	defer f.writer.Flush()
 
 	for {
 		select {
-		case m := <-s.bufferCh:
-			s.write(m)
+		case m := <-f.bufferCh:
+			f.write(m)
 		default:
 			return
 		}
diff --git a/internal/io/dlog/loggers/fout.go b/internal/io/dlog/loggers/fout.go
index 603dbe9..60c318d 100644
--- a/internal/io/dlog/loggers/fout.go
+++ b/internal/io/dlog/loggers/fout.go
@@ -12,8 +12,8 @@ type fout struct {
 }
 
 // Logs to both, a file and stdout
-func newFout() *fout {
-	return &fout{file: newFile(), stdout: newStdout()}
+func newFout(strategy Strategy) *fout {
+	return &fout{file: newFile(strategy), stdout: newStdout()}
 }
 
 func (f *fout) Start(ctx context.Context, wg *sync.WaitGroup) {
diff --git a/internal/io/dlog/strategy.go b/internal/io/dlog/strategy.go
deleted file mode 100644
index 32d8298..0000000
--- a/internal/io/dlog/strategy.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package dlog
-
-import "github.com/mimecast/dtail/internal/config"
-
-// Strategy allows to specify a log rotation strategy.
-type Strategy int
-
-// Possible log strategies.
-const (
-	NormalStrategy Strategy = iota
-	DailyStrategy  Strategy = iota
-	StdoutStrategy Strategy = iota
-)
-
-func logStrategy() Strategy {
-	switch config.Common.LogStrategy {
-	case "daily":
-		return DailyStrategy
-	default:
-	}
-	return StdoutStrategy
-}
-- 
cgit v1.2.3


From 764ef99a3d779a0db1fb60679292af52425ba2f6 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 10:46:47 +0300
Subject: add more default fields to MAPREDUCE

---
 internal/io/dlog/dlog.go         | 38 ++++++++++++++++++++++++++++++++++++--
 internal/io/dlog/loggers/file.go |  1 -
 2 files changed, 36 insertions(+), 3 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 49533a5..bc9b2f8 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -3,7 +3,11 @@ package dlog
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -211,8 +215,38 @@ func (d *DLog) Raw(message string) string {
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
-	// TODO: mC compatible SERVER mapreduce fields, no MAPREDUCE keyword in CLIENT mode
-	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
+	if d.sourceProcess == SERVER {
+		// level|date-time|process|caller|cpus|goroutines|cgocalls|loadavg|uptime|MAPREDUCE:TABLE|key=value|...
+
+		var loadAvg string
+		if loadAvgBytes, err := ioutil.ReadFile("/proc/loadavg"); err == nil {
+			tmp := string(loadAvgBytes)
+			s := strings.SplitN(tmp, " ", 2)
+			loadAvg = s[0]
+		}
+
+		var uptime string
+		if uptimeBytes, err := ioutil.ReadFile("/proc/uptime"); err == nil {
+			tmp := string(uptimeBytes)
+			s := strings.SplitN(tmp, ".", 2)
+			i, _ := strconv.ParseInt(s[0], 10, 64)
+			t := time.Duration(i) * time.Second
+			uptime = fmt.Sprintf("%v", t)
+		}
+
+		_, file, line, _ := runtime.Caller(1)
+		args[0] = fmt.Sprintf("%d|%s:%d|%d|%d|%d|%s|%s|MAPREDUCE:%s",
+			os.Getpid(),
+			filepath.Base(file), line,
+			runtime.NumCPU(),
+			runtime.NumGoroutine(),
+			runtime.NumCgoCall(),
+			loadAvg,
+			uptime,
+			strings.ToUpper(table))
+	} else {
+		args[0] = fmt.Sprintf("STATS:%s", strings.ToUpper(table))
+	}
 
 	i := 1
 	for k, v := range data {
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index dcdd7d0..6e692a3 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -100,7 +100,6 @@ func (f *file) Pause()  { f.pauseCh <- struct{}{} }
 func (f *file) Resume() { f.resumeCh <- struct{}{} }
 func (f *file) Flush()  { f.flushCh <- struct{}{} }
 
-// TODO: Test that Rotate() actually works.
 func (f *file) Rotate()            { f.rotateCh <- struct{}{} }
 func (*file) SupportsColors() bool { return false }
 
-- 
cgit v1.2.3


From 6e1af993924bc7bebe898b403962db5a6b3505d1 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 11:23:08 +0300
Subject: Client default log dir is ~/log

---
 internal/io/dlog/dlog.go             | 27 ++++++++++++++-------------
 internal/io/dlog/loggers/strategy.go | 27 +++++++++++++++++++++++++++
 internal/io/dlog/source.go           | 19 -------------------
 3 files changed, 41 insertions(+), 32 deletions(-)
 create mode 100644 internal/io/dlog/loggers/strategy.go
 delete mode 100644 internal/io/dlog/source.go

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index bc9b2f8..3de3120 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -17,6 +17,7 @@ import (
 	"github.com/mimecast/dtail/internal/io/dlog/loggers"
 	"github.com/mimecast/dtail/internal/io/pool"
 	"github.com/mimecast/dtail/internal/protocol"
+	"github.com/mimecast/dtail/internal/source"
 )
 
 // Client is the log handler for the client packages.
@@ -32,7 +33,7 @@ var mutex sync.Mutex
 var started bool
 
 // Start logger(s).
-func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLevel string) {
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source, logLevel string) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
@@ -44,17 +45,17 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 	level := newLevel(logLevel)
 
 	switch sourceProcess {
-	case CLIENT:
+	case source.Client:
 		// This is a DTail client process running.
 		impl := loggers.FOUT
-		Client = New(CLIENT, CLIENT, level, impl, strategy)
-		Server = New(CLIENT, SERVER, level, impl, strategy)
+		Client = New(source.Client, source.Client, level, impl, strategy)
+		Server = New(source.Client, source.Server, level, impl, strategy)
 		Common = Client
-	case SERVER:
+	case source.Server:
 		// This is a DTail server process running.
 		impl := loggers.FILE
-		Client = New(SERVER, CLIENT, level, impl, strategy)
-		Server = New(SERVER, SERVER, level, impl, strategy)
+		Client = New(source.Server, source.Client, level, impl, strategy)
+		Server = New(source.Server, source.Server, level, impl, strategy)
 		Common = Server
 	}
 
@@ -75,10 +76,10 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source, logLev
 type DLog struct {
 	logger loggers.Logger
 	// Is this a DTail server or client process logging?
-	sourceProcess source
+	sourceProcess source.Source
 	// Is this a DTail server or client package logging? In serverless mode
 	// the client can also execute code from the server package.
-	sourcePackage source
+	sourcePackage source.Source
 	// Max log level to log.
 	maxLevel level
 	// Current hostname.
@@ -86,7 +87,7 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
+func New(sourceProcess, sourcePackage source.Source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
@@ -120,7 +121,7 @@ func (d *DLog) log(level level, args []interface{}) string {
 	now := time.Now()
 
 	switch d.sourceProcess {
-	case CLIENT:
+	case source.Client:
 		sb.WriteString(d.sourcePackage.String())
 		sb.WriteString(protocol.FieldDelimiter)
 		sb.WriteString(d.hostname)
@@ -179,7 +180,7 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	if d.sourcePackage == SERVER && d.sourceProcess != CLIENT {
+	if d.sourcePackage == source.Server && d.sourceProcess != source.Client {
 		// This can be dtail client in serverless mode. In this case log all
 		// info server messages as verbose.
 		return d.log(VERBOSE, args)
@@ -215,7 +216,7 @@ func (d *DLog) Raw(message string) string {
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
-	if d.sourceProcess == SERVER {
+	if d.sourceProcess == source.Server {
 		// level|date-time|process|caller|cpus|goroutines|cgocalls|loadavg|uptime|MAPREDUCE:TABLE|key=value|...
 
 		var loadAvg string
diff --git a/internal/io/dlog/loggers/strategy.go b/internal/io/dlog/loggers/strategy.go
new file mode 100644
index 0000000..a1b9355
--- /dev/null
+++ b/internal/io/dlog/loggers/strategy.go
@@ -0,0 +1,27 @@
+package loggers
+
+import (
+	"os"
+	"path/filepath"
+)
+
+type Rotation int
+
+const (
+	DailyRotation  Rotation = iota
+	SignalRotation Rotation = iota
+)
+
+type Strategy struct {
+	Rotation Rotation
+	FileBase string
+}
+
+func GetStrategy(name string) Strategy {
+	switch name {
+	case "daily":
+		return Strategy{DailyRotation, ""}
+	default:
+		return Strategy{SignalRotation, filepath.Base(os.Args[0])}
+	}
+}
diff --git a/internal/io/dlog/source.go b/internal/io/dlog/source.go
deleted file mode 100644
index 265885e..0000000
--- a/internal/io/dlog/source.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package dlog
-
-type source int
-
-const (
-	CLIENT source = iota
-	SERVER source = iota
-)
-
-func (s source) String() string {
-	switch s {
-	case CLIENT:
-		return "CLIENT"
-	case SERVER:
-		return "SERVER"
-	}
-
-	panic("Unknown log source type")
-}
-- 
cgit v1.2.3


From 12c79f68bb5bda6673819d7b754820ecfe6d08ff Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 11:54:07 +0300
Subject: reduce logging in serverless mode

---
 internal/io/dlog/dlog.go | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 3de3120..6cacfe2 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -180,11 +180,6 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	if d.sourcePackage == source.Server && d.sourceProcess != source.Client {
-		// This can be dtail client in serverless mode. In this case log all
-		// info server messages as verbose.
-		return d.log(VERBOSE, args)
-	}
 	return d.log(INFO, args)
 }
 
-- 
cgit v1.2.3


From 86ec83754e0ee7153ad55091f7b6da448bc529c5 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 2 Oct 2021 13:44:27 +0300
Subject: add dcat test

---
 internal/io/dlog/loggers/stdout.go | 59 +++++++++++++-------------------------
 1 file changed, 20 insertions(+), 39 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/loggers/stdout.go b/internal/io/dlog/loggers/stdout.go
index 9738323..05485c6 100644
--- a/internal/io/dlog/loggers/stdout.go
+++ b/internal/io/dlog/loggers/stdout.go
@@ -8,66 +8,47 @@ import (
 )
 
 type stdout struct {
-	bufferCh chan string
 	pauseCh  chan struct{}
 	resumeCh chan struct{}
+	mutex    sync.Mutex
 }
 
 func newStdout() *stdout {
 	return &stdout{
-		bufferCh: make(chan string, 100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
 	}
 }
 
 func (s *stdout) Start(ctx context.Context, wg *sync.WaitGroup) {
-	pause := func(ctx context.Context) {
-		select {
-		case <-s.resumeCh:
-			return
-		case <-ctx.Done():
-			return
-		}
-	}
-
-	go func() {
-		defer wg.Done()
-
-		for {
-			select {
-			case message := <-s.bufferCh:
-				fmt.Println(message)
-			case <-s.pauseCh:
-				pause(ctx)
-			case <-ctx.Done():
-				s.Flush()
-				return
-			}
-		}
-	}()
+	wg.Done()
 }
 
 func (s *stdout) Log(now time.Time, message string) {
-	s.bufferCh <- message
+	s.log(message)
 }
 
 func (s *stdout) LogWithColors(now time.Time, message, coloredMessage string) {
-	s.bufferCh <- coloredMessage
+	s.log(coloredMessage)
 }
 
-func (s *stdout) Flush() {
-	for {
-		select {
-		case message := <-s.bufferCh:
-			fmt.Println(message)
-		default:
-			return
-		}
+func (s *stdout) log(message string) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	select {
+	case <-s.pauseCh:
+		// Pause until resumed.
+		<-s.resumeCh
+	default:
 	}
+
+	fmt.Println(message)
 }
 
-func (s *stdout) Pause()            { s.pauseCh <- struct{}{} }
-func (s *stdout) Resume()           { s.resumeCh <- struct{}{} }
-func (s *stdout) Rotate()           {}
+func (s *stdout) Pause()  { s.pauseCh <- struct{}{} }
+func (s *stdout) Resume() { s.resumeCh <- struct{}{} }
+func (s *stdout) Flush()  {}
+func (s *stdout) Rotate() {}
+
 func (stdout) SupportsColors() bool { return true }
-- 
cgit v1.2.3


From 07e1470892beacf0722276f94bacbd822b002540 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 3 Oct 2021 13:09:32 +0300
Subject: add dmap tests

---
 internal/io/dlog/dlog.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 6cacfe2..2beda75 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -163,8 +163,11 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 
 func (d *DLog) FatalPanic(args ...interface{}) {
 	d.log(FATAL, args)
-	d.logger.Flush()
-	panic("Not recovering from this fatal error...")
+	d.Flush()
+
+	var sb strings.Builder
+	d.writeArgStrings(&sb, args)
+	panic(sb.String())
 }
 
 func (d *DLog) Fatal(args ...interface{}) string {
-- 
cgit v1.2.3


From f70622f307629a2542ea5eb128dea8c1043d3a40 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Tue, 5 Oct 2021 10:00:38 +0300
Subject: more on this

---
 internal/io/dlog/dlog.go | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 2beda75..db99307 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -57,6 +57,12 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Client = New(source.Server, source.Client, level, impl, strategy)
 		Server = New(source.Server, source.Server, level, impl, strategy)
 		Common = Server
+	case source.HealthCheck:
+		// Health check isn't logging anything.
+		impl := loggers.STDOUT
+		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
+		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
+		Common = Client
 	}
 
 	var wg2 sync.WaitGroup
-- 
cgit v1.2.3


From fab5dc3e70434ea0abc7a0976487a1973b662331 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 6 Oct 2021 09:50:41 +0300
Subject: enable faster shutdown - useful for dgrep/dmap and dcat commands

---
 internal/io/dlog/dlog.go | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index db99307..a17d6e9 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -201,6 +201,8 @@ func (d *DLog) Debug(args ...interface{}) string {
 }
 
 func (d *DLog) Trace(args ...interface{}) string {
+	_, file, line, _ := runtime.Caller(1)
+	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(TRACE, args)
 }
 
-- 
cgit v1.2.3


From 7306afe9ab073c424ddca0ddc57950f237948118 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Wed, 6 Oct 2021 10:55:50 +0300
Subject: move health check to separate client binary

---
 internal/io/dlog/dlog.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index a17d6e9..2ae3b04 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -59,7 +59,7 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Common = Server
 	case source.HealthCheck:
 		// Health check isn't logging anything.
-		impl := loggers.STDOUT
+		impl := loggers.NONE
 		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
 		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
 		Common = Client
-- 
cgit v1.2.3


From 2d7ddbeae8286373ac19787dc7dde598a7cb0598 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Fri, 8 Oct 2021 11:43:43 +0300
Subject: refactor

---
 internal/io/dlog/dlog.go            | 53 ++++++++++-----------
 internal/io/dlog/level.go           | 95 ++++++++++++++++---------------------
 internal/io/dlog/loggers/factory.go | 36 ++++++--------
 3 files changed, 80 insertions(+), 104 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 2ae3b04..f3774ba 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -33,7 +33,7 @@ var mutex sync.Mutex
 var started bool
 
 // Start logger(s).
-func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source, logLevel string) {
+func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
@@ -41,27 +41,18 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source,
 		Common.FatalPanic("Logger already started")
 	}
 
-	strategy := loggers.GetStrategy(config.Common.LogStrategy)
-	level := newLevel(logLevel)
-
 	switch sourceProcess {
 	case source.Client:
-		// This is a DTail client process running.
-		impl := loggers.FOUT
-		Client = New(source.Client, source.Client, level, impl, strategy)
-		Server = New(source.Client, source.Server, level, impl, strategy)
+		Client = New(source.Client, source.Client)
+		Server = New(source.Client, source.Server)
 		Common = Client
 	case source.Server:
-		// This is a DTail server process running.
-		impl := loggers.FILE
-		Client = New(source.Server, source.Client, level, impl, strategy)
-		Server = New(source.Server, source.Server, level, impl, strategy)
+		Client = New(source.Server, source.Client)
+		Server = New(source.Server, source.Server)
 		Common = Server
 	case source.HealthCheck:
-		// Health check isn't logging anything.
-		impl := loggers.NONE
-		Client = New(source.HealthCheck, source.Client, level, impl, strategy)
-		Server = New(source.HealthCheck, source.Server, level, impl, strategy)
+		Client = New(source.HealthCheck, source.Client)
+		Server = New(source.HealthCheck, source.Server)
 		Common = Client
 	}
 
@@ -93,16 +84,20 @@ type DLog struct {
 }
 
 // New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source.Source, maxLevel level, impl loggers.Impl, strategy loggers.Strategy) *DLog {
+func New(sourceProcess, sourcePackage source.Source) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
 	}
+	strategy := loggers.GetStrategy(config.Common.LogStrategy)
+	loggerName := config.Common.Logger
+	level := newLevel(config.Common.LogLevel)
+
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), impl, strategy),
+		logger:        loggers.Factory(sourceProcess.String(), loggerName, strategy),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
-		maxLevel:      maxLevel,
+		maxLevel:      level,
 		hostname:      hostname,
 	}
 }
@@ -168,7 +163,7 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 }
 
 func (d *DLog) FatalPanic(args ...interface{}) {
-	d.log(FATAL, args)
+	d.log(Fatal, args)
 	d.Flush()
 
 	var sb strings.Builder
@@ -177,37 +172,37 @@ func (d *DLog) FatalPanic(args ...interface{}) {
 }
 
 func (d *DLog) Fatal(args ...interface{}) string {
-	return d.log(FATAL, args)
+	return d.log(Fatal, args)
 }
 
 func (d *DLog) Error(args ...interface{}) string {
-	return d.log(ERROR, args)
+	return d.log(Error, args)
 }
 
 func (d *DLog) Warn(args ...interface{}) string {
-	return d.log(WARN, args)
+	return d.log(Warn, args)
 }
 
 func (d *DLog) Info(args ...interface{}) string {
-	return d.log(INFO, args)
+	return d.log(Info, args)
 }
 
 func (d *DLog) Verbose(args ...interface{}) string {
-	return d.log(VERBOSE, args)
+	return d.log(Verbose, args)
 }
 
 func (d *DLog) Debug(args ...interface{}) string {
-	return d.log(DEBUG, args)
+	return d.log(Debug, args)
 }
 
 func (d *DLog) Trace(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
-	return d.log(TRACE, args)
+	return d.log(Trace, args)
 }
 
 func (d *DLog) Devel(args ...interface{}) string {
-	return d.log(DEVEL, args)
+	return d.log(Devel, args)
 }
 
 func (d *DLog) Raw(message string) string {
@@ -260,7 +255,7 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 		args[i] = fmt.Sprintf("%s=%v", k, v)
 		i++
 	}
-	return d.log(INFO, args)
+	return d.log(Info, args)
 }
 
 func (d *DLog) Flush()  { d.logger.Flush() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 84550f0..248ad83 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -8,80 +8,69 @@ import (
 type level int
 
 const (
-	FATAL   level = iota
-	ERROR   level = iota
-	WARN    level = iota
-	INFO    level = iota
-	DEFAULT level = iota
-	VERBOSE level = iota
-	DEBUG   level = iota
-	DEVEL   level = iota
-	TRACE   level = iota
-	ALL     level = iota
+	Fatal   level = iota
+	Error   level = iota
+	Warn    level = iota
+	Info    level = iota
+	Default level = iota
+	Verbose level = iota
+	Debug   level = iota
+	Devel   level = iota
+	Trace   level = iota
+	All     level = iota
 )
 
-var allLevels = []level{
-	FATAL,
-	ERROR,
-	WARN,
-	INFO,
-	DEFAULT,
-	VERBOSE,
-	DEBUG,
-	DEVEL,
-	TRACE,
-	ALL,
-}
+var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug, Devel, Trace, All}
 
 func newLevel(l string) level {
-	switch strings.ToUpper(l) {
-	case "FATAL":
-		return FATAL
-	case "ERROR":
-		return ERROR
-	case "WARN":
-		return WARN
-	case "INFO":
-		return INFO
+	switch strings.ToLower(l) {
+	case "fatal":
+		return Fatal
+	case "error":
+		return Error
+	case "warn":
+		return Warn
+	case "info":
+		return Info
 	case "":
 		fallthrough
-	case "DEFAULT":
-		return DEFAULT
-	case "VERBOSE":
-		return VERBOSE
-	case "DEBUG":
-		return DEBUG
-	case "DEVEL":
-		return DEVEL
-	case "TRACE":
-		return TRACE
-	case "ALL":
-		return ALL
+	case "default":
+		return Default
+	case "verbose":
+		return Verbose
+	case "debug":
+		return Debug
+	case "devel":
+		return Devel
+	case "trace":
+		return Trace
+	case "all":
+		return All
 	}
 	panic(fmt.Sprintf("Unknown log level %s, must be one of: %v", l, allLevels))
 }
 
 func (l level) String() string {
 	switch l {
-	case FATAL:
+	case Fatal:
 		return "FATAL"
-	case ERROR:
+	case Error:
 		return "ERROR"
-	case WARN:
+	case Warn:
 		return "WARN"
-	case INFO:
+	case Info:
 		return "INFO"
-	case DEFAULT:
+	case Default:
 		return "DEFAULT"
-	case VERBOSE:
+	case Verbose:
 		return "VERBOSE"
-	case DEBUG:
+	case Debug:
 		return "DEBUG"
-	case DEVEL:
+	case Devel:
 		return "DEVEL"
-	case TRACE:
+	case Trace:
 		return "TRACE"
-	case ALL:
+	case All:
 		return "ALL"
 	}
 
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 8697dc4..dda3ee6 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -2,45 +2,38 @@ package loggers
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 )
 
-type Impl int
-
-const (
-	NONE   Impl = iota
-	STDOUT Impl = iota
-	FILE   Impl = iota
-	FOUT   Impl = iota
-)
-
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
-func Factory(name string, impl Impl, strategy Strategy) Logger {
+func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("name:%s,fileBase:%s,impl:%v", name, strategy.FileBase, impl)
-
+	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName, rotationStrategy.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
 
 	singleton, ok := factoryMap[id]
 	if !ok {
-		switch impl {
-		case NONE:
+		switch strings.ToLower(loggerName) {
+		case "none":
 			singleton = none{}
-		case STDOUT:
+		case "stdout":
 			singleton = newStdout()
 			factoryMap[id] = singleton
-		case FILE:
-			singleton = newFile(strategy)
+		case "file":
+			singleton = newFile(rotationStrategy)
 			factoryMap[id] = singleton
-		case FOUT:
-			singleton = newFout(strategy)
+		case "fout":
+			singleton = newFout(rotationStrategy)
 			factoryMap[id] = singleton
+		default:
+			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
 		}
 	}
 
@@ -53,8 +46,7 @@ func FactoryRotate() {
 	if factoryMap == nil {
 		return
 	}
-
-	for _, impl := range factoryMap {
-		impl.Rotate()
+	for _, logger := range factoryMap {
+		logger.Rotate()
 	}
 }
-- 
cgit v1.2.3


From 7a7169791a64190e1002e38bc9c04ad0d5c1ce1f Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 9 Oct 2021 16:44:28 +0300
Subject: add dtail health check unit test.

---
 internal/io/dlog/dlog.go         | 43 +++++++++++++++++-----------------------
 internal/io/dlog/loggers/file.go | 10 ++++++----
 2 files changed, 24 insertions(+), 29 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index f3774ba..28e6882 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -41,32 +41,25 @@ func Start(ctx context.Context, wg *sync.WaitGroup, sourceProcess source.Source)
 		Common.FatalPanic("Logger already started")
 	}
 
-	switch sourceProcess {
-	case source.Client:
-		Client = New(source.Client, source.Client)
-		Server = New(source.Client, source.Server)
-		Common = Client
-	case source.Server:
-		Client = New(source.Server, source.Client)
-		Server = New(source.Server, source.Server)
+	Client = new(sourceProcess, source.Client)
+	Server = new(sourceProcess, source.Server)
+	Common = Client
+	if sourceProcess == source.Server {
 		Common = Server
-	case source.HealthCheck:
-		Client = New(source.HealthCheck, source.Client)
-		Server = New(source.HealthCheck, source.Server)
-		Common = Client
 	}
 
 	var wg2 sync.WaitGroup
 	wg2.Add(2)
-	Client.start(ctx, &wg2)
-	Server.start(ctx, &wg2)
-	started = true
+	go Client.start(ctx, &wg2)
+	go Server.start(ctx, &wg2)
 
 	go rotation(ctx)
 	go func() {
 		wg2.Wait()
 		wg.Done()
 	}()
+
+	started = true
 }
 
 // DLog is the DTail logger.
@@ -83,8 +76,8 @@ type DLog struct {
 	hostname string
 }
 
-// New creates a new DTail logger.
-func New(sourceProcess, sourcePackage source.Source) *DLog {
+// new creates a new DTail logger.
+func new(sourceProcess, sourcePackage source.Source) *DLog {
 	hostname, err := os.Hostname()
 	if err != nil {
 		panic(err)
@@ -103,14 +96,12 @@ func New(sourceProcess, sourcePackage source.Source) *DLog {
 }
 
 func (d *DLog) start(ctx context.Context, wg *sync.WaitGroup) {
-	go func() {
-		defer wg.Done()
-		var wg2 sync.WaitGroup
-		wg2.Add(1)
-		d.logger.Start(ctx, &wg2)
-		<-ctx.Done()
-		wg2.Wait()
-	}()
+	defer wg.Done()
+	var wg2 sync.WaitGroup
+	wg2.Add(1)
+	d.logger.Start(ctx, &wg2)
+	<-ctx.Done()
+	wg2.Wait()
 }
 
 func (d *DLog) log(level level, args []interface{}) string {
@@ -202,6 +193,8 @@ func (d *DLog) Trace(args ...interface{}) string {
 }
 
 func (d *DLog) Devel(args ...interface{}) string {
+	_, file, line, _ := runtime.Caller(1)
+	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Devel, args)
 }
 
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 6e692a3..87280fd 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -126,7 +126,6 @@ func (f *file) getWriter(name string) *bufio.Writer {
 	if f.lastFileName == name {
 		return f.writer
 	}
-
 	if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) {
 		if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil {
 			panic(err)
@@ -144,7 +143,7 @@ func (f *file) getWriter(name string) *bufio.Writer {
 		f.writer.Flush()
 		f.fd.Close()
 	}
-
+	// Set new writer.
 	f.fd = newFd
 	f.writer = bufio.NewWriterSize(f.fd, 1)
 	f.lastFileName = name
@@ -153,8 +152,11 @@ func (f *file) getWriter(name string) *bufio.Writer {
 }
 
 func (f *file) flush() {
-	defer f.writer.Flush()
-
+	defer func() {
+		if f.writer != nil {
+			f.writer.Flush()
+		}
+	}()
 	for {
 		select {
 		case m := <-f.bufferCh:
-- 
cgit v1.2.3


From 97747ea0f3178f7f5890512d483fdccaa82846b0 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 9 Oct 2021 21:10:29 +0300
Subject: vetting and linting and some code restyling

---
 internal/io/dlog/dlog.go             | 22 +++++++++++++++++++---
 internal/io/dlog/level.go            |  5 +++--
 internal/io/dlog/loggers/factory.go  |  6 ++++--
 internal/io/dlog/loggers/file.go     | 17 +++++++----------
 internal/io/dlog/loggers/logger.go   |  1 +
 internal/io/dlog/loggers/strategy.go | 11 +++++++++--
 6 files changed, 43 insertions(+), 19 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 28e6882..da67585 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -82,7 +82,7 @@ func new(sourceProcess, sourcePackage source.Source) *DLog {
 	if err != nil {
 		panic(err)
 	}
-	strategy := loggers.GetStrategy(config.Common.LogStrategy)
+	strategy := loggers.NewStrategy(config.Common.LogStrategy)
 	loggerName := config.Common.Logger
 	level := newLevel(config.Common.LogLevel)
 
@@ -153,6 +153,7 @@ func (d *DLog) writeArgStrings(sb *strings.Builder, args []interface{}) {
 	}
 }
 
+// FatalPanic terminates the process with a fatal error.
 func (d *DLog) FatalPanic(args ...interface{}) {
 	d.log(Fatal, args)
 	d.Flush()
@@ -162,42 +163,51 @@ func (d *DLog) FatalPanic(args ...interface{}) {
 	panic(sb.String())
 }
 
+// Fatal logs a fatal error.
 func (d *DLog) Fatal(args ...interface{}) string {
 	return d.log(Fatal, args)
 }
 
+// Error logging.
 func (d *DLog) Error(args ...interface{}) string {
 	return d.log(Error, args)
 }
 
+// Warn logs a warning message.
 func (d *DLog) Warn(args ...interface{}) string {
 	return d.log(Warn, args)
 }
 
+// Info logging.
 func (d *DLog) Info(args ...interface{}) string {
 	return d.log(Info, args)
 }
 
+// Verbose logging.
 func (d *DLog) Verbose(args ...interface{}) string {
 	return d.log(Verbose, args)
 }
 
+// Debug logging.
 func (d *DLog) Debug(args ...interface{}) string {
 	return d.log(Debug, args)
 }
 
+// Trace logging.
 func (d *DLog) Trace(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Trace, args)
 }
 
+// Devel used for development purpose only logging (e.g. "print" debugging).
 func (d *DLog) Devel(args ...interface{}) string {
 	_, file, line, _ := runtime.Caller(1)
 	args = append(args, fmt.Sprintf("at %s:%d", file, line))
 	return d.log(Devel, args)
 }
 
+// Raw message logging.
 func (d *DLog) Raw(message string) string {
 	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
 		d.logger.Log(time.Now(), message)
@@ -207,6 +217,7 @@ func (d *DLog) Raw(message string) string {
 	return message
 }
 
+// Mapreduce logging.
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
 
@@ -251,6 +262,11 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	return d.log(Info, args)
 }
 
-func (d *DLog) Flush()  { d.logger.Flush() }
-func (d *DLog) Pause()  { d.logger.Pause() }
+// Flush the log buffers.
+func (d *DLog) Flush() { d.logger.Flush() }
+
+// Pause the logging.
+func (d *DLog) Pause() { d.logger.Pause() }
+
+// Resume the logging.
 func (d *DLog) Resume() { d.logger.Resume() }
diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 248ad83..0971094 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -7,6 +7,7 @@ import (
 
 type level int
 
+// Available log levels.
 const (
 	Fatal   level = iota
 	Error   level = iota
@@ -20,7 +21,8 @@ const (
 	All     level = iota
 )
 
-var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug, Devel, Trace, All}
+var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug,
+	Devel, Trace, All}
 
 func newLevel(l string) level {
 	switch strings.ToLower(l) {
@@ -73,6 +75,5 @@ func (l level) String() string {
 	case All:
 		return "ALL"
 	}
-
 	panic("Unknown log level " + fmt.Sprintf("%d", l))
 }
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index dda3ee6..415d7fb 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -9,11 +9,13 @@ import (
 var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
+// Factory is there to retrieve a logger based on various settings.
 func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
-	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName, rotationStrategy.FileBase, loggerName)
+	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName,
+		rotationStrategy.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
@@ -36,10 +38,10 @@ func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
 		}
 	}
-
 	return singleton
 }
 
+// FactoryRotate invokes a log rotation of all loggers.
 func FactoryRotate() {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 87280fd..94824fe 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -12,8 +12,7 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 )
 
-type fileWriter struct {
-}
+type fileWriter struct{}
 
 type fileMessageBuf struct {
 	now     time.Time
@@ -35,7 +34,7 @@ type file struct {
 }
 
 func newFile(strategy Strategy) *file {
-	f := file{
+	return &file{
 		bufferCh: make(chan *fileMessageBuf, runtime.NumCPU()*100),
 		pauseCh:  make(chan struct{}),
 		resumeCh: make(chan struct{}),
@@ -43,16 +42,17 @@ func newFile(strategy Strategy) *file {
 		flushCh:  make(chan struct{}),
 		strategy: strategy,
 	}
-
-	return &f
 }
 
 func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 	f.mutex.Lock()
-	defer f.mutex.Unlock()
+	defer func() {
+		f.started = true
+		f.mutex.Unlock()
+	}()
 
-	// Logger already started from another Goroutine.
 	if f.started {
+		// Logger already started from another Goroutine.
 		wg.Done()
 		return
 	}
@@ -68,7 +68,6 @@ func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 
 	go func() {
 		defer wg.Done()
-
 		for {
 			select {
 			case m := <-f.bufferCh:
@@ -84,8 +83,6 @@ func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 			}
 		}
 	}()
-
-	f.started = true
 }
 
 func (f *file) Log(now time.Time, message string) {
diff --git a/internal/io/dlog/loggers/logger.go b/internal/io/dlog/loggers/logger.go
index c88900d..d4e85de 100644
--- a/internal/io/dlog/loggers/logger.go
+++ b/internal/io/dlog/loggers/logger.go
@@ -6,6 +6,7 @@ import (
 	"time"
 )
 
+// Logger is there to plug in your own log implementation.
 type Logger interface {
 	Log(now time.Time, message string)
 	LogWithColors(now time.Time, message, messageWithColors string)
diff --git a/internal/io/dlog/loggers/strategy.go b/internal/io/dlog/loggers/strategy.go
index a1b9355..25d10f0 100644
--- a/internal/io/dlog/loggers/strategy.go
+++ b/internal/io/dlog/loggers/strategy.go
@@ -5,19 +5,26 @@ import (
 	"path/filepath"
 )
 
+// Rotation is the actual strategy used for log rotation..
 type Rotation int
 
 const (
-	DailyRotation  Rotation = iota
+	// DailyRotation tells DTail to rotate its logs on a daily basis or on SIGHUP.
+	DailyRotation Rotation = iota
+	// SignalRotation tells DTail to rotate its logs only on SIGHUP.
 	SignalRotation Rotation = iota
 )
 
+// Strategy is a pair of the rotation and the file base.
 type Strategy struct {
+	// Rotation is the actual rotation strategy used.
 	Rotation Rotation
+	// FileBase can be a name (e.g. "dserver", "dmap") when signal rotation is used.
 	FileBase string
 }
 
-func GetStrategy(name string) Strategy {
+// NewStrategy returns the stratey based on its name.
+func NewStrategy(name string) Strategy {
 	switch name {
 	case "daily":
 		return Strategy{DailyRotation, ""}
-- 
cgit v1.2.3


From f44792c9102488774c9993b080f35c65287a64b1 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sun, 10 Oct 2021 14:02:12 +0300
Subject: add another dmap test - reading 100 source files at once

fix a data race when reading multiple files on one server from the same session at once
---
 internal/io/dlog/level.go | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/level.go b/internal/io/dlog/level.go
index 0971094..05d9ed9 100644
--- a/internal/io/dlog/level.go
+++ b/internal/io/dlog/level.go
@@ -9,6 +9,7 @@ type level int
 
 // Available log levels.
 const (
+	None    level = iota
 	Fatal   level = iota
 	Error   level = iota
 	Warn    level = iota
@@ -26,6 +27,8 @@ var allLevels = []level{Fatal, Error, Warn, Info, Default, Verbose, Debug,
 
 func newLevel(l string) level {
 	switch strings.ToLower(l) {
+	case "none":
+		return None
 	case "fatal":
 		return Fatal
 	case "error":
@@ -54,6 +57,8 @@ func newLevel(l string) level {
 
 func (l level) String() string {
 	switch l {
+	case None:
+		return "NONE"
 	case Fatal:
 		return "FATAL"
 	case Error:
-- 
cgit v1.2.3


From a6098084f7150df34edecf1519386bd28a527361 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 11 Oct 2021 17:42:37 +0300
Subject: Update JSON-schema to reflect all recent config file changes.

---
 internal/io/dlog/dlog.go            | 4 ++--
 internal/io/dlog/loggers/factory.go | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index da67585..5e0c3a1 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -82,12 +82,12 @@ func new(sourceProcess, sourcePackage source.Source) *DLog {
 	if err != nil {
 		panic(err)
 	}
-	strategy := loggers.NewStrategy(config.Common.LogStrategy)
+	logRotation := loggers.NewStrategy(config.Common.LogRotation)
 	loggerName := config.Common.Logger
 	level := newLevel(config.Common.LogLevel)
 
 	return &DLog{
-		logger:        loggers.Factory(sourceProcess.String(), loggerName, strategy),
+		logger:        loggers.Factory(sourceProcess.String(), loggerName, logRotation),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      level,
diff --git a/internal/io/dlog/loggers/factory.go b/internal/io/dlog/loggers/factory.go
index 415d7fb..a5cc7cf 100644
--- a/internal/io/dlog/loggers/factory.go
+++ b/internal/io/dlog/loggers/factory.go
@@ -10,12 +10,12 @@ var factoryMap map[string]Logger
 var factoryMutex sync.Mutex
 
 // Factory is there to retrieve a logger based on various settings.
-func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
+func Factory(sourceName, loggerName string, logRotation Strategy) Logger {
 	factoryMutex.Lock()
 	defer factoryMutex.Unlock()
 
 	id := fmt.Sprintf("sourceName:%s,fileBase:%s,loggerName:%s", sourceName,
-		rotationStrategy.FileBase, loggerName)
+		logRotation.FileBase, loggerName)
 	if factoryMap == nil {
 		factoryMap = make(map[string]Logger)
 	}
@@ -29,10 +29,10 @@ func Factory(sourceName, loggerName string, rotationStrategy Strategy) Logger {
 			singleton = newStdout()
 			factoryMap[id] = singleton
 		case "file":
-			singleton = newFile(rotationStrategy)
+			singleton = newFile(logRotation)
 			factoryMap[id] = singleton
 		case "fout":
-			singleton = newFout(rotationStrategy)
+			singleton = newFout(logRotation)
 			factoryMap[id] = singleton
 		default:
 			panic(fmt.Sprintf("Unsupported logger type '%s'", loggerName))
-- 
cgit v1.2.3


From ddfdb8e01aea4f8b6127834a14b74c5c534820d1 Mon Sep 17 00:00:00 2001
From: Paul Buetow <pbuetow@mimecast.com>
Date: Tue, 19 Oct 2021 19:50:38 +0300
Subject: lowercase log rotation comparison

---
 internal/io/dlog/loggers/strategy.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'internal/io/dlog')

diff --git a/internal/io/dlog/loggers/strategy.go b/internal/io/dlog/loggers/strategy.go
index 25d10f0..48e7d44 100644
--- a/internal/io/dlog/loggers/strategy.go
+++ b/internal/io/dlog/loggers/strategy.go
@@ -3,6 +3,7 @@ package loggers
 import (
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // Rotation is the actual strategy used for log rotation..
@@ -25,7 +26,7 @@ type Strategy struct {
 
 // NewStrategy returns the stratey based on its name.
 func NewStrategy(name string) Strategy {
-	switch name {
+	switch strings.ToLower(name) {
 	case "daily":
 		return Strategy{DailyRotation, ""}
 	default:
-- 
cgit v1.2.3