From f46abf36e80a67d13d12dbe4ba8c899026e53961 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 31 Jul 2021 18:20:03 +0300 Subject: more on configurable colors --- internal/io/logger/logger.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'internal/io/logger') diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go index 4254eef..bef5293 100644 --- a/internal/io/logger/logger.go +++ b/internal/io/logger/logger.go @@ -12,7 +12,7 @@ import ( "syscall" "time" - "github.com/mimecast/dtail/internal/color" + "github.com/mimecast/dtail/internal/color/brush" "github.com/mimecast/dtail/internal/config" ) @@ -207,8 +207,8 @@ func write(what, severity, message string) { if Mode.logToStdout { line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message) - if color.Colored { - line = color.Colorfy(line) + if config.Client.TermColorsEnabled { + line = brush.Colorfy(line) } stdoutBufCh <- line @@ -262,8 +262,8 @@ func Raw(message string) { } if Mode.logToStdout { - if color.Colored { - message = color.Colorfy(message) + if config.Client.TermColorsEnabled { + message = brush.Colorfy(message) } stdoutBufCh <- message } -- cgit v1.2.3 From 3cc8887885f24a3f0d607af24197bc364ab16b8d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 12 Aug 2021 10:56:36 +0300 Subject: add missing brush and also add color client configs plus jsonschema --- internal/io/logger/logger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'internal/io/logger') diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go index bef5293..bb9dc02 100644 --- a/internal/io/logger/logger.go +++ b/internal/io/logger/logger.go @@ -207,7 +207,7 @@ func write(what, severity, message string) { if Mode.logToStdout { line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message) - if config.Client.TermColorsEnabled { + if config.Client.TermColorsEnable { line = brush.Colorfy(line) } @@ -262,7 +262,7 @@ func Raw(message string) { } if Mode.logToStdout { - if config.Client.TermColorsEnabled { + if config.Client.TermColorsEnable { message = brush.Colorfy(message) } stdoutBufCh <- message -- cgit v1.2.3 From c2522ffb59514443816a96386c16bb7527cbe57c Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 21 Aug 2021 14:54:24 +0300 Subject: read files bytewise for more control of whats happening - change transport protocol for more control over newlines --- internal/io/logger/logger.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'internal/io/logger') diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go index bb9dc02..3a3935d 100644 --- a/internal/io/logger/logger.go +++ b/internal/io/logger/logger.go @@ -205,7 +205,7 @@ func Trace(args ...interface{}) string { // Write log line to buffer and/or log file. func write(what, severity, message string) { if Mode.logToStdout { - line := fmt.Sprintf("%s|%s|%s|%s\n", what, hostname, severity, message) + line := fmt.Sprintf("%s|%s|%s|%s", what, hostname, severity, message) if config.Client.TermColorsEnable { line = brush.Colorfy(line) @@ -219,7 +219,7 @@ func write(what, severity, message string) { timeStr := t.Format("20060102-150405") fileLogBufCh <- buf{ time: t, - message: fmt.Sprintf("%s|%s|%s|%s\n", severity, timeStr, what, message), + message: fmt.Sprintf("%s|%s|%s|%s", severity, timeStr, what, message), } } } @@ -326,6 +326,7 @@ func Flush() { select { case message := <-stdoutBufCh: stdoutWriter.WriteString(message) + stdoutWriter.WriteString("\n") default: stdoutWriter.Flush() return @@ -338,6 +339,7 @@ func writeToStdout(ctx context.Context) { select { case message := <-stdoutBufCh: stdoutWriter.WriteString(message) + stdoutWriter.WriteString("\n") case <-time.After(time.Millisecond * 100): stdoutWriter.Flush() case <-pauseCh: @@ -365,6 +367,7 @@ func writeToFile(ctx context.Context) { dateStr := buf.time.Format("20060102") w := fileWriter(dateStr) w.WriteString(buf.message) + w.WriteString("\n") case <-pauseCh: PAUSE: for { -- cgit v1.2.3 From f74a9e4b35feb8c07d8a70b5a581088a0a59889d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 7 Sep 2021 10:01:32 +0300 Subject: Produce MAPREDUCE lines, can aggregate these via default log format --- internal/io/logger/logger.go | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) (limited to 'internal/io/logger') diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go index 3a3935d..6890201 100644 --- a/internal/io/logger/logger.go +++ b/internal/io/logger/logger.go @@ -14,6 +14,8 @@ import ( "github.com/mimecast/dtail/internal/color/brush" "github.com/mimecast/dtail/internal/config" + "github.com/mimecast/dtail/internal/io/pool" + "github.com/mimecast/dtail/internal/protocol" ) const ( @@ -132,6 +134,24 @@ func Info(args ...interface{}) string { return log(clientStr, infoStr, args) } +// Mapreduce message logging. +func Mapreduce(table string, data map[string]interface{}) string { + args := make([]interface{}, len(data)+1) + + args[0] = fmt.Sprintf("MAPREDUCE:%s", strings.ToUpper(table)) + i := 1 + for k, v := range data { + args[i] = fmt.Sprintf("%s=%v", k, v) + i++ + } + + if Mode.Server { + return log(serverStr, infoStr, args) + } + + return log(clientStr, infoStr, args) +} + // Warn message logging. func Warn(args ...interface{}) string { if !Mode.Quiet { @@ -230,24 +250,28 @@ func log(what string, severity string, args []interface{}) string { return "" } - messages := []string{} + sb := pool.BuilderBuffer.Get().(*strings.Builder) + + for i, arg := range args { + if i > 0 { + sb.WriteString(protocol.FieldDelimiter) + } - for _, arg := range args { switch v := arg.(type) { case string: - messages = append(messages, v) + sb.WriteString(v) case int: - messages = append(messages, fmt.Sprintf("%d", v)) + sb.WriteString(fmt.Sprintf("%d", v)) case error: - messages = append(messages, v.Error()) + sb.WriteString(v.Error()) default: - messages = append(messages, fmt.Sprintf("%v", v)) + sb.WriteString(fmt.Sprintf("%v", v)) } } - message := strings.Join(messages, "|") + message := sb.String() + pool.RecycleBuilderBuffer(sb) write(what, severity, message) - return fmt.Sprintf("%s|%s", severity, message) } -- cgit v1.2.3 From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 19 Sep 2021 13:22:59 +0300 Subject: move args to config package logger package rewrite as dlog --- internal/io/logger/logger.go | 23 +++++++++++++++++++---- internal/io/logger/modes.go | 9 +++++---- 2 files changed, 24 insertions(+), 8 deletions(-) (limited to 'internal/io/logger') diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go index 6890201..6a6b5ec 100644 --- a/internal/io/logger/logger.go +++ b/internal/io/logger/logger.go @@ -1,5 +1,7 @@ package logger +// TODO: Rewrite this logger + import ( "bufio" "context" @@ -64,12 +66,25 @@ var resumeCh chan struct{} // Tell the logger about logrotation var rotateCh chan os.Signal +// Override the logger with a custom callack (e.g. for the t.Log for unit tests) +type unitTestCallback func(message string) + +var unitTestOkCb unitTestCallback +var unitTestErrorCb unitTestCallback + // Helper type to make logging non-blocking. type buf struct { time time.Time message string } +// StartUnitTests enables to log all messages to the unit tests. +func StartUnitTests(ctx context.Context, okCb, errCb unitTestCallback) { + unitTestOkCb = okCb + unitTestErrorCb = errCb + Start(ctx, Modes{UnitTest: true}) +} + // Start logging. func Start(ctx context.Context, mode Modes) { Mode = mode @@ -91,12 +106,12 @@ func Start(ctx context.Context, mode Modes) { switch strategy { case DailyStrategy: _, err := os.Stat(config.Common.LogDir) - Mode.logToFile = !os.IsNotExist(err) + Mode.logToFile = !os.IsNotExist(err) && !Mode.UnitTest Mode.logToStdout = !Mode.Server || Mode.Debug || Mode.Trace || Mode.Quiet case StdoutStrategy: fallthrough default: - Mode.logToFile = !Mode.Server + Mode.logToFile = !Mode.Server && !Mode.UnitTest Mode.logToStdout = true } @@ -182,8 +197,8 @@ func Fatal(args ...interface{}) string { return log(clientStr, fatalStr, args) } -// FatalExit logs an error and exists the process. -func FatalExit(args ...interface{}) { +// FatalPanic logs an error and exists the process. +func FatalPanic(args ...interface{}) { what := clientStr if Mode.Server { what = serverStr diff --git a/internal/io/logger/modes.go b/internal/io/logger/modes.go index 8864179..85f90a5 100644 --- a/internal/io/logger/modes.go +++ b/internal/io/logger/modes.go @@ -2,11 +2,12 @@ package logger // Modes specifies the logging mode. type Modes struct { - Server bool - Trace bool Debug bool + logToFile bool + logToStdout bool Nothing bool Quiet bool - logToStdout bool - logToFile bool + Server bool + Trace bool + UnitTest bool } -- cgit v1.2.3 From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 26 Sep 2021 16:42:47 +0300 Subject: refactor config reader - also looks in additional search paths for config file unless NONE is specified --- internal/io/logger/logger.go | 2 -- 1 file changed, 2 deletions(-) (limited to 'internal/io/logger') diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go index 6a6b5ec..905d1cf 100644 --- a/internal/io/logger/logger.go +++ b/internal/io/logger/logger.go @@ -1,7 +1,5 @@ package logger -// TODO: Rewrite this logger - import ( "bufio" "context" -- cgit v1.2.3 From 609921f9c783941eaa9019a92b78ec45b49d681c Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 28 Sep 2021 21:11:50 +0300 Subject: can have daily and normal file log rotation --- internal/io/logger/logger.go | 443 ----------------------------------------- internal/io/logger/modes.go | 13 -- internal/io/logger/strategy.go | 22 -- 3 files changed, 478 deletions(-) delete mode 100644 internal/io/logger/logger.go delete mode 100644 internal/io/logger/modes.go delete mode 100644 internal/io/logger/strategy.go (limited to 'internal/io/logger') diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go deleted file mode 100644 index 905d1cf..0000000 --- a/internal/io/logger/logger.go +++ /dev/null @@ -1,443 +0,0 @@ -package logger - -import ( - "bufio" - "context" - "fmt" - "os" - "os/signal" - "runtime" - "strings" - "sync" - "syscall" - "time" - - "github.com/mimecast/dtail/internal/color/brush" - "github.com/mimecast/dtail/internal/config" - "github.com/mimecast/dtail/internal/io/pool" - "github.com/mimecast/dtail/internal/protocol" -) - -const ( - clientStr string = "CLIENT" - serverStr string = "SERVER" - infoStr string = "INFO" - warnStr string = "WARN" - errorStr string = "ERROR" - fatalStr string = "FATAL" - debugStr string = "DEBUG" - traceStr string = "TRACE" -) - -// Mode specifies the configured logging mode(s) -var Mode Modes - -// Strategy is the current log strattegy used. -var strategy Strategy - -// Synchronise access to logging. -var mutex sync.Mutex - -// File descriptor of log file when Mode.logToFile enabled. -var fd *os.File - -// File write buffer of log file when Mode.logToFile enabled. -var writer *bufio.Writer - -// File write buffer of stdout when Mode.logToStdout enabled. -var stdoutWriter *bufio.Writer - -// Current hostname. -var hostname string - -// Used to detect change of day (create one log file per day0 -var lastDateStr string - -// Used to make logging non-blocking. -var fileLogBufCh chan buf -var stdoutBufCh chan string - -// Stdout channel, required to pause output -var pauseCh chan struct{} -var resumeCh chan struct{} - -// Tell the logger about logrotation -var rotateCh chan os.Signal - -// Override the logger with a custom callack (e.g. for the t.Log for unit tests) -type unitTestCallback func(message string) - -var unitTestOkCb unitTestCallback -var unitTestErrorCb unitTestCallback - -// Helper type to make logging non-blocking. -type buf struct { - time time.Time - message string -} - -// StartUnitTests enables to log all messages to the unit tests. -func StartUnitTests(ctx context.Context, okCb, errCb unitTestCallback) { - unitTestOkCb = okCb - unitTestErrorCb = errCb - Start(ctx, Modes{UnitTest: true}) -} - -// Start logging. -func Start(ctx context.Context, mode Modes) { - Mode = mode - - switch { - case Mode.Nothing: - return - case Mode.Quiet: - Mode.Trace = false - Mode.Debug = false - case Mode.Trace: - Mode.Debug = true - default: - } - - strategy := logStrategy() - stdoutWriter = bufio.NewWriter(os.Stdout) - - switch strategy { - case DailyStrategy: - _, err := os.Stat(config.Common.LogDir) - Mode.logToFile = !os.IsNotExist(err) && !Mode.UnitTest - Mode.logToStdout = !Mode.Server || Mode.Debug || Mode.Trace || Mode.Quiet - case StdoutStrategy: - fallthrough - default: - Mode.logToFile = !Mode.Server && !Mode.UnitTest - Mode.logToStdout = true - } - - fqdn, err := os.Hostname() - if err != nil { - panic(err) - } - s := strings.Split(fqdn, ".") - hostname = s[0] - - pauseCh = make(chan struct{}) - resumeCh = make(chan struct{}) - - // Setup logrotation - rotateCh = make(chan os.Signal, 1) - signal.Notify(rotateCh, syscall.SIGHUP) - - if Mode.logToStdout { - stdoutBufCh = make(chan string, runtime.NumCPU()*100) - go writeToStdout(ctx) - } - - if Mode.logToFile { - fileLogBufCh = make(chan buf, runtime.NumCPU()*100) - go writeToFile(ctx) - } -} - -// Info message logging. -func Info(args ...interface{}) string { - if Mode.Server { - return log(serverStr, infoStr, args) - } - - return log(clientStr, infoStr, args) -} - -// Mapreduce message logging. -func Mapreduce(table string, data map[string]interface{}) string { - args := make([]interface{}, len(data)+1) - - args[0] = fmt.Sprintf("MAPREDUCE:%s", strings.ToUpper(table)) - i := 1 - for k, v := range data { - args[i] = fmt.Sprintf("%s=%v", k, v) - i++ - } - - if Mode.Server { - return log(serverStr, infoStr, args) - } - - return log(clientStr, infoStr, args) -} - -// Warn message logging. -func Warn(args ...interface{}) string { - if !Mode.Quiet { - if Mode.Server { - return log(serverStr, warnStr, args) - } - return log(clientStr, warnStr, args) - } - - return "" -} - -// Error message logging. -func Error(args ...interface{}) string { - if Mode.Server { - return log(serverStr, errorStr, args) - } - - return log(clientStr, errorStr, args) -} - -// Fatal message logging. -func Fatal(args ...interface{}) string { - if Mode.Server { - return log(serverStr, fatalStr, args) - } - - return log(clientStr, fatalStr, args) -} - -// FatalPanic logs an error and exists the process. -func FatalPanic(args ...interface{}) { - what := clientStr - if Mode.Server { - what = serverStr - } - log(what, fatalStr, args) - - time.Sleep(time.Second) - mutex.Lock() - defer mutex.Unlock() - - closeWriter() - os.Exit(3) -} - -// Debug message logging. -func Debug(args ...interface{}) string { - if Mode.Debug { - if Mode.Server { - return log(serverStr, debugStr, args) - } - return log(clientStr, debugStr, args) - } - - return "" -} - -// Trace message logging. -func Trace(args ...interface{}) string { - if Mode.Trace { - if Mode.Server { - return log(serverStr, traceStr, args) - } - return log(clientStr, traceStr, args) - } - - return "" -} - -// Write log line to buffer and/or log file. -func write(what, severity, message string) { - if Mode.logToStdout { - line := fmt.Sprintf("%s|%s|%s|%s", what, hostname, severity, message) - - if config.Client.TermColorsEnable { - line = brush.Colorfy(line) - } - - stdoutBufCh <- line - } - - if Mode.logToFile { - t := time.Now() - timeStr := t.Format("20060102-150405") - fileLogBufCh <- buf{ - time: t, - message: fmt.Sprintf("%s|%s|%s|%s", severity, timeStr, what, message), - } - } -} - -// Generig log message. -func log(what string, severity string, args []interface{}) string { - if Mode.Nothing { - return "" - } - - sb := pool.BuilderBuffer.Get().(*strings.Builder) - - for i, arg := range args { - if i > 0 { - sb.WriteString(protocol.FieldDelimiter) - } - - switch v := arg.(type) { - case string: - sb.WriteString(v) - case int: - sb.WriteString(fmt.Sprintf("%d", v)) - case error: - sb.WriteString(v.Error()) - default: - sb.WriteString(fmt.Sprintf("%v", v)) - } - } - - message := sb.String() - pool.RecycleBuilderBuffer(sb) - write(what, severity, message) - return fmt.Sprintf("%s|%s", severity, message) -} - -// Raw message logging. -func Raw(message string) { - if Mode.Nothing { - return - } - - if Mode.logToFile { - fileLogBufCh <- buf{time.Now(), message} - } - - if Mode.logToStdout { - if config.Client.TermColorsEnable { - message = brush.Colorfy(message) - } - stdoutBufCh <- message - } -} - -// Close log writer (e.g. on change of day). -func closeWriter() { - if writer != nil { - writer.Flush() - fd.Close() - } -} - -// Return the correct log file writer -func fileWriter(dateStr string) *bufio.Writer { - if dateStr != lastDateStr { - return updateFileWriter(dateStr) - } - - // Check for log rotation signal - select { - case <-rotateCh: - stdoutWriter.WriteString("Received signal for logrotation\n") - return updateFileWriter(dateStr) - default: - } - - return writer -} - -// Update log file writer -func updateFileWriter(dateStr string) *bufio.Writer { - // Detected change of day. Close current writer and create a new one. - mutex.Lock() - defer mutex.Unlock() - closeWriter() - - if _, err := os.Stat(config.Common.LogDir); os.IsNotExist(err) { - if err = os.MkdirAll(config.Common.LogDir, 0755); err != nil { - panic(err) - } - } - - logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, dateStr) - newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644) - if err != nil { - panic(err) - } - - fd = newFd - writer = bufio.NewWriterSize(fd, 1) - lastDateStr = dateStr - - return writer -} - -// Flush all outstanding lines. -func Flush() { - for { - select { - case message := <-stdoutBufCh: - stdoutWriter.WriteString(message) - stdoutWriter.WriteString("\n") - default: - stdoutWriter.Flush() - return - } - } -} - -func writeToStdout(ctx context.Context) { - for { - select { - case message := <-stdoutBufCh: - stdoutWriter.WriteString(message) - stdoutWriter.WriteString("\n") - case <-time.After(time.Millisecond * 100): - stdoutWriter.Flush() - case <-pauseCh: - PAUSE: - for { - select { - case <-stdoutBufCh: - case <-resumeCh: - break PAUSE - case <-ctx.Done(): - return - } - } - case <-ctx.Done(): - Flush() - return - } - } -} - -func writeToFile(ctx context.Context) { - for { - select { - case buf := <-fileLogBufCh: - dateStr := buf.time.Format("20060102") - w := fileWriter(dateStr) - w.WriteString(buf.message) - w.WriteString("\n") - case <-pauseCh: - PAUSE: - for { - select { - case <-stdoutBufCh: - case <-resumeCh: - break PAUSE - case <-ctx.Done(): - return - } - } - case <-ctx.Done(): - return - } - } -} - -// Pause logging. -func Pause() { - if Mode.logToStdout { - pauseCh <- struct{}{} - } - if Mode.logToFile { - pauseCh <- struct{}{} - } -} - -// Resume logging (after pausing). -func Resume() { - if Mode.logToStdout { - resumeCh <- struct{}{} - } - if Mode.logToFile { - resumeCh <- struct{}{} - } -} diff --git a/internal/io/logger/modes.go b/internal/io/logger/modes.go deleted file mode 100644 index 85f90a5..0000000 --- a/internal/io/logger/modes.go +++ /dev/null @@ -1,13 +0,0 @@ -package logger - -// Modes specifies the logging mode. -type Modes struct { - Debug bool - logToFile bool - logToStdout bool - Nothing bool - Quiet bool - Server bool - Trace bool - UnitTest bool -} diff --git a/internal/io/logger/strategy.go b/internal/io/logger/strategy.go deleted file mode 100644 index 44bf393..0000000 --- a/internal/io/logger/strategy.go +++ /dev/null @@ -1,22 +0,0 @@ -package logger - -import "github.com/mimecast/dtail/internal/config" - -// Strategy allows to specify a log rotation strategy. -type Strategy int - -// Possible log strategies. -const ( - NormalStrategy Strategy = iota - DailyStrategy Strategy = iota - StdoutStrategy Strategy = iota -) - -func logStrategy() Strategy { - switch config.Common.LogStrategy { - case "daily": - return DailyStrategy - default: - } - return StdoutStrategy -} -- cgit v1.2.3