From 4ebff5064251d24c7d80873074c9a614311b027d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 30 Jul 2021 08:51:05 +0300 Subject: initial color config support --- internal/config/client.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 1515aae..3d8c45a 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -1,8 +1,25 @@ package config +// ClientColorConfig allows to override the default terminal color codes. +type ClientColorConfig struct { + OkBgColor Color +} + +/* + splitted := strings.Split(line, "|") + if splitted[2] == "100" { + splitted[2] = Paint(BgGreen, splitted[2]) + } else { + splitted[2] = Paint(BgRed, splitted[2]) + } + info := strings.Join(splitted[0:5], "|") + log := strings.Join(splitted[5:], "|") +*/ + // ClientConfig represents a DTail client configuration (empty as of now as there // are no available config options yet, but that may changes in the future). type ClientConfig struct { + TerminalColors ClientColorConfig `json:",omitempty"` } // Create a new default client configuration. -- cgit v1.2.3 From f46abf36e80a67d13d12dbe4ba8c899026e53961 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 31 Jul 2021 18:20:03 +0300 Subject: more on configurable colors --- internal/config/client.go | 51 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 16 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 3d8c45a..d6d106f 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -1,28 +1,47 @@ package config -// ClientColorConfig allows to override the default terminal color codes. -type ClientColorConfig struct { - OkBgColor Color -} +import "github.com/mimecast/dtail/internal/color" -/* - splitted := strings.Split(line, "|") - if splitted[2] == "100" { - splitted[2] = Paint(BgGreen, splitted[2]) - } else { - splitted[2] = Paint(BgRed, splitted[2]) - } - info := strings.Join(splitted[0:5], "|") - log := strings.Join(splitted[5:], "|") -*/ +// ClientColorConfig allows to override the default terminal color color. +type termColors struct { + RemoteTextFg color.Color + RemoteStatsOkBg color.Color + RemoteStatsWarnBg color.Color + RemoteTraceBg color.Color + RemoteDebugBg color.Color + RemoteWarnBg color.Color + RemoteErrorBg color.Color + RemoteFatalBg color.Color + RemoteFatalAttr color.Attribute + ClientStatsBg color.Color + ClientWarnFg color.Color + ClientErrorFg color.Color +} // ClientConfig represents a DTail client configuration (empty as of now as there // are no available config options yet, but that may changes in the future). type ClientConfig struct { - TerminalColors ClientColorConfig `json:",omitempty"` + TermColorsEnabled bool + TermColors termColors `json:",omitempty"` } // Create a new default client configuration. func newDefaultClientConfig() *ClientConfig { - return &ClientConfig{} + return &ClientConfig{ + TermColorsEnabled: true, + TermColors: termColors{ + RemoteTextFg: color.LightGray, + RemoteStatsOkBg: color.BgGreen, + RemoteStatsWarnBg: color.BgRed, + RemoteTraceBg: color.BgYellow, + RemoteDebugBg: color.BgYellow, + RemoteWarnBg: color.BgYellow, + RemoteErrorBg: color.BgRed, + RemoteFatalBg: color.BgRed, + RemoteFatalAttr: color.Bold, + ClientStatsBg: color.BgBlue, + ClientWarnFg: color.Purple, + ClientErrorFg: color.BgRed, + }, + } } -- cgit v1.2.3 From f3e08046badcbc852a6875b5553ee0a648021552 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 10 Aug 2021 12:05:40 +0300 Subject: can compile with new color codes --- internal/config/client.go | 108 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 23 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index d6d106f..84ad037 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -4,18 +4,49 @@ import "github.com/mimecast/dtail/internal/color" // ClientColorConfig allows to override the default terminal color color. type termColors struct { - RemoteTextFg color.Color - RemoteStatsOkBg color.Color - RemoteStatsWarnBg color.Color - RemoteTraceBg color.Color - RemoteDebugBg color.Color - RemoteWarnBg color.Color - RemoteErrorBg color.Color - RemoteFatalBg color.Color - RemoteFatalAttr color.Attribute - ClientStatsBg color.Color - ClientWarnFg color.Color - ClientErrorFg color.Color + ClientErrorAttr color.Attribute + ClientErrorBg color.BgColor + ClientErrorFg color.FgColor + + ClientStatsAttr color.Attribute + ClientStatsBg color.BgColor + ClientStatsFg color.FgColor + + ClientWarnAttr color.Attribute + ClientWarnBg color.BgColor + ClientWarnFg color.FgColor + + RemoteDebugAttr color.Attribute + RemoteDebugBg color.BgColor + RemoteDebugFg color.FgColor + + RemoteErrorAttr color.Attribute + RemoteErrorBg color.BgColor + RemoteErrorFg color.FgColor + + RemoteFatalAttr color.Attribute + RemoteFatalBg color.BgColor + RemoteFatalFg color.FgColor + + RemoteStatsOkAttr color.Attribute + RemoteStatsOkBg color.BgColor + RemoteStatsOkFg color.FgColor + + RemoteStatsWarnAttr color.Attribute + RemoteStatsWarnBg color.BgColor + RemoteStatsWarnFg color.FgColor + + RemoteTextAttr color.Attribute + RemoteTextBg color.BgColor + RemoteTextFg color.FgColor + + RemoteTraceAttr color.Attribute + RemoteTraceBg color.BgColor + RemoteTraceFg color.FgColor + + RemoteWarnAttr color.Attribute + RemoteWarnBg color.BgColor + RemoteWarnFg color.FgColor } // ClientConfig represents a DTail client configuration (empty as of now as there @@ -30,18 +61,49 @@ func newDefaultClientConfig() *ClientConfig { return &ClientConfig{ TermColorsEnabled: true, TermColors: termColors{ - RemoteTextFg: color.LightGray, + ClientErrorAttr: color.AttrBold, + ClientErrorBg: color.BgBlack, + ClientErrorFg: color.FgRed, + + ClientStatsAttr: color.AttrDim, + ClientStatsBg: color.BgBlue, + ClientStatsFg: color.FgWhite, + + ClientWarnAttr: color.AttrNone, + ClientWarnBg: color.BgBlack, + ClientWarnFg: color.FgMagenta, + + RemoteDebugAttr: color.AttrNone, + RemoteDebugBg: color.BgGreen, + RemoteDebugFg: color.FgWhite, + + RemoteErrorAttr: color.AttrBold, + RemoteErrorBg: color.BgRed, + RemoteErrorFg: color.FgWhite, + + RemoteFatalAttr: color.AttrBlink, + RemoteFatalBg: color.BgRed, + RemoteFatalFg: color.FgBlue, + + RemoteStatsOkAttr: color.AttrNone, RemoteStatsOkBg: color.BgGreen, - RemoteStatsWarnBg: color.BgRed, - RemoteTraceBg: color.BgYellow, - RemoteDebugBg: color.BgYellow, - RemoteWarnBg: color.BgYellow, - RemoteErrorBg: color.BgRed, - RemoteFatalBg: color.BgRed, - RemoteFatalAttr: color.Bold, - ClientStatsBg: color.BgBlue, - ClientWarnFg: color.Purple, - ClientErrorFg: color.BgRed, + RemoteStatsOkFg: color.FgWhite, + + RemoteStatsWarnAttr: color.AttrNone, + RemoteStatsWarnBg: color.BgRed, + RemoteStatsWarnFg: color.FgWhite, + + RemoteTextAttr: color.AttrNone, + RemoteTextBg: color.BgBlack, + RemoteTextFg: color.FgWhite, + + RemoteTraceAttr: color.AttrBold, + RemoteTraceBg: color.BgGreen, + RemoteTraceFg: color.FgWhite, + + RemoteWarnAttr: color.AttrBold, + RemoteWarnBg: color.BgYellow, + RemoteWarnFg: color.FgWhite, }, } } -- cgit v1.2.3 From 9bb96433be61d89f8e86f6f9ded8e35f74156a63 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 11 Aug 2021 09:32:13 +0300 Subject: add colorTable option --- internal/config/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 84ad037..5386576 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -75,7 +75,7 @@ func newDefaultClientConfig() *ClientConfig { RemoteDebugAttr: color.AttrNone, RemoteDebugBg: color.BgGreen, - RemoteDebugFg: color.FgWhite, + RemoteDebugFg: color.FgBlack, RemoteErrorAttr: color.AttrBold, RemoteErrorBg: color.BgRed, @@ -83,11 +83,11 @@ func newDefaultClientConfig() *ClientConfig { RemoteFatalAttr: color.AttrBlink, RemoteFatalBg: color.BgRed, - RemoteFatalFg: color.FgBlue, + RemoteFatalFg: color.FgWhite, RemoteStatsOkAttr: color.AttrNone, RemoteStatsOkBg: color.BgGreen, - RemoteStatsOkFg: color.FgWhite, + RemoteStatsOkFg: color.FgBlack, RemoteStatsWarnAttr: color.AttrNone, RemoteStatsWarnBg: color.BgRed, -- cgit v1.2.3 From 3cc8887885f24a3f0d607af24197bc364ab16b8d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 12 Aug 2021 10:56:36 +0300 Subject: add missing brush and also add color client configs plus jsonschema --- internal/config/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 5386576..c1b488c 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -52,14 +52,14 @@ type termColors struct { // ClientConfig represents a DTail client configuration (empty as of now as there // are no available config options yet, but that may changes in the future). type ClientConfig struct { - TermColorsEnabled bool - TermColors termColors `json:",omitempty"` + TermColorsEnable bool + TermColors termColors `json:",omitempty"` } // Create a new default client configuration. func newDefaultClientConfig() *ClientConfig { return &ClientConfig{ - TermColorsEnabled: true, + TermColorsEnable: true, TermColors: termColors{ ClientErrorAttr: color.AttrBold, ClientErrorBg: color.BgBlack, -- cgit v1.2.3 From bfcb0f159a4835cc6a0326ff46de7934e0363aed Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 5 Sep 2021 15:51:26 +0300 Subject: finalize new default color schema --- internal/config/client.go | 164 ++++++++++++++++++++++++---------------------- 1 file changed, 84 insertions(+), 80 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index c1b488c..05e4f93 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -4,49 +4,51 @@ import "github.com/mimecast/dtail/internal/color" // ClientColorConfig allows to override the default terminal color color. type termColors struct { - ClientErrorAttr color.Attribute - ClientErrorBg color.BgColor - ClientErrorFg color.FgColor - - ClientStatsAttr color.Attribute - ClientStatsBg color.BgColor - ClientStatsFg color.FgColor - - ClientWarnAttr color.Attribute - ClientWarnBg color.BgColor - ClientWarnFg color.FgColor - - RemoteDebugAttr color.Attribute - RemoteDebugBg color.BgColor - RemoteDebugFg color.FgColor - - RemoteErrorAttr color.Attribute - RemoteErrorBg color.BgColor - RemoteErrorFg color.FgColor - - RemoteFatalAttr color.Attribute - RemoteFatalBg color.BgColor - RemoteFatalFg color.FgColor - - RemoteStatsOkAttr color.Attribute - RemoteStatsOkBg color.BgColor - RemoteStatsOkFg color.FgColor - + ClientErrorAttr color.Attribute + ClientErrorBg color.BgColor + ClientErrorFg color.FgColor + ClientWarnAttr color.Attribute + ClientWarnBg color.BgColor + ClientWarnFg color.FgColor + DelimiterAttr color.Attribute + DelimiterBg color.BgColor + DelimiterFg color.FgColor + RemoteCountAttr color.Attribute + RemoteCountBg color.BgColor + RemoteCountFg color.FgColor + RemoteDebugAttr color.Attribute + RemoteDebugBg color.BgColor + RemoteDebugFg color.FgColor + RemoteErrorAttr color.Attribute + RemoteErrorBg color.BgColor + RemoteErrorFg color.FgColor + RemoteFatalAttr color.Attribute + RemoteFatalBg color.BgColor + RemoteFatalFg color.FgColor + RemoteIdAttr color.Attribute + RemoteIdBg color.BgColor + RemoteIdFg color.FgColor + RemoteServerAttr color.Attribute + RemoteServerBg color.BgColor + RemoteServerFg color.FgColor + RemoteStatsOkAttr color.Attribute + RemoteStatsOkBg color.BgColor + RemoteStatsOkFg color.FgColor RemoteStatsWarnAttr color.Attribute RemoteStatsWarnBg color.BgColor RemoteStatsWarnFg color.FgColor - - RemoteTextAttr color.Attribute - RemoteTextBg color.BgColor - RemoteTextFg color.FgColor - - RemoteTraceAttr color.Attribute - RemoteTraceBg color.BgColor - RemoteTraceFg color.FgColor - - RemoteWarnAttr color.Attribute - RemoteWarnBg color.BgColor - RemoteWarnFg color.FgColor + RemoteStrAttr color.Attribute + RemoteStrBg color.BgColor + RemoteStrFg color.FgColor + RemoteTextAttr color.Attribute + RemoteTextBg color.BgColor + RemoteTextFg color.FgColor + RemoteTraceAttr color.Attribute + RemoteTraceBg color.BgColor + RemoteTraceFg color.FgColor + RemoteWarnAttr color.Attribute + RemoteWarnBg color.BgColor + RemoteWarnFg color.FgColor } // ClientConfig represents a DTail client configuration (empty as of now as there @@ -61,49 +63,51 @@ func newDefaultClientConfig() *ClientConfig { return &ClientConfig{ TermColorsEnable: true, TermColors: termColors{ - ClientErrorAttr: color.AttrBold, - ClientErrorBg: color.BgBlack, - ClientErrorFg: color.FgRed, - - ClientStatsAttr: color.AttrDim, - ClientStatsBg: color.BgBlue, - ClientStatsFg: color.FgWhite, - - ClientWarnAttr: color.AttrNone, - ClientWarnBg: color.BgBlack, - ClientWarnFg: color.FgMagenta, - - RemoteDebugAttr: color.AttrNone, - RemoteDebugBg: color.BgGreen, - RemoteDebugFg: color.FgBlack, - - RemoteErrorAttr: color.AttrBold, - RemoteErrorBg: color.BgRed, - RemoteErrorFg: color.FgWhite, - - RemoteFatalAttr: color.AttrBlink, - RemoteFatalBg: color.BgRed, - RemoteFatalFg: color.FgWhite, - - RemoteStatsOkAttr: color.AttrNone, - RemoteStatsOkBg: color.BgGreen, - RemoteStatsOkFg: color.FgBlack, - + ClientErrorAttr: color.AttrBold, + ClientErrorBg: color.BgBlack, + ClientErrorFg: color.FgRed, + ClientWarnAttr: color.AttrNone, + ClientWarnBg: color.BgBlack, + ClientWarnFg: color.FgMagenta, + DelimiterAttr: color.AttrDim, + DelimiterBg: color.BgBlue, + DelimiterFg: color.FgCyan, + RemoteCountAttr: color.AttrDim, + RemoteCountBg: color.BgBlue, + RemoteCountFg: color.FgGreen, + RemoteDebugAttr: color.AttrBold, + RemoteDebugBg: color.BgBlack, + RemoteDebugFg: color.FgGreen, + RemoteErrorAttr: color.AttrBold, + RemoteErrorBg: color.BgRed, + RemoteErrorFg: color.FgWhite, + RemoteFatalAttr: color.AttrBlink, + RemoteFatalBg: color.BgRed, + RemoteFatalFg: color.FgWhite, + RemoteIdAttr: color.AttrDim, + RemoteIdBg: color.BgBlue, + RemoteIdFg: color.FgWhite, + RemoteServerAttr: color.AttrBold, + RemoteServerBg: color.BgBlue, + RemoteServerFg: color.FgWhite, + RemoteStatsOkAttr: color.AttrNone, + RemoteStatsOkBg: color.BgGreen, + RemoteStatsOkFg: color.FgBlue, RemoteStatsWarnAttr: color.AttrNone, RemoteStatsWarnBg: color.BgRed, RemoteStatsWarnFg: color.FgWhite, - - RemoteTextAttr: color.AttrNone, - RemoteTextBg: color.BgBlack, - RemoteTextFg: color.FgWhite, - - RemoteTraceAttr: color.AttrBold, - RemoteTraceBg: color.BgGreen, - RemoteTraceFg: color.FgWhite, - - RemoteWarnAttr: color.AttrBold, - RemoteWarnBg: color.BgYellow, - RemoteWarnFg: color.FgWhite, + RemoteStrAttr: color.AttrDim, + RemoteStrBg: color.BgBlue, + RemoteStrFg: color.FgWhite, + RemoteTextAttr: color.AttrNone, + RemoteTextBg: color.BgBlack, + RemoteTextFg: color.FgWhite, + RemoteTraceAttr: color.AttrBold, + RemoteTraceBg: color.BgGreen, + RemoteTraceFg: color.FgWhite, + RemoteWarnAttr: color.AttrBold, + RemoteWarnBg: color.BgYellow, + RemoteWarnFg: color.FgWhite, }, } } -- cgit v1.2.3 From 2c1c70313bb03cf2b2d7e7afadb07a48ff6bb690 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 6 Sep 2021 09:22:21 +0300 Subject: REMOTE and CLIENT colors are brushed correctly too now --- internal/config/client.go | 229 ++++++++++++++++++++++++++++------------------ 1 file changed, 138 insertions(+), 91 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 05e4f93..837c7d6 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -2,53 +2,80 @@ package config import "github.com/mimecast/dtail/internal/color" -// ClientColorConfig allows to override the default terminal color color. +type remoteTermColors struct { + DelimiterAttr color.Attribute + DelimiterBg color.BgColor + DelimiterFg color.FgColor + RemoteAttr color.Attribute + RemoteBg color.BgColor + RemoteFg color.FgColor + CountAttr color.Attribute + CountBg color.BgColor + CountFg color.FgColor + HostnameAttr color.Attribute + HostnameBg color.BgColor + HostnameFg color.FgColor + IdAttr color.Attribute + IdBg color.BgColor + IdFg color.FgColor + StatsOkAttr color.Attribute + StatsOkBg color.BgColor + StatsOkFg color.FgColor + StatsWarnAttr color.Attribute + StatsWarnBg color.BgColor + StatsWarnFg color.FgColor + TextAttr color.Attribute + TextBg color.BgColor + TextFg color.FgColor +} + +type clientTermColors struct { + DelimiterAttr color.Attribute + DelimiterBg color.BgColor + DelimiterFg color.FgColor + ClientAttr color.Attribute + ClientBg color.BgColor + ClientFg color.FgColor + HostnameAttr color.Attribute + HostnameBg color.BgColor + HostnameFg color.FgColor + TextAttr color.Attribute + TextBg color.BgColor + TextFg color.FgColor +} + +type serverTermColors struct { + DelimiterAttr color.Attribute + DelimiterBg color.BgColor + DelimiterFg color.FgColor + ServerAttr color.Attribute + ServerBg color.BgColor + ServerFg color.FgColor + HostnameAttr color.Attribute + HostnameBg color.BgColor + HostnameFg color.FgColor + TextAttr color.Attribute + TextBg color.BgColor + TextFg color.FgColor +} + +type commonTermColors struct { + SeverityErrorAttr color.Attribute + SeverityErrorBg color.BgColor + SeverityErrorFg color.FgColor + SeverityFatalAttr color.Attribute + SeverityFatalBg color.BgColor + SeverityFatalFg color.FgColor + SeverityWarnAttr color.Attribute + SeverityWarnBg color.BgColor + SeverityWarnFg color.FgColor +} + type termColors struct { - ClientErrorAttr color.Attribute - ClientErrorBg color.BgColor - ClientErrorFg color.FgColor - ClientWarnAttr color.Attribute - ClientWarnBg color.BgColor - ClientWarnFg color.FgColor - DelimiterAttr color.Attribute - DelimiterBg color.BgColor - DelimiterFg color.FgColor - RemoteCountAttr color.Attribute - RemoteCountBg color.BgColor - RemoteCountFg color.FgColor - RemoteDebugAttr color.Attribute - RemoteDebugBg color.BgColor - RemoteDebugFg color.FgColor - RemoteErrorAttr color.Attribute - RemoteErrorBg color.BgColor - RemoteErrorFg color.FgColor - RemoteFatalAttr color.Attribute - RemoteFatalBg color.BgColor - RemoteFatalFg color.FgColor - RemoteIdAttr color.Attribute - RemoteIdBg color.BgColor - RemoteIdFg color.FgColor - RemoteServerAttr color.Attribute - RemoteServerBg color.BgColor - RemoteServerFg color.FgColor - RemoteStatsOkAttr color.Attribute - RemoteStatsOkBg color.BgColor - RemoteStatsOkFg color.FgColor - RemoteStatsWarnAttr color.Attribute - RemoteStatsWarnBg color.BgColor - RemoteStatsWarnFg color.FgColor - RemoteStrAttr color.Attribute - RemoteStrBg color.BgColor - RemoteStrFg color.FgColor - RemoteTextAttr color.Attribute - RemoteTextBg color.BgColor - RemoteTextFg color.FgColor - RemoteTraceAttr color.Attribute - RemoteTraceBg color.BgColor - RemoteTraceFg color.FgColor - RemoteWarnAttr color.Attribute - RemoteWarnBg color.BgColor - RemoteWarnFg color.FgColor + Remote remoteTermColors + Client clientTermColors + Server serverTermColors + Common commonTermColors } // ClientConfig represents a DTail client configuration (empty as of now as there @@ -63,51 +90,71 @@ func newDefaultClientConfig() *ClientConfig { return &ClientConfig{ TermColorsEnable: true, TermColors: termColors{ - ClientErrorAttr: color.AttrBold, - ClientErrorBg: color.BgBlack, - ClientErrorFg: color.FgRed, - ClientWarnAttr: color.AttrNone, - ClientWarnBg: color.BgBlack, - ClientWarnFg: color.FgMagenta, - DelimiterAttr: color.AttrDim, - DelimiterBg: color.BgBlue, - DelimiterFg: color.FgCyan, - RemoteCountAttr: color.AttrDim, - RemoteCountBg: color.BgBlue, - RemoteCountFg: color.FgGreen, - RemoteDebugAttr: color.AttrBold, - RemoteDebugBg: color.BgBlack, - RemoteDebugFg: color.FgGreen, - RemoteErrorAttr: color.AttrBold, - RemoteErrorBg: color.BgRed, - RemoteErrorFg: color.FgWhite, - RemoteFatalAttr: color.AttrBlink, - RemoteFatalBg: color.BgRed, - RemoteFatalFg: color.FgWhite, - RemoteIdAttr: color.AttrDim, - RemoteIdBg: color.BgBlue, - RemoteIdFg: color.FgWhite, - RemoteServerAttr: color.AttrBold, - RemoteServerBg: color.BgBlue, - RemoteServerFg: color.FgWhite, - RemoteStatsOkAttr: color.AttrNone, - RemoteStatsOkBg: color.BgGreen, - RemoteStatsOkFg: color.FgBlue, - RemoteStatsWarnAttr: color.AttrNone, - RemoteStatsWarnBg: color.BgRed, - RemoteStatsWarnFg: color.FgWhite, - RemoteStrAttr: color.AttrDim, - RemoteStrBg: color.BgBlue, - RemoteStrFg: color.FgWhite, - RemoteTextAttr: color.AttrNone, - RemoteTextBg: color.BgBlack, - RemoteTextFg: color.FgWhite, - RemoteTraceAttr: color.AttrBold, - RemoteTraceBg: color.BgGreen, - RemoteTraceFg: color.FgWhite, - RemoteWarnAttr: color.AttrBold, - RemoteWarnBg: color.BgYellow, - RemoteWarnFg: color.FgWhite, + Remote: remoteTermColors{ + DelimiterAttr: color.AttrDim, + DelimiterBg: color.BgBlue, + DelimiterFg: color.FgCyan, + RemoteAttr: color.AttrDim, + RemoteBg: color.BgBlue, + RemoteFg: color.FgWhite, + CountAttr: color.AttrDim, + CountBg: color.BgBlue, + CountFg: color.FgGreen, + HostnameAttr: color.AttrBold, + HostnameBg: color.BgBlue, + HostnameFg: color.FgWhite, + IdAttr: color.AttrDim, + IdBg: color.BgBlue, + IdFg: color.FgWhite, + StatsOkAttr: color.AttrNone, + StatsOkBg: color.BgGreen, + StatsOkFg: color.FgBlue, + StatsWarnAttr: color.AttrNone, + StatsWarnBg: color.BgRed, + StatsWarnFg: color.FgWhite, + TextAttr: color.AttrNone, + TextBg: color.BgBlack, + TextFg: color.FgWhite, + }, + Client: clientTermColors{ + DelimiterAttr: color.AttrDim, + DelimiterBg: color.BgYellow, + DelimiterFg: color.FgBlack, + ClientAttr: color.AttrDim, + ClientBg: color.BgYellow, + ClientFg: color.FgBlack, + HostnameAttr: color.AttrDim, + HostnameBg: color.BgYellow, + HostnameFg: color.FgBlack, + TextAttr: color.AttrNone, + TextBg: color.BgYellow, + TextFg: color.FgBlack, + }, + Server: serverTermColors{ + DelimiterAttr: color.AttrDim, + DelimiterBg: color.BgCyan, + DelimiterFg: color.FgBlack, + ServerAttr: color.AttrDim, + ServerBg: color.BgCyan, + ServerFg: color.FgBlack, + HostnameAttr: color.AttrBold, + HostnameBg: color.BgCyan, + HostnameFg: color.FgBlack, + TextAttr: color.AttrNone, + TextBg: color.BgCyan, + TextFg: color.FgBlack, + }, + Common: commonTermColors{ + SeverityErrorAttr: color.AttrBold, + SeverityErrorBg: color.BgRed, + SeverityErrorFg: color.FgWhite, + SeverityFatalAttr: color.AttrBlink, + SeverityFatalBg: color.BgRed, + SeverityFatalFg: color.FgWhite, + SeverityWarnAttr: color.AttrNone, + SeverityWarnBg: color.BgBlack, + SeverityWarnFg: color.FgRed, + }, }, } } -- cgit v1.2.3 From cc89d3fb8be2465af276d7ef03ea2a8affd87b2e Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 6 Sep 2021 13:48:55 +0300 Subject: Print out client/server update notice even from dtail server 4 to dtail client 3. --- internal/config/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 837c7d6..8bde7a4 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -127,8 +127,8 @@ func newDefaultClientConfig() *ClientConfig { HostnameBg: color.BgYellow, HostnameFg: color.FgBlack, TextAttr: color.AttrNone, - TextBg: color.BgYellow, - TextFg: color.FgBlack, + TextBg: color.BgBlack, + TextFg: color.FgWhite, }, Server: serverTermColors{ DelimiterAttr: color.AttrDim, @@ -151,9 +151,9 @@ func newDefaultClientConfig() *ClientConfig { SeverityFatalAttr: color.AttrBlink, SeverityFatalBg: color.BgRed, SeverityFatalFg: color.FgWhite, - SeverityWarnAttr: color.AttrNone, + SeverityWarnAttr: color.AttrBold, SeverityWarnBg: color.BgBlack, - SeverityWarnFg: color.FgRed, + SeverityWarnFg: color.FgWhite, }, }, } -- cgit v1.2.3 From 6ae75e8f106d3eee18ea61e6c4d6925c6f514460 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 6 Sep 2021 14:07:07 +0300 Subject: fine tweak color schema --- internal/config/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 8bde7a4..3c2f7de 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -99,7 +99,7 @@ func newDefaultClientConfig() *ClientConfig { RemoteFg: color.FgWhite, CountAttr: color.AttrDim, CountBg: color.BgBlue, - CountFg: color.FgGreen, + CountFg: color.FgWhite, HostnameAttr: color.AttrBold, HostnameBg: color.BgBlue, HostnameFg: color.FgWhite, @@ -108,7 +108,7 @@ func newDefaultClientConfig() *ClientConfig { IdFg: color.FgWhite, StatsOkAttr: color.AttrNone, StatsOkBg: color.BgGreen, - StatsOkFg: color.FgBlue, + StatsOkFg: color.FgBlack, StatsWarnAttr: color.AttrNone, StatsWarnBg: color.BgRed, StatsWarnFg: color.FgWhite, -- cgit v1.2.3 From 16dc57e1e1c28e9d762424e596223a980770e059 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 8 Sep 2021 19:10:50 +0300 Subject: mapreduce tables are in colors now too --- internal/config/client.go | 48 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 3c2f7de..795c4a4 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -71,11 +71,32 @@ type commonTermColors struct { SeverityWarnFg color.FgColor } +type maprTableTermColors struct { + DataAttr color.Attribute + DataBg color.BgColor + DataFg color.FgColor + DelimiterAttr color.Attribute + DelimiterBg color.BgColor + DelimiterFg color.FgColor + HeaderAttr color.Attribute + HeaderBg color.BgColor + HeaderDelimiterAttr color.Attribute + HeaderDelimiterBg color.BgColor + HeaderDelimiterFg color.FgColor + HeaderFg color.FgColor + HeaderGroupKeyAttr color.Attribute + HeaderSortKeyAttr color.Attribute + RawQueryAttr color.Attribute + RawQueryBg color.BgColor + RawQueryFg color.FgColor +} + type termColors struct { - Remote remoteTermColors - Client clientTermColors - Server serverTermColors - Common commonTermColors + Remote remoteTermColors + Client clientTermColors + Server serverTermColors + Common commonTermColors + MaprTable maprTableTermColors } // ClientConfig represents a DTail client configuration (empty as of now as there @@ -155,6 +176,25 @@ func newDefaultClientConfig() *ClientConfig { SeverityWarnBg: color.BgBlack, SeverityWarnFg: color.FgWhite, }, + MaprTable: maprTableTermColors{ + DataAttr: color.AttrNone, + DataBg: color.BgBlue, + DataFg: color.FgWhite, + DelimiterAttr: color.AttrDim, + DelimiterBg: color.BgBlue, + DelimiterFg: color.FgWhite, + HeaderAttr: color.AttrBold, + HeaderBg: color.BgBlue, + HeaderFg: color.FgWhite, + HeaderDelimiterAttr: color.AttrDim, + HeaderDelimiterBg: color.BgBlue, + HeaderDelimiterFg: color.FgWhite, + HeaderSortKeyAttr: color.AttrUnderline, + HeaderGroupKeyAttr: color.AttrReverse, + RawQueryAttr: color.AttrDim, + RawQueryBg: color.BgBlack, + RawQueryFg: color.FgCyan, + }, }, } } -- cgit v1.2.3 From 5e717af91e8012c72ec7dc0204420dea46f187db Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 18 Sep 2021 11:57:09 +0300 Subject: new docker test cases - also change default FATAL bg color to magenta --- internal/config/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 795c4a4..152ab15 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -169,8 +169,8 @@ func newDefaultClientConfig() *ClientConfig { SeverityErrorAttr: color.AttrBold, SeverityErrorBg: color.BgRed, SeverityErrorFg: color.FgWhite, - SeverityFatalAttr: color.AttrBlink, - SeverityFatalBg: color.BgRed, + SeverityFatalAttr: color.AttrBold, + SeverityFatalBg: color.BgMagenta, SeverityFatalFg: color.FgWhite, SeverityWarnAttr: color.AttrBold, SeverityWarnBg: color.BgBlack, -- cgit v1.2.3 From 6506e20f6c80f4acb7434eb9dd14f784a67189cd Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 18 Sep 2021 14:41:25 +0300 Subject: add spartan mode --- internal/config/read.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'internal/config') diff --git a/internal/config/read.go b/internal/config/read.go index a4e605b..ea358f8 100644 --- a/internal/config/read.go +++ b/internal/config/read.go @@ -5,7 +5,7 @@ import ( ) // Read the DTail configuration. -func Read(configFile string, sshPort int) { +func Read(configFile string, sshPort int, noColor bool) { initializer := configInitializer{ Common: newDefaultCommonConfig(), Server: newDefaultServerConfig(), @@ -34,4 +34,7 @@ func Read(configFile string, sshPort int) { if sshPort != 2222 { Common.SSHPort = sshPort } + if noColor { + Client.TermColorsEnable = false + } } -- cgit v1.2.3 From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 19 Sep 2021 13:22:59 +0300 Subject: move args to config package logger package rewrite as dlog --- internal/config/args.go | 105 ++++++++++++++++++++++++++++++++++++++++++++++ internal/config/common.go | 8 +--- internal/config/config.go | 3 ++ internal/config/read.go | 40 ------------------ internal/config/server.go | 1 + internal/config/setup.go | 35 ++++++++++++++++ 6 files changed, 146 insertions(+), 46 deletions(-) create mode 100644 internal/config/args.go delete mode 100644 internal/config/read.go create mode 100644 internal/config/setup.go (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go new file mode 100644 index 0000000..89e4bc9 --- /dev/null +++ b/internal/config/args.go @@ -0,0 +1,105 @@ +package config + +import ( + "flag" + "fmt" + "strings" + + "github.com/mimecast/dtail/internal/omode" + + gossh "golang.org/x/crypto/ssh" +) + +// Args is a helper struct to summarize common client arguments. +type Args struct { + Arguments []string + ConfigFile string + ConnectionsPerCPU int + Discovery string + LogLevel string + Mode omode.Mode + NoColor bool + PrivateKeyPathFile string + Quiet bool + RegexInvert bool + RegexStr string + Serverless bool + ServersStr string + Spartan bool + SSHAuthMethods []gossh.AuthMethod + SSHHostKeyCallback gossh.HostKeyCallback + SSHPort int + Timeout int + TrustAllHosts bool + UserName string + What string +} + +func (a *Args) String() string { + var sb strings.Builder + + sb.WriteString("Args(") + sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) + sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile)) + sb.WriteString(fmt.Sprintf("%s:%v,", "ConnectionsPerCPU", a.ConnectionsPerCPU)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Discovery", a.Discovery)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode)) + sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor)) + sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet)) + sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert)) + sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless)) + sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Spartan", a.Spartan)) + sb.WriteString(fmt.Sprintf("%s:%v,", "SSHAuthMethods", a.SSHAuthMethods)) + sb.WriteString(fmt.Sprintf("%s:%v,", "SSHHostKeyCallback", a.SSHHostKeyCallback)) + sb.WriteString(fmt.Sprintf("%s:%v,", "SSHPort", a.SSHPort)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Timeout", a.Timeout)) + sb.WriteString(fmt.Sprintf("%s:%v,", "TrustAllHosts", a.TrustAllHosts)) + sb.WriteString(fmt.Sprintf("%s:%v,", "UserName", a.UserName)) + sb.WriteString(fmt.Sprintf("%s:%v", "What", a.What)) + sb.WriteString(")") + + return sb.String() +} + +// Based on the argument list, transform/manipulate some of the arguments. +func (a *Args) transform(args []string) { + if a.LogLevel != "" { + Common.LogLevel = a.LogLevel + } + + if a.SSHPort != 2222 { + Common.SSHPort = a.SSHPort + } + if a.NoColor { + Client.TermColorsEnable = false + } + + if a.Spartan { + a.Quiet = true + a.NoColor = true + } + + if a.Discovery == "" && a.ServersStr == "" { + a.Serverless = true + } + + // Interpret additional args as file list. + if a.What == "" { + var files []string + for _, file := range flag.Args() { + files = append(files, file) + } + a.What = strings.Join(files, ",") + } +} + +// SerializeOptions returns a string ready to be sent over the wire to the server. +func (a *Args) SerializeOptions() string { + return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan) +} + +// TODO: Put the DeseializeOptions function here (move it away from the internal/server package) diff --git a/internal/config/common.go b/internal/config/common.go index c3e203e..7d45261 100644 --- a/internal/config/common.go +++ b/internal/config/common.go @@ -6,10 +6,8 @@ type CommonConfig struct { SSHPort int // Enable experimental features (mainly for dev purposes) ExperimentalFeaturesEnable bool `json:",omitempty"` - // Enable debug logging. Don't enable in production. - DebugEnable bool `json:",omitempty"` - // Enable trace logging. Don't enable in production. - TraceEnable bool `json:",omitempty"` + // LogLevel defines how much is logged. TODO: Adjust JSONschema + LogLevel string `json:",omitempty"` // The log strategy to use, one of // stdout: only log to stdout (useful when used with systemd) // daily: create a log file for every day @@ -26,8 +24,6 @@ type CommonConfig struct { func newDefaultCommonConfig() *CommonConfig { return &CommonConfig{ SSHPort: 2222, - DebugEnable: false, - TraceEnable: false, ExperimentalFeaturesEnable: false, LogDir: "log", CacheDir: "cache", diff --git a/internal/config/config.go b/internal/config/config.go index 276ddcf..2d77041 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,6 +15,9 @@ const ScheduleUser string = "DTAIL-SCHEDULE" // ContinuousUser is used for non-interactive continuous mapreduce queries. const ContinuousUser string = "DTAIL-CONTINUOUS" +// TestUser is used for unit tests and potentially also for integration tests. +const TestUser string = "DTAIL-TEST" + // InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval. const InterruptTimeoutS int = 3 diff --git a/internal/config/read.go b/internal/config/read.go deleted file mode 100644 index ea358f8..0000000 --- a/internal/config/read.go +++ /dev/null @@ -1,40 +0,0 @@ -package config - -import ( - "os" -) - -// Read the DTail configuration. -func Read(configFile string, sshPort int, noColor bool) { - initializer := configInitializer{ - Common: newDefaultCommonConfig(), - Server: newDefaultServerConfig(), - Client: newDefaultClientConfig(), - } - - if configFile == "" { - configFile = "./cfg/dtail.json" - } - - if _, err := os.Stat(configFile); !os.IsNotExist(err) { - initializer.parseConfig(configFile) - } - - // Assign pointers to global variables, so that we can access the - // configuration from any place of the program. - Common = initializer.Common - Server = initializer.Server - Client = initializer.Client - - if Server.MapreduceLogFormat == "" { - Server.MapreduceLogFormat = "default" - } - - // If non-standard port specified, overwrite config - if sshPort != 2222 { - Common.SSHPort = sshPort - } - if noColor { - Client.TermColorsEnable = false - } -} diff --git a/internal/config/server.go b/internal/config/server.go index dc0d587..8bbb394 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -76,6 +76,7 @@ func newDefaultServerConfig() *ServerConfig { MaxConcurrentTails: 50, HostKeyFile: "./cache/ssh_host_key", HostKeyBits: 4096, + MapreduceLogFormat: "default", Permissions: Permissions{ Default: defaultPermissions, }, diff --git a/internal/config/setup.go b/internal/config/setup.go new file mode 100644 index 0000000..3c4bcc4 --- /dev/null +++ b/internal/config/setup.go @@ -0,0 +1,35 @@ +package config + +import ( + "os" +) + +const NoConfigFile string = "Don't read a config file - use defaults only" + +// Setup the DTail configuration. +func Setup(args *Args, additionalArgs []string) { + initializer := configInitializer{ + Common: newDefaultCommonConfig(), + Server: newDefaultServerConfig(), + Client: newDefaultClientConfig(), + } + + if args.ConfigFile == "" { + // TODO: Search more paths for config file (e.g. in /etc and in ~/.config/... + args.ConfigFile = "./cfg/dtail.json" + } + + if args.ConfigFile != NoConfigFile { + if _, err := os.Stat(args.ConfigFile); !os.IsNotExist(err) { + initializer.parseConfig(args.ConfigFile) + } + } + + // Assign pointers to global variables, so that we can access the + // configuration from any place of the program. + Common = initializer.Common + Server = initializer.Server + Client = initializer.Client + + args.transform(additionalArgs) +} -- cgit v1.2.3 From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 26 Sep 2021 16:42:47 +0300 Subject: refactor config reader - also looks in additional search paths for config file unless NONE is specified --- internal/config/args.go | 25 ++++++++++++++----- internal/config/common.go | 2 +- internal/config/config.go | 61 ++++++++++++++++++++++++++++++++--------------- internal/config/setup.go | 32 ++++++------------------- 4 files changed, 69 insertions(+), 51 deletions(-) (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go index 89e4bc9..767cc65 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -16,6 +16,7 @@ type Args struct { ConfigFile string ConnectionsPerCPU int Discovery string + LogDir string LogLevel string Mode omode.Mode NoColor bool @@ -39,6 +40,8 @@ func (a *Args) String() string { var sb strings.Builder sb.WriteString("Args(") + // TODO: All commands should make use of this + sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir)) sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile)) @@ -66,16 +69,24 @@ func (a *Args) String() string { } // Based on the argument list, transform/manipulate some of the arguments. -func (a *Args) transform(args []string) { +func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { + if a.LogDir != "" { + common.LogDir = a.LogDir + if common.LogStrategy == "" { + // TODO: Implement the other (not-daily) log strategy for the server. + common.LogStrategy = "daily" + } + } + if a.LogLevel != "" { - Common.LogLevel = a.LogLevel + common.LogLevel = a.LogLevel } - if a.SSHPort != 2222 { - Common.SSHPort = a.SSHPort + if a.SSHPort != DefaultSSHPort { + common.SSHPort = a.SSHPort } if a.NoColor { - Client.TermColorsEnable = false + client.TermColorsEnable = false } if a.Spartan { @@ -95,6 +106,8 @@ func (a *Args) transform(args []string) { } a.What = strings.Join(files, ",") } + + return client, server, common } // SerializeOptions returns a string ready to be sent over the wire to the server. @@ -102,4 +115,4 @@ func (a *Args) SerializeOptions() string { return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan) } -// TODO: Put the DeseializeOptions function here (move it away from the internal/server package) +// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package) diff --git a/internal/config/common.go b/internal/config/common.go index 7d45261..acc8e6a 100644 --- a/internal/config/common.go +++ b/internal/config/common.go @@ -23,7 +23,7 @@ type CommonConfig struct { // Create a new default configuration. func newDefaultCommonConfig() *CommonConfig { return &CommonConfig{ - SSHPort: 2222, + SSHPort: DefaultSSHPort, ExperimentalFeaturesEnable: false, LogDir: "log", CacheDir: "cache", diff --git a/internal/config/config.go b/internal/config/config.go index 2d77041..c9f411c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,24 +2,26 @@ package config import ( "encoding/json" + "fmt" "io/ioutil" "os" + "strings" ) -// ControlUser is used for various DTail specific operations. -const ControlUser string = "DTAIL-CONTROL" - -// ScheduleUser is used for non-interactive scheduled mapreduce queries. -const ScheduleUser string = "DTAIL-SCHEDULE" - -// ContinuousUser is used for non-interactive continuous mapreduce queries. -const ContinuousUser string = "DTAIL-CONTINUOUS" - -// TestUser is used for unit tests and potentially also for integration tests. -const TestUser string = "DTAIL-TEST" - -// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval. -const InterruptTimeoutS int = 3 +const ( + // ControlUser is used for various DTail specific operations. + ControlUser string = "DTAIL-CONTROL" + // ScheduleUser is used for non-interactive scheduled mapreduce queries. + ScheduleUser string = "DTAIL-SCHEDULE" + // ContinuousUser is used for non-interactive continuous mapreduce queries. + ContinuousUser string = "DTAIL-CONTINUOUS" + // InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval. + InterruptTimeoutS int = 3 + // ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start) + DefaultConnectionsPerCPU int = 10 + // DTailSSHServerDefaultPort is the default DServer port. + DefaultSSHPort int = 2222 +) // Client holds a DTail client configuration. var Client *ClientConfig @@ -37,21 +39,42 @@ type configInitializer struct { Client *ClientConfig } -// Parse and read a given config file in JSON format. -func (c *configInitializer) parseConfig(configFile string) { +func (c *configInitializer) parseConfig(args *Args) { + if strings.ToUpper(args.ConfigFile) == "NONE" { + return + } + + if args.ConfigFile != "" { + c.parseSpecificConfig(args.ConfigFile) + return + } + + if homeDir, err := os.UserHomeDir(); err != nil { + var paths []string + paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir)) + paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir)) + for _, configPath := range paths { + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + c.parseSpecificConfig(configPath) + } + } + } +} + +func (c *configInitializer) parseSpecificConfig(configFile string) { fd, err := os.Open(configFile) if err != nil { - panic(err) + panic(fmt.Sprintf("Unable to read config file: %v", err)) } defer fd.Close() cfgBytes, err := ioutil.ReadAll(fd) if err != nil { - panic(err) + panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err)) } err = json.Unmarshal([]byte(cfgBytes), c) if err != nil { - panic(err) + panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err)) } } diff --git a/internal/config/setup.go b/internal/config/setup.go index 3c4bcc4..be8e867 100644 --- a/internal/config/setup.go +++ b/internal/config/setup.go @@ -1,11 +1,5 @@ package config -import ( - "os" -) - -const NoConfigFile string = "Don't read a config file - use defaults only" - // Setup the DTail configuration. func Setup(args *Args, additionalArgs []string) { initializer := configInitializer{ @@ -13,23 +7,11 @@ func Setup(args *Args, additionalArgs []string) { Server: newDefaultServerConfig(), Client: newDefaultClientConfig(), } - - if args.ConfigFile == "" { - // TODO: Search more paths for config file (e.g. in /etc and in ~/.config/... - args.ConfigFile = "./cfg/dtail.json" - } - - if args.ConfigFile != NoConfigFile { - if _, err := os.Stat(args.ConfigFile); !os.IsNotExist(err) { - initializer.parseConfig(args.ConfigFile) - } - } - - // Assign pointers to global variables, so that we can access the - // configuration from any place of the program. - Common = initializer.Common - Server = initializer.Server - Client = initializer.Client - - args.transform(additionalArgs) + initializer.parseConfig(args) + Client, Server, Common = args.transformConfig( + additionalArgs, + initializer.Client, + initializer.Server, + initializer.Common, + ) } -- cgit v1.2.3 From 609921f9c783941eaa9019a92b78ec45b49d681c Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 28 Sep 2021 21:11:50 +0300 Subject: can have daily and normal file log rotation --- internal/config/args.go | 43 ------------------------------------------- internal/config/config.go | 43 +++++++++++++++++++++++++++++++++++++++++++ internal/config/setup.go | 4 ++-- 3 files changed, 45 insertions(+), 45 deletions(-) (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go index 767cc65..7f24348 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -1,7 +1,6 @@ package config import ( - "flag" "fmt" "strings" @@ -68,48 +67,6 @@ func (a *Args) String() string { return sb.String() } -// Based on the argument list, transform/manipulate some of the arguments. -func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { - if a.LogDir != "" { - common.LogDir = a.LogDir - if common.LogStrategy == "" { - // TODO: Implement the other (not-daily) log strategy for the server. - common.LogStrategy = "daily" - } - } - - if a.LogLevel != "" { - common.LogLevel = a.LogLevel - } - - if a.SSHPort != DefaultSSHPort { - common.SSHPort = a.SSHPort - } - if a.NoColor { - client.TermColorsEnable = false - } - - if a.Spartan { - a.Quiet = true - a.NoColor = true - } - - if a.Discovery == "" && a.ServersStr == "" { - a.Serverless = true - } - - // Interpret additional args as file list. - if a.What == "" { - var files []string - for _, file := range flag.Args() { - files = append(files, file) - } - a.What = strings.Join(files, ",") - } - - return client, server, common -} - // SerializeOptions returns a string ready to be sent over the wire to the server. func (a *Args) SerializeOptions() string { return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan) diff --git a/internal/config/config.go b/internal/config/config.go index c9f411c..76dcc65 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "encoding/json" + "flag" "fmt" "io/ioutil" "os" @@ -78,3 +79,45 @@ func (c *configInitializer) parseSpecificConfig(configFile string) { panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err)) } } + +func (c *configInitializer) transformConfig(args *Args, additionalArgs []string, + client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { + if args.LogDir != "" { + common.LogDir = args.LogDir + if common.LogStrategy == "" { + // TODO: Implement the other (not-daily) log strategy for the server. + common.LogStrategy = "daily" + } + } + + if args.LogLevel != "" { + common.LogLevel = args.LogLevel + } + + if args.SSHPort != DefaultSSHPort { + common.SSHPort = args.SSHPort + } + if args.NoColor { + client.TermColorsEnable = false + } + + if args.Spartan { + args.Quiet = true + args.NoColor = true + } + + if args.Discovery == "" && args.ServersStr == "" { + args.Serverless = true + } + + // Interpret additional args as file list. + if args.What == "" { + var files []string + for _, file := range flag.Args() { + files = append(files, file) + } + args.What = strings.Join(files, ",") + } + + return client, server, common +} diff --git a/internal/config/setup.go b/internal/config/setup.go index be8e867..7800914 100644 --- a/internal/config/setup.go +++ b/internal/config/setup.go @@ -8,8 +8,8 @@ func Setup(args *Args, additionalArgs []string) { Client: newDefaultClientConfig(), } initializer.parseConfig(args) - Client, Server, Common = args.transformConfig( - additionalArgs, + Client, Server, Common = initializer.transformConfig( + args, additionalArgs, initializer.Client, initializer.Server, initializer.Common, -- cgit v1.2.3 From 764ef99a3d779a0db1fb60679292af52425ba2f6 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 10:46:47 +0300 Subject: add more default fields to MAPREDUCE --- internal/config/args.go | 2 +- internal/config/config.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go index 7f24348..484aa8b 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -39,7 +39,7 @@ func (a *Args) String() string { var sb strings.Builder sb.WriteString("Args(") - // TODO: All commands should make use of this + sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir)) sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) diff --git a/internal/config/config.go b/internal/config/config.go index 76dcc65..3d05a11 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -85,7 +85,6 @@ func (c *configInitializer) transformConfig(args *Args, additionalArgs []string, if args.LogDir != "" { common.LogDir = args.LogDir if common.LogStrategy == "" { - // TODO: Implement the other (not-daily) log strategy for the server. common.LogStrategy = "daily" } } -- cgit v1.2.3 From 6e1af993924bc7bebe898b403962db5a6b3505d1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 11:23:08 +0300 Subject: Client default log dir is ~/log --- internal/config/config.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'internal/config') diff --git a/internal/config/config.go b/internal/config/config.go index 3d05a11..6d4730a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -84,9 +84,16 @@ func (c *configInitializer) transformConfig(args *Args, additionalArgs []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { if args.LogDir != "" { common.LogDir = args.LogDir - if common.LogStrategy == "" { - common.LogStrategy = "daily" + } + if strings.Contains(common.LogDir, "~/") { + homeDir, err := os.UserHomeDir() + if err != nil { + panic(err) } + common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) + } + if common.LogStrategy == "" { + common.LogStrategy = "daily" } if args.LogLevel != "" { -- cgit v1.2.3 From 12c79f68bb5bda6673819d7b754820ecfe6d08ff Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 11:54:07 +0300 Subject: reduce logging in serverless mode --- internal/config/args.go | 30 ++++++++++- internal/config/config.go | 116 +++++------------------------------------ internal/config/initializer.go | 109 ++++++++++++++++++++++++++++++++++++++ internal/config/setup.go | 17 ------ 4 files changed, 151 insertions(+), 121 deletions(-) create mode 100644 internal/config/initializer.go delete mode 100644 internal/config/setup.go (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go index 484aa8b..3e2eb1f 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -1,6 +1,7 @@ package config import ( + "encoding/base64" "fmt" "strings" @@ -69,7 +70,32 @@ func (a *Args) String() string { // SerializeOptions returns a string ready to be sent over the wire to the server. func (a *Args) SerializeOptions() string { - return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan) + return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, a.Serverless) } -// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package) +// DeserializeOptions deserializes the options, but into a map. +func DeserializeOptions(opts []string) (map[string]string, error) { + options := make(map[string]string, len(opts)) + + for _, o := range opts { + kv := strings.SplitN(o, "=", 2) + if len(kv) != 2 { + return options, fmt.Errorf("Unable to parse options: %v", kv) + } + key := kv[0] + val := kv[1] + + if strings.HasPrefix(val, "base64%") { + s := strings.SplitN(val, "%", 2) + decoded, err := base64.StdEncoding.DecodeString(s[1]) + if err != nil { + return options, err + } + val = string(decoded) + } + + options[key] = val + } + + return options, nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 6d4730a..09ae994 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,14 +1,5 @@ package config -import ( - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "os" - "strings" -) - const ( // ControlUser is used for various DTail specific operations. ControlUser string = "DTAIL-CONTROL" @@ -33,97 +24,18 @@ var Server *ServerConfig // Common holds common configs of both both, client and server. var Common *CommonConfig -// Used to initialize the configuration. -type configInitializer struct { - Common *CommonConfig - Server *ServerConfig - Client *ClientConfig -} - -func (c *configInitializer) parseConfig(args *Args) { - if strings.ToUpper(args.ConfigFile) == "NONE" { - return - } - - if args.ConfigFile != "" { - c.parseSpecificConfig(args.ConfigFile) - return - } - - if homeDir, err := os.UserHomeDir(); err != nil { - var paths []string - paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir)) - paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir)) - for _, configPath := range paths { - if _, err := os.Stat(configPath); !os.IsNotExist(err) { - c.parseSpecificConfig(configPath) - } - } - } -} - -func (c *configInitializer) parseSpecificConfig(configFile string) { - fd, err := os.Open(configFile) - if err != nil { - panic(fmt.Sprintf("Unable to read config file: %v", err)) - } - defer fd.Close() - - cfgBytes, err := ioutil.ReadAll(fd) - if err != nil { - panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err)) - } - - err = json.Unmarshal([]byte(cfgBytes), c) - if err != nil { - panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err)) - } -} - -func (c *configInitializer) transformConfig(args *Args, additionalArgs []string, - client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { - if args.LogDir != "" { - common.LogDir = args.LogDir - } - if strings.Contains(common.LogDir, "~/") { - homeDir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) - } - if common.LogStrategy == "" { - common.LogStrategy = "daily" - } - - if args.LogLevel != "" { - common.LogLevel = args.LogLevel - } - - if args.SSHPort != DefaultSSHPort { - common.SSHPort = args.SSHPort - } - if args.NoColor { - client.TermColorsEnable = false - } - - if args.Spartan { - args.Quiet = true - args.NoColor = true - } - - if args.Discovery == "" && args.ServersStr == "" { - args.Serverless = true - } - - // Interpret additional args as file list. - if args.What == "" { - var files []string - for _, file := range flag.Args() { - files = append(files, file) - } - args.What = strings.Join(files, ",") - } - - return client, server, common +// Setup the DTail configuration. +func Setup(args *Args, additionalArgs []string) { + initializer := initializer{ + Common: newDefaultCommonConfig(), + Server: newDefaultServerConfig(), + Client: newDefaultClientConfig(), + } + initializer.parseConfig(args) + Client, Server, Common = initializer.transformConfig( + args, additionalArgs, + initializer.Client, + initializer.Server, + initializer.Common, + ) } diff --git a/internal/config/initializer.go b/internal/config/initializer.go new file mode 100644 index 0000000..f1a9ec4 --- /dev/null +++ b/internal/config/initializer.go @@ -0,0 +1,109 @@ +package config + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "strings" +) + +// Used to initialize the configuration. +type initializer struct { + Common *CommonConfig + Server *ServerConfig + Client *ClientConfig +} + +func (c *initializer) parseConfig(args *Args) { + if strings.ToUpper(args.ConfigFile) == "NONE" { + return + } + + if args.ConfigFile != "" { + c.parseSpecificConfig(args.ConfigFile) + return + } + + if homeDir, err := os.UserHomeDir(); err != nil { + var paths []string + paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir)) + paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir)) + for _, configPath := range paths { + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + c.parseSpecificConfig(configPath) + } + } + } +} + +func (c *initializer) parseSpecificConfig(configFile string) { + fd, err := os.Open(configFile) + if err != nil { + panic(fmt.Sprintf("Unable to read config file: %v", err)) + } + defer fd.Close() + + cfgBytes, err := ioutil.ReadAll(fd) + if err != nil { + panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err)) + } + + err = json.Unmarshal([]byte(cfgBytes), c) + if err != nil { + panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err)) + } +} + +func (c *initializer) transformConfig(args *Args, additionalArgs []string, + client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { + if args.LogDir != "" { + common.LogDir = args.LogDir + } + if strings.Contains(common.LogDir, "~/") { + homeDir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) + } + if common.LogStrategy == "" { + common.LogStrategy = "daily" + } + + if args.LogLevel != "" { + common.LogLevel = args.LogLevel + } else if args.ServersStr == "" && args.Discovery == "" { + // We are in serverless mode. Default log level is WARN. + common.LogLevel = "WARN" + } + + if args.SSHPort != DefaultSSHPort { + common.SSHPort = args.SSHPort + } + if args.NoColor { + client.TermColorsEnable = false + } + + if args.Spartan { + args.Quiet = true + args.NoColor = true + } + + if args.Discovery == "" && args.ServersStr == "" { + // We are not connecting to any servers. + args.Serverless = true + } + + // Interpret additional args as file list. + if args.What == "" { + var files []string + for _, file := range flag.Args() { + files = append(files, file) + } + args.What = strings.Join(files, ",") + } + + return client, server, common +} diff --git a/internal/config/setup.go b/internal/config/setup.go deleted file mode 100644 index 7800914..0000000 --- a/internal/config/setup.go +++ /dev/null @@ -1,17 +0,0 @@ -package config - -// Setup the DTail configuration. -func Setup(args *Args, additionalArgs []string) { - initializer := configInitializer{ - Common: newDefaultCommonConfig(), - Server: newDefaultServerConfig(), - Client: newDefaultClientConfig(), - } - initializer.parseConfig(args) - Client, Server, Common = initializer.transformConfig( - args, additionalArgs, - initializer.Client, - initializer.Server, - initializer.Common, - ) -} -- cgit v1.2.3 From 86ec83754e0ee7153ad55091f7b6da448bc529c5 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 13:44:27 +0300 Subject: add dcat test --- internal/config/common.go | 2 +- internal/config/initializer.go | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) (limited to 'internal/config') diff --git a/internal/config/common.go b/internal/config/common.go index acc8e6a..255bd28 100644 --- a/internal/config/common.go +++ b/internal/config/common.go @@ -6,7 +6,7 @@ type CommonConfig struct { SSHPort int // Enable experimental features (mainly for dev purposes) ExperimentalFeaturesEnable bool `json:",omitempty"` - // LogLevel defines how much is logged. TODO: Adjust JSONschema + // LogLevel defines how much is logged. LogLevel string `json:",omitempty"` // The log strategy to use, one of // stdout: only log to stdout (useful when used with systemd) diff --git a/internal/config/initializer.go b/internal/config/initializer.go index f1a9ec4..0e725a6 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -72,6 +72,17 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string, common.LogStrategy = "daily" } + if args.Spartan { + args.Quiet = true + args.NoColor = true + if args.LogLevel == "" { + args.LogLevel = "ERROR" + } + } + if args.NoColor { + client.TermColorsEnable = false + } + if args.LogLevel != "" { common.LogLevel = args.LogLevel } else if args.ServersStr == "" && args.Discovery == "" { @@ -82,14 +93,6 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string, if args.SSHPort != DefaultSSHPort { common.SSHPort = args.SSHPort } - if args.NoColor { - client.TermColorsEnable = false - } - - if args.Spartan { - args.Quiet = true - args.NoColor = true - } if args.Discovery == "" && args.ServersStr == "" { // We are not connecting to any servers. -- cgit v1.2.3 From 07e1470892beacf0722276f94bacbd822b002540 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Oct 2021 13:09:32 +0300 Subject: add dmap tests --- internal/config/common.go | 1 + internal/config/config.go | 5 ++++- internal/config/initializer.go | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) (limited to 'internal/config') diff --git a/internal/config/common.go b/internal/config/common.go index 255bd28..5e81bc9 100644 --- a/internal/config/common.go +++ b/internal/config/common.go @@ -26,6 +26,7 @@ func newDefaultCommonConfig() *CommonConfig { SSHPort: DefaultSSHPort, ExperimentalFeaturesEnable: false, LogDir: "log", + LogLevel: "INFO", CacheDir: "cache", TmpDir: "/tmp", } diff --git a/internal/config/config.go b/internal/config/config.go index 09ae994..d58162f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,5 +1,7 @@ package config +import "github.com/mimecast/dtail/internal/source" + const ( // ControlUser is used for various DTail specific operations. ControlUser string = "DTAIL-CONTROL" @@ -25,7 +27,7 @@ var Server *ServerConfig var Common *CommonConfig // Setup the DTail configuration. -func Setup(args *Args, additionalArgs []string) { +func Setup(sourceProcess source.Source, args *Args, additionalArgs []string) { initializer := initializer{ Common: newDefaultCommonConfig(), Server: newDefaultServerConfig(), @@ -33,6 +35,7 @@ func Setup(args *Args, additionalArgs []string) { } initializer.parseConfig(args) Client, Server, Common = initializer.transformConfig( + sourceProcess, args, additionalArgs, initializer.Client, initializer.Server, diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 0e725a6..a58f82a 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "os" "strings" + + "github.com/mimecast/dtail/internal/source" ) // Used to initialize the configuration. @@ -56,7 +58,7 @@ func (c *initializer) parseSpecificConfig(configFile string) { } } -func (c *initializer) transformConfig(args *Args, additionalArgs []string, +func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { if args.LogDir != "" { common.LogDir = args.LogDir @@ -85,7 +87,7 @@ func (c *initializer) transformConfig(args *Args, additionalArgs []string, if args.LogLevel != "" { common.LogLevel = args.LogLevel - } else if args.ServersStr == "" && args.Discovery == "" { + } else if sourceProcess == source.Client && args.ServersStr == "" && args.Discovery == "" { // We are in serverless mode. Default log level is WARN. common.LogLevel = "WARN" } -- cgit v1.2.3 From 9f395a03f25941d8ed98ec43035688daa1e8877f Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Oct 2021 22:39:58 +0300 Subject: more on this --- internal/config/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 152ab15..ecd05c5 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -162,8 +162,8 @@ func newDefaultClientConfig() *ClientConfig { HostnameBg: color.BgCyan, HostnameFg: color.FgBlack, TextAttr: color.AttrNone, - TextBg: color.BgCyan, - TextFg: color.FgBlack, + TextBg: color.BgBlack, + TextFg: color.FgWhite, }, Common: commonTermColors{ SeverityErrorAttr: color.AttrBold, -- cgit v1.2.3 From fab5dc3e70434ea0abc7a0976487a1973b662331 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 6 Oct 2021 09:50:41 +0300 Subject: enable faster shutdown - useful for dgrep/dmap and dcat commands --- internal/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'internal/config') diff --git a/internal/config/config.go b/internal/config/config.go index d58162f..f216688 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,8 +3,8 @@ package config import "github.com/mimecast/dtail/internal/source" const ( - // ControlUser is used for various DTail specific operations. - ControlUser string = "DTAIL-CONTROL" + // HealthUser is used for the health check + HealthUser string = "DTAIL-HEALTH" // ScheduleUser is used for non-interactive scheduled mapreduce queries. ScheduleUser string = "DTAIL-SCHEDULE" // ContinuousUser is used for non-interactive continuous mapreduce queries. -- cgit v1.2.3 From 7306afe9ab073c424ddca0ddc57950f237948118 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 6 Oct 2021 10:55:50 +0300 Subject: move health check to separate client binary --- internal/config/initializer.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'internal/config') diff --git a/internal/config/initializer.go b/internal/config/initializer.go index a58f82a..ec758c8 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -96,11 +96,19 @@ func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, a common.SSHPort = args.SSHPort } - if args.Discovery == "" && args.ServersStr == "" { + if args.Discovery == "" && (args.ServersStr == "" || + strings.ToLower(args.ServersStr) == "serverless") { // We are not connecting to any servers. args.Serverless = true } + if sourceProcess == source.HealthCheck { + args.TrustAllHosts = true + if !args.Serverless && strings.ToLower(args.ServersStr) == "" { + args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort) + } + } + // Interpret additional args as file list. if args.What == "" { var files []string -- cgit v1.2.3 From 2d7ddbeae8286373ac19787dc7dde598a7cb0598 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 8 Oct 2021 11:43:43 +0300 Subject: refactor --- internal/config/args.go | 4 ++ internal/config/common.go | 14 +++-- internal/config/config.go | 27 ++++++--- internal/config/initializer.go | 131 ++++++++++++++++++++++++++--------------- 4 files changed, 114 insertions(+), 62 deletions(-) (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go index 3e2eb1f..a671ae3 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -17,10 +17,12 @@ type Args struct { ConnectionsPerCPU int Discovery string LogDir string + Logger string LogLevel string Mode omode.Mode NoColor bool PrivateKeyPathFile string + QueryStr string Quiet bool RegexInvert bool RegexStr string @@ -42,6 +44,7 @@ func (a *Args) String() string { sb.WriteString("Args(") sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir)) + sb.WriteString(fmt.Sprintf("%s:%s,", "Logger", a.Logger)) sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile)) @@ -50,6 +53,7 @@ func (a *Args) String() string { sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode)) sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor)) sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile)) + sb.WriteString(fmt.Sprintf("%s:%v,", "QueryStr", a.QueryStr)) sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet)) sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert)) sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr)) diff --git a/internal/config/common.go b/internal/config/common.go index 5e81bc9..9d22c95 100644 --- a/internal/config/common.go +++ b/internal/config/common.go @@ -6,14 +6,14 @@ type CommonConfig struct { SSHPort int // Enable experimental features (mainly for dev purposes) ExperimentalFeaturesEnable bool `json:",omitempty"` + // LogDir defines the log directory. + LogDir string + // Logger defines the name of the logger implementation. + Logger string // LogLevel defines how much is logged. LogLevel string `json:",omitempty"` - // The log strategy to use, one of - // stdout: only log to stdout (useful when used with systemd) - // daily: create a log file for every day + // LogStrategy defines the log rotation strategy. LogStrategy string - // The log directory - LogDir string // The cache directory CacheDir string // The temp directory @@ -26,7 +26,9 @@ func newDefaultCommonConfig() *CommonConfig { SSHPort: DefaultSSHPort, ExperimentalFeaturesEnable: false, LogDir: "log", - LogLevel: "INFO", + Logger: "stdout", + LogLevel: DefaultLogLevel, + LogStrategy: "daily", CacheDir: "cache", TmpDir: "/tmp", } diff --git a/internal/config/config.go b/internal/config/config.go index f216688..077a658 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,6 +15,14 @@ const ( DefaultConnectionsPerCPU int = 10 // DTailSSHServerDefaultPort is the default DServer port. DefaultSSHPort int = 2222 + // DefaultLogLevel specifies the default log level (obviously) + DefaultLogLevel string = "INFO" + // DefaultClientLogger specifies the default logger for the client commands. + DefaultClientLogger string = "fout" + // DefaultServerLogger specifies the default logger for dtail server. + DefaultServerLogger string = "file" + // DefaultHealthCheckLogger specifies the default logger used for health checks. + DefaultHealthCheckLogger string = "none" ) // Client holds a DTail client configuration. @@ -33,12 +41,15 @@ func Setup(sourceProcess source.Source, args *Args, additionalArgs []string) { Server: newDefaultServerConfig(), Client: newDefaultClientConfig(), } - initializer.parseConfig(args) - Client, Server, Common = initializer.transformConfig( - sourceProcess, - args, additionalArgs, - initializer.Client, - initializer.Server, - initializer.Common, - ) + if err := initializer.parseConfig(args); err != nil { + panic(err) + } + if err := initializer.transformConfig(sourceProcess, args, additionalArgs); err != nil { + panic(err) + } + + // Make config accessible globally + Server = initializer.Server + Client = initializer.Client + Common = initializer.Common } diff --git a/internal/config/initializer.go b/internal/config/initializer.go index ec758c8..5247699 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -18,14 +18,15 @@ type initializer struct { Client *ClientConfig } -func (c *initializer) parseConfig(args *Args) { +type transformCb func(*initializer, *Args, []string) error + +func (c *initializer) parseConfig(args *Args) error { if strings.ToUpper(args.ConfigFile) == "NONE" { - return + return nil } if args.ConfigFile != "" { - c.parseSpecificConfig(args.ConfigFile) - return + return c.parseSpecificConfig(args.ConfigFile) } if homeDir, err := os.UserHomeDir(); err != nil { @@ -38,85 +39,119 @@ func (c *initializer) parseConfig(args *Args) { } } } + + return nil } -func (c *initializer) parseSpecificConfig(configFile string) { +func (c *initializer) parseSpecificConfig(configFile string) error { fd, err := os.Open(configFile) if err != nil { - panic(fmt.Sprintf("Unable to read config file: %v", err)) + return fmt.Errorf("Unable to read config file: %v", err) } defer fd.Close() cfgBytes, err := ioutil.ReadAll(fd) if err != nil { - panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err)) + return fmt.Errorf("Unable to read config file %s: %v", configFile, err) } - err = json.Unmarshal([]byte(cfgBytes), c) - if err != nil { - panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err)) + if err := json.Unmarshal([]byte(cfgBytes), c); err != nil { + return fmt.Errorf("Unable to parse config file %s: %v", configFile, err) } + + return nil } -func (c *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string, - client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { - if args.LogDir != "" { - common.LogDir = args.LogDir - } - if strings.Contains(common.LogDir, "~/") { - homeDir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) - } - if common.LogStrategy == "" { - common.LogStrategy = "daily" +func (i *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string) error { + + switch sourceProcess { + case source.Server: + return i.optimusPrime(transformServer, args, additionalArgs) + case source.Client: + return i.optimusPrime(transformClient, args, additionalArgs) + case source.HealthCheck: + return i.optimusPrime(transformHealthCheck, args, additionalArgs) + default: + return fmt.Errorf("Unable to transform config, unknown source '%s'", sourceProcess) } +} - if args.Spartan { - args.Quiet = true - args.NoColor = true - if args.LogLevel == "" { - args.LogLevel = "ERROR" - } +func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalArgs []string) error { + // Copy args to config objects. + if args.SSHPort != DefaultSSHPort { + i.Common.SSHPort = args.SSHPort + } + if args.LogLevel != DefaultLogLevel { + i.Common.LogLevel = args.LogLevel } if args.NoColor { - client.TermColorsEnable = false + i.Client.TermColorsEnable = false } - - if args.LogLevel != "" { - common.LogLevel = args.LogLevel - } else if sourceProcess == source.Client && args.ServersStr == "" && args.Discovery == "" { - // We are in serverless mode. Default log level is WARN. - common.LogLevel = "WARN" + if args.LogDir != "" { + i.Common.LogDir = args.LogDir + } + if args.Logger != "" { + i.Common.Logger = args.Logger } - if args.SSHPort != DefaultSSHPort { - common.SSHPort = args.SSHPort + // Setup log directory. + if strings.Contains(i.Common.LogDir, "~/") { + homeDir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) } + // Serverless mode. if args.Discovery == "" && (args.ServersStr == "" || strings.ToLower(args.ServersStr) == "serverless") { // We are not connecting to any servers. args.Serverless = true + i.Common.LogLevel = "WARN" } - if sourceProcess == source.HealthCheck { - args.TrustAllHosts = true - if !args.Serverless && strings.ToLower(args.ServersStr) == "" { - args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort) + // Source type specific transormations. + sourceCb(i, args, additionalArgs) + + // Spartan mode. + if args.Spartan { + args.Quiet = true + args.NoColor = true + i.Client.TermColorsEnable = false + if args.LogLevel == "" { + args.LogLevel = "ERROR" + i.Common.LogLevel = "ERROR" } } - - // Interpret additional args as file list. + // Interpret additional args as file list or as query. if args.What == "" { var files []string - for _, file := range flag.Args() { - files = append(files, file) + for _, arg := range flag.Args() { + if args.QueryStr == "" && strings.Contains(strings.ToLower(arg), "select ") { + args.QueryStr = arg + continue + } + files = append(files, arg) } args.What = strings.Join(files, ",") } - return client, server, common + return nil +} + +func transformClient(i *initializer, args *Args, additionalArgs []string) error { + return nil +} + +func transformServer(i *initializer, args *Args, additionalArgs []string) error { + return nil +} + +func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error { + args.TrustAllHosts = true + if !args.Serverless && args.ServersStr == "" { + args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort) + } + return nil } -- cgit v1.2.3 From 7a7169791a64190e1002e38bc9c04ad0d5c1ce1f Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 16:44:28 +0300 Subject: add dtail health check unit test. --- internal/config/initializer.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) (limited to 'internal/config') diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 5247699..e4cbeaf 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -93,6 +93,9 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA if args.Logger != "" { i.Common.Logger = args.Logger } + if args.ConnectionsPerCPU == 0 { + args.ConnectionsPerCPU = DefaultConnectionsPerCPU + } // Setup log directory. if strings.Contains(i.Common.LogDir, "~/") { @@ -103,14 +106,6 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) } - // Serverless mode. - if args.Discovery == "" && (args.ServersStr == "" || - strings.ToLower(args.ServersStr) == "serverless") { - // We are not connecting to any servers. - args.Serverless = true - i.Common.LogLevel = "WARN" - } - // Source type specific transormations. sourceCb(i, args, additionalArgs) @@ -141,6 +136,14 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA } func transformClient(i *initializer, args *Args, additionalArgs []string) error { + // Serverless mode. + if args.Discovery == "" && (args.ServersStr == "" || + strings.ToLower(args.ServersStr) == "serverless") { + // We are not connecting to any servers. + args.Serverless = true + i.Common.LogLevel = "warn" + } + return nil } @@ -149,9 +152,13 @@ func transformServer(i *initializer, args *Args, additionalArgs []string) error } func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error { - args.TrustAllHosts = true - if !args.Serverless && args.ServersStr == "" { - args.ServersStr = fmt.Sprintf("localhost:%d", DefaultSSHPort) + // Serverless mode. + if args.Discovery == "" && (args.ServersStr == "" || + strings.ToLower(args.ServersStr) == "serverless") { + // We are not connecting to any servers. + args.Serverless = true + i.Common.LogLevel = "warn" } + args.TrustAllHosts = true return nil } -- cgit v1.2.3 From 97747ea0f3178f7f5890512d483fdccaa82846b0 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 21:10:29 +0300 Subject: vetting and linting and some code restyling --- internal/config/args.go | 6 ++--- internal/config/client.go | 12 ++++----- internal/config/config.go | 6 ++--- internal/config/initializer.go | 59 +++++++++++++++++++++++------------------- internal/config/server.go | 7 ++--- 5 files changed, 45 insertions(+), 45 deletions(-) (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go index a671ae3..f721390 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -74,13 +74,13 @@ func (a *Args) String() string { // SerializeOptions returns a string ready to be sent over the wire to the server. func (a *Args) SerializeOptions() string { - return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, a.Serverless) + return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, + a.Serverless) } // DeserializeOptions deserializes the options, but into a map. func DeserializeOptions(opts []string) (map[string]string, error) { options := make(map[string]string, len(opts)) - for _, o := range opts { kv := strings.SplitN(o, "=", 2) if len(kv) != 2 { @@ -97,9 +97,7 @@ func DeserializeOptions(opts []string) (map[string]string, error) { } val = string(decoded) } - options[key] = val } - return options, nil } diff --git a/internal/config/client.go b/internal/config/client.go index ecd05c5..8227c68 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -15,9 +15,9 @@ type remoteTermColors struct { HostnameAttr color.Attribute HostnameBg color.BgColor HostnameFg color.FgColor - IdAttr color.Attribute - IdBg color.BgColor - IdFg color.FgColor + IDAttr color.Attribute + IDBg color.BgColor + IDFg color.FgColor StatsOkAttr color.Attribute StatsOkBg color.BgColor StatsOkFg color.FgColor @@ -124,9 +124,9 @@ func newDefaultClientConfig() *ClientConfig { HostnameAttr: color.AttrBold, HostnameBg: color.BgBlue, HostnameFg: color.FgWhite, - IdAttr: color.AttrDim, - IdBg: color.BgBlue, - IdFg: color.FgWhite, + IDAttr: color.AttrDim, + IDBg: color.BgBlue, + IDFg: color.FgWhite, StatsOkAttr: color.AttrNone, StatsOkBg: color.BgGreen, StatsOkFg: color.FgBlack, diff --git a/internal/config/config.go b/internal/config/config.go index 077a658..b99b22b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,11 +9,11 @@ const ( ScheduleUser string = "DTAIL-SCHEDULE" // ContinuousUser is used for non-interactive continuous mapreduce queries. ContinuousUser string = "DTAIL-CONTINUOUS" - // InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval. + // InterruptTimeoutS specifies the Ctrl+C log pause interval. InterruptTimeoutS int = 3 - // ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start) + // DefaultConnectionsPerCPU controls how many connections are established concurrently. DefaultConnectionsPerCPU int = 10 - // DTailSSHServerDefaultPort is the default DServer port. + // DefaultSSHPort is the default DServer port. DefaultSSHPort int = 2222 // DefaultLogLevel specifies the default log level (obviously) DefaultLogLevel string = "INFO" diff --git a/internal/config/initializer.go b/internal/config/initializer.go index e4cbeaf..0a913db 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -20,13 +20,13 @@ type initializer struct { type transformCb func(*initializer, *Args, []string) error -func (c *initializer) parseConfig(args *Args) error { +func (in *initializer) parseConfig(args *Args) error { if strings.ToUpper(args.ConfigFile) == "NONE" { return nil } if args.ConfigFile != "" { - return c.parseSpecificConfig(args.ConfigFile) + return in.parseSpecificConfig(args.ConfigFile) } if homeDir, err := os.UserHomeDir(); err != nil { @@ -35,7 +35,7 @@ func (c *initializer) parseConfig(args *Args) error { paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir)) for _, configPath := range paths { if _, err := os.Stat(configPath); !os.IsNotExist(err) { - c.parseSpecificConfig(configPath) + in.parseSpecificConfig(configPath) } } } @@ -43,7 +43,7 @@ func (c *initializer) parseConfig(args *Args) error { return nil } -func (c *initializer) parseSpecificConfig(configFile string) error { +func (in *initializer) parseSpecificConfig(configFile string) error { fd, err := os.Open(configFile) if err != nil { return fmt.Errorf("Unable to read config file: %v", err) @@ -55,68 +55,74 @@ func (c *initializer) parseSpecificConfig(configFile string) error { return fmt.Errorf("Unable to read config file %s: %v", configFile, err) } - if err := json.Unmarshal([]byte(cfgBytes), c); err != nil { + if err := json.Unmarshal([]byte(cfgBytes), in); err != nil { return fmt.Errorf("Unable to parse config file %s: %v", configFile, err) } return nil } -func (i *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string) error { +func (in *initializer) transformConfig(sourceProcess source.Source, args *Args, + additionalArgs []string) error { switch sourceProcess { case source.Server: - return i.optimusPrime(transformServer, args, additionalArgs) + return in.optimusPrime(transformServer, args, additionalArgs) case source.Client: - return i.optimusPrime(transformClient, args, additionalArgs) + return in.optimusPrime(transformClient, args, additionalArgs) case source.HealthCheck: - return i.optimusPrime(transformHealthCheck, args, additionalArgs) + return in.optimusPrime(transformHealthCheck, args, additionalArgs) default: - return fmt.Errorf("Unable to transform config, unknown source '%s'", sourceProcess) + return fmt.Errorf("Unable to transform config, unknown source '%s'", + sourceProcess) } } -func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalArgs []string) error { +func (in *initializer) optimusPrime(sourceCb transformCb, args *Args, + additionalArgs []string) error { + // Copy args to config objects. + // NEXT: Maybe unify args and config structs? if args.SSHPort != DefaultSSHPort { - i.Common.SSHPort = args.SSHPort + in.Common.SSHPort = args.SSHPort } if args.LogLevel != DefaultLogLevel { - i.Common.LogLevel = args.LogLevel + in.Common.LogLevel = args.LogLevel } if args.NoColor { - i.Client.TermColorsEnable = false + in.Client.TermColorsEnable = false } if args.LogDir != "" { - i.Common.LogDir = args.LogDir + in.Common.LogDir = args.LogDir } if args.Logger != "" { - i.Common.Logger = args.Logger + in.Common.Logger = args.Logger } if args.ConnectionsPerCPU == 0 { args.ConnectionsPerCPU = DefaultConnectionsPerCPU } // Setup log directory. - if strings.Contains(i.Common.LogDir, "~/") { + if strings.Contains(in.Common.LogDir, "~/") { homeDir, err := os.UserHomeDir() if err != nil { panic(err) } - i.Common.LogDir = strings.ReplaceAll(i.Common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) + in.Common.LogDir = strings.ReplaceAll(in.Common.LogDir, "~/", + fmt.Sprintf("%s/", homeDir)) } // Source type specific transormations. - sourceCb(i, args, additionalArgs) + sourceCb(in, args, additionalArgs) // Spartan mode. if args.Spartan { args.Quiet = true args.NoColor = true - i.Client.TermColorsEnable = false + in.Client.TermColorsEnable = false if args.LogLevel == "" { args.LogLevel = "ERROR" - i.Common.LogLevel = "ERROR" + in.Common.LogLevel = "ERROR" } } // Interpret additional args as file list or as query. @@ -135,29 +141,28 @@ func (i *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalA return nil } -func transformClient(i *initializer, args *Args, additionalArgs []string) error { +func transformClient(in *initializer, args *Args, additionalArgs []string) error { // Serverless mode. if args.Discovery == "" && (args.ServersStr == "" || strings.ToLower(args.ServersStr) == "serverless") { // We are not connecting to any servers. args.Serverless = true - i.Common.LogLevel = "warn" + in.Common.LogLevel = "warn" } - return nil } -func transformServer(i *initializer, args *Args, additionalArgs []string) error { +func transformServer(in *initializer, args *Args, additionalArgs []string) error { return nil } -func transformHealthCheck(i *initializer, args *Args, additionalArgs []string) error { +func transformHealthCheck(in *initializer, args *Args, additionalArgs []string) error { // Serverless mode. if args.Discovery == "" && (args.ServersStr == "" || strings.ToLower(args.ServersStr) == "serverless") { // We are not connecting to any servers. args.Serverless = true - i.Common.LogLevel = "warn" + in.Common.LogLevel = "warn" } args.TrustAllHosts = true return nil diff --git a/internal/config/server.go b/internal/config/server.go index 8bbb394..677f5ac 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -4,8 +4,8 @@ import ( "errors" ) -// Permissions map. Each SSH user has a list of permissions which -// log files it is allowed to follow and which ones not. +// Permissions map. Each SSH user has a list of permissions which log files it +// is allowed to follow and which ones not. type Permissions struct { // The default user permissions. Default []string @@ -68,7 +68,6 @@ var ServerRelaxedAuthEnable bool func newDefaultServerConfig() *ServerConfig { defaultPermissions := []string{"^/.*"} defaultBindAddress := "0.0.0.0" - return &ServerConfig{ SSHBindAddress: defaultBindAddress, MaxConnections: 10, @@ -89,10 +88,8 @@ func ServerUserPermissions(userName string) (permissions []string, err error) { if p, ok := Server.Permissions.Users[userName]; ok { permissions = p } - if len(permissions) == 0 { err = errors.New("Empty set of permission, user won't be able to open any files") } - return } -- cgit v1.2.3 From f44792c9102488774c9993b080f35c65287a64b1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 10 Oct 2021 14:02:12 +0300 Subject: add another dmap test - reading 100 source files at once fix a data race when reading multiple files on one server from the same session at once --- internal/config/config.go | 2 +- internal/config/initializer.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'internal/config') diff --git a/internal/config/config.go b/internal/config/config.go index b99b22b..ee23829 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,7 +16,7 @@ const ( // DefaultSSHPort is the default DServer port. DefaultSSHPort int = 2222 // DefaultLogLevel specifies the default log level (obviously) - DefaultLogLevel string = "INFO" + DefaultLogLevel string = "info" // DefaultClientLogger specifies the default logger for the client commands. DefaultClientLogger string = "fout" // DefaultServerLogger specifies the default logger for dtail server. diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 0a913db..0c6dfdf 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -147,7 +147,9 @@ func transformClient(in *initializer, args *Args, additionalArgs []string) error strings.ToLower(args.ServersStr) == "serverless") { // We are not connecting to any servers. args.Serverless = true - in.Common.LogLevel = "warn" + if args.LogLevel == DefaultLogLevel { + in.Common.LogLevel = "warn" + } } return nil } -- cgit v1.2.3 From a6098084f7150df34edecf1519386bd28a527361 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 11 Oct 2021 17:42:37 +0300 Subject: Update JSON-schema to reflect all recent config file changes. --- internal/config/client.go | 2 +- internal/config/common.go | 9 +++------ internal/config/server.go | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 8227c68..9f4df97 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -102,7 +102,7 @@ type termColors struct { // ClientConfig represents a DTail client configuration (empty as of now as there // are no available config options yet, but that may changes in the future). type ClientConfig struct { - TermColorsEnable bool + TermColorsEnable bool `json:",omitempty"` TermColors termColors `json:",omitempty"` } diff --git a/internal/config/common.go b/internal/config/common.go index 9d22c95..7a72cfe 100644 --- a/internal/config/common.go +++ b/internal/config/common.go @@ -12,12 +12,10 @@ type CommonConfig struct { Logger string // LogLevel defines how much is logged. LogLevel string `json:",omitempty"` - // LogStrategy defines the log rotation strategy. - LogStrategy string + // LogRotation strategy to be used. + LogRotation string // The cache directory CacheDir string - // The temp directory - TmpDir string `json:",omitempty"` } // Create a new default configuration. @@ -28,8 +26,7 @@ func newDefaultCommonConfig() *CommonConfig { LogDir: "log", Logger: "stdout", LogLevel: DefaultLogLevel, - LogStrategy: "daily", + LogRotation: "daily", CacheDir: "cache", - TmpDir: "/tmp", } } diff --git a/internal/config/server.go b/internal/config/server.go index 677f5ac..254ea0c 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -47,7 +47,7 @@ type ServerConfig struct { MaxConcurrentCats int // The max amount of concurrent tails per server. MaxConcurrentTails int - // The user permissions. + // The user permissions. TODO: Add to JSON schema Permissions Permissions `json:",omitempty"` // The mapr log format MapreduceLogFormat string `json:",omitempty"` -- cgit v1.2.3 From 1dead22129a26e4f532e68c2c63fe4122b519506 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 13 Oct 2021 21:10:28 +0300 Subject: Merging grep context from master --- internal/config/args.go | 73 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 7 deletions(-) (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go index f721390..f94e0a9 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -3,8 +3,10 @@ package config import ( "encoding/base64" "fmt" + "strconv" "strings" + "github.com/mimecast/dtail/internal/lcontext" "github.com/mimecast/dtail/internal/omode" gossh "golang.org/x/crypto/ssh" @@ -12,6 +14,7 @@ import ( // Args is a helper struct to summarize common client arguments. type Args struct { + lcontext.LContext Arguments []string ConfigFile string ConnectionsPerCPU int @@ -74,17 +77,50 @@ func (a *Args) String() string { // SerializeOptions returns a string ready to be sent over the wire to the server. func (a *Args) SerializeOptions() string { - return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, - a.Serverless) + options := make(map[string]string) + + if a.Quiet { + options["quiet"] = fmt.Sprintf("%v", a.Quiet) + } + if a.Spartan { + options["spartan"] = fmt.Sprintf("%v", a.Spartan) + } + if a.Serverless { + options["serverless"] = fmt.Sprintf("%v", a.Serverless) + } + if a.LContext.MaxCount != 0 { + options["max"] = fmt.Sprintf("%d", a.LContext.MaxCount) + } + if a.LContext.BeforeContext != 0 { + options["before"] = fmt.Sprintf("%d", a.LContext.BeforeContext) + } + if a.LContext.AfterContext != 0 { + options["after"] = fmt.Sprintf("%d", a.LContext.AfterContext) + } + + var sb strings.Builder + var i int + for k, v := range options { + if i > 0 { + sb.WriteString(":") + } + sb.WriteString(k) + sb.WriteString("=") + sb.WriteString(v) + i++ + } + return sb.String() } // DeserializeOptions deserializes the options, but into a map. -func DeserializeOptions(opts []string) (map[string]string, error) { +func DeserializeOptions(opts []string) (map[string]string, lcontext.LContext, error) { options := make(map[string]string, len(opts)) + var ltx lcontext.LContext + for _, o := range opts { kv := strings.SplitN(o, "=", 2) if len(kv) != 2 { - return options, fmt.Errorf("Unable to parse options: %v", kv) + return options, ltx, fmt.Errorf("Unable to parse options: %v", kv) } key := kv[0] val := kv[1] @@ -93,11 +129,34 @@ func DeserializeOptions(opts []string) (map[string]string, error) { s := strings.SplitN(val, "%", 2) decoded, err := base64.StdEncoding.DecodeString(s[1]) if err != nil { - return options, err + return options, ltx, err } val = string(decoded) } - options[key] = val + + switch key { + case "before": + iVal, err := strconv.Atoi(val) + if err != nil { + return options, ltx, err + } + ltx.BeforeContext = iVal + case "after": + iVal, err := strconv.Atoi(val) + if err != nil { + return options, ltx, err + } + ltx.AfterContext = iVal + case "max": + iVal, err := strconv.Atoi(val) + if err != nil { + return options, ltx, err + } + ltx.MaxCount = iVal + default: + options[key] = val + } } - return options, nil + + return options, ltx, nil } -- cgit v1.2.3 From b27fc108ecd6eead5c97cf6e894bf8d639fff75c Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Oct 2021 13:06:18 +0300 Subject: Execute test directories individually --- internal/config/args.go | 14 ++++++++------ internal/config/initializer.go | 3 +++ 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'internal/config') diff --git a/internal/config/args.go b/internal/config/args.go index f94e0a9..3d7ac7d 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -33,6 +33,7 @@ type Args struct { ServersStr string Spartan bool SSHAuthMethods []gossh.AuthMethod + SSHBindAddress string SSHHostKeyCallback gossh.HostKeyCallback SSHPort int Timeout int @@ -46,13 +47,13 @@ func (a *Args) String() string { sb.WriteString("Args(") - sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir)) - sb.WriteString(fmt.Sprintf("%s:%s,", "Logger", a.Logger)) - sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile)) sb.WriteString(fmt.Sprintf("%s:%v,", "ConnectionsPerCPU", a.ConnectionsPerCPU)) sb.WriteString(fmt.Sprintf("%s:%v,", "Discovery", a.Discovery)) + sb.WriteString(fmt.Sprintf("%s:%v,", "LogDir", a.LogDir)) + sb.WriteString(fmt.Sprintf("%s:%v,", "LogLevel", a.LogLevel)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Logger", a.Logger)) sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode)) sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor)) sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile)) @@ -60,12 +61,13 @@ func (a *Args) String() string { sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet)) sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert)) sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr)) - sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless)) - sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr)) - sb.WriteString(fmt.Sprintf("%s:%v,", "Spartan", a.Spartan)) sb.WriteString(fmt.Sprintf("%s:%v,", "SSHAuthMethods", a.SSHAuthMethods)) + sb.WriteString(fmt.Sprintf("%s:%v,", "SSHBindAddress", a.SSHBindAddress)) sb.WriteString(fmt.Sprintf("%s:%v,", "SSHHostKeyCallback", a.SSHHostKeyCallback)) sb.WriteString(fmt.Sprintf("%s:%v,", "SSHPort", a.SSHPort)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless)) + sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Spartan", a.Spartan)) sb.WriteString(fmt.Sprintf("%s:%v,", "Timeout", a.Timeout)) sb.WriteString(fmt.Sprintf("%s:%v,", "TrustAllHosts", a.TrustAllHosts)) sb.WriteString(fmt.Sprintf("%s:%v,", "UserName", a.UserName)) diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 0c6dfdf..8215891 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -155,6 +155,9 @@ func transformClient(in *initializer, args *Args, additionalArgs []string) error } func transformServer(in *initializer, args *Args, additionalArgs []string) error { + if args.SSHBindAddress != "" { + in.Server.SSHBindAddress = args.SSHBindAddress + } return nil } -- cgit v1.2.3 From 10314cef906fd9b73e003be69c2f6b7b3d66570c Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Oct 2021 13:20:48 +0300 Subject: Can configure DTail client not to mess with ~/.ssh/known_hosts via env var - this is useful for running unit and integration tests in jenkins --- internal/config/client.go | 3 +++ internal/config/initializer.go | 10 ++++++++++ 2 files changed, 13 insertions(+) (limited to 'internal/config') diff --git a/internal/config/client.go b/internal/config/client.go index 9f4df97..86f97f0 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -104,6 +104,9 @@ type termColors struct { type ClientConfig struct { TermColorsEnable bool `json:",omitempty"` TermColors termColors `json:",omitempty"` + // When unit testing in Jenkins you don't want to touch files in ~jenkins + // during integration tests really. + SSHDontAddHostsToKnownHostsFile bool `json:",omitempty"` } // Create a new default client configuration. diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 8215891..35105bf 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -65,6 +65,8 @@ func (in *initializer) parseSpecificConfig(configFile string) error { func (in *initializer) transformConfig(sourceProcess source.Source, args *Args, additionalArgs []string) error { + in.readEnvironmentVars() + switch sourceProcess { case source.Server: return in.optimusPrime(transformServer, args, additionalArgs) @@ -78,6 +80,14 @@ func (in *initializer) transformConfig(sourceProcess source.Source, args *Args, } } +// There are some special options which can be set by environment variable. +func (in *initializer) readEnvironmentVars() { + if len(os.Getenv("DTAIL_SSH_DONT_ADD_HOSTS_TO_KNOWNHOSTS_FILE")) != 0 || + len(os.Getenv("DTAIL_JENKINS")) != 0 { + in.Client.SSHDontAddHostsToKnownHostsFile = true + } +} + func (in *initializer) optimusPrime(sourceCb transformCb, args *Args, additionalArgs []string) error { -- cgit v1.2.3 From 860c41441b3bcf542b5701ae2257812879ce47b4 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 19 Oct 2021 19:01:12 +0300 Subject: Set DTAIL_RUN_INTEGRATIONT_TEST to yes for integration tests --- internal/config/env.go | 7 +++++++ internal/config/initializer.go | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 internal/config/env.go (limited to 'internal/config') diff --git a/internal/config/env.go b/internal/config/env.go new file mode 100644 index 0000000..88b831d --- /dev/null +++ b/internal/config/env.go @@ -0,0 +1,7 @@ +package config + +import "os" + +func Env(env string) bool { + return "yes" == os.Getenv(env) +} diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 35105bf..4d6a73b 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -82,8 +82,8 @@ func (in *initializer) transformConfig(sourceProcess source.Source, args *Args, // There are some special options which can be set by environment variable. func (in *initializer) readEnvironmentVars() { - if len(os.Getenv("DTAIL_SSH_DONT_ADD_HOSTS_TO_KNOWNHOSTS_FILE")) != 0 || - len(os.Getenv("DTAIL_JENKINS")) != 0 { + if Env("DTAIL_SSH_DONT_ADD_HOSTS_TO_KNOWNHOSTS_FILE") || + Env("DTAIL_JENKINS") { in.Client.SSHDontAddHostsToKnownHostsFile = true } } -- cgit v1.2.3