From f46abf36e80a67d13d12dbe4ba8c899026e53961 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 31 Jul 2021 18:20:03 +0300 Subject: more on configurable colors --- cmd/dmap/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index 1a05549..8a38069 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -6,7 +6,6 @@ import ( "os" "github.com/mimecast/dtail/internal/clients" - "github.com/mimecast/dtail/internal/color" "github.com/mimecast/dtail/internal/config" "github.com/mimecast/dtail/internal/io/logger" "github.com/mimecast/dtail/internal/io/signal" @@ -49,7 +48,9 @@ func main() { flag.Parse() config.Read(cfgFile, sshPort) - color.Colored = !noColor + if noColor { + config.Client.TermColorsEnabled = false + } if displayVersion { version.PrintAndExit() -- cgit v1.2.3 From 3cc8887885f24a3f0d607af24197bc364ab16b8d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 12 Aug 2021 10:56:36 +0300 Subject: add missing brush and also add color client configs plus jsonschema --- cmd/dmap/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index 8a38069..f99be52 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -49,12 +49,13 @@ func main() { config.Read(cfgFile, sshPort) if noColor { - config.Client.TermColorsEnabled = false + config.Client.TermColorsEnable = false } if displayVersion { version.PrintAndExit() } + version.Print() ctx := context.TODO() logger.Start(ctx, logger.Modes{ -- cgit v1.2.3 From 7fbea88cf55af9b3354b4a1334e49c38d0d920fc Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 18 Sep 2021 13:16:36 +0300 Subject: additional flags can be interpreted as file list --- cmd/dmap/main.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index f99be52..ab710ac 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -4,6 +4,7 @@ import ( "context" "flag" "os" + "strings" "github.com/mimecast/dtail/internal/clients" "github.com/mimecast/dtail/internal/config" @@ -47,6 +48,15 @@ func main() { flag.Parse() + if args.What == "" { + // Interpret additional args as file list. + var files []string + for _, file := range flag.Args() { + files = append(files, file) + } + args.What = strings.Join(files, ",") + } + config.Read(cfgFile, sshPort) if noColor { config.Client.TermColorsEnable = false @@ -65,7 +75,7 @@ func main() { client, err := clients.NewMaprClient(args, queryStr, clients.DefaultMode) if err != nil { - panic(err) + logger.FatalExit(err) } status := client.Start(ctx, signal.InterruptCh(ctx)) -- cgit v1.2.3 From 6506e20f6c80f4acb7434eb9dd14f784a67189cd Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 18 Sep 2021 14:41:25 +0300 Subject: add spartan mode --- cmd/dmap/main.go | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index ab710ac..4525503 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -4,7 +4,6 @@ import ( "context" "flag" "os" - "strings" "github.com/mimecast/dtail/internal/clients" "github.com/mimecast/dtail/internal/config" @@ -20,7 +19,6 @@ func main() { var cfgFile string var debugEnable bool var displayVersion bool - var noColor bool var queryStr string var sshPort int @@ -30,11 +28,12 @@ func main() { userName := user.Name() + flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode") + flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode") flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys") flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages") flag.BoolVar(&displayVersion, "version", false, "Display version") - flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors") - flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode") + flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors") flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently") flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection") flag.IntVar(&sshPort, "port", 2222, "SSH server port") @@ -47,25 +46,16 @@ func main() { flag.StringVar(&queryStr, "query", "", "Map reduce query") flag.Parse() - - if args.What == "" { - // Interpret additional args as file list. - var files []string - for _, file := range flag.Args() { - files = append(files, file) - } - args.What = strings.Join(files, ",") - } - - config.Read(cfgFile, sshPort) - if noColor { - config.Client.TermColorsEnable = false - } + args.Transform(flag.Args()) + config.Read(cfgFile, sshPort, args.NoColor) + args.TransformAfterConfigFile() if displayVersion { version.PrintAndExit() } - version.Print() + if !args.Spartan { + version.Print() + } ctx := context.TODO() logger.Start(ctx, logger.Modes{ -- cgit v1.2.3 From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 19 Sep 2021 13:22:59 +0300 Subject: move args to config package logger package rewrite as dlog --- cmd/dmap/main.go | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index 4525503..061d859 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -4,10 +4,11 @@ import ( "context" "flag" "os" + "sync" "github.com/mimecast/dtail/internal/clients" "github.com/mimecast/dtail/internal/config" - "github.com/mimecast/dtail/internal/io/logger" + "github.com/mimecast/dtail/internal/io/dlog" "github.com/mimecast/dtail/internal/io/signal" "github.com/mimecast/dtail/internal/omode" "github.com/mimecast/dtail/internal/user" @@ -16,39 +17,36 @@ import ( // The evil begins here. func main() { - var cfgFile string - var debugEnable bool var displayVersion bool var queryStr string - var sshPort int - args := clients.Args{ + args := config.Args{ Mode: omode.MapClient, } userName := user.Name() + flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors") flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode") flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode") - flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Auto trust all unknown host keys") - flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages") + flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys") flag.BoolVar(&displayVersion, "version", false, "Display version") - flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors") + // TODO: Make ConnectionsPerCPU default value as a constant in config package. flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently") + // TODO: make default ssh port a constant in the config package. + flag.IntVar(&args.SSHPort, "port", 2222, "SSH server port") flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection") - flag.IntVar(&sshPort, "port", 2222, "SSH server port") + flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path") flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method") + flag.StringVar(&args.LogLevel, "logLevel", "", "Log level") flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") flag.StringVar(&args.UserName, "user", userName, "Your system user name") flag.StringVar(&args.What, "files", "", "File(s) to read") - flag.StringVar(&cfgFile, "cfg", "", "Config file path") flag.StringVar(&queryStr, "query", "", "Map reduce query") flag.Parse() - args.Transform(flag.Args()) - config.Read(cfgFile, sshPort, args.NoColor) - args.TransformAfterConfigFile() + config.Setup(&args, flag.Args()) if displayVersion { version.PrintAndExit() @@ -57,18 +55,19 @@ func main() { version.Print() } - ctx := context.TODO() - logger.Start(ctx, logger.Modes{ - Debug: debugEnable || config.Common.DebugEnable, - Quiet: args.Quiet, - }) + ctx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + wg.Add(1) + dlog.Start(ctx, &wg, dlog.CLIENT, config.Common.LogLevel) client, err := clients.NewMaprClient(args, queryStr, clients.DefaultMode) if err != nil { - logger.FatalExit(err) + dlog.Client.FatalPanic(err) } status := client.Start(ctx, signal.InterruptCh(ctx)) - logger.Flush() + cancel() + + wg.Wait() os.Exit(status) } -- cgit v1.2.3 From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 26 Sep 2021 16:42:47 +0300 Subject: refactor config reader - also looks in additional search paths for config file unless NONE is specified --- cmd/dmap/main.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index 061d859..b895964 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -31,13 +31,12 @@ func main() { flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode") flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys") flag.BoolVar(&displayVersion, "version", false, "Display version") - // TODO: Make ConnectionsPerCPU default value as a constant in config package. - flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently") - // TODO: make default ssh port a constant in the config package. - flag.IntVar(&args.SSHPort, "port", 2222, "SSH server port") + flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU, "How many connections established per CPU core concurrently") + flag.IntVar(&args.SSHPort, "port", config.DefaultSSHPort, "SSH server port") flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection") flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path") flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method") + flag.StringVar(&args.LogDir, "logDir", "", "Log dir") flag.StringVar(&args.LogLevel, "logLevel", "", "Log level") flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") -- cgit v1.2.3 From 6e1af993924bc7bebe898b403962db5a6b3505d1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 11:23:08 +0300 Subject: Client default log dir is ~/log --- cmd/dmap/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index b895964..cc8a158 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -11,6 +11,7 @@ import ( "github.com/mimecast/dtail/internal/io/dlog" "github.com/mimecast/dtail/internal/io/signal" "github.com/mimecast/dtail/internal/omode" + "github.com/mimecast/dtail/internal/source" "github.com/mimecast/dtail/internal/user" "github.com/mimecast/dtail/internal/version" ) @@ -36,7 +37,7 @@ func main() { flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection") flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path") flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method") - flag.StringVar(&args.LogDir, "logDir", "", "Log dir") + flag.StringVar(&args.LogDir, "logDir", "~/log", "Log dir") flag.StringVar(&args.LogLevel, "logLevel", "", "Log level") flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") @@ -57,7 +58,7 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup wg.Add(1) - dlog.Start(ctx, &wg, dlog.CLIENT, config.Common.LogLevel) + dlog.Start(ctx, &wg, source.Client, config.Common.LogLevel) client, err := clients.NewMaprClient(args, queryStr, clients.DefaultMode) if err != nil { -- cgit v1.2.3 From 07e1470892beacf0722276f94bacbd822b002540 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Oct 2021 13:09:32 +0300 Subject: add dmap tests --- cmd/dmap/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index cc8a158..d32ccb0 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -46,7 +46,7 @@ func main() { flag.StringVar(&queryStr, "query", "", "Map reduce query") flag.Parse() - config.Setup(&args, flag.Args()) + config.Setup(source.Client, &args, flag.Args()) if displayVersion { version.PrintAndExit() -- cgit v1.2.3 From 2d7ddbeae8286373ac19787dc7dde598a7cb0598 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 8 Oct 2021 11:43:43 +0300 Subject: refactor --- cmd/dmap/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index d32ccb0..acc1dc6 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -19,7 +19,6 @@ import ( // The evil begins here. func main() { var displayVersion bool - var queryStr string args := config.Args{ Mode: omode.MapClient, @@ -38,12 +37,13 @@ func main() { flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path") flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method") flag.StringVar(&args.LogDir, "logDir", "~/log", "Log dir") + flag.StringVar(&args.Logger, "logger", config.DefaultClientLogger, "Logger name") flag.StringVar(&args.LogLevel, "logLevel", "", "Log level") flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") + flag.StringVar(&args.QueryStr, "query", "", "Map reduce query") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") flag.StringVar(&args.UserName, "user", userName, "Your system user name") flag.StringVar(&args.What, "files", "", "File(s) to read") - flag.StringVar(&queryStr, "query", "", "Map reduce query") flag.Parse() config.Setup(source.Client, &args, flag.Args()) @@ -58,9 +58,9 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup wg.Add(1) - dlog.Start(ctx, &wg, source.Client, config.Common.LogLevel) + dlog.Start(ctx, &wg, source.Client) - client, err := clients.NewMaprClient(args, queryStr, clients.DefaultMode) + client, err := clients.NewMaprClient(args, clients.DefaultMode) if err != nil { dlog.Client.FatalPanic(err) } -- cgit v1.2.3 From 97747ea0f3178f7f5890512d483fdccaa82846b0 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 21:10:29 +0300 Subject: vetting and linting and some code restyling --- cmd/dmap/main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index acc1dc6..1f44076 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -19,11 +19,9 @@ import ( // The evil begins here. func main() { var displayVersion bool - args := config.Args{ Mode: omode.MapClient, } - userName := user.Name() flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors") @@ -31,7 +29,8 @@ func main() { flag.BoolVar(&args.Spartan, "spartan", false, "Spartan output mode") flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys") flag.BoolVar(&displayVersion, "version", false, "Display version") - flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU, "How many connections established per CPU core concurrently") + flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU, + "How many connections established per CPU core concurrently") flag.IntVar(&args.SSHPort, "port", config.DefaultSSHPort, "SSH server port") flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection") flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path") -- cgit v1.2.3 From f44792c9102488774c9993b080f35c65287a64b1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 10 Oct 2021 14:02:12 +0300 Subject: add another dmap test - reading 100 source files at once fix a data race when reading multiple files on one server from the same session at once --- cmd/dmap/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index 1f44076..89ca5d0 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -37,7 +37,7 @@ func main() { flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method") flag.StringVar(&args.LogDir, "logDir", "~/log", "Log dir") flag.StringVar(&args.Logger, "logger", config.DefaultClientLogger, "Logger name") - flag.StringVar(&args.LogLevel, "logLevel", "", "Log level") + flag.StringVar(&args.LogLevel, "logLevel", config.DefaultLogLevel, "Log level") flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") flag.StringVar(&args.QueryStr, "query", "", "Map reduce query") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") -- cgit v1.2.3 From c0332e8fa13f4939de17c856b2e71fd093847253 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Oct 2021 09:04:40 +0300 Subject: add dcat color output test --- cmd/dmap/main.go | 3 --- 1 file changed, 3 deletions(-) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index 89ca5d0..7591c6c 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -50,9 +50,6 @@ func main() { if displayVersion { version.PrintAndExit() } - if !args.Spartan { - version.Print() - } ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup -- cgit v1.2.3 From b679462fa687b375ccefd3c11fec3fd96b9b4d60 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 20 Oct 2021 06:41:08 +0300 Subject: make pprof bind address configurable --- cmd/dmap/main.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'cmd/dmap') diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index 7591c6c..2c941f3 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -6,6 +6,10 @@ import ( "os" "sync" + "net/http" + _ "net/http" + _ "net/http/pprof" + "github.com/mimecast/dtail/internal/clients" "github.com/mimecast/dtail/internal/config" "github.com/mimecast/dtail/internal/io/dlog" @@ -19,6 +23,8 @@ import ( // The evil begins here. func main() { var displayVersion bool + var pprof string + args := config.Args{ Mode: omode.MapClient, } @@ -43,6 +49,7 @@ func main() { flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") flag.StringVar(&args.UserName, "user", userName, "Your system user name") flag.StringVar(&args.What, "files", "", "File(s) to read") + flag.StringVar(&pprof, "pprof", "", "Start PProf server this address") flag.Parse() config.Setup(source.Client, &args, flag.Args()) @@ -56,6 +63,11 @@ func main() { wg.Add(1) dlog.Start(ctx, &wg, source.Client) + if pprof != "" { + go http.ListenAndServe(pprof, nil) + dlog.Client.Info("Started PProf", pprof) + } + client, err := clients.NewMaprClient(args, clients.DefaultMode) if err != nil { dlog.Client.FatalPanic(err) -- cgit v1.2.3