From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 19 Sep 2021 13:22:59 +0300 Subject: move args to config package logger package rewrite as dlog --- internal/config/args.go | 105 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 internal/config/args.go (limited to 'internal/config/args.go') diff --git a/internal/config/args.go b/internal/config/args.go new file mode 100644 index 0000000..89e4bc9 --- /dev/null +++ b/internal/config/args.go @@ -0,0 +1,105 @@ +package config + +import ( + "flag" + "fmt" + "strings" + + "github.com/mimecast/dtail/internal/omode" + + gossh "golang.org/x/crypto/ssh" +) + +// Args is a helper struct to summarize common client arguments. +type Args struct { + Arguments []string + ConfigFile string + ConnectionsPerCPU int + Discovery string + LogLevel string + Mode omode.Mode + NoColor bool + PrivateKeyPathFile string + Quiet bool + RegexInvert bool + RegexStr string + Serverless bool + ServersStr string + Spartan bool + SSHAuthMethods []gossh.AuthMethod + SSHHostKeyCallback gossh.HostKeyCallback + SSHPort int + Timeout int + TrustAllHosts bool + UserName string + What string +} + +func (a *Args) String() string { + var sb strings.Builder + + sb.WriteString("Args(") + sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) + sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile)) + sb.WriteString(fmt.Sprintf("%s:%v,", "ConnectionsPerCPU", a.ConnectionsPerCPU)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Discovery", a.Discovery)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode)) + sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor)) + sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet)) + sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert)) + sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless)) + sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Spartan", a.Spartan)) + sb.WriteString(fmt.Sprintf("%s:%v,", "SSHAuthMethods", a.SSHAuthMethods)) + sb.WriteString(fmt.Sprintf("%s:%v,", "SSHHostKeyCallback", a.SSHHostKeyCallback)) + sb.WriteString(fmt.Sprintf("%s:%v,", "SSHPort", a.SSHPort)) + sb.WriteString(fmt.Sprintf("%s:%v,", "Timeout", a.Timeout)) + sb.WriteString(fmt.Sprintf("%s:%v,", "TrustAllHosts", a.TrustAllHosts)) + sb.WriteString(fmt.Sprintf("%s:%v,", "UserName", a.UserName)) + sb.WriteString(fmt.Sprintf("%s:%v", "What", a.What)) + sb.WriteString(")") + + return sb.String() +} + +// Based on the argument list, transform/manipulate some of the arguments. +func (a *Args) transform(args []string) { + if a.LogLevel != "" { + Common.LogLevel = a.LogLevel + } + + if a.SSHPort != 2222 { + Common.SSHPort = a.SSHPort + } + if a.NoColor { + Client.TermColorsEnable = false + } + + if a.Spartan { + a.Quiet = true + a.NoColor = true + } + + if a.Discovery == "" && a.ServersStr == "" { + a.Serverless = true + } + + // Interpret additional args as file list. + if a.What == "" { + var files []string + for _, file := range flag.Args() { + files = append(files, file) + } + a.What = strings.Join(files, ",") + } +} + +// SerializeOptions returns a string ready to be sent over the wire to the server. +func (a *Args) SerializeOptions() string { + return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan) +} + +// TODO: Put the DeseializeOptions function here (move it away from the internal/server package) -- cgit v1.2.3 From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 26 Sep 2021 16:42:47 +0300 Subject: refactor config reader - also looks in additional search paths for config file unless NONE is specified --- internal/config/args.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'internal/config/args.go') diff --git a/internal/config/args.go b/internal/config/args.go index 89e4bc9..767cc65 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -16,6 +16,7 @@ type Args struct { ConfigFile string ConnectionsPerCPU int Discovery string + LogDir string LogLevel string Mode omode.Mode NoColor bool @@ -39,6 +40,8 @@ func (a *Args) String() string { var sb strings.Builder sb.WriteString("Args(") + // TODO: All commands should make use of this + sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir)) sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile)) @@ -66,16 +69,24 @@ func (a *Args) String() string { } // Based on the argument list, transform/manipulate some of the arguments. -func (a *Args) transform(args []string) { +func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { + if a.LogDir != "" { + common.LogDir = a.LogDir + if common.LogStrategy == "" { + // TODO: Implement the other (not-daily) log strategy for the server. + common.LogStrategy = "daily" + } + } + if a.LogLevel != "" { - Common.LogLevel = a.LogLevel + common.LogLevel = a.LogLevel } - if a.SSHPort != 2222 { - Common.SSHPort = a.SSHPort + if a.SSHPort != DefaultSSHPort { + common.SSHPort = a.SSHPort } if a.NoColor { - Client.TermColorsEnable = false + client.TermColorsEnable = false } if a.Spartan { @@ -95,6 +106,8 @@ func (a *Args) transform(args []string) { } a.What = strings.Join(files, ",") } + + return client, server, common } // SerializeOptions returns a string ready to be sent over the wire to the server. @@ -102,4 +115,4 @@ func (a *Args) SerializeOptions() string { return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan) } -// TODO: Put the DeseializeOptions function here (move it away from the internal/server package) +// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package) -- cgit v1.2.3 From 609921f9c783941eaa9019a92b78ec45b49d681c Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 28 Sep 2021 21:11:50 +0300 Subject: can have daily and normal file log rotation --- internal/config/args.go | 43 ------------------------------------------- 1 file changed, 43 deletions(-) (limited to 'internal/config/args.go') diff --git a/internal/config/args.go b/internal/config/args.go index 767cc65..7f24348 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -1,7 +1,6 @@ package config import ( - "flag" "fmt" "strings" @@ -68,48 +67,6 @@ func (a *Args) String() string { return sb.String() } -// Based on the argument list, transform/manipulate some of the arguments. -func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { - if a.LogDir != "" { - common.LogDir = a.LogDir - if common.LogStrategy == "" { - // TODO: Implement the other (not-daily) log strategy for the server. - common.LogStrategy = "daily" - } - } - - if a.LogLevel != "" { - common.LogLevel = a.LogLevel - } - - if a.SSHPort != DefaultSSHPort { - common.SSHPort = a.SSHPort - } - if a.NoColor { - client.TermColorsEnable = false - } - - if a.Spartan { - a.Quiet = true - a.NoColor = true - } - - if a.Discovery == "" && a.ServersStr == "" { - a.Serverless = true - } - - // Interpret additional args as file list. - if a.What == "" { - var files []string - for _, file := range flag.Args() { - files = append(files, file) - } - a.What = strings.Join(files, ",") - } - - return client, server, common -} - // SerializeOptions returns a string ready to be sent over the wire to the server. func (a *Args) SerializeOptions() string { return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan) -- cgit v1.2.3 From 764ef99a3d779a0db1fb60679292af52425ba2f6 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 10:46:47 +0300 Subject: add more default fields to MAPREDUCE --- internal/config/args.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'internal/config/args.go') diff --git a/internal/config/args.go b/internal/config/args.go index 7f24348..484aa8b 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -39,7 +39,7 @@ func (a *Args) String() string { var sb strings.Builder sb.WriteString("Args(") - // TODO: All commands should make use of this + sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir)) sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) -- cgit v1.2.3 From 12c79f68bb5bda6673819d7b754820ecfe6d08ff Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 11:54:07 +0300 Subject: reduce logging in serverless mode --- internal/config/args.go | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) (limited to 'internal/config/args.go') diff --git a/internal/config/args.go b/internal/config/args.go index 484aa8b..3e2eb1f 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -1,6 +1,7 @@ package config import ( + "encoding/base64" "fmt" "strings" @@ -69,7 +70,32 @@ func (a *Args) String() string { // SerializeOptions returns a string ready to be sent over the wire to the server. func (a *Args) SerializeOptions() string { - return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan) + return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, a.Serverless) } -// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package) +// DeserializeOptions deserializes the options, but into a map. +func DeserializeOptions(opts []string) (map[string]string, error) { + options := make(map[string]string, len(opts)) + + for _, o := range opts { + kv := strings.SplitN(o, "=", 2) + if len(kv) != 2 { + return options, fmt.Errorf("Unable to parse options: %v", kv) + } + key := kv[0] + val := kv[1] + + if strings.HasPrefix(val, "base64%") { + s := strings.SplitN(val, "%", 2) + decoded, err := base64.StdEncoding.DecodeString(s[1]) + if err != nil { + return options, err + } + val = string(decoded) + } + + options[key] = val + } + + return options, nil +} -- cgit v1.2.3 From 2d7ddbeae8286373ac19787dc7dde598a7cb0598 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 8 Oct 2021 11:43:43 +0300 Subject: refactor --- internal/config/args.go | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'internal/config/args.go') diff --git a/internal/config/args.go b/internal/config/args.go index 3e2eb1f..a671ae3 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -17,10 +17,12 @@ type Args struct { ConnectionsPerCPU int Discovery string LogDir string + Logger string LogLevel string Mode omode.Mode NoColor bool PrivateKeyPathFile string + QueryStr string Quiet bool RegexInvert bool RegexStr string @@ -42,6 +44,7 @@ func (a *Args) String() string { sb.WriteString("Args(") sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir)) + sb.WriteString(fmt.Sprintf("%s:%s,", "Logger", a.Logger)) sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel)) sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments)) sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile)) @@ -50,6 +53,7 @@ func (a *Args) String() string { sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode)) sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor)) sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile)) + sb.WriteString(fmt.Sprintf("%s:%v,", "QueryStr", a.QueryStr)) sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet)) sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert)) sb.WriteString(fmt.Sprintf("%s:%v,", "RegexStr", a.RegexStr)) -- cgit v1.2.3 From 97747ea0f3178f7f5890512d483fdccaa82846b0 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 21:10:29 +0300 Subject: vetting and linting and some code restyling --- internal/config/args.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'internal/config/args.go') diff --git a/internal/config/args.go b/internal/config/args.go index a671ae3..f721390 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -74,13 +74,13 @@ func (a *Args) String() string { // SerializeOptions returns a string ready to be sent over the wire to the server. func (a *Args) SerializeOptions() string { - return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, a.Serverless) + return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, + a.Serverless) } // DeserializeOptions deserializes the options, but into a map. func DeserializeOptions(opts []string) (map[string]string, error) { options := make(map[string]string, len(opts)) - for _, o := range opts { kv := strings.SplitN(o, "=", 2) if len(kv) != 2 { @@ -97,9 +97,7 @@ func DeserializeOptions(opts []string) (map[string]string, error) { } val = string(decoded) } - options[key] = val } - return options, nil } -- cgit v1.2.3 From 1dead22129a26e4f532e68c2c63fe4122b519506 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 13 Oct 2021 21:10:28 +0300 Subject: Merging grep context from master --- internal/config/args.go | 73 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 7 deletions(-) (limited to 'internal/config/args.go') diff --git a/internal/config/args.go b/internal/config/args.go index f721390..f94e0a9 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -3,8 +3,10 @@ package config import ( "encoding/base64" "fmt" + "strconv" "strings" + "github.com/mimecast/dtail/internal/lcontext" "github.com/mimecast/dtail/internal/omode" gossh "golang.org/x/crypto/ssh" @@ -12,6 +14,7 @@ import ( // Args is a helper struct to summarize common client arguments. type Args struct { + lcontext.LContext Arguments []string ConfigFile string ConnectionsPerCPU int @@ -74,17 +77,50 @@ func (a *Args) String() string { // SerializeOptions returns a string ready to be sent over the wire to the server. func (a *Args) SerializeOptions() string { - return fmt.Sprintf("quiet=%v:spartan=%v:serverless=%v", a.Quiet, a.Spartan, - a.Serverless) + options := make(map[string]string) + + if a.Quiet { + options["quiet"] = fmt.Sprintf("%v", a.Quiet) + } + if a.Spartan { + options["spartan"] = fmt.Sprintf("%v", a.Spartan) + } + if a.Serverless { + options["serverless"] = fmt.Sprintf("%v", a.Serverless) + } + if a.LContext.MaxCount != 0 { + options["max"] = fmt.Sprintf("%d", a.LContext.MaxCount) + } + if a.LContext.BeforeContext != 0 { + options["before"] = fmt.Sprintf("%d", a.LContext.BeforeContext) + } + if a.LContext.AfterContext != 0 { + options["after"] = fmt.Sprintf("%d", a.LContext.AfterContext) + } + + var sb strings.Builder + var i int + for k, v := range options { + if i > 0 { + sb.WriteString(":") + } + sb.WriteString(k) + sb.WriteString("=") + sb.WriteString(v) + i++ + } + return sb.String() } // DeserializeOptions deserializes the options, but into a map. -func DeserializeOptions(opts []string) (map[string]string, error) { +func DeserializeOptions(opts []string) (map[string]string, lcontext.LContext, error) { options := make(map[string]string, len(opts)) + var ltx lcontext.LContext + for _, o := range opts { kv := strings.SplitN(o, "=", 2) if len(kv) != 2 { - return options, fmt.Errorf("Unable to parse options: %v", kv) + return options, ltx, fmt.Errorf("Unable to parse options: %v", kv) } key := kv[0] val := kv[1] @@ -93,11 +129,34 @@ func DeserializeOptions(opts []string) (map[string]string, error) { s := strings.SplitN(val, "%", 2) decoded, err := base64.StdEncoding.DecodeString(s[1]) if err != nil { - return options, err + return options, ltx, err } val = string(decoded) } - options[key] = val + + switch key { + case "before": + iVal, err := strconv.Atoi(val) + if err != nil { + return options, ltx, err + } + ltx.BeforeContext = iVal + case "after": + iVal, err := strconv.Atoi(val) + if err != nil { + return options, ltx, err + } + ltx.AfterContext = iVal + case "max": + iVal, err := strconv.Atoi(val) + if err != nil { + return options, ltx, err + } + ltx.MaxCount = iVal + default: + options[key] = val + } } - return options, nil + + return options, ltx, nil } -- cgit v1.2.3