From fe3e68afd99d8ea246be52893730f987e138ec24 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 19 Sep 2021 13:22:59 +0300 Subject: move args to config package logger package rewrite as dlog --- internal/config/config.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index 276ddcf..2d77041 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,6 +15,9 @@ const ScheduleUser string = "DTAIL-SCHEDULE" // ContinuousUser is used for non-interactive continuous mapreduce queries. const ContinuousUser string = "DTAIL-CONTINUOUS" +// TestUser is used for unit tests and potentially also for integration tests. +const TestUser string = "DTAIL-TEST" + // InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval. const InterruptTimeoutS int = 3 -- cgit v1.2.3 From fcaa94c7453efa0d74e330128c0f5c2cde8f11b3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 26 Sep 2021 16:42:47 +0300 Subject: refactor config reader - also looks in additional search paths for config file unless NONE is specified --- internal/config/config.go | 61 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 19 deletions(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index 2d77041..c9f411c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,24 +2,26 @@ package config import ( "encoding/json" + "fmt" "io/ioutil" "os" + "strings" ) -// ControlUser is used for various DTail specific operations. -const ControlUser string = "DTAIL-CONTROL" - -// ScheduleUser is used for non-interactive scheduled mapreduce queries. -const ScheduleUser string = "DTAIL-SCHEDULE" - -// ContinuousUser is used for non-interactive continuous mapreduce queries. -const ContinuousUser string = "DTAIL-CONTINUOUS" - -// TestUser is used for unit tests and potentially also for integration tests. -const TestUser string = "DTAIL-TEST" - -// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval. -const InterruptTimeoutS int = 3 +const ( + // ControlUser is used for various DTail specific operations. + ControlUser string = "DTAIL-CONTROL" + // ScheduleUser is used for non-interactive scheduled mapreduce queries. + ScheduleUser string = "DTAIL-SCHEDULE" + // ContinuousUser is used for non-interactive continuous mapreduce queries. + ContinuousUser string = "DTAIL-CONTINUOUS" + // InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval. + InterruptTimeoutS int = 3 + // ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start) + DefaultConnectionsPerCPU int = 10 + // DTailSSHServerDefaultPort is the default DServer port. + DefaultSSHPort int = 2222 +) // Client holds a DTail client configuration. var Client *ClientConfig @@ -37,21 +39,42 @@ type configInitializer struct { Client *ClientConfig } -// Parse and read a given config file in JSON format. -func (c *configInitializer) parseConfig(configFile string) { +func (c *configInitializer) parseConfig(args *Args) { + if strings.ToUpper(args.ConfigFile) == "NONE" { + return + } + + if args.ConfigFile != "" { + c.parseSpecificConfig(args.ConfigFile) + return + } + + if homeDir, err := os.UserHomeDir(); err != nil { + var paths []string + paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir)) + paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir)) + for _, configPath := range paths { + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + c.parseSpecificConfig(configPath) + } + } + } +} + +func (c *configInitializer) parseSpecificConfig(configFile string) { fd, err := os.Open(configFile) if err != nil { - panic(err) + panic(fmt.Sprintf("Unable to read config file: %v", err)) } defer fd.Close() cfgBytes, err := ioutil.ReadAll(fd) if err != nil { - panic(err) + panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err)) } err = json.Unmarshal([]byte(cfgBytes), c) if err != nil { - panic(err) + panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err)) } } -- cgit v1.2.3 From 609921f9c783941eaa9019a92b78ec45b49d681c Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 28 Sep 2021 21:11:50 +0300 Subject: can have daily and normal file log rotation --- internal/config/config.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index c9f411c..76dcc65 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "encoding/json" + "flag" "fmt" "io/ioutil" "os" @@ -78,3 +79,45 @@ func (c *configInitializer) parseSpecificConfig(configFile string) { panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err)) } } + +func (c *configInitializer) transformConfig(args *Args, additionalArgs []string, + client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { + if args.LogDir != "" { + common.LogDir = args.LogDir + if common.LogStrategy == "" { + // TODO: Implement the other (not-daily) log strategy for the server. + common.LogStrategy = "daily" + } + } + + if args.LogLevel != "" { + common.LogLevel = args.LogLevel + } + + if args.SSHPort != DefaultSSHPort { + common.SSHPort = args.SSHPort + } + if args.NoColor { + client.TermColorsEnable = false + } + + if args.Spartan { + args.Quiet = true + args.NoColor = true + } + + if args.Discovery == "" && args.ServersStr == "" { + args.Serverless = true + } + + // Interpret additional args as file list. + if args.What == "" { + var files []string + for _, file := range flag.Args() { + files = append(files, file) + } + args.What = strings.Join(files, ",") + } + + return client, server, common +} -- cgit v1.2.3 From 764ef99a3d779a0db1fb60679292af52425ba2f6 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 10:46:47 +0300 Subject: add more default fields to MAPREDUCE --- internal/config/config.go | 1 - 1 file changed, 1 deletion(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index 76dcc65..3d05a11 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -85,7 +85,6 @@ func (c *configInitializer) transformConfig(args *Args, additionalArgs []string, if args.LogDir != "" { common.LogDir = args.LogDir if common.LogStrategy == "" { - // TODO: Implement the other (not-daily) log strategy for the server. common.LogStrategy = "daily" } } -- cgit v1.2.3 From 6e1af993924bc7bebe898b403962db5a6b3505d1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 11:23:08 +0300 Subject: Client default log dir is ~/log --- internal/config/config.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index 3d05a11..6d4730a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -84,9 +84,16 @@ func (c *configInitializer) transformConfig(args *Args, additionalArgs []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { if args.LogDir != "" { common.LogDir = args.LogDir - if common.LogStrategy == "" { - common.LogStrategy = "daily" + } + if strings.Contains(common.LogDir, "~/") { + homeDir, err := os.UserHomeDir() + if err != nil { + panic(err) } + common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) + } + if common.LogStrategy == "" { + common.LogStrategy = "daily" } if args.LogLevel != "" { -- cgit v1.2.3 From 12c79f68bb5bda6673819d7b754820ecfe6d08ff Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 2 Oct 2021 11:54:07 +0300 Subject: reduce logging in serverless mode --- internal/config/config.go | 116 ++++++---------------------------------------- 1 file changed, 14 insertions(+), 102 deletions(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index 6d4730a..09ae994 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,14 +1,5 @@ package config -import ( - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "os" - "strings" -) - const ( // ControlUser is used for various DTail specific operations. ControlUser string = "DTAIL-CONTROL" @@ -33,97 +24,18 @@ var Server *ServerConfig // Common holds common configs of both both, client and server. var Common *CommonConfig -// Used to initialize the configuration. -type configInitializer struct { - Common *CommonConfig - Server *ServerConfig - Client *ClientConfig -} - -func (c *configInitializer) parseConfig(args *Args) { - if strings.ToUpper(args.ConfigFile) == "NONE" { - return - } - - if args.ConfigFile != "" { - c.parseSpecificConfig(args.ConfigFile) - return - } - - if homeDir, err := os.UserHomeDir(); err != nil { - var paths []string - paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir)) - paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir)) - for _, configPath := range paths { - if _, err := os.Stat(configPath); !os.IsNotExist(err) { - c.parseSpecificConfig(configPath) - } - } - } -} - -func (c *configInitializer) parseSpecificConfig(configFile string) { - fd, err := os.Open(configFile) - if err != nil { - panic(fmt.Sprintf("Unable to read config file: %v", err)) - } - defer fd.Close() - - cfgBytes, err := ioutil.ReadAll(fd) - if err != nil { - panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err)) - } - - err = json.Unmarshal([]byte(cfgBytes), c) - if err != nil { - panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err)) - } -} - -func (c *configInitializer) transformConfig(args *Args, additionalArgs []string, - client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) { - if args.LogDir != "" { - common.LogDir = args.LogDir - } - if strings.Contains(common.LogDir, "~/") { - homeDir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - common.LogDir = strings.ReplaceAll(common.LogDir, "~/", fmt.Sprintf("%s/", homeDir)) - } - if common.LogStrategy == "" { - common.LogStrategy = "daily" - } - - if args.LogLevel != "" { - common.LogLevel = args.LogLevel - } - - if args.SSHPort != DefaultSSHPort { - common.SSHPort = args.SSHPort - } - if args.NoColor { - client.TermColorsEnable = false - } - - if args.Spartan { - args.Quiet = true - args.NoColor = true - } - - if args.Discovery == "" && args.ServersStr == "" { - args.Serverless = true - } - - // Interpret additional args as file list. - if args.What == "" { - var files []string - for _, file := range flag.Args() { - files = append(files, file) - } - args.What = strings.Join(files, ",") - } - - return client, server, common +// Setup the DTail configuration. +func Setup(args *Args, additionalArgs []string) { + initializer := initializer{ + Common: newDefaultCommonConfig(), + Server: newDefaultServerConfig(), + Client: newDefaultClientConfig(), + } + initializer.parseConfig(args) + Client, Server, Common = initializer.transformConfig( + args, additionalArgs, + initializer.Client, + initializer.Server, + initializer.Common, + ) } -- cgit v1.2.3 From 07e1470892beacf0722276f94bacbd822b002540 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Oct 2021 13:09:32 +0300 Subject: add dmap tests --- internal/config/config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index 09ae994..d58162f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,5 +1,7 @@ package config +import "github.com/mimecast/dtail/internal/source" + const ( // ControlUser is used for various DTail specific operations. ControlUser string = "DTAIL-CONTROL" @@ -25,7 +27,7 @@ var Server *ServerConfig var Common *CommonConfig // Setup the DTail configuration. -func Setup(args *Args, additionalArgs []string) { +func Setup(sourceProcess source.Source, args *Args, additionalArgs []string) { initializer := initializer{ Common: newDefaultCommonConfig(), Server: newDefaultServerConfig(), @@ -33,6 +35,7 @@ func Setup(args *Args, additionalArgs []string) { } initializer.parseConfig(args) Client, Server, Common = initializer.transformConfig( + sourceProcess, args, additionalArgs, initializer.Client, initializer.Server, -- cgit v1.2.3 From fab5dc3e70434ea0abc7a0976487a1973b662331 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 6 Oct 2021 09:50:41 +0300 Subject: enable faster shutdown - useful for dgrep/dmap and dcat commands --- internal/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index d58162f..f216688 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,8 +3,8 @@ package config import "github.com/mimecast/dtail/internal/source" const ( - // ControlUser is used for various DTail specific operations. - ControlUser string = "DTAIL-CONTROL" + // HealthUser is used for the health check + HealthUser string = "DTAIL-HEALTH" // ScheduleUser is used for non-interactive scheduled mapreduce queries. ScheduleUser string = "DTAIL-SCHEDULE" // ContinuousUser is used for non-interactive continuous mapreduce queries. -- cgit v1.2.3 From 2d7ddbeae8286373ac19787dc7dde598a7cb0598 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 8 Oct 2021 11:43:43 +0300 Subject: refactor --- internal/config/config.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index f216688..077a658 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,6 +15,14 @@ const ( DefaultConnectionsPerCPU int = 10 // DTailSSHServerDefaultPort is the default DServer port. DefaultSSHPort int = 2222 + // DefaultLogLevel specifies the default log level (obviously) + DefaultLogLevel string = "INFO" + // DefaultClientLogger specifies the default logger for the client commands. + DefaultClientLogger string = "fout" + // DefaultServerLogger specifies the default logger for dtail server. + DefaultServerLogger string = "file" + // DefaultHealthCheckLogger specifies the default logger used for health checks. + DefaultHealthCheckLogger string = "none" ) // Client holds a DTail client configuration. @@ -33,12 +41,15 @@ func Setup(sourceProcess source.Source, args *Args, additionalArgs []string) { Server: newDefaultServerConfig(), Client: newDefaultClientConfig(), } - initializer.parseConfig(args) - Client, Server, Common = initializer.transformConfig( - sourceProcess, - args, additionalArgs, - initializer.Client, - initializer.Server, - initializer.Common, - ) + if err := initializer.parseConfig(args); err != nil { + panic(err) + } + if err := initializer.transformConfig(sourceProcess, args, additionalArgs); err != nil { + panic(err) + } + + // Make config accessible globally + Server = initializer.Server + Client = initializer.Client + Common = initializer.Common } -- cgit v1.2.3 From 97747ea0f3178f7f5890512d483fdccaa82846b0 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 9 Oct 2021 21:10:29 +0300 Subject: vetting and linting and some code restyling --- internal/config/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index 077a658..b99b22b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,11 +9,11 @@ const ( ScheduleUser string = "DTAIL-SCHEDULE" // ContinuousUser is used for non-interactive continuous mapreduce queries. ContinuousUser string = "DTAIL-CONTINUOUS" - // InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval. + // InterruptTimeoutS specifies the Ctrl+C log pause interval. InterruptTimeoutS int = 3 - // ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start) + // DefaultConnectionsPerCPU controls how many connections are established concurrently. DefaultConnectionsPerCPU int = 10 - // DTailSSHServerDefaultPort is the default DServer port. + // DefaultSSHPort is the default DServer port. DefaultSSHPort int = 2222 // DefaultLogLevel specifies the default log level (obviously) DefaultLogLevel string = "INFO" -- cgit v1.2.3 From f44792c9102488774c9993b080f35c65287a64b1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 10 Oct 2021 14:02:12 +0300 Subject: add another dmap test - reading 100 source files at once fix a data race when reading multiple files on one server from the same session at once --- internal/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'internal/config/config.go') diff --git a/internal/config/config.go b/internal/config/config.go index b99b22b..ee23829 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,7 +16,7 @@ const ( // DefaultSSHPort is the default DServer port. DefaultSSHPort int = 2222 // DefaultLogLevel specifies the default log level (obviously) - DefaultLogLevel string = "INFO" + DefaultLogLevel string = "info" // DefaultClientLogger specifies the default logger for the client commands. DefaultClientLogger string = "fout" // DefaultServerLogger specifies the default logger for dtail server. -- cgit v1.2.3