refactor config reader - also looks in additional search paths for config file unless NONE is specified

15 files changed, 140 insertions, 90 deletions

diff --git a/internal/clients/connectors/serverless.go b/internal/clients/connectors/serverless.go
index c7b5f62..7740aab 100644
--- a/internal/clients/connectors/serverless.go
+++ b/internal/clients/connectors/serverless.go
@@ -53,8 +53,13 @@ func (s *Serverless) Start(ctx context.Context, cancel context.CancelFunc, throt
 func (s *Serverless) handle(ctx context.Context, cancel context.CancelFunc) error {
 	dlog.Client.Debug("Creating server handler for a serverless session")
 
+	user, err := user.New(s.userName, s.Server())
+	if err != nil {
+		return err
+	}
+
 	serverHandler := serverHandlers.NewServerHandler(
-		user.New(s.userName, s.Server()),
+		user,
 		make(chan struct{}, config.Server.MaxConcurrentCats),
 		make(chan struct{}, config.Server.MaxConcurrentTails),
 	)
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index 3291b43..8acb45f 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -69,7 +69,7 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		switch b {
 		/*
-			// TODO: Next DTail version make it so that '\n' gets ignored. For now
+			// NEXT: Next DTail version make it so that '\n' gets ignored. For now
 			// leave it for compatibility with older DTail server + ability to display
 			// the protocol mismatch warn message.
 			case '\n' {
diff --git a/internal/config/args.go b/internal/config/args.go
index 89e4bc9..767cc65 100644
--- a/internal/config/args.go
+++ b/internal/config/args.go
@@ -16,6 +16,7 @@ type Args struct {
 	ConfigFile         string
 	ConnectionsPerCPU  int
 	Discovery          string
+	LogDir             string
 	LogLevel           string
 	Mode               omode.Mode
 	NoColor            bool
@@ -39,6 +40,8 @@ func (a *Args) String() string {
 	var sb strings.Builder
 
 	sb.WriteString("Args(")
+	// TODO: All commands should make use of this
+	sb.WriteString(fmt.Sprintf("%s:%s,", "LogDir", a.LogDir))
 	sb.WriteString(fmt.Sprintf("%s:%s,", "LogLevel", a.LogLevel))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "Arguments", a.Arguments))
 	sb.WriteString(fmt.Sprintf("%s:%v,", "ConfigFile", a.ConfigFile))
@@ -66,16 +69,24 @@ func (a *Args) String() string {
 }
 
 // Based on the argument list, transform/manipulate some of the arguments.
-func (a *Args) transform(args []string) {
+func (a *Args) transformConfig(args []string, client *ClientConfig, server *ServerConfig, common *CommonConfig) (*ClientConfig, *ServerConfig, *CommonConfig) {
+	if a.LogDir != "" {
+		common.LogDir = a.LogDir
+		if common.LogStrategy == "" {
+			// TODO: Implement the other (not-daily) log strategy for the server.
+			common.LogStrategy = "daily"
+		}
+	}
+
 	if a.LogLevel != "" {
-		Common.LogLevel = a.LogLevel
+		common.LogLevel = a.LogLevel
 	}
 
-	if a.SSHPort != 2222 {
-		Common.SSHPort = a.SSHPort
+	if a.SSHPort != DefaultSSHPort {
+		common.SSHPort = a.SSHPort
 	}
 	if a.NoColor {
-		Client.TermColorsEnable = false
+		client.TermColorsEnable = false
 	}
 
 	if a.Spartan {
@@ -95,6 +106,8 @@ func (a *Args) transform(args []string) {
 		}
 		a.What = strings.Join(files, ",")
 	}
+
+	return client, server, common
 }
 
 // SerializeOptions returns a string ready to be sent over the wire to the server.
@@ -102,4 +115,4 @@ func (a *Args) SerializeOptions() string {
 	return fmt.Sprintf("quiet=%v:spartan=%v", a.Quiet, a.Spartan)
 }
 
-// TODO: Put the DeseializeOptions function here (move it away from the internal/server package)
+// NEXT: Put the DeseializeOptions function here (move it away from the internal/server package)
diff --git a/internal/config/common.go b/internal/config/common.go
index 7d45261..acc8e6a 100644
--- a/internal/config/common.go
+++ b/internal/config/common.go
@@ -23,7 +23,7 @@ type CommonConfig struct {
 // Create a new default configuration.
 func newDefaultCommonConfig() *CommonConfig {
 	return &CommonConfig{
-		SSHPort:                    2222,
+		SSHPort:                    DefaultSSHPort,
 		ExperimentalFeaturesEnable: false,
 		LogDir:                     "log",
 		CacheDir:                   "cache",
diff --git a/internal/config/config.go b/internal/config/config.go
index 2d77041..c9f411c 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -2,24 +2,26 @@ package config
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 )
 
-// ControlUser is used for various DTail specific operations.
-const ControlUser string = "DTAIL-CONTROL"
-
-// ScheduleUser is used for non-interactive scheduled mapreduce queries.
-const ScheduleUser string = "DTAIL-SCHEDULE"
-
-// ContinuousUser is used for non-interactive continuous mapreduce queries.
-const ContinuousUser string = "DTAIL-CONTINUOUS"
-
-// TestUser is used for unit tests and potentially also for integration tests.
-const TestUser string = "DTAIL-TEST"
-
-// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
-const InterruptTimeoutS int = 3
+const (
+	// ControlUser is used for various DTail specific operations.
+	ControlUser string = "DTAIL-CONTROL"
+	// ScheduleUser is used for non-interactive scheduled mapreduce queries.
+	ScheduleUser string = "DTAIL-SCHEDULE"
+	// ContinuousUser is used for non-interactive continuous mapreduce queries.
+	ContinuousUser string = "DTAIL-CONTINUOUS"
+	// InterruptTimeoutS is used to terminate DTail when Ctrl+C was pressed twice within a given interval.
+	InterruptTimeoutS int = 3
+	// ConnectionsPerCPU controls how many connections are established concurrently as a start (slow start)
+	DefaultConnectionsPerCPU int = 10
+	// DTailSSHServerDefaultPort is the default DServer port.
+	DefaultSSHPort int = 2222
+)
 
 // Client holds a DTail client configuration.
 var Client *ClientConfig
@@ -37,21 +39,42 @@ type configInitializer struct {
 	Client *ClientConfig
 }
 
-// Parse and read a given config file in JSON format.
-func (c *configInitializer) parseConfig(configFile string) {
+func (c *configInitializer) parseConfig(args *Args) {
+	if strings.ToUpper(args.ConfigFile) == "NONE" {
+		return
+	}
+
+	if args.ConfigFile != "" {
+		c.parseSpecificConfig(args.ConfigFile)
+		return
+	}
+
+	if homeDir, err := os.UserHomeDir(); err != nil {
+		var paths []string
+		paths = append(paths, fmt.Sprintf("%s/.config/dtail/dtail.conf", homeDir))
+		paths = append(paths, fmt.Sprintf("%s/.dtail.conf", homeDir))
+		for _, configPath := range paths {
+			if _, err := os.Stat(configPath); !os.IsNotExist(err) {
+				c.parseSpecificConfig(configPath)
+			}
+		}
+	}
+}
+
+func (c *configInitializer) parseSpecificConfig(configFile string) {
 	fd, err := os.Open(configFile)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to read config file: %v", err))
 	}
 	defer fd.Close()
 
 	cfgBytes, err := ioutil.ReadAll(fd)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to read config file %s: %v", configFile, err))
 	}
 
 	err = json.Unmarshal([]byte(cfgBytes), c)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("Unable to parse config file %s: %v", configFile, err))
 	}
 }
diff --git a/internal/config/setup.go b/internal/config/setup.go
index 3c4bcc4..be8e867 100644
--- a/internal/config/setup.go
+++ b/internal/config/setup.go
@@ -1,11 +1,5 @@
 package config
 
-import (
-	"os"
-)
-
-const NoConfigFile string = "Don't read a config file - use defaults only"
-
 // Setup the DTail configuration.
 func Setup(args *Args, additionalArgs []string) {
 	initializer := configInitializer{
@@ -13,23 +7,11 @@ func Setup(args *Args, additionalArgs []string) {
 		Server: newDefaultServerConfig(),
 		Client: newDefaultClientConfig(),
 	}
-
-	if args.ConfigFile == "" {
-		// TODO: Search more paths for config file (e.g. in /etc and in ~/.config/...
-		args.ConfigFile = "./cfg/dtail.json"
-	}
-
-	if args.ConfigFile != NoConfigFile {
-		if _, err := os.Stat(args.ConfigFile); !os.IsNotExist(err) {
-			initializer.parseConfig(args.ConfigFile)
-		}
-	}
-
-	// Assign pointers to global variables, so that we can access the
-	// configuration from any place of the program.
-	Common = initializer.Common
-	Server = initializer.Server
-	Client = initializer.Client
-
-	args.transform(additionalArgs)
+	initializer.parseConfig(args)
+	Client, Server, Common = args.transformConfig(
+		additionalArgs,
+		initializer.Client,
+		initializer.Server,
+		initializer.Common,
+	)
 }
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index 7282741..49b405d 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -3,6 +3,7 @@ package dlog
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -21,7 +22,6 @@ var Client *DLog
 var Server *DLog
 
 // Common is the log handler for all other packages.
-// TODO: Rename Common to Common
 var Common *DLog
 
 var mutex sync.Mutex
@@ -75,15 +75,22 @@ type DLog struct {
 	sourcePackage source
 	// Max log level to log.
 	maxLevel level
+	// Current hostname.
+	hostname string
 }
 
 // New creates a new DTail logger.
 func New(sourceProcess, sourcePackage source, impl loggers.Impl, maxLevel level) *DLog {
+	hostname, err := os.Hostname()
+	if err != nil {
+		panic(err)
+	}
 	return &DLog{
 		logger:        loggers.Factory(sourceProcess.String(), impl),
 		sourceProcess: sourceProcess,
 		sourcePackage: sourcePackage,
 		maxLevel:      maxLevel,
+		hostname:      hostname,
 	}
 }
 
@@ -106,11 +113,18 @@ func (d *DLog) log(level level, args []interface{}) string {
 	defer pool.RecycleBuilderBuffer(sb)
 	now := time.Now()
 
-	sb.WriteString(d.sourcePackage.String())
-	sb.WriteString(protocol.FieldDelimiter)
-	sb.WriteString(now.Format("20060102-150405"))
-	sb.WriteString(protocol.FieldDelimiter)
-	sb.WriteString(level.String())
+	switch d.sourceProcess {
+	case CLIENT:
+		sb.WriteString(d.sourcePackage.String())
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(d.hostname)
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(level.String())
+	default:
+		sb.WriteString(level.String())
+		sb.WriteString(protocol.FieldDelimiter)
+		sb.WriteString(now.Format("20060102-150405"))
+	}
 	sb.WriteString(protocol.FieldDelimiter)
 	d.writeArgStrings(sb, args)
 
@@ -159,6 +173,11 @@ func (d *DLog) Warn(args ...interface{}) string {
 }
 
 func (d *DLog) Info(args ...interface{}) string {
+	if d.sourcePackage == SERVER && d.sourceProcess != CLIENT {
+		// This can be dtail client in serverless mode. In this case log all
+		// info server messages as verbose.
+		return d.log(VERBOSE, args)
+	}
 	return d.log(INFO, args)
 }
 
@@ -183,13 +202,14 @@ func (d *DLog) Raw(message string) string {
 		d.logger.Log(time.Now(), message)
 		return message
 	}
-
-	d.logger.Log(time.Now(), brush.Colorfy(message))
+	d.logger.LogWithColors(time.Now(), message, brush.Colorfy(message))
 	return message
 }
 
 func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 	args := make([]interface{}, len(data)+1)
+
+	// TODO: mC compatible SERVER mapreduce fields, no MAPREDUCE keyword in CLIENT mode
 	args[0] = fmt.Sprintf("%s:%s", "MAPREDUCE", strings.ToUpper(table))
 
 	i := 1
@@ -197,7 +217,6 @@ func (d *DLog) Mapreduce(table string, data map[string]interface{}) string {
 		args[i] = fmt.Sprintf("%s=%v", k, v)
 		i++
 	}
-
 	return d.log(INFO, args)
 }
 
diff --git a/internal/io/logger/logger.go b/internal/io/logger/logger.go
index 6a6b5ec..905d1cf 100644
--- a/internal/io/logger/logger.go
+++ b/internal/io/logger/logger.go
@@ -1,7 +1,5 @@
 package logger
 
-// TODO: Rewrite this logger
-
 import (
 	"bufio"
 	"context"
diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go
index e0bbc30..7fb1700 100644
--- a/internal/mapr/logformat/default.go
+++ b/internal/mapr/logformat/default.go
@@ -27,7 +27,7 @@ func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) {
 
 	fields["$severity"] = splitted[0]
 	fields["$loglevel"] = splitted[0]
-	// TODO: Parse time like we do at Mimecast
+	// NEXT: Parse time like we do at Mimecast
 	fields["$time"] = splitted[1]
 
 	for _, kv := range splitted[4:] {
diff --git a/internal/server/continuous.go b/internal/server/continuous.go
index 5f4c454..87c8889 100644
--- a/internal/server/continuous.go
+++ b/internal/server/continuous.go
@@ -61,7 +61,7 @@ func (c *continuous) runJob(ctx context.Context, job config.Continuous) {
 	}
 
 	args := config.Args{
-		ConnectionsPerCPU: 10,
+		ConnectionsPerCPU: config.DefaultConnectionsPerCPU,
 		Discovery:         job.Discovery,
 		ServersStr:        servers,
 		What:              files,
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 60ad2a0..c76ae2a 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -7,9 +7,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/fs"
 	"github.com/mimecast/dtail/internal/io/line"
-	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/omode"
 	"github.com/mimecast/dtail/internal/regex"
 )
diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go
index f474cc8..64e6573 100644
--- a/internal/server/scheduler.go
+++ b/internal/server/scheduler.go
@@ -72,7 +72,7 @@ func (s *scheduler) runJobs(ctx context.Context) {
 		}
 
 		args := config.Args{
-			ConnectionsPerCPU: 10,
+			ConnectionsPerCPU: config.DefaultConnectionsPerCPU,
 			Discovery:         job.Discovery,
 			ServersStr:        servers,
 			What:              files,
diff --git a/internal/server/server.go b/internal/server/server.go
index a8f541b..d1cd57d 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -124,7 +124,12 @@ func (s *Server) handleConnection(ctx context.Context, conn net.Conn) {
 }
 
 func (s *Server) handleChannel(ctx context.Context, sshConn gossh.Conn, newChannel gossh.NewChannel) {
-	user := user.New(sshConn.User(), sshConn.RemoteAddr().String())
+	user, err := user.New(sshConn.User(), sshConn.RemoteAddr().String())
+	if err != nil {
+		dlog.Server.Error(user, err)
+		newChannel.Reject(gossh.Prohibited, err.Error())
+		return
+	}
 	dlog.Server.Info(user, "Invoking channel handler")
 
 	if newChannel.ChannelType() != "session" {
@@ -213,7 +218,10 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch
 
 // Callback for SSH authentication.
 func (s *Server) Callback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Permissions, error) {
-	user := user.New(c.User(), c.RemoteAddr().String())
+	user, err := user.New(c.User(), c.RemoteAddr().String())
+	if err != nil {
+		return nil, err
+	}
 
 	if config.ServerRelaxedAuthEnable {
 		dlog.Server.Fatal(user, "Granting permissions via relaxed-auth")
diff --git a/internal/ssh/server/publickeycallback.go b/internal/ssh/server/publickeycallback.go
index 65ecdd1..59d1f31 100644
--- a/internal/ssh/server/publickeycallback.go
+++ b/internal/ssh/server/publickeycallback.go
@@ -15,7 +15,10 @@ import (
 
 // PublicKeyCallback is for the server to check whether a public SSH key is authorized ot not.
 func PublicKeyCallback(c gossh.ConnMetadata, offeredPubKey gossh.PublicKey) (*gossh.Permissions, error) {
-	user := user.New(c.User(), c.RemoteAddr().String())
+	user, err := user.New(c.User(), c.RemoteAddr().String())
+	if err != nil {
+		return nil, err
+	}
 	dlog.Common.Info(user, "Incoming authorization")
 
 	cwd, err := os.Getwd()
diff --git a/internal/user/server/user.go b/internal/user/server/user.go
index 99cd211..70ead1c 100644
--- a/internal/user/server/user.go
+++ b/internal/user/server/user.go
@@ -8,8 +8,8 @@ import (
 	"strings"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/fs/permissions"
 	"github.com/mimecast/dtail/internal/io/dlog"
+	"github.com/mimecast/dtail/internal/io/fs/permissions"
 )
 
 const maxLinkDepth int = 100
@@ -25,11 +25,16 @@ type User struct {
 }
 
 // New returns a new user.
-func New(name, remoteAddress string) *User {
+func New(name, remoteAddress string) (*User, error) {
+	permissions, err := config.ServerUserPermissions(name)
+	if err != nil {
+		return nil, err
+	}
 	return &User{
 		Name:          name,
 		remoteAddress: remoteAddress,
-	}
+		permissions:   permissions,
+	}, nil
 }
 
 // String representation of the user.
@@ -39,9 +44,9 @@ func (u *User) String() string {
 
 // HasFilePermission is used to determine whether user is alowed to read a file.
 func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission bool) {
-	dlog.Common.Debug(u, filePath, permissionType, "Checking config permissions")
+	dlog.Server.Debug(u, filePath, permissionType, "Checking config permissions")
 	if config.ServerRelaxedAuthEnable {
-		dlog.Common.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
+		dlog.Server.Fatal(u, filePath, permissionType, "Server releaxed auth enabled")
 		return true
 	}
 
@@ -52,25 +57,25 @@ func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission
 
 	cleanPath, err := filepath.EvalSymlinks(filePath)
 	if err != nil {
-		dlog.Common.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
+		dlog.Server.Error(u, filePath, permissionType, "Unable to evaluate symlinks", err)
 		hasPermission = false
 		return
 	}
 
 	cleanPath, err = filepath.Abs(cleanPath)
 	if err != nil {
-		dlog.Common.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
+		dlog.Server.Error(u, cleanPath, permissionType, "Unable to make file path absolute", err)
 		hasPermission = false
 		return
 	}
 
 	if cleanPath != filePath {
-		dlog.Common.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
+		dlog.Server.Info(u, filePath, cleanPath, permissionType, "Calculated new clean path from original file path (possibly symlink)")
 	}
 
 	hasPermission, err = u.hasFilePermission(cleanPath, permissionType)
 	if err != nil {
-		dlog.Common.Warn(u, cleanPath, err)
+		dlog.Server.Warn(u, cleanPath, err)
 	}
 
 	return
@@ -81,7 +86,7 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 	if _, err := permissions.ToRead(u.Name, cleanPath); err != nil {
 		return false, fmt.Errorf("User without OS file system permissions to read path: '%v'", err)
 	}
-	dlog.Common.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
+	dlog.Server.Info(u, cleanPath, permissionType, "User with OS file system permissions to path")
 
 	// Only allow to follow regular files or symlinks.
 	info, err := os.Lstat(cleanPath)
@@ -93,12 +98,6 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error)
 		return false, fmt.Errorf("Can only open regular files or follow symlinks")
 	}
 
-	permissions, err := config.ServerUserPermissions(u.Name)
-	if err != nil {
-		return false, err
-	}
-	u.permissions = permissions
-
 	hasPermission, err := u.iteratePaths(cleanPath, permissionType)
 	if err != nil {
 		return false, err
@@ -123,7 +122,7 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 			permission = strings.Join(splitted[1:], ":")
 		}
 
-		dlog.Common.Debug(u, cleanPath, typeStr, permission)
+		dlog.Server.Debug(u, cleanPath, typeStr, permission)
 
 		if typeStr != permissionType {
 			continue
@@ -141,12 +140,12 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) {
 		}
 
 		if negate && re.MatchString(cleanPath) {
-			dlog.Common.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission)
 			hasPermission = false
 		}
 
 		if !negate && re.MatchString(cleanPath) {
-			dlog.Common.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
+			dlog.Server.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission)
 			hasPermission = true
 		}
 	}