From 40728693d9cfbd99458f12d5183119ec699b8907 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 27 Feb 2020 16:28:10 +0000 Subject: can run scheduled queries across a storage volume --- internal/config/server.go | 5 +++-- internal/server/scheduler.go | 22 ++++++++-------------- internal/server/server.go | 36 +++++++++++++++++++++++++++++++++--- internal/version/version.go | 2 +- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/internal/config/server.go b/internal/config/server.go index aa1a02e..09eb2c4 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -20,9 +20,10 @@ type Scheduled struct { Files string Query string Outfile string - Discovery string `json:",omitempty"` - Servers string + Discovery string `json:",omitempty"` + Servers []string `json:",omitempty"` TimeRange [2]int + AllowFrom []string `json:",omitempty"` } // ServerConfig represents the server configuration. diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go index e0cecfd..d2657d0 100644 --- a/internal/server/scheduler.go +++ b/internal/server/scheduler.go @@ -3,7 +3,6 @@ package server import ( "context" "fmt" - "math/rand" "os" "strconv" "strings" @@ -21,23 +20,17 @@ const authLength = 64 const authCharset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@$%^&*()_+[]" type scheduler struct { - authPayload string } func newScheduler() *scheduler { - seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) - - b := make([]byte, authLength) - for i := range b { - b[i] = authCharset[seededRand.Intn(len(authCharset))] - } - - return &scheduler{ - authPayload: string(b), - } + return &scheduler{} } func (s *scheduler) start(ctx context.Context) { + // First run after just 10s! + time.Sleep(time.Second * 10) + s.runJobs(ctx) + for { select { case <-time.After(time.Minute): @@ -75,7 +68,7 @@ func (s *scheduler) runJobs(ctx context.Context) { continue } - servers := scheduled.Servers + servers := strings.Join(scheduled.Servers, ",") if servers == "" { servers = config.Server.SSHBindAddress } @@ -88,7 +81,8 @@ func (s *scheduler) runJobs(ctx context.Context) { Mode: omode.MapClient, UserName: config.ScheduledUser, } - args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(s.authPayload)) + + args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(scheduled.Name)) tmpOutfile := fmt.Sprintf("%s.tmp", outfile) query := fmt.Sprintf("%s outfile %s", scheduled.Query, tmpOutfile) diff --git a/internal/server/server.go b/internal/server/server.go index 5c46147..34d0d88 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "strings" "github.com/mimecast/dtail/internal/config" "github.com/mimecast/dtail/internal/io/logger" @@ -198,16 +199,45 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, in <-ch func (s *Server) backgroundUserCallback(c gossh.ConnMetadata, authPayload []byte) (*gossh.Permissions, error) { user := user.New(c.User(), c.RemoteAddr().String()) + authInfo := string(authPayload) - if user.Name == config.ControlUser && string(authPayload) == config.ControlUser { + if user.Name == config.ControlUser && authInfo == config.ControlUser { logger.Debug(user, "Granting permissions to control user") return nil, nil } - if user.Name == config.ScheduledUser && string(authPayload) == s.sched.authPayload { - logger.Debug(user, "Granting permissions to schedule user") + if user.Name == config.ScheduledUser && s.canRunScheduledJob(c.RemoteAddr().String(), user, authInfo) { + logger.Debug(user, "Schedule user can run scheduled job remotely") return nil, nil } return nil, fmt.Errorf("user %s not authorized", user) } + +func (s *Server) canRunScheduledJob(addr string, user *user.User, jobName string) bool { + logger.Debug("canRunScheduledJob", user, jobName) + splitted := strings.Split(addr, ":") + ip := splitted[0] + + for _, job := range config.Server.Schedule { + if job.Name != jobName { + continue + } + for _, myAddr := range job.AllowFrom { + myIps, err := net.LookupIP(myAddr) + if err != nil { + logger.Error(user, myAddr, err) + continue + } + + for _, myIp := range myIps { + logger.Debug("canRunScheduledJob", "Comparing IP addresses", ip, myIp.String()) + if ip == myIp.String() { + return true + } + } + } + } + + return false +} diff --git a/internal/version/version.go b/internal/version/version.go index b3614c5..66918f6 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -11,7 +11,7 @@ const ( // Name of DTail. Name string = "DTail" // Version of DTail. - Version string = "2.1.0" + Version string = "2.1.1" // Additional information for DTail Additional string = "" // ProtocolCompat -ibility version. -- cgit v1.2.3 From cb2de4109632932bccf5d13e77f40157d4e84c18 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 27 Feb 2020 16:54:42 +0000 Subject: outfile is optional --- internal/config/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/server.go b/internal/config/server.go index 09eb2c4..bc2f40a 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -19,7 +19,7 @@ type Scheduled struct { Enable bool Files string Query string - Outfile string + Outfile string `json:",omitempty"` Discovery string `json:",omitempty"` Servers []string `json:",omitempty"` TimeRange [2]int -- cgit v1.2.3 From 71d2786aeaa793d9cf51c389d2e91cf472d25f7a Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 28 Feb 2020 13:14:29 +0000 Subject: more on scheduled commands --- internal/config/config.go | 2 +- internal/mapr/query.go | 3 ++- internal/server/handlers/mapcommand.go | 10 ++++------ internal/server/handlers/readcommand.go | 4 ++-- internal/server/handlers/serverhandler.go | 2 ++ internal/server/scheduler.go | 2 +- internal/server/server.go | 10 +++++----- internal/user/server/user.go | 2 +- 8 files changed, 18 insertions(+), 17 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index d20a480..166e143 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,7 +10,7 @@ import ( const ControlUser string = "DTAIL-CONTROL-USER" // ScheduledUser is used for scheduled queries. -const ScheduledUser string = "DTAIL-SCHEDULED-USER" +const ScheduleUser string = "DTAIL-SCHEDULED-USER" // Client holds a DTail client configuration. var Client *ClientConfig diff --git a/internal/mapr/query.go b/internal/mapr/query.go index 0127be3..ec93e14 100644 --- a/internal/mapr/query.go +++ b/internal/mapr/query.go @@ -3,10 +3,11 @@ package mapr import ( "errors" "fmt" - "github.com/mimecast/dtail/internal/io/logger" "strconv" "strings" "time" + + "github.com/mimecast/dtail/internal/io/logger" ) const ( diff --git a/internal/server/handlers/mapcommand.go b/internal/server/handlers/mapcommand.go index 10372da..c3e600e 100644 --- a/internal/server/handlers/mapcommand.go +++ b/internal/server/handlers/mapcommand.go @@ -15,18 +15,16 @@ type mapCommand struct { // NewMapCommand returns a new server side mapreduce command. func newMapCommand(serverHandler *ServerHandler, argc int, args []string) (mapCommand, *server.Aggregate, error) { - mapCommand := mapCommand{ - server: serverHandler, - } + m := mapCommand{server: serverHandler} queryStr := strings.Join(args[1:], " ") aggregate, err := server.NewAggregate(queryStr) if err != nil { - return mapCommand, nil, err + return m, nil, err } - mapCommand.aggregate = aggregate - return mapCommand, aggregate, nil + m.aggregate = aggregate + return m, aggregate, nil } diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go index 6ab41cd..63158f6 100644 --- a/internal/server/handlers/readcommand.go +++ b/internal/server/handlers/readcommand.go @@ -27,8 +27,8 @@ func newReadCommand(server *ServerHandler, mode omode.Mode) *readCommand { func (r *readCommand) Start(ctx context.Context, argc int, args []string) { regex := "." if argc >= 4 { - regex = strings.Join(args[3:], " ") - logger.Debug("Joined regex", regex) + regex = strings.Join(args[3:], " ") + logger.Debug("Joined regex", regex) } if argc < 3 { r.server.sendServerMessage(logger.Warn(r.server.user, commandParseWarning, args, argc)) diff --git a/internal/server/handlers/serverhandler.go b/internal/server/handlers/serverhandler.go index b840c77..2979dd5 100644 --- a/internal/server/handlers/serverhandler.go +++ b/internal/server/handlers/serverhandler.go @@ -229,6 +229,7 @@ func (h *ServerHandler) handleUserCommand(ctx context.Context, argc int, args [] if err != nil { h.sendServerMessage(err.Error()) logger.Error(h.user, err) + h.shutdown() return } @@ -334,6 +335,7 @@ func (h *ServerHandler) incrementActiveReaders() { defer h.mutex.Unlock() h.activeReaders++ } + func (h *ServerHandler) decrementActiveReaders() int { h.mutex.Lock() defer h.mutex.Unlock() diff --git a/internal/server/scheduler.go b/internal/server/scheduler.go index d2657d0..db49d1b 100644 --- a/internal/server/scheduler.go +++ b/internal/server/scheduler.go @@ -79,7 +79,7 @@ func (s *scheduler) runJobs(ctx context.Context) { ServersStr: servers, What: files, Mode: omode.MapClient, - UserName: config.ScheduledUser, + UserName: config.ScheduleUser, } args.SSHAuthMethods = append(args.SSHAuthMethods, gossh.Password(scheduled.Name)) diff --git a/internal/server/server.go b/internal/server/server.go index 34d0d88..0b93537 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -206,16 +206,16 @@ func (s *Server) backgroundUserCallback(c gossh.ConnMetadata, authPayload []byte return nil, nil } - if user.Name == config.ScheduledUser && s.canRunScheduledJob(c.RemoteAddr().String(), user, authInfo) { - logger.Debug(user, "Schedule user can run scheduled job remotely") + if user.Name == config.ScheduleUser && s.schedueleUserCanHaveSSHSession(c.RemoteAddr().String(), user, authInfo) { + logger.Debug(user, "Granting SSH connection to schedule user") return nil, nil } return nil, fmt.Errorf("user %s not authorized", user) } -func (s *Server) canRunScheduledJob(addr string, user *user.User, jobName string) bool { - logger.Debug("canRunScheduledJob", user, jobName) +func (s *Server) schedueleUserCanHaveSSHSession(addr string, user *user.User, jobName string) bool { + logger.Debug("schedueleUserCanHaveSSHSession", user, jobName) splitted := strings.Split(addr, ":") ip := splitted[0] @@ -231,7 +231,7 @@ func (s *Server) canRunScheduledJob(addr string, user *user.User, jobName string } for _, myIp := range myIps { - logger.Debug("canRunScheduledJob", "Comparing IP addresses", ip, myIp.String()) + logger.Debug("schedueleUserCanHaveSSHSession", "Comparing IP addresses", ip, myIp.String()) if ip == myIp.String() { return true } diff --git a/internal/user/server/user.go b/internal/user/server/user.go index 7389d74..2c1df64 100644 --- a/internal/user/server/user.go +++ b/internal/user/server/user.go @@ -41,7 +41,7 @@ func (u *User) String() string { func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission bool) { logger.Debug(u, filePath, permissionType, "Checking config permissions") - if u.Name == config.ScheduledUser { + if u.Name == config.ScheduleUser { // Schedule user has same permissions as dtail process itself. return true } -- cgit v1.2.3 From 1922e448e84e218cc39d4394e9b4becfa6f0a83d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 28 Feb 2020 17:22:17 +0000 Subject: change whitelisting to blacklisting --- internal/user/server/user.go | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/internal/user/server/user.go b/internal/user/server/user.go index 2c1df64..00cc611 100644 --- a/internal/user/server/user.go +++ b/internal/user/server/user.go @@ -79,23 +79,6 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error) } logger.Info(u, cleanPath, permissionType, "User with OS file system permissions to path") - // If file system permission is given, also check permissions - // as configured in DTail config file. - if len(u.permissions) == 0 { - p, err := config.ServerUserPermissions(u.Name) - if err != nil { - return false, err - } - u.permissions = p - } - - var hasPermission bool - var err error - - if hasPermission, err = u.iteratePaths(cleanPath, permissionType); err != nil { - return false, err - } - // Only allow to follow regular files or symlinks. info, err := os.Lstat(cleanPath) if err != nil { @@ -106,10 +89,24 @@ func (u *User) hasFilePermission(cleanPath, permissionType string) (bool, error) return false, fmt.Errorf("Can only open regular files or follow symlinks") } + permissions, err := config.ServerUserPermissions(u.Name) + if err != nil { + return false, err + } + u.permissions = permissions + + hasPermission, err := u.iteratePaths(cleanPath, permissionType) + if err != nil { + return false, err + } + return hasPermission, nil } func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) { + // By default assume no permissions + hasPermission := false + for _, permission := range u.permissions { typeStr := "readfiles" // Assume ReadFiles by default. @@ -138,13 +135,15 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) { } if negate && re.MatchString(cleanPath) { - return false, fmt.Errorf("Permission test failed, matching negative pattern '%s'", permission) + logger.Info(u, cleanPath, "Permission test failed partially, matching negative pattern '%s'", permission) + hasPermission = false } if !negate && re.MatchString(cleanPath) { logger.Info(u, cleanPath, "Permission test passed partially, matching positive pattern", permission) + hasPermission = true } } - return true, nil + return hasPermission, nil } -- cgit v1.2.3