// Package main provides the DMap (Distributed MapReduce) command-line tool. // DMap is a specialized client for running MapReduce-style queries across // distributed log files on multiple servers simultaneously via SSH. // // Key features: // - SQL-like query syntax (SELECT...FROM...WHERE...GROUP BY) // - Distributed log processing and aggregation // - Server-side local aggregation with client-side final aggregation // - Pluggable log format parsers (CSV, JSON, custom formats) // - SSH-based secure connections // - Configurable connection pooling and timeouts // - Color-coded output for better readability // - Built-in profiling support // // DMap is particularly useful for analyzing log patterns, generating // reports, and performing statistical analysis across large distributed // log datasets without having to collect all logs to a central location. package main import ( "context" "flag" "os" "sync" "net/http" _ "net/http" _ "net/http/pprof" "github.com/mimecast/dtail/internal/clients" "github.com/mimecast/dtail/internal/config" "github.com/mimecast/dtail/internal/io/dlog" "github.com/mimecast/dtail/internal/io/signal" "github.com/mimecast/dtail/internal/omode" "github.com/mimecast/dtail/internal/source" "github.com/mimecast/dtail/internal/user" "github.com/mimecast/dtail/internal/version" ) // main is the entry point for the DMap application. // It parses command-line arguments, initializes logging, creates a MaprClient // for executing MapReduce queries, and processes results from distributed // servers. The function handles graceful shutdown and waits for all // operations to complete. func main() { var displayVersion bool var pprof string args := config.Args{ Mode: omode.MapClient, } userName := user.Name() flag.BoolVar(&args.NoColor, "noColor", false, "Disable ANSII terminal colors") flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode") flag.BoolVar(&args.Plain, "plain", false, "Plain output mode") flag.BoolVar(&args.TrustAllHosts, "trustAllHosts", false, "Trust all unknown host keys") flag.BoolVar(&displayVersion, "version", false, "Display version") flag.IntVar(&args.ConnectionsPerCPU, "cpc", config.DefaultConnectionsPerCPU, "How many connections established per CPU core concurrently") flag.IntVar(&args.SSHPort, "port", config.DefaultSSHPort, "SSH server port") flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection") flag.StringVar(&args.ConfigFile, "cfg", "", "Config file path") flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method") flag.StringVar(&args.LogDir, "logDir", "~/log", "Log dir") flag.StringVar(&args.Logger, "logger", config.DefaultClientLogger, "Logger name") flag.StringVar(&args.LogLevel, "logLevel", config.DefaultLogLevel, "Log level") flag.StringVar(&args.SSHPrivateKeyFilePath, "key", "", "Path to private key") flag.StringVar(&args.QueryStr, "query", "", "Map reduce query") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") flag.StringVar(&args.UserName, "user", userName, "Your system user name") flag.StringVar(&args.What, "files", "", "File(s) to read") flag.StringVar(&pprof, "pprof", "", "Start PProf server this address") flag.Parse() config.Setup(source.Client, &args, flag.Args()) if displayVersion { version.PrintAndExit() } ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup wg.Add(1) dlog.Start(ctx, &wg, source.Client) if pprof != "" { go http.ListenAndServe(pprof, nil) dlog.Client.Info("Started PProf", pprof) } client, err := clients.NewMaprClient(args, clients.DefaultMode) if err != nil { dlog.Client.FatalPanic(err) } status := client.Start(ctx, signal.InterruptCh(ctx)) cancel() wg.Wait() os.Exit(status) }