From 06fc0aba4eaa5906b7883d305e9b4116840e8871 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 24 Oct 2021 18:13:36 +0300 Subject: Bump up RC version --- .gitignore | 1 + internal/version/version.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1ccedb6..20e162c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ tags /dmap /dserver /dtailhealth +known_hosts diff --git a/internal/version/version.go b/internal/version/version.go index 68b9e6e..5872980 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -13,7 +13,7 @@ const ( // Name of DTail. Name string = "DTail" // Version of DTail. - Version string = "4.0.0-RC1" + Version string = "4.0.0-RC2" // Additional information for DTail Additional string = "Have a lot of fun!" ) -- cgit v1.2.3 From 7c927e7e6d913168798d245a7ceea32a9ca85643 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 24 Oct 2021 18:24:46 +0300 Subject: Use different ssh host key file path for integration tests --- internal/config/initializer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 024464e..936df8a 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -85,6 +85,7 @@ func (in *initializer) readEnvironmentVars() { if Env("DTAIL_RUN_INTEGRATION_TESTS") { os.Setenv("DTAIL_HOSTNAME_OVERRIDE", "integrationtest") os.Setenv("DTAIL_SSH_KNOWN_HOSTS_FILE", "./known_hosts") + in.Server.HostKeyFile = "./ssh_host_key" } } -- cgit v1.2.3 From d0492e3f63f86ce053f59362a55d23cf6397d526 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 27 Oct 2021 09:54:06 +0300 Subject: integration tests use separate ssh private key file --- .gitignore | 3 ++ cmd/dcat/main.go | 2 +- cmd/dgrep/main.go | 2 +- cmd/dmap/main.go | 2 +- cmd/dtail/main.go | 2 +- integrationtests/dcat_test.go | 6 +-- integrationtests/dgrep_test.go | 8 ++-- integrationtests/dmap_test.go | 6 +-- integrationtests/dtail_test.go | 4 +- integrationtests/dtailhealth_test.go | 6 +-- internal/clients/baseclient.go | 2 +- internal/config/args.go | 52 ++++++++++----------- internal/config/initializer.go | 2 +- internal/ssh/client/authmethods.go | 40 ++++++++++------ internal/ssh/client/clientkeypair.go | 91 ++++++++++++++++++++++++++++++++++++ 15 files changed, 167 insertions(+), 61 deletions(-) create mode 100644 internal/ssh/client/clientkeypair.go diff --git a/.gitignore b/.gitignore index 20e162c..44dc08e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ tags /dserver /dtailhealth known_hosts +id_rsa +id_rsa.pub +ssh_host_key diff --git a/cmd/dcat/main.go b/cmd/dcat/main.go index 5fd22ea..8ac5eda 100644 --- a/cmd/dcat/main.go +++ b/cmd/dcat/main.go @@ -40,7 +40,7 @@ func main() { flag.StringVar(&args.LogDir, "logDir", "~/log", "Log dir") flag.StringVar(&args.Logger, "logger", config.DefaultClientLogger, "Logger name") flag.StringVar(&args.LogLevel, "logLevel", config.DefaultLogLevel, "Log level") - flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") + flag.StringVar(&args.SSHPrivateKeyPathFile, "key", "", "Path to private key") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") flag.StringVar(&args.UserName, "user", userName, "Your system user name") flag.StringVar(&args.What, "files", "", "File(s) to read") diff --git a/cmd/dgrep/main.go b/cmd/dgrep/main.go index 02b2463..8657fa9 100644 --- a/cmd/dgrep/main.go +++ b/cmd/dgrep/main.go @@ -44,7 +44,7 @@ func main() { flag.StringVar(&args.LogDir, "logDir", "~/log", "Log dir") flag.StringVar(&args.Logger, "logger", config.DefaultClientLogger, "Logger name") flag.StringVar(&args.LogLevel, "logLevel", config.DefaultLogLevel, "Log level") - flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") + flag.StringVar(&args.SSHPrivateKeyPathFile, "key", "", "Path to private key") flag.StringVar(&args.RegexStr, "regex", ".", "Regular expression") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") flag.StringVar(&args.UserName, "user", userName, "Your system user name") diff --git a/cmd/dmap/main.go b/cmd/dmap/main.go index 2c941f3..79a53ba 100644 --- a/cmd/dmap/main.go +++ b/cmd/dmap/main.go @@ -44,7 +44,7 @@ func main() { flag.StringVar(&args.LogDir, "logDir", "~/log", "Log dir") flag.StringVar(&args.Logger, "logger", config.DefaultClientLogger, "Logger name") flag.StringVar(&args.LogLevel, "logLevel", config.DefaultLogLevel, "Log level") - flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") + flag.StringVar(&args.SSHPrivateKeyPathFile, "key", "", "Path to private key") flag.StringVar(&args.QueryStr, "query", "", "Map reduce query") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") flag.StringVar(&args.UserName, "user", userName, "Your system user name") diff --git a/cmd/dtail/main.go b/cmd/dtail/main.go index ff0cea9..2b46141 100644 --- a/cmd/dtail/main.go +++ b/cmd/dtail/main.go @@ -57,7 +57,7 @@ func main() { flag.StringVar(&args.LogDir, "logDir", "~/log", "Log dir") flag.StringVar(&args.Logger, "logger", config.DefaultClientLogger, "Logger name") flag.StringVar(&args.LogLevel, "logLevel", config.DefaultLogLevel, "Log level") - flag.StringVar(&args.PrivateKeyPathFile, "key", "", "Path to private key") + flag.StringVar(&args.SSHPrivateKeyPathFile, "key", "", "Path to private key") flag.StringVar(&args.QueryStr, "query", "", "Map reduce query") flag.StringVar(&args.RegexStr, "regex", ".", "Regular expression") flag.StringVar(&args.ServersStr, "servers", "", "Remote servers to connect") diff --git a/integrationtests/dcat_test.go b/integrationtests/dcat_test.go index 777e835..6928afa 100644 --- a/integrationtests/dcat_test.go +++ b/integrationtests/dcat_test.go @@ -9,7 +9,7 @@ import ( ) func TestDCat(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -33,7 +33,7 @@ func TestDCat(t *testing.T) { } func TestDCat2(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { return } testdataFile := "dcat2.txt" @@ -62,7 +62,7 @@ func TestDCat2(t *testing.T) { } func TestDCatColors(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { return } diff --git a/integrationtests/dgrep_test.go b/integrationtests/dgrep_test.go index 26abc2f..35c3ff5 100644 --- a/integrationtests/dgrep_test.go +++ b/integrationtests/dgrep_test.go @@ -9,7 +9,7 @@ import ( ) func TestDGrep(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -38,7 +38,7 @@ func TestDGrep(t *testing.T) { } func TestDGrep2(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -68,7 +68,7 @@ func TestDGrep2(t *testing.T) { } func TestDGrepContext(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -98,7 +98,7 @@ func TestDGrepContext(t *testing.T) { } func TestDGrepContext2(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index 53b8574..6a93b7b 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -10,7 +10,7 @@ import ( ) func TestDMap(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -56,7 +56,7 @@ func TestDMap(t *testing.T) { } func TestDMap2(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -93,7 +93,7 @@ func TestDMap2(t *testing.T) { } func TestDMap3(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } diff --git a/integrationtests/dtail_test.go b/integrationtests/dtail_test.go index e9cf257..2f2708e 100644 --- a/integrationtests/dtail_test.go +++ b/integrationtests/dtail_test.go @@ -12,7 +12,7 @@ import ( ) func TestDTailWithServer(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -131,7 +131,7 @@ func TestDTailWithServer(t *testing.T) { } func TestDTailColorTable(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } diff --git a/integrationtests/dtailhealth_test.go b/integrationtests/dtailhealth_test.go index 271f11d..b53c425 100644 --- a/integrationtests/dtailhealth_test.go +++ b/integrationtests/dtailhealth_test.go @@ -10,7 +10,7 @@ import ( ) func TestDTailHealthCheck(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -32,7 +32,7 @@ func TestDTailHealthCheck(t *testing.T) { } func TestDTailHealthCheck2(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } @@ -57,7 +57,7 @@ func TestDTailHealthCheck2(t *testing.T) { } func TestDTailHealthCheck3(t *testing.T) { - if !config.Env("DTAIL_RUN_INTEGRATION_TESTS") { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { t.Log("Skipping") return } diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go index 4a7bd84..41521ea 100644 --- a/internal/clients/baseclient.go +++ b/internal/clients/baseclient.go @@ -56,7 +56,7 @@ func (c *baseClient) init() { } c.sshAuthMethods, c.hostKeyCallback = client.InitSSHAuthMethods( c.Args.SSHAuthMethods, c.Args.SSHHostKeyCallback, c.Args.TrustAllHosts, - c.throttleCh, c.Args.PrivateKeyPathFile) + c.throttleCh, c.Args.SSHPrivateKeyPathFile) } func (c *baseClient) makeConnections(maker maker) { diff --git a/internal/config/args.go b/internal/config/args.go index 3d7ac7d..859166d 100644 --- a/internal/config/args.go +++ b/internal/config/args.go @@ -15,31 +15,31 @@ import ( // Args is a helper struct to summarize common client arguments. type Args struct { lcontext.LContext - Arguments []string - ConfigFile string - ConnectionsPerCPU int - Discovery string - LogDir string - Logger string - LogLevel string - Mode omode.Mode - NoColor bool - PrivateKeyPathFile string - QueryStr string - Quiet bool - RegexInvert bool - RegexStr string - Serverless bool - ServersStr string - Spartan bool - SSHAuthMethods []gossh.AuthMethod - SSHBindAddress string - SSHHostKeyCallback gossh.HostKeyCallback - SSHPort int - Timeout int - TrustAllHosts bool - UserName string - What string + Arguments []string + ConfigFile string + ConnectionsPerCPU int + Discovery string + LogDir string + Logger string + LogLevel string + Mode omode.Mode + NoColor bool + QueryStr string + Quiet bool + RegexInvert bool + RegexStr string + SSHAuthMethods []gossh.AuthMethod + SSHBindAddress string + SSHHostKeyCallback gossh.HostKeyCallback + SSHPort int + SSHPrivateKeyPathFile string + Serverless bool + ServersStr string + Spartan bool + Timeout int + TrustAllHosts bool + UserName string + What string } func (a *Args) String() string { @@ -56,7 +56,6 @@ func (a *Args) String() string { sb.WriteString(fmt.Sprintf("%s:%v,", "Logger", a.Logger)) sb.WriteString(fmt.Sprintf("%s:%v,", "Mode", a.Mode)) sb.WriteString(fmt.Sprintf("%s:%v,", "NoColor", a.NoColor)) - sb.WriteString(fmt.Sprintf("%s:%v,", "PrivateKeyPathFile", a.PrivateKeyPathFile)) sb.WriteString(fmt.Sprintf("%s:%v,", "QueryStr", a.QueryStr)) sb.WriteString(fmt.Sprintf("%s:%v,", "Quiet", a.Quiet)) sb.WriteString(fmt.Sprintf("%s:%v,", "RegexInvert", a.RegexInvert)) @@ -64,6 +63,7 @@ func (a *Args) String() string { sb.WriteString(fmt.Sprintf("%s:%v,", "SSHAuthMethods", a.SSHAuthMethods)) sb.WriteString(fmt.Sprintf("%s:%v,", "SSHBindAddress", a.SSHBindAddress)) sb.WriteString(fmt.Sprintf("%s:%v,", "SSHHostKeyCallback", a.SSHHostKeyCallback)) + sb.WriteString(fmt.Sprintf("%s:%v,", "SSHPrivateKeyPathFile", a.SSHPrivateKeyPathFile)) sb.WriteString(fmt.Sprintf("%s:%v,", "SSHPort", a.SSHPort)) sb.WriteString(fmt.Sprintf("%s:%v,", "Serverless", a.Serverless)) sb.WriteString(fmt.Sprintf("%s:%v,", "ServersStr", a.ServersStr)) diff --git a/internal/config/initializer.go b/internal/config/initializer.go index 936df8a..74d0289 100644 --- a/internal/config/initializer.go +++ b/internal/config/initializer.go @@ -82,7 +82,7 @@ func (in *initializer) transformConfig(sourceProcess source.Source, args *Args, // There are some special options which can be set by environment variable. func (in *initializer) readEnvironmentVars() { - if Env("DTAIL_RUN_INTEGRATION_TESTS") { + if Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { os.Setenv("DTAIL_HOSTNAME_OVERRIDE", "integrationtest") os.Setenv("DTAIL_SSH_KNOWN_HOSTS_FILE", "./known_hosts") in.Server.HostKeyFile = "./ssh_host_key" diff --git a/internal/ssh/client/authmethods.go b/internal/ssh/client/authmethods.go index 2ee32ad..49cf938 100644 --- a/internal/ssh/client/authmethods.go +++ b/internal/ssh/client/authmethods.go @@ -25,27 +25,44 @@ func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod, return initKnownHostsAuthMethods(trustAllHosts, throttleCh, privateKeyPath) } +func initIntegrationTestKnownHostsAuthMethods() []gossh.AuthMethod { + var sshAuthMethods []gossh.AuthMethod + privateKeyPath := "./id_rsa" + + GeneratePrivatePublicKeyPairIfNotExists(privateKeyPath, 4096) + authMethod, err := ssh.PrivateKey(privateKeyPath) + if err != nil { + dlog.Client.FatalPanic("Unable to use private SSH key", privateKeyPath, err) + } + + sshAuthMethods = append(sshAuthMethods, authMethod) + dlog.Client.Debug("initKnownHostsAuthMethods", + "Added path to list of auth methods, not adding further methods", privateKeyPath) + return sshAuthMethods +} + func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) { - var sshAuthMethods []gossh.AuthMethod knownHostsFile := config.SSHKnownHostsFile() - knownHostsCallback, err := NewKnownHostsCallback(knownHostsFile, trustAllHosts, - throttleCh) + knownHostsCallback, err := NewKnownHostsCallback(knownHostsFile, trustAllHosts, throttleCh) if err != nil { dlog.Client.FatalPanic(knownHostsFile, err) } - dlog.Client.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsFile) + if config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { + return initIntegrationTestKnownHostsAuthMethods(), knownHostsCallback + } + + var sshAuthMethods []gossh.AuthMethod // First try to read custom private key path. if privateKeyPath != "" { authMethod, err := ssh.PrivateKey(privateKeyPath) if err == nil { sshAuthMethods = append(sshAuthMethods, authMethod) dlog.Client.Debug("initKnownHostsAuthMethods", - "Added path to list of auth methods, not adding further methods", - privateKeyPath) + "Added path to list of auth methods, not adding further methods", privateKeyPath) return sshAuthMethods, knownHostsCallback } dlog.Client.FatalPanic("Unable to use private SSH key", privateKeyPath, err) @@ -59,8 +76,7 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, "to list of auth methods, not adding further methods") return sshAuthMethods, knownHostsCallback } - dlog.Client.Debug("initKnownHostsAuthMethods", - "Unable to init SSH Agent auth method", err) + dlog.Client.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err) // Third, try Linux/UNIX default key paths privateKeyPath = os.Getenv("HOME") + "/.ssh/id_rsa" @@ -71,8 +87,7 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, "Added path to list of auth methods, not adding further methods", privateKeyPath) return sshAuthMethods, knownHostsCallback } - dlog.Client.Debug("initKnownHostsAuthMethods", "Unable to use private key", - privateKeyPath, err) + dlog.Client.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err) privateKeyPath = os.Getenv("HOME") + "/.ssh/id_dsa" authMethod, err = ssh.PrivateKey(privateKeyPath) @@ -92,10 +107,7 @@ func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, return sshAuthMethods, knownHostsCallback } - dlog.Client.Debug("initKnownHostsAuthMethods", "Unable to use private key", - privateKeyPath, err) - - dlog.Client.FatalPanic("Unable to find private SSH key information") + dlog.Client.FatalPanic("Unable to find private SSH key information", privateKeyPath, err) // Never reach this point. return sshAuthMethods, knownHostsCallback } diff --git a/internal/ssh/client/clientkeypair.go b/internal/ssh/client/clientkeypair.go new file mode 100644 index 0000000..0e21d0c --- /dev/null +++ b/internal/ssh/client/clientkeypair.go @@ -0,0 +1,91 @@ +package client + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "io/ioutil" + "os" + + "github.com/mimecast/dtail/internal/io/dlog" + "golang.org/x/crypto/ssh" +) + +// GeneratePrivatePublicKeyPairIfNotExists generates a SSH key pair (used by the integration tests) +func GeneratePrivatePublicKeyPairIfNotExists(keyPath string, bitSize int) { + if _, err := os.Stat(keyPath); err == nil { + dlog.Common.Debug("Private/public key pair already exists", keyPath) + return + } + GeneratePrivatePublicKeyPair(keyPath, bitSize) +} + +// GeneratePrivatePublicKeyPair generates a SSH key pair (used by the integration tests) +func GeneratePrivatePublicKeyPair(keyPath string, bitSize int) { + privateKeyPath := keyPath + publicKeyPath := fmt.Sprintf("%s.pub", keyPath) + + dlog.Common.Debug("Generating private/public key pair", privateKeyPath, publicKeyPath) + + privateKey, err := generatePrivateKey(bitSize) + if err != nil { + dlog.Common.FatalPanic(err) + } + publicKeyBytes, err := generatePublicKey(&privateKey.PublicKey) + if err != nil { + dlog.Common.FatalPanic(err) + } + privateKeyBytes := encodePrivateKeyToPEM(privateKey) + err = writeKey(privateKeyBytes, privateKeyPath) + if err != nil { + dlog.Common.FatalPanic(err) + } + err = writeKey([]byte(publicKeyBytes), publicKeyPath) + if err != nil { + dlog.Common.FatalPanic(err) + } + + dlog.Common.Debug("Done generating private/public key pair", privateKeyPath, publicKeyPath) +} + +func generatePrivateKey(bitSize int) (*rsa.PrivateKey, error) { + privateKey, err := rsa.GenerateKey(rand.Reader, bitSize) + if err != nil { + return nil, err + } + err = privateKey.Validate() + if err != nil { + return nil, err + } + return privateKey, nil +} + +func encodePrivateKeyToPEM(privateKey *rsa.PrivateKey) []byte { + privDER := x509.MarshalPKCS1PrivateKey(privateKey) + privBlock := pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: privDER, + } + privatePEM := pem.EncodeToMemory(&privBlock) + return privatePEM +} + +func generatePublicKey(privatekey *rsa.PublicKey) ([]byte, error) { + publicRsaKey, err := ssh.NewPublicKey(privatekey) + if err != nil { + return nil, err + } + pubKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey) + return pubKeyBytes, nil +} + +func writeKey(keyBytes []byte, saveFileTo string) error { + err := ioutil.WriteFile(saveFileTo, keyBytes, 0600) + if err != nil { + return err + } + return nil +} -- cgit v1.2.3 From f6bcd5be51a427747be8058d7a7b9887bc2670ca Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 27 Oct 2021 10:07:29 +0300 Subject: Fix --shutdownAfter client switch --- internal/clients/baseclient.go | 9 +++++++++ internal/io/fs/readfile.go | 1 - 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go index 41521ea..3cd85fe 100644 --- a/internal/clients/baseclient.go +++ b/internal/clients/baseclient.go @@ -112,10 +112,19 @@ func (c *baseClient) startConnection(ctx context.Context, i int, // Retrieve status code from handler (dtail client will exit with that status) status = conn.Handler().Status() + // Do we want to retry? if !c.retry { + // No, we don't. return } + select { + case <-ctx.Done(): + // No, context is done, so no retry. + return + default: + } + // Yes, we want to retry. time.Sleep(time.Second * 2) dlog.Client.Debug(conn.Server(), "Reconnecting") conn = c.makeConnection(conn.Server(), c.sshAuthMethods, c.hostKeyCallback) diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go index 5815aa3..a42fc53 100644 --- a/internal/io/fs/readfile.go +++ b/internal/io/fs/readfile.go @@ -233,7 +233,6 @@ func (f readFile) filterWithoutLContext(ctx context.Context, rawLines <-chan *by return } if filteredLine, ok := f.transmittable(line, len(lines), cap(lines), re); ok { - //dlog.Common.Trace("TODO", "lines", lines, len(lines), cap(lines)) select { case lines <- filteredLine: case <-ctx.Done(): -- cgit v1.2.3 From dadbaab24d66685db0a2a6655bd75cdbb19eb929 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 27 Oct 2021 10:35:36 +0300 Subject: Update docs to reflect some of the changes made in the past --- doc/examples.md | 37 +++++++++++++++++++++++++++++++++++++ doc/installation.md | 4 ++-- doc/quickstart.md | 20 ++++++++++++-------- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/doc/examples.md b/doc/examples.md index 91ab7f2..6c23120 100644 --- a/doc/examples.md +++ b/doc/examples.md @@ -15,6 +15,12 @@ The following example demonstrates how to follow logs of multiple servers at onc ![dtail](dtail.gif "Tail example") +You can also use the shorthand version: + +```shell +% dtail --servers serverlist.txt --regex STAT "/var/log/service/*.log" +``` + ## Aggregating logs To run ad-hoc MapReduce aggregations on newly written log lines, you also must add a query. The following example follows all remote log lines and prints out every 5 seconds the top 10 servers with the most average free memory. To run a MapReduce query across log lines written in the past, please use the ``dmap`` command instead. @@ -29,6 +35,14 @@ For MapReduce queries to work, you have to ensure that DTail supports your log f ![dtail-map](dtail-map.gif "Tail mapreduce example") +You can also use the shorthand version: + +```shell +% dtail --servers serverlist.txt \ + 'select avg(memfree), $hostname from MCVMSTATS group by $hostname order by avg(memfree) limit 10 interval 5' \ + '/var/log/service/*.log' +``` + # How to use ``dcat`` The following example demonstrates how to cat files (display the full content of the files) of multiple servers at once. The servers are provided as a comma-separated list this time. @@ -40,6 +54,13 @@ The following example demonstrates how to cat files (display the full content of ![dcat](dcat.gif "Cat example") +You can also use the shorthand version: + +```shell +% dcat --servers serv-011.lan.example.org,serv-012.lan.example.org,serv-013.lan.example.org \ + /etc/hostname +``` + # How to use ``dgrep`` The following example demonstrates how to grep files (display only the lines which match a given regular expression) of multiple servers at once. In this example, we look after the swap partition in ``/etc/fstab``. We do that only on the first 20 servers from ``serverlist.txt``. ``dgrep`` is also very useful for searching log files of the past. @@ -52,6 +73,14 @@ The following example demonstrates how to grep files (display only the lines whi ![dgrep](dgrep.gif "Grep example") +You can also use the shorthand version: + +TODO: Auto detect that swap is a regex. +```shell +% dgrep --servers <(head -n 20 serverlist.txt) \ + /etc/fstab swap +``` + # How to use ``dmap`` To run a MapReduce aggregation over logs written in the past, the ``dmap`` command can be used. For example, the following command aggregates all MapReduce fields of all the records and calculates the average memory free grouped by day of the month, hour, minute and the server hostname. ``dmap`` will print interim results every few seconds. The final product, however, will be written to file ``mapreduce.csv``. @@ -65,3 +94,11 @@ To run a MapReduce aggregation over logs written in the past, the ``dmap`` comma Remember: For that to work, you have to make sure that DTail supports your log format. You can either use the ones already defined in ``internal/mapr/log format`` or add an extension to support a custom log format. ![dmap](dmap.gif "DMap example") + +You can also use the shorthand version: + +```shell +% dmap --servers serv-011.lan.example.org,serv-012.lan.example.org,serv-013.lan.example.org,serv-021.lan.example.org,serv-022.lan.example.org,serv-023.lan.example.org \ + 'select avg(memfree), $day, $hour, $minute, $hostname from MCVMSTATS group by $day, $hour, $minute, $hostname order by avg(memfree) limit 10 outfile mapreduce.csv' \ + "/var/log/service/*.log" +``` diff --git a/doc/installation.md b/doc/installation.md index 8f3892c..0f6143b 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -19,10 +19,10 @@ This is optional, but it gives you better security. On Linux, you have the optio ### 2. Enable ACL via a Go build flag -Set the `USE_ACL` environment variable before invoking the make command. +Set the `DTAIL_USE_ACL` environment variable before invoking the make command. ```console -% export USE_ACL=yes +% export DTAIL_USE_ACL=yes ``` Alternatively, you could add `-tags linuxacl` to the Go compiler. diff --git a/doc/quickstart.md b/doc/quickstart.md index 21274ff..ea943d3 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -10,7 +10,7 @@ This guide assumes that you know how to generate and configure a public/private To compile and install all DTail binaries directly from GitHub run: ```console -% for cmd in dcat dgrep dmap dtail dserver; do +% for cmd in dcat dgrep dmap dtail dserver dtailhealth; do go get github.com/mimecast/dtail/cmd/$cmd; done ``` @@ -21,6 +21,7 @@ It produces the following executables in ``$GOPATH/bin``: * ``dgrep``: Client for searching whole files remotely using a regex (distributed grep) * ``dmap``: Client for executing distributed MapReduce queries (may consume a lot of RAM and CPU) * ``dtail``: Client for tailing/following log files remotely (distributed tail) +* ``dtailhealth``: Client for dserver health checks * ``dserver``: The DTail server # Start DTail server @@ -28,12 +29,15 @@ It produces the following executables in ``$GOPATH/bin``: Copy the ``dserver`` binary to the remote server machines of your choice (e.g. ``serv-001.lan.example.org`` and ``serv-002.lan.example.org``) and start it on each of the servers as follows: ```console -% ./dserver -SERVER|serv-001|INFO|Launching server|server|DTail 1.0.0 -SERVER|serv-001|INFO|Creating server|DTail 1.0.0 -SERVER|serv-001|INFO|Generating private server RSA host key -SERVER|serv-001|INFO|Starting server -SERVER|serv-001|INFO|Binding server|0.0.0.0:2222 +❯ ./dserver --logger Stdout --logLevel debug --bindAddress $(hostname) --port 2222 +DTail 4.0.0 Protocol 4 Have a lot of fun! +INFO|20211027-102513|Creating server|DTail 4.0.0-RC2 Protocol 4 Have a lot of fun! +INFO|20211027-102513|Reading private server RSA host key from file|./ssh_host_key +INFO|20211027-102513|Starting server +INFO|20211027-102513|Binding server|X.Y.Z.W:2222 +INFO|20211027-102513|Starting continuous job runner after 10s +DEBUG|20211027-102513|Starting listener loop +INFO|20211027-102513|Starting scheduled job runner after 10s ``` ``dserver`` is now listening on TCP port 2222 and waiting for incoming connections. All SSH keys listed in ``~/.ssh/authorized_keys`` are now respected by the DTail server for authorization. @@ -79,7 +83,7 @@ Now it is time to connect to the DTail servers through the DTail client: ```console % dtail --servers serv-001.lan.example.org,server-002.lan.example.org --files "/var/log/service/*.log" -CLIENT|workstation01|INFO|Launching client|tail|DTail 1.0.0 +CLIENT|workstation01|INFO|Launching client|tail|DTail 4.0.0 CLIENT|workstation01|INFO|Initiating base client CLIENT|workstation01|INFO|Added SSH Agent to list of auth methods CLIENT|workstation01|INFO|Deduped server list|1|1 -- cgit v1.2.3