merge 4.0.0-RC

4 files changed, 150 insertions, 49 deletions

diff --git a/internal/ssh/client/authmethods.go b/internal/ssh/client/authmethods.go
index bbfb7be..6128018 100644
--- a/internal/ssh/client/authmethods.go
+++ b/internal/ssh/client/authmethods.go
@@ -1,93 +1,116 @@
 package client
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/mimecast/dtail/internal/config"
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/ssh"
 
 	gossh "golang.org/x/crypto/ssh"
 )
 
+const addedPathStr string = "Added path to list of auth methods, not adding further methods"
+
 // InitSSHAuthMethods initialises all known SSH auth methods on the client side.
-func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod, hostKeyCallback gossh.HostKeyCallback, trustAllHosts bool, throttleCh chan struct{}, privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
+func InitSSHAuthMethods(sshAuthMethods []gossh.AuthMethod,
+	hostKeyCallback gossh.HostKeyCallback, trustAllHosts bool, throttleCh chan struct{},
+	privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
+
 	if len(sshAuthMethods) > 0 {
 		simpleCallback, err := NewSimpleCallback()
 		if err != nil {
-			logger.FatalExit(err)
+			dlog.Client.FatalPanic(err)
 		}
 		return sshAuthMethods, simpleCallback
 	}
-
 	return initKnownHostsAuthMethods(trustAllHosts, throttleCh, privateKeyPath)
 }
 
-func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{}, privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
+func initIntegrationTestKnownHostsAuthMethods() []gossh.AuthMethod {
 	var sshAuthMethods []gossh.AuthMethod
+	privateKeyPath := "./id_rsa"
 
-	knownHostsPath := os.Getenv("HOME") + "/.ssh/known_hosts"
-	knownHostsCallback, err := NewKnownHostsCallback(knownHostsPath, trustAllHosts, throttleCh)
+	GeneratePrivatePublicKeyPairIfNotExists(privateKeyPath, 4096)
+	authMethod, err := ssh.PrivateKey(privateKeyPath)
 	if err != nil {
-		logger.FatalExit(knownHostsPath, err)
+		dlog.Client.FatalPanic("Unable to use private SSH key", privateKeyPath, err)
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsPath)
 
-	if config.Common.ExperimentalFeaturesEnable {
-		sshAuthMethods = append(sshAuthMethods, gossh.Password("experimental feature test"))
-		logger.Debug("initKnownHostsAuthMethods", "Added experimental method to list of auth methods")
+	sshAuthMethods = append(sshAuthMethods, authMethod)
+	dlog.Client.Debug("initKnownHostsAuthMethods", addedPathStr, privateKeyPath)
+	return sshAuthMethods
+}
+
+func initKnownHostsAuthMethods(trustAllHosts bool, throttleCh chan struct{},
+	privateKeyPath string) ([]gossh.AuthMethod, HostKeyCallback) {
+
+	var sshAuthMethods []gossh.AuthMethod
+	knownHostsFile := fmt.Sprintf("%s/.ssh/known_hosts", os.Getenv("HOME"))
+	if config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
+		// In case of integration test, override known hosts file path.
+		knownHostsFile = "./known_hosts"
 	}
 
-	// First try to read custom private key path.
+	knownHostsCallback, err := NewKnownHostsCallback(knownHostsFile, trustAllHosts, throttleCh)
+	if err != nil {
+		dlog.Client.FatalPanic(knownHostsFile, err)
+	}
+	dlog.Client.Debug("initKnownHostsAuthMethods", "Added known hosts file path", knownHostsFile)
+
+	if config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
+		return initIntegrationTestKnownHostsAuthMethods(), knownHostsCallback
+	}
+
+	// Try to read custom private key path.
 	if privateKeyPath != "" {
 		authMethod, err := ssh.PrivateKey(privateKeyPath)
 		if err == nil {
 			sshAuthMethods = append(sshAuthMethods, authMethod)
-			logger.Debug("initKnownHostsAuthMethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+			dlog.Client.Debug("initKnownHostsAuthMethods", addedPathStr, privateKeyPath)
 			return sshAuthMethods, knownHostsCallback
 		}
-		logger.FatalExit("Unable to use private SSH key", privateKeyPath, err)
+		dlog.Client.FatalPanic("Unable to use private SSH key", privateKeyPath, err)
 	}
 
 	// Second, try SSH Agent
 	authMethod, err := ssh.Agent()
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK) to list of auth methods, not adding further methods")
+		dlog.Client.Debug("initKnownHostsAuthMethods", "Added SSH Agent (SSH_AUTH_SOCK)"+
+			"to list of auth methods, not adding further methods")
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err)
+	dlog.Client.Debug("initKnownHostsAuthMethods", "Unable to init SSH Agent auth method", err)
 
 	// Third, try Linux/UNIX default key paths
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_rsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Client.Debug("initKnownHostsAuthmethods", addedPathStr, privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
+	dlog.Client.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
 
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_dsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Client.Debug("initKnownHostsAuthmethods", addedPathStr, privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
 
 	privateKeyPath = os.Getenv("HOME") + "/.ssh/id_ecdsa"
 	authMethod, err = ssh.PrivateKey(privateKeyPath)
 	if err == nil {
 		sshAuthMethods = append(sshAuthMethods, authMethod)
-		logger.Debug("initKnownHostsAuthmethods", "Added path to list of auth methods, not adding further methods", privateKeyPath)
+		dlog.Client.Debug("initKnownHostsAuthmethods", addedPathStr, privateKeyPath)
 		return sshAuthMethods, knownHostsCallback
 	}
-	logger.Debug("initKnownHostsAuthMethods", "Unable to use private key", privateKeyPath, err)
-
-	logger.FatalExit("Unable to find private SSH key information")
 
+	dlog.Client.FatalPanic("Unable to find private SSH key information", privateKeyPath, err)
 	// Never reach this point.
 	return sshAuthMethods, knownHostsCallback
 }
diff --git a/internal/ssh/client/clientkeypair.go b/internal/ssh/client/clientkeypair.go
new file mode 100644
index 0000000..b35b25d
--- /dev/null
+++ b/internal/ssh/client/clientkeypair.go
@@ -0,0 +1,91 @@
+package client
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/mimecast/dtail/internal/io/dlog"
+	"golang.org/x/crypto/ssh"
+)
+
+// GeneratePrivatePublicKeyPairIfNotExists generates a SSH key pair (used by the integration tests)
+func GeneratePrivatePublicKeyPairIfNotExists(keyPath string, bitSize int) {
+	if _, err := os.Stat(keyPath); err == nil {
+		dlog.Client.Debug("Private/public key pair already exists", keyPath)
+		return
+	}
+	GeneratePrivatePublicKeyPair(keyPath, bitSize)
+}
+
+// GeneratePrivatePublicKeyPair generates a SSH key pair (used by the integration tests)
+func GeneratePrivatePublicKeyPair(keyPath string, bitSize int) {
+	privateKeyPath := keyPath
+	publicKeyPath := fmt.Sprintf("%s.pub", keyPath)
+
+	dlog.Client.Debug("Generating private/public key pair", privateKeyPath, publicKeyPath)
+
+	privateKey, err := generatePrivateKey(bitSize)
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
+	publicKeyBytes, err := generatePublicKey(&privateKey.PublicKey)
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
+	privateKeyBytes := encodePrivateKeyToPEM(privateKey)
+	err = writeKey(privateKeyBytes, privateKeyPath)
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
+	err = writeKey([]byte(publicKeyBytes), publicKeyPath)
+	if err != nil {
+		dlog.Client.FatalPanic(err)
+	}
+
+	dlog.Client.Debug("Done generating private/public key pair", privateKeyPath, publicKeyPath)
+}
+
+func generatePrivateKey(bitSize int) (*rsa.PrivateKey, error) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, bitSize)
+	if err != nil {
+		return nil, err
+	}
+	err = privateKey.Validate()
+	if err != nil {
+		return nil, err
+	}
+	return privateKey, nil
+}
+
+func encodePrivateKeyToPEM(privateKey *rsa.PrivateKey) []byte {
+	privDER := x509.MarshalPKCS1PrivateKey(privateKey)
+	privBlock := pem.Block{
+		Type:    "RSA PRIVATE KEY",
+		Headers: nil,
+		Bytes:   privDER,
+	}
+	privatePEM := pem.EncodeToMemory(&privBlock)
+	return privatePEM
+}
+
+func generatePublicKey(privatekey *rsa.PublicKey) ([]byte, error) {
+	publicRsaKey, err := ssh.NewPublicKey(privatekey)
+	if err != nil {
+		return nil, err
+	}
+	pubKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey)
+	return pubKeyBytes, nil
+}
+
+func writeKey(keyBytes []byte, saveFileTo string) error {
+	err := ioutil.WriteFile(saveFileTo, keyBytes, 0600)
+	if err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/internal/ssh/client/customkeycallback.go b/internal/ssh/client/customkeycallback.go
index 73e5289..53b8e3c 100644
--- a/internal/ssh/client/customkeycallback.go
+++ b/internal/ssh/client/customkeycallback.go
@@ -7,8 +7,7 @@ import (
 )
 
 // CustomCallback is a custom host key callback wrapper.
-type CustomCallback struct {
-}
+type CustomCallback struct{}
 
 // NewCustomCallback returns a new wrapper.
 func NewCustomCallback() (*CustomCallback, error) {
diff --git a/internal/ssh/client/knownhostscallback.go b/internal/ssh/client/knownhostscallback.go
index 1ccf6c6..393c4c7 100644
--- a/internal/ssh/client/knownhostscallback.go
+++ b/internal/ssh/client/knownhostscallback.go
@@ -10,7 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mimecast/dtail/internal/io/logger"
+	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/prompt"
 
 	"golang.org/x/crypto/ssh"
@@ -46,8 +46,9 @@ type KnownHostsCallback struct {
 }
 
 // NewKnownHostsCallback returns a new wrapper.
-func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool, throttleCh chan struct{}) (HostKeyCallback, error) {
-	// Ensure file exists
+func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool,
+	throttleCh chan struct{}) (HostKeyCallback, error) {
+
 	os.OpenFile(knownHostsPath, os.O_RDONLY|os.O_CREATE, 0666)
 	untrustedHosts := make(map[string]bool)
 
@@ -59,11 +60,9 @@ func NewKnownHostsCallback(knownHostsPath string, trustAllHosts bool, throttleCh
 		untrustedHosts:  untrustedHosts,
 		mutex:           &sync.Mutex{},
 	}
-
 	if trustAllHosts {
 		close(c.trustAllHostsCh)
 	}
-
 	return c, nil
 }
 
@@ -75,14 +74,12 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 		if err != nil {
 			return err
 		}
-
 		// Check for valid entry in known_hosts file
 		err = knownHostsCb(server, remote, key)
 		if err == nil {
 			// OK
 			return nil
 		}
-
 		// Make sure that interactive user callback does not interfere with
 		// SSH connection throttler.
 		<-c.throttleCh
@@ -96,11 +93,9 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 			ipLine:     knownhosts.Line([]string{remote.String()}, key),
 			responseCh: make(chan response),
 		}
-
-		logger.Warn("Encountered unknown host", unknown)
+		dlog.Client.Warn("Encountered unknown host", unknown)
 		// Notify user that there is an unknown host
 		c.unknownCh <- unknown
-
 		// Wait for user input.
 		switch <-unknown.responseCh {
 		case trustHost:
@@ -112,7 +107,6 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 		c.mutex.Lock()
 		defer c.mutex.Unlock()
 		c.untrustedHosts[server] = true
-
 		return err
 	}
 }
@@ -121,7 +115,6 @@ func (c KnownHostsCallback) Wrap() ssh.HostKeyCallback {
 // be added to the known hosts or not.
 func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 	var hosts []unknownHost
-
 	for {
 		// Check whether there is a unknown host
 		select {
@@ -139,7 +132,7 @@ func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 				hosts = []unknownHost{}
 			}
 		case <-ctx.Done():
-			logger.Debug("Stopping goroutine prompting new hosts...")
+			dlog.Client.Debug("Stopping goroutine prompting new hosts...")
 			return
 		}
 	}
@@ -147,14 +140,13 @@ func (c KnownHostsCallback) PromptAddHosts(ctx context.Context) {
 
 func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 	var servers []string
-
 	for _, host := range hosts {
 		servers = append(servers, host.server)
 	}
 
 	select {
 	case <-c.trustAllHostsCh:
-		logger.Warn("Trusting host keys of servers", servers)
+		dlog.Client.Warn("Trusting host keys of servers", servers)
 		c.trustHosts(hosts)
 		return
 	default:
@@ -165,7 +157,6 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 		strings.Join(servers, ","),
 		"Do you want to trust these hosts?",
 	)
-
 	p := prompt.New(question)
 
 	a := prompt.Answer{
@@ -175,7 +166,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.trustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Added hosts to known hosts file", c.knownHostsPath)
+			dlog.Client.Info("Added hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
@@ -188,7 +179,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.trustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Added hosts to known hosts file", c.knownHostsPath)
+			dlog.Client.Info("Added hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
@@ -200,7 +191,7 @@ func (c KnownHostsCallback) promptAddHosts(hosts []unknownHost) {
 			c.dontTrustHosts(hosts)
 		},
 		EndCallback: func() {
-			logger.Info("Didn't add hosts to known hosts file", c.knownHostsPath)
+			dlog.Client.Info("Didn't add hosts to known hosts file", c.knownHostsPath)
 		},
 	}
 	p.Add(a)
@@ -232,7 +223,6 @@ func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 
 	// Newly trusted hosts in normalized form
 	addresses := make(map[string]struct{})
-
 	// First write to new known hosts file, and keep track of addresses
 	for _, unknown := range hosts {
 		unknown.responseCh <- trustHost
@@ -255,7 +245,6 @@ func (c KnownHostsCallback) trustHosts(hosts []unknownHost) {
 	defer oldFd.Close()
 
 	scanner := bufio.NewScanner(oldFd)
-
 	// Now, append all still valid old entries to the new host file
 	for scanner.Scan() {
 		line := scanner.Text()
@@ -283,6 +272,5 @@ func (c KnownHostsCallback) Untrusted(server string) bool {
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
 	_, ok := c.untrustedHosts[server]
-
 	return ok
 }