Add integration test for long line splitting - Also fixed a bug regarding this along the way

3 files changed, 12 insertions, 8 deletions

diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index f8af8bc..1a7822c 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -83,6 +83,7 @@ func (in *initializer) transformConfig(sourceProcess source.Source, args *Args,
 func (in *initializer) processEnvVars() {
 	if Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		os.Setenv("DTAIL_HOSTNAME_OVERRIDE", "integrationtest")
+		in.Server.MaxLineLength = 1024
 	}
 }
 
diff --git a/internal/config/server.go b/internal/config/server.go
index 254ea0c..8285bdf 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -47,6 +47,8 @@ type ServerConfig struct {
 	MaxConcurrentCats int
 	// The max amount of concurrent tails per server.
 	MaxConcurrentTails int
+	// The max line length until it's split up into multiple smaller lines.
+	MaxLineLength int
 	// The user permissions. TODO: Add to JSON schema
 	Permissions Permissions `json:",omitempty"`
 	// The mapr log format
@@ -69,13 +71,14 @@ func newDefaultServerConfig() *ServerConfig {
 	defaultPermissions := []string{"^/.*"}
 	defaultBindAddress := "0.0.0.0"
 	return &ServerConfig{
-		SSHBindAddress:     defaultBindAddress,
-		MaxConnections:     10,
-		MaxConcurrentCats:  2,
-		MaxConcurrentTails: 50,
-		HostKeyFile:        "./cache/ssh_host_key",
 		HostKeyBits:        4096,
+		HostKeyFile:        "./cache/ssh_host_key",
 		MapreduceLogFormat: "default",
+		MaxConcurrentCats:  2,
+		MaxConcurrentTails: 50,
+		MaxConnections:     10,
+		MaxLineLength:      1024 * 1024,
+		SSHBindAddress:     defaultBindAddress,
 		Permissions: Permissions{
 			Default: defaultPermissions,
 		},
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index e499853..806cd32 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
@@ -167,7 +168,6 @@ func (f readFile) read(ctx context.Context, fd *os.File, reader *bufio.Reader,
 	rawLines chan *bytes.Buffer, truncate <-chan struct{}) error {
 
 	var offset uint64
-	lineLengthThreshold := 1024 * 1024 // 1mb
 	warnedAboutLongLine := false
 	message := pool.BytesBuffer.Get().(*bytes.Buffer)
 
@@ -214,13 +214,13 @@ func (f readFile) read(ctx context.Context, fd *os.File, reader *bufio.Reader,
 				return nil
 			}
 		default:
-			// TODO: Add integration test with input file having a very long line.
-			if message.Len() >= lineLengthThreshold {
+			if message.Len() >= config.Server.MaxLineLength {
 				if !warnedAboutLongLine {
 					f.serverMessages <- dlog.Common.Warn(f.filePath,
 						"Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
+				message.WriteByte('\n')
 				select {
 				case rawLines <- message:
 					message = pool.BytesBuffer.Get().(*bytes.Buffer)