Add integration test for long line splitting - Also fixed a bug regarding this along the way

8 files changed, 53 insertions, 8 deletions

diff --git a/integrationtests/dcat3.txt b/integrationtests/dcat3.txt
new file mode 100644
index 0000000..f57b70b
--- /dev/null
+++ b/integrationtests/dcat3.txt
@@ -0,0 +1,3 @@
+A short line
+A long line (dcat will split it up into multiple smaller lines): Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here!
+Another short line
diff --git a/integrationtests/dcat3.txt.expected b/integrationtests/dcat3.txt.expected
new file mode 100644
index 0000000..730f31a
--- /dev/null
+++ b/integrationtests/dcat3.txt.expected
@@ -0,0 +1,5 @@
+A short line
+A long line (dcat will split it up into multiple smaller lines): Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some rand
+om content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some ran
+dom content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here Some random content here!
+Another short line
diff --git a/integrationtests/dcat_test.go b/integrationtests/dcat_test.go
index bef5db2..d07279a 100644
--- a/integrationtests/dcat_test.go
+++ b/integrationtests/dcat_test.go
@@ -68,6 +68,33 @@ func TestDCat2(t *testing.T) {
 	os.Remove(outFile)
 }
 
+func TestDCat3(t *testing.T) {
+	if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
+		return
+	}
+	inFile := "dcat3.txt"
+	expectedFile := "dcat3.txt.expected"
+	outFile := "dcat3.out"
+
+	args := []string{"--plain", "--logLevel", "error", "--cfg", "none", inFile}
+
+	// Split up long lines to smaller ones.
+	os.Setenv("DTAIL_MAX_LINE_LENGTH", "1000")
+
+	_, err := runCommand(context.TODO(), t, outFile, "../dcat", args...)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := compareFilesContents(t, outFile, expectedFile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(outFile)
+}
+
 func TestDCatColors(t *testing.T) {
 	if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		return
diff --git a/internal/config/initializer.go b/internal/config/initializer.go
index f8af8bc..1a7822c 100644
--- a/internal/config/initializer.go
+++ b/internal/config/initializer.go
@@ -83,6 +83,7 @@ func (in *initializer) transformConfig(sourceProcess source.Source, args *Args,
 func (in *initializer) processEnvVars() {
 	if Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		os.Setenv("DTAIL_HOSTNAME_OVERRIDE", "integrationtest")
+		in.Server.MaxLineLength = 1024
 	}
 }
 
diff --git a/internal/config/server.go b/internal/config/server.go
index 254ea0c..8285bdf 100644
--- a/internal/config/server.go
+++ b/internal/config/server.go
@@ -47,6 +47,8 @@ type ServerConfig struct {
 	MaxConcurrentCats int
 	// The max amount of concurrent tails per server.
 	MaxConcurrentTails int
+	// The max line length until it's split up into multiple smaller lines.
+	MaxLineLength int
 	// The user permissions. TODO: Add to JSON schema
 	Permissions Permissions `json:",omitempty"`
 	// The mapr log format
@@ -69,13 +71,14 @@ func newDefaultServerConfig() *ServerConfig {
 	defaultPermissions := []string{"^/.*"}
 	defaultBindAddress := "0.0.0.0"
 	return &ServerConfig{
-		SSHBindAddress:     defaultBindAddress,
-		MaxConnections:     10,
-		MaxConcurrentCats:  2,
-		MaxConcurrentTails: 50,
-		HostKeyFile:        "./cache/ssh_host_key",
 		HostKeyBits:        4096,
+		HostKeyFile:        "./cache/ssh_host_key",
 		MapreduceLogFormat: "default",
+		MaxConcurrentCats:  2,
+		MaxConcurrentTails: 50,
+		MaxConnections:     10,
+		MaxLineLength:      1024 * 1024,
+		SSHBindAddress:     defaultBindAddress,
 		Permissions: Permissions{
 			Default: defaultPermissions,
 		},
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index e499853..806cd32 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mimecast/dtail/internal/config"
 	"github.com/mimecast/dtail/internal/io/dlog"
 	"github.com/mimecast/dtail/internal/io/line"
 	"github.com/mimecast/dtail/internal/io/pool"
@@ -167,7 +168,6 @@ func (f readFile) read(ctx context.Context, fd *os.File, reader *bufio.Reader,
 	rawLines chan *bytes.Buffer, truncate <-chan struct{}) error {
 
 	var offset uint64
-	lineLengthThreshold := 1024 * 1024 // 1mb
 	warnedAboutLongLine := false
 	message := pool.BytesBuffer.Get().(*bytes.Buffer)
 
@@ -214,13 +214,13 @@ func (f readFile) read(ctx context.Context, fd *os.File, reader *bufio.Reader,
 				return nil
 			}
 		default:
-			// TODO: Add integration test with input file having a very long line.
-			if message.Len() >= lineLengthThreshold {
+			if message.Len() >= config.Server.MaxLineLength {
 				if !warnedAboutLongLine {
 					f.serverMessages <- dlog.Common.Warn(f.filePath,
 						"Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
+				message.WriteByte('\n')
 				select {
 				case rawLines <- message:
 					message = pool.BytesBuffer.Get().(*bytes.Buffer)
diff --git a/samples/dtail.json.sample b/samples/dtail.json.sample
index 20a1359..26eb8a1 100644
--- a/samples/dtail.json.sample
+++ b/samples/dtail.json.sample
@@ -96,6 +96,7 @@
     "MaxConcurrentCats": 2,
     "MaxConcurrentTails": 50,
     "MaxConnections": 50,
+    "MaxLineLength": 1048576,
     "Permissions": {
       "Default": [
         "readfiles:^/.*$"
diff --git a/samples/dtail.schema.json b/samples/dtail.schema.json
index bf07525..7551449 100755
--- a/samples/dtail.schema.json
+++ b/samples/dtail.schema.json
@@ -358,6 +358,11 @@
           "minimum": 1,
           "maximum": 100
         },
+        "MaxLineLength": {
+          "type": "integer",
+          "minimum": 1024,
+          "maximum": 10240000
+        },
         "Permissions": {
           "type": "object",
           "properties": {}