Bugfix: Dealing correctly with files without newline characters, also add more tests

18 files changed, 127 insertions, 61 deletions

diff --git a/integrationtests/dcat.txt b/integrationtests/dcat1a.txt
index 9e80424..9e80424 100644
--- a/integrationtests/dcat.txt
+++ b/integrationtests/dcat1a.txt
diff --git a/integrationtests/dcat1b.txt b/integrationtests/dcat1b.txt
new file mode 100644
index 0000000..658ee3b
--- /dev/null
+++ b/integrationtests/dcat1b.txt
@@ -0,0 +1,4 @@
+2 empty lines:
+
+
+line without newline
diff --git a/integrationtests/dcat1c.txt b/integrationtests/dcat1c.txt
new file mode 100644
index 0000000..f952bf8
--- /dev/null
+++ b/integrationtests/dcat1c.txt
@@ -0,0 +1,10 @@
+1 Sat  2 Oct 13:46:45 EEST 2021

+2 Sat  2 Oct 13:46:45 EEST 2021

+3 Sat  2 Oct 13:46:45 EEST 2021

+4 Sat  2 Oct 13:46:45 EEST 2021

+5 Sat  2 Oct 13:46:45 EEST 2021

+6 Sat  2 Oct 13:46:45 EEST 2021

+7 Sat  2 Oct 13:46:45 EEST 2021

+8 Sat  2 Oct 13:46:45 EEST 2021

+9 Sat  2 Oct 13:46:45 EEST 2021

+10 Sat  2 Oct 13:46:45 EEST 2021

diff --git a/integrationtests/dcat1d.txt b/integrationtests/dcat1d.txt
new file mode 100644
index 0000000..074c277
--- /dev/null
+++ b/integrationtests/dcat1d.txt
@@ -0,0 +1 @@
+single line without newline
\ No newline at end of file
diff --git a/integrationtests/dcat_test.go b/integrationtests/dcat_test.go
index 124cb62..bef5db2 100644
--- a/integrationtests/dcat_test.go
+++ b/integrationtests/dcat_test.go
@@ -8,57 +8,64 @@ import (
 	"github.com/mimecast/dtail/internal/config"
 )
 
-func TestDCat(t *testing.T) {
+func TestDCat1(t *testing.T) {
 	if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		t.Log("Skipping")
 		return
 	}
-	testdataFile := "dcat.txt"
-	stdoutFile := "dcat.out"
 
-	_, err := runCommand(context.TODO(), t, stdoutFile,
-		"../dcat", "--plain", "--cfg", "none", testdataFile)
+	inFiles := []string{"dcat1a.txt", "dcat1b.txt", "dcat1c.txt", "dcat1d.txt"}
+	for _, inFile := range inFiles {
+		if err := testDCat1(t, inFile); err != nil {
+			t.Error(err)
+			return
+		}
+	}
+}
 
+func testDCat1(t *testing.T, inFile string) error {
+	outFile := "dcat1.out"
+
+	_, err := runCommand(context.TODO(), t, outFile,
+		"../dcat", "--plain", "--cfg", "none", inFile)
 	if err != nil {
-		t.Error(err)
-		return
+		return err
 	}
-
-	if err := compareFiles(t, stdoutFile, testdataFile); err != nil {
-		t.Error(err)
-		return
+	if err := compareFiles(t, outFile, inFile); err != nil {
+		return err
 	}
 
-	os.Remove(stdoutFile)
+	os.Remove(outFile)
+	return nil
 }
 
 func TestDCat2(t *testing.T) {
 	if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") {
 		return
 	}
-	testdataFile := "dcat2.txt"
+	inFile := "dcat2.txt"
 	expectedFile := "dcat2.txt.expected"
-	stdoutFile := "dcat2.out"
+	outFile := "dcat2.out"
 
 	args := []string{"--plain", "--logLevel", "error", "--cfg", "none"}
 
 	// Cat file 100 times in one session.
 	for i := 0; i < 100; i++ {
-		args = append(args, testdataFile)
+		args = append(args, inFile)
 	}
 
-	_, err := runCommand(context.TODO(), t, stdoutFile, "../dcat", args...)
+	_, err := runCommand(context.TODO(), t, outFile, "../dcat", args...)
 	if err != nil {
 		t.Error(err)
 		return
 	}
 
-	if err := compareFilesContents(t, stdoutFile, expectedFile); err != nil {
+	if err := compareFilesContents(t, outFile, expectedFile); err != nil {
 		t.Error(err)
 		return
 	}
 
-	os.Remove(stdoutFile)
+	os.Remove(outFile)
 }
 
 func TestDCatColors(t *testing.T) {
@@ -66,22 +73,22 @@ func TestDCatColors(t *testing.T) {
 		return
 	}
 
-	testdataFile := "dcatcolors.txt"
-	stdoutFile := "dcatcolors.out"
+	inFile := "dcatcolors.txt"
+	outFile := "dcatcolors.out"
 	expectedFile := "dcatcolors.expected"
 
-	_, err := runCommand(context.TODO(), t, stdoutFile,
-		"../dcat", "--logLevel", "error", "--cfg", "none", testdataFile)
+	_, err := runCommand(context.TODO(), t, outFile,
+		"../dcat", "--logLevel", "error", "--cfg", "none", inFile)
 
 	if err != nil {
 		t.Error(err)
 		return
 	}
 
-	if err := compareFiles(t, stdoutFile, expectedFile); err != nil {
+	if err := compareFiles(t, outFile, expectedFile); err != nil {
 		t.Error(err)
 		return
 	}
 
-	os.Remove(stdoutFile)
+	os.Remove(outFile)
 }
diff --git a/internal/clients/handlers/basehandler.go b/internal/clients/handlers/basehandler.go
index b520c25..3ffea82 100644
--- a/internal/clients/handlers/basehandler.go
+++ b/internal/clients/handlers/basehandler.go
@@ -60,14 +60,7 @@ func (h *baseHandler) SendMessage(command string) error {
 func (h *baseHandler) Write(p []byte) (n int, err error) {
 	for _, b := range p {
 		switch b {
-		/*
-			// NEXT: Next DTail version make it so that '\n' gets ignored. For now
-			// leave it for compatibility with older DTail server + ability to display
-			// the protocol mismatch warn message.
-			case '\n' {
-				continue
-		*/
-		case '\n', protocol.MessageDelimiter:
+		case protocol.MessageDelimiter:
 			message := h.receiveBuf.String()
 			h.handleMessage(message)
 			h.receiveBuf.Reset()
@@ -75,7 +68,6 @@ func (h *baseHandler) Write(p []byte) (n int, err error) {
 			h.receiveBuf.WriteByte(b)
 		}
 	}
-
 	return len(p), nil
 }
 
diff --git a/internal/color/paint.go b/internal/color/paint.go
index 7735d87..4c9d2bc 100644
--- a/internal/color/paint.go
+++ b/internal/color/paint.go
@@ -39,9 +39,13 @@ func PaintStrAttr(text string, attr Attribute) string {
 func Paint(sb *strings.Builder, text string, fg FgColor, bg BgColor) {
 	sb.WriteString(string(fg))
 	sb.WriteString(string(bg))
-	sb.WriteString(text)
+	trimmed := strings.TrimSuffix(text, "\n")
+	sb.WriteString(trimmed)
 	sb.WriteString(string(BgDefault))
 	sb.WriteString(string(FgDefault))
+	if trimmed != text {
+		sb.WriteByte('\n')
+	}
 }
 
 // Reset background and foreground colors.
@@ -62,10 +66,14 @@ func PaintWithAttr(sb *strings.Builder, text string, fg FgColor, bg BgColor,
 	sb.WriteString(string(fg))
 	sb.WriteString(string(bg))
 	sb.WriteString(string(attr))
-	sb.WriteString(text)
+	trimmed := strings.TrimSuffix(text, "\n")
+	sb.WriteString(trimmed)
 	sb.WriteString(string(AttrReset))
 	sb.WriteString(string(BgDefault))
 	sb.WriteString(string(FgDefault))
+	if trimmed != text {
+		sb.WriteByte('\n')
+	}
 }
 
 // PaintWithAttrs is similar to PaintWithAttr, but it takes multiple attributes.
@@ -77,10 +85,14 @@ func PaintWithAttrs(sb *strings.Builder, text string, fg FgColor, bg BgColor,
 	for _, attr := range attrs {
 		sb.WriteString(string(attr))
 	}
-	sb.WriteString(text)
+	trimmed := strings.TrimSuffix(text, "\n")
+	sb.WriteString(trimmed)
 	sb.WriteString(string(AttrReset))
 	sb.WriteString(string(BgDefault))
 	sb.WriteString(string(FgDefault))
+	if trimmed != text {
+		sb.WriteByte('\n')
+	}
 }
 
 // ResetWithAttr resets background, foreground and attributes.
diff --git a/internal/io/dlog/dlog.go b/internal/io/dlog/dlog.go
index ff2cef4..5713c1a 100644
--- a/internal/io/dlog/dlog.go
+++ b/internal/io/dlog/dlog.go
@@ -210,10 +210,10 @@ func (d *DLog) Devel(args ...interface{}) string {
 // Raw message logging.
 func (d *DLog) Raw(message string) string {
 	if !config.Client.TermColorsEnable || !d.logger.SupportsColors() {
-		d.logger.Log(time.Now(), message)
+		d.logger.Raw(time.Now(), message)
 		return message
 	}
-	d.logger.LogWithColors(time.Now(), message, brush.Colorfy(message))
+	d.logger.RawWithColors(time.Now(), message, brush.Colorfy(message))
 	return message
 }
 
diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go
index 94824fe..9dce251 100644
--- a/internal/io/dlog/loggers/file.go
+++ b/internal/io/dlog/loggers/file.go
@@ -17,6 +17,7 @@ type fileWriter struct{}
 type fileMessageBuf struct {
 	now     time.Time
 	message string
+	nl      bool
 }
 
 type file struct {
@@ -86,10 +87,18 @@ func (f *file) Start(ctx context.Context, wg *sync.WaitGroup) {
 }
 
 func (f *file) Log(now time.Time, message string) {
-	f.bufferCh <- &fileMessageBuf{now, message}
+	f.bufferCh <- &fileMessageBuf{now, message, true}
 }
 
 func (f *file) LogWithColors(now time.Time, message, coloredMessage string) {
+	f.RawWithColors(now, message, coloredMessage)
+}
+
+func (f *file) Raw(now time.Time, message string) {
+	f.bufferCh <- &fileMessageBuf{now, message, false}
+}
+
+func (f *file) RawWithColors(now time.Time, message, coloredMessage string) {
 	panic("Colors not supported in file logger")
 }
 
@@ -116,7 +125,9 @@ func (f *file) write(m *fileMessageBuf) {
 	}
 
 	writer.WriteString(m.message)
-	writer.WriteByte('\n')
+	if m.nl {
+		writer.WriteByte('\n')
+	}
 }
 
 func (f *file) getWriter(name string) *bufio.Writer {
diff --git a/internal/io/dlog/loggers/fout.go b/internal/io/dlog/loggers/fout.go
index 60c318d..6888d40 100644
--- a/internal/io/dlog/loggers/fout.go
+++ b/internal/io/dlog/loggers/fout.go
@@ -38,6 +38,16 @@ func (f *fout) LogWithColors(now time.Time, message, coloredMessage string) {
 	f.file.Log(now, message)
 }
 
+func (f *fout) Raw(now time.Time, message string) {
+	f.stdout.Raw(now, message)
+	f.file.Raw(now, message)
+}
+
+func (f *fout) RawWithColors(now time.Time, message, coloredMessage string) {
+	f.stdout.RawWithColors(now, "", coloredMessage)
+	f.file.Raw(now, message)
+}
+
 func (f *fout) Flush()  { f.stdout.Flush(); f.file.Flush() }
 func (f *fout) Pause()  { f.stdout.Pause(); f.file.Pause() }
 func (f *fout) Resume() { f.stdout.Resume(); f.file.Resume() }
diff --git a/internal/io/dlog/loggers/logger.go b/internal/io/dlog/loggers/logger.go
index d4e85de..195108b 100644
--- a/internal/io/dlog/loggers/logger.go
+++ b/internal/io/dlog/loggers/logger.go
@@ -10,6 +10,8 @@ import (
 type Logger interface {
 	Log(now time.Time, message string)
 	LogWithColors(now time.Time, message, messageWithColors string)
+	Raw(now time.Time, message string)
+	RawWithColors(now time.Time, message, messageWithColors string)
 	Start(ctx context.Context, wg *sync.WaitGroup)
 	Flush()
 	Pause()
diff --git a/internal/io/dlog/loggers/none.go b/internal/io/dlog/loggers/none.go
index 270027f..973ae3c 100644
--- a/internal/io/dlog/loggers/none.go
+++ b/internal/io/dlog/loggers/none.go
@@ -9,13 +9,13 @@ import (
 // don't log anything
 type none struct{}
 
-func (none) Start(ctx context.Context, wg *sync.WaitGroup) { wg.Done() }
-func (none) Log(now time.Time, message string)             {}
-
+func (none) Start(ctx context.Context, wg *sync.WaitGroup)               { wg.Done() }
+func (none) Log(now time.Time, message string)                           {}
 func (none) LogWithColors(now time.Time, message, coloredMessage string) {}
-
-func (none) Flush()               {}
-func (none) Pause()               {}
-func (none) Resume()              {}
-func (none) Rotate()              {}
-func (none) SupportsColors() bool { return false }
+func (none) Raw(now time.Time, message string)                           {}
+func (none) RawWithColors(now time.Time, message, coloredMessage string) {}
+func (none) Flush()                                                      {}
+func (none) Pause()                                                      {}
+func (none) Resume()                                                     {}
+func (none) Rotate()                                                     {}
+func (none) SupportsColors() bool                                        { return false }
diff --git a/internal/io/dlog/loggers/stdout.go b/internal/io/dlog/loggers/stdout.go
index 05485c6..0369ed7 100644
--- a/internal/io/dlog/loggers/stdout.go
+++ b/internal/io/dlog/loggers/stdout.go
@@ -25,14 +25,22 @@ func (s *stdout) Start(ctx context.Context, wg *sync.WaitGroup) {
 }
 
 func (s *stdout) Log(now time.Time, message string) {
-	s.log(message)
+	s.log(message, true)
 }
 
 func (s *stdout) LogWithColors(now time.Time, message, coloredMessage string) {
-	s.log(coloredMessage)
+	s.log(coloredMessage, true)
 }
 
-func (s *stdout) log(message string) {
+func (s *stdout) Raw(now time.Time, message string) {
+	s.log(message, false)
+}
+
+func (s *stdout) RawWithColors(now time.Time, message, coloredMessage string) {
+	s.log(coloredMessage, false)
+}
+
+func (s *stdout) log(message string, nl bool) {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 
@@ -43,7 +51,11 @@ func (s *stdout) log(message string) {
 	default:
 	}
 
-	fmt.Println(message)
+	if nl {
+		fmt.Println(message)
+		return
+	}
+	fmt.Print(message)
 }
 
 func (s *stdout) Pause()  { s.pauseCh <- struct{}{} }
diff --git a/internal/io/fs/readfile.go b/internal/io/fs/readfile.go
index 18c20c0..e499853 100644
--- a/internal/io/fs/readfile.go
+++ b/internal/io/fs/readfile.go
@@ -167,7 +167,6 @@ func (f readFile) read(ctx context.Context, fd *os.File, reader *bufio.Reader,
 	rawLines chan *bytes.Buffer, truncate <-chan struct{}) error {
 
 	var offset uint64
-
 	lineLengthThreshold := 1024 * 1024 // 1mb
 	warnedAboutLongLine := false
 	message := pool.BytesBuffer.Get().(*bytes.Buffer)
@@ -190,31 +189,38 @@ func (f readFile) read(ctx context.Context, fd *os.File, reader *bufio.Reader,
 			}
 			if !f.seekEOF {
 				dlog.Common.Info(f.FilePath(), "End of file reached")
+				if len(message.Bytes()) > 0 {
+					select {
+					case rawLines <- message:
+					case <-ctx.Done():
+					}
+				}
 				return nil
 			}
 			time.Sleep(time.Millisecond * 100)
 			continue
 		}
+
 		offset++
+		message.WriteByte(b)
 
 		switch b {
 		case '\n':
 			select {
 			case rawLines <- message:
 				message = pool.BytesBuffer.Get().(*bytes.Buffer)
-				//fmt.Printf("%d %d %p\n", message.Len(), message.Cap(), message)
 				warnedAboutLongLine = false
 			case <-ctx.Done():
 				return nil
 			}
 		default:
+			// TODO: Add integration test with input file having a very long line.
 			if message.Len() >= lineLengthThreshold {
 				if !warnedAboutLongLine {
 					f.serverMessages <- dlog.Common.Warn(f.filePath,
 						"Long log line, splitting into multiple lines")
 					warnedAboutLongLine = true
 				}
-				message.WriteString("\n")
 				select {
 				case rawLines <- message:
 					message = pool.BytesBuffer.Get().(*bytes.Buffer)
@@ -222,7 +228,6 @@ func (f readFile) read(ctx context.Context, fd *os.File, reader *bufio.Reader,
 					return nil
 				}
 			}
-			message.WriteByte(b)
 		}
 	}
 }
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index d29706c..2a95a00 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -2,7 +2,7 @@ package protocol
 
 const (
 	// ProtocolCompat -ibility version
-	ProtocolCompat string = "4"
+	ProtocolCompat string = "5"
 	// MessageDelimiter delimits separate messages.
 	MessageDelimiter byte = '¬'
 	// FieldDelimiter delimits messagefields.
diff --git a/internal/server/handlers/basehandler.go b/internal/server/handlers/basehandler.go
index 897ff81..f068944 100644
--- a/internal/server/handlers/basehandler.go
+++ b/internal/server/handlers/basehandler.go
@@ -44,7 +44,7 @@ type baseHandler struct {
 	once       sync.Once
 	mutex      sync.Mutex
 	quiet      bool
-	plain    bool
+	plain      bool
 	serverless bool
 }
 
diff --git a/internal/server/server.go b/internal/server/server.go
index 0cb5e27..fffa560 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -36,7 +36,7 @@ type Server struct {
 
 // New returns a new server.
 func New() *Server {
-	dlog.Server.Info("Creating server", version.String())
+	dlog.Server.Info("Starting server", version.String())
 
 	s := Server{
 		sshServerConfig: &gossh.ServerConfig{},
diff --git a/internal/version/version.go b/internal/version/version.go
index 60e44c2..e2388f5 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -13,7 +13,7 @@ const (
 	// Name of DTail.
 	Name string = "DTail"
 	// Version of DTail.
-	Version string = "4.0.0-RC4"
+	Version string = "4.0.0-RC5"
 	// Additional information for DTail
 	Additional string = "Have a lot of fun!"
 )