From 111fb5753d416214c680abb288d31c595dcdcea1 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Mon, 16 Jun 2025 23:23:26 +0300
Subject: implement true Profile-Guided Optimization with Go compiler -pgo flag
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Refactor PGO script to use actual Go compiler PGO instead of just profiling
- Add proper baseline vs PGO-optimized binary comparison
- Break script into maintainable functions for better organization
- Update Makefile and documentation to reflect PGO process
- Generate comprehensive performance reports with before/after analysis

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 internal/io/fs/chunkedreader_bench_test.go | 101 +++++++++++++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 internal/io/fs/chunkedreader_bench_test.go

(limited to 'internal/io')

diff --git a/internal/io/fs/chunkedreader_bench_test.go b/internal/io/fs/chunkedreader_bench_test.go
new file mode 100644
index 0000000..a909a2c
--- /dev/null
+++ b/internal/io/fs/chunkedreader_bench_test.go
@@ -0,0 +1,101 @@
+package fs
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mimecast/dtail/internal/io/pool"
+)
+
+// BenchmarkChunkedReader tests the performance of the chunked reader
+func BenchmarkChunkedReader(b *testing.B) {
+	// Create test data - simulate a log file with many lines
+	var testData strings.Builder
+	for i := 0; i < 10000; i++ {
+		testData.WriteString("INFO|1002-071143|1|stats.go:56|8|13|7|0.21|471h0m21s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=1\n")
+	}
+	data := testData.String()
+
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	for i := 0; i < b.N; i++ {
+		reader := strings.NewReader(data)
+		chunkedReader := NewChunkedReader(reader, 64*1024)
+		
+		rawLines := make(chan *bytes.Buffer, 100)
+		serverMessages := make(chan string, 10)
+		
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		
+		go func() {
+			defer cancel()
+			err := chunkedReader.ProcessLines(ctx, rawLines, 1000, "test.log", serverMessages, false)
+			if err != nil {
+				b.Errorf("ProcessLines error: %v", err)
+			}
+			close(rawLines)
+		}()
+		
+		// Consume all lines
+		lineCount := 0
+		for line := range rawLines {
+			lineCount++
+			pool.BytesBuffer.Put(line)
+		}
+		
+		cancel()
+		
+		if lineCount != 10000 {
+			b.Errorf("Expected 10000 lines, got %d", lineCount)
+		}
+	}
+}
+
+// BenchmarkChunkedReaderSmall tests performance with smaller chunks
+func BenchmarkChunkedReaderSmall(b *testing.B) {
+	// Create smaller test data
+	var testData strings.Builder
+	for i := 0; i < 1000; i++ {
+		testData.WriteString("INFO|1002-071143|1|stats.go:56|8|13|7|0.21|471h0m21s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=1\n")
+	}
+	data := testData.String()
+
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	for i := 0; i < b.N; i++ {
+		reader := strings.NewReader(data)
+		chunkedReader := NewChunkedReader(reader, 4*1024) // 4KB chunks
+		
+		rawLines := make(chan *bytes.Buffer, 100)
+		serverMessages := make(chan string, 10)
+		
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		
+		go func() {
+			defer cancel()
+			err := chunkedReader.ProcessLines(ctx, rawLines, 1000, "test.log", serverMessages, false)
+			if err != nil {
+				b.Errorf("ProcessLines error: %v", err)
+			}
+			close(rawLines)
+		}()
+		
+		// Consume all lines
+		lineCount := 0
+		for line := range rawLines {
+			lineCount++
+			pool.BytesBuffer.Put(line)
+		}
+		
+		cancel()
+		
+		if lineCount != 1000 {
+			b.Errorf("Expected 1000 lines, got %d", lineCount)
+		}
+	}
+}
\ No newline at end of file
-- 
cgit v1.2.3