Add comprehensive test logging infrastructure to integration tests

- Add test logging infrastructure to track command execution and file comparisons
- Generate .log files for each test with command history and manual verification commands
- Ensure all temporary test files use .tmp suffix for consistency
- Clean up .tmp files before each test run (not after) for clean test starts
- Update .gitignore to exclude generated test artifacts (.log, .query files)
- Fix dserver test configurations to use .tmp suffix for output files
- Fix expected test outputs for dgrep context tests

This change improves test debugging and verification by providing detailed logs
of what each test does and allows manual verification of test results.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

1 files changed, 64 insertions, 39 deletions

diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go
index 235f55c..c1339b5 100644
--- a/integrationtests/dmap_test.go
+++ b/integrationtests/dmap_test.go
@@ -1,12 +1,16 @@
 package integrationtests
 
 import (
+	"context"
 	"fmt"
 	"testing"
 )
 
 func TestDMap1(t *testing.T) {
 	skipIfNotIntegrationTest(t)
+	cleanupTmpFiles(t)
+	testLogger := NewTestLogger("TestDMap1")
+	defer testLogger.WriteLogFile()
 
 	testTable := map[string]string{
 		"a": "from STATS select count($line),last($time)," +
@@ -28,10 +32,10 @@ func TestDMap1(t *testing.T) {
 		for subtestName, query := range testTable {
 			t.Run(subtestName, func(t *testing.T) {
 				t.Log("Testing dmap with input file")
-				testDmap1Serverless(t, query, subtestName, false)
+				testDmap1Serverless(t, testLogger, query, subtestName, false)
 				
 				t.Log("Testing dmap with stdin input pipe")
-				testDmap1Serverless(t, query, subtestName, true)
+				testDmap1Serverless(t, testLogger, query, subtestName, true)
 			})
 		}
 	})
@@ -41,13 +45,13 @@ func TestDMap1(t *testing.T) {
 		for subtestName, query := range testTable {
 			t.Run(subtestName, func(t *testing.T) {
 				t.Log("Testing dmap with input file in server mode")
-				testDmap1WithServer(t, query, subtestName)
+				testDmap1WithServer(t, testLogger, query, subtestName)
 			})
 		}
 	})
 }
 
-func testDmap1Serverless(t *testing.T, query, subtestName string, usePipe bool) {
+func testDmap1Serverless(t *testing.T, logger *TestLogger, query, subtestName string, usePipe bool) {
 	paths := GetStandardTestPaths()
 	csvFile := fmt.Sprintf("dmap1%s.csv.tmp", subtestName)
 	expectedCsvFile := fmt.Sprintf("dmap1%s.csv.expected", subtestName)
@@ -56,7 +60,8 @@ func testDmap1Serverless(t *testing.T, query, subtestName string, usePipe bool)
 	
 	cleanupFiles(t, csvFile, queryFile)
 
-	ctx, cancel := createTestContextWithTimeout(t)
+	ctxTimeout, cancel := createTestContextWithTimeout(t)
+	ctx := WithTestLogger(ctxTimeout, logger)
 	defer cancel()
 
 	var stdoutCh, stderrCh <-chan string
@@ -84,7 +89,7 @@ func testDmap1Serverless(t *testing.T, query, subtestName string, usePipe bool)
 
 	waitForCommand(ctx, t, stdoutCh, stderrCh, cmdErrCh)
 
-	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+	if err := compareFilesContentsWithContext(ctx, t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 	}
 	if err := verifyQueryFile(t, queryFile, query); err != nil {
@@ -92,7 +97,8 @@ func testDmap1Serverless(t *testing.T, query, subtestName string, usePipe bool)
 	}
 }
 
-func testDmap1WithServer(t *testing.T, query, subtestName string) {
+func testDmap1WithServer(t *testing.T, logger *TestLogger, query, subtestName string) {
+	ctx := WithTestLogger(context.Background(), logger)
 	paths := GetStandardTestPaths()
 	csvFile := fmt.Sprintf("dmap1%s.csv.tmp", subtestName)
 	expectedCsvFile := fmt.Sprintf("dmap1%s.csv.expected", subtestName)
@@ -125,7 +131,7 @@ func testDmap1WithServer(t *testing.T, query, subtestName string) {
 
 	waitForCommand(server.ctx, t, stdoutCh, stderrCh, cmdErrCh)
 
-	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+	if err := compareFilesContentsWithContext(ctx, t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 	}
 	if err := verifyQueryFile(t, queryFile, query); err != nil {
@@ -134,14 +140,17 @@ func testDmap1WithServer(t *testing.T, query, subtestName string) {
 }
 
 func TestDMap2(t *testing.T) {
+	cleanupTmpFiles(t)
+	testLogger := NewTestLogger("TestDMap2")
+	defer testLogger.WriteLogFile()
 	runDualModeTest(t, DualModeTest{
 		Name:           "TestDMap2",
-		ServerlessTest: testDMap2Serverless,
-		ServerTest:     testDMap2WithServer,
+		ServerlessTest: func(t *testing.T) { testDMap2Serverless(t, testLogger) },
+		ServerTest:     func(t *testing.T) { testDMap2WithServer(t, testLogger) },
 	})
 }
 
-func testDMap2Serverless(t *testing.T) {
+func testDMap2Serverless(t *testing.T, logger *TestLogger) {
 	paths := GetStandardTestPaths()
 	outFile := "dmap2_serverless.stdout.tmp"
 	csvFile := "dmap2_serverless.csv.tmp"
@@ -153,7 +162,8 @@ func testDMap2Serverless(t *testing.T) {
 		"avg($goroutines),min($goroutines) group by $time order by count($time) "+
 		"outfile %s", csvFile)
 
-	ctx, cancel := createTestContextWithTimeout(t)
+	ctxTimeout, cancel := createTestContextWithTimeout(t)
+	ctx := WithTestLogger(ctxTimeout, logger)
 	defer cancel()
 	_, err := runCommand(ctx, t, outFile,
 		"../dmap", "--query", query, "--cfg", "none", paths.MaprTestData)
@@ -162,7 +172,7 @@ func testDMap2Serverless(t *testing.T) {
 		return
 	}
 
-	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+	if err := compareFilesContentsWithContext(ctx, t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 	}
 	if err := verifyQueryFile(t, queryFile, query); err != nil {
@@ -170,7 +180,8 @@ func testDMap2Serverless(t *testing.T) {
 	}
 }
 
-func testDMap2WithServer(t *testing.T) {
+func testDMap2WithServer(t *testing.T, logger *TestLogger) {
+	ctx := WithTestLogger(context.Background(), logger)
 	paths := GetStandardTestPaths()
 	outFile := "dmap2_server.stdout.tmp"
 	csvFile := "dmap2_server.csv.tmp"
@@ -202,7 +213,7 @@ func testDMap2WithServer(t *testing.T) {
 		return
 	}
 
-	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+	if err := compareFilesContentsWithContext(ctx, t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 	}
 	if err := verifyQueryFile(t, queryFile, query); err != nil {
@@ -211,14 +222,17 @@ func testDMap2WithServer(t *testing.T) {
 }
 
 func TestDMap3(t *testing.T) {
+	cleanupTmpFiles(t)
+	testLogger := NewTestLogger("TestDMap3")
+	defer testLogger.WriteLogFile()
 	runDualModeTest(t, DualModeTest{
 		Name:           "TestDMap3",
-		ServerlessTest: testDMap3Serverless,
-		ServerTest:     testDMap3WithServer,
+		ServerlessTest: func(t *testing.T) { testDMap3Serverless(t, testLogger) },
+		ServerTest:     func(t *testing.T) { testDMap3WithServer(t, testLogger) },
 	})
 }
 
-func testDMap3Serverless(t *testing.T) {
+func testDMap3Serverless(t *testing.T, logger *TestLogger) {
 	paths := GetStandardTestPaths()
 	outFile := "dmap3_serverless.stdout.tmp"
 	csvFile := "dmap3_serverless.csv.tmp"
@@ -237,7 +251,8 @@ func testDMap3Serverless(t *testing.T) {
 	}
 
 	// Simply run dmap with multiple input files directly
-	ctx, cancel := createTestContextWithTimeout(t)
+	ctxTimeout, cancel := createTestContextWithTimeout(t)
+	ctx := WithTestLogger(ctxTimeout, logger)
 	defer cancel()
 
 	args := NewCommandArgs()
@@ -250,7 +265,7 @@ func testDMap3Serverless(t *testing.T) {
 		return
 	}
 
-	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+	if err := compareFilesContentsWithContext(ctx, t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 	}
 	if err := verifyQueryFile(t, queryFile, query); err != nil {
@@ -258,7 +273,8 @@ func testDMap3Serverless(t *testing.T) {
 	}
 }
 
-func testDMap3WithServer(t *testing.T) {
+func testDMap3WithServer(t *testing.T, logger *TestLogger) {
+	ctx := WithTestLogger(context.Background(), logger)
 	paths := GetStandardTestPaths()
 	outFile := "dmap3_server.stdout.tmp"
 	csvFile := "dmap3_server.csv.tmp"
@@ -296,7 +312,7 @@ func testDMap3WithServer(t *testing.T) {
 		return
 	}
 
-	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+	if err := compareFilesContentsWithContext(ctx, t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 	}
 	if err := verifyQueryFile(t, queryFile, query); err != nil {
@@ -305,14 +321,17 @@ func testDMap3WithServer(t *testing.T) {
 }
 
 func TestDMap4Append(t *testing.T) {
+	cleanupTmpFiles(t)
+	testLogger := NewTestLogger("TestDMap4Append")
+	defer testLogger.WriteLogFile()
 	runDualModeTest(t, DualModeTest{
 		Name:           "TestDMap4Append",
-		ServerlessTest: testDMap4AppendServerless,
-		ServerTest:     testDMap4AppendWithServer,
+		ServerlessTest: func(t *testing.T) { testDMap4AppendServerless(t, testLogger) },
+		ServerTest:     func(t *testing.T) { testDMap4AppendWithServer(t, testLogger) },
 	})
 }
 
-func testDMap4AppendServerless(t *testing.T) {
+func testDMap4AppendServerless(t *testing.T, logger *TestLogger) {
 	paths := GetStandardTestPaths()
 	csvFile := "dmap4_serverless.csv.tmp"
 	queryFile := fmt.Sprintf("%s.query", csvFile)
@@ -339,7 +358,7 @@ func testDMap4AppendServerless(t *testing.T) {
 		}
 		
 		// Verify the CSV output
-		if err := compareFilesContents(t, csvFile, "dmap4_query1.csv.expected"); err != nil {
+		if err := compareFilesContentsWithContext(ctx, t, csvFile, "dmap4_query1.csv.expected"); err != nil {
 			t.Error(err)
 		}
 		
@@ -368,7 +387,7 @@ func testDMap4AppendServerless(t *testing.T) {
 		}
 		
 		// Verify the CSV output (should still be the first query result - append doesn't change existing file)
-		if err := compareFilesContents(t, csvFile, "dmap4_query1.csv.expected"); err != nil {
+		if err := compareFilesContentsWithContext(ctx, t, csvFile, "dmap4_query1.csv.expected"); err != nil {
 			t.Error(err)
 		}
 	})
@@ -392,7 +411,7 @@ func testDMap4AppendServerless(t *testing.T) {
 		}
 		
 		// Verify the CSV output (should still be the first query result - append doesn't change existing file)
-		if err := compareFilesContents(t, csvFile, "dmap4_query1.csv.expected"); err != nil {
+		if err := compareFilesContentsWithContext(ctx, t, csvFile, "dmap4_query1.csv.expected"); err != nil {
 			t.Error(err)
 		}
 		
@@ -406,7 +425,8 @@ func testDMap4AppendServerless(t *testing.T) {
 	})
 }
 
-func testDMap4AppendWithServer(t *testing.T) {
+func testDMap4AppendWithServer(t *testing.T, logger *TestLogger) {
+	ctx := WithTestLogger(context.Background(), logger)
 	paths := GetStandardTestPaths()
 	csvFile := "dmap4_server.csv.tmp"
 	queryFile := fmt.Sprintf("%s.query", csvFile)
@@ -446,7 +466,7 @@ func testDMap4AppendWithServer(t *testing.T) {
 		}
 		
 		// Verify the CSV output
-		if err := compareFilesContents(t, csvFile, "dmap4_query1.csv.expected"); err != nil {
+		if err := compareFilesContentsWithContext(ctx, t, csvFile, "dmap4_query1.csv.expected"); err != nil {
 			t.Error(err)
 		}
 		
@@ -476,7 +496,7 @@ func testDMap4AppendWithServer(t *testing.T) {
 		}
 		
 		// Verify the CSV output (should still be the first query result - append doesn't change existing file)
-		if err := compareFilesContents(t, csvFile, "dmap4_query1.csv.expected"); err != nil {
+		if err := compareFilesContentsWithContext(ctx, t, csvFile, "dmap4_query1.csv.expected"); err != nil {
 			t.Error(err)
 		}
 	})
@@ -501,7 +521,7 @@ func testDMap4AppendWithServer(t *testing.T) {
 		}
 		
 		// Verify the CSV output (should still be the first query result - append doesn't change existing file)
-		if err := compareFilesContents(t, csvFile, "dmap4_query1.csv.expected"); err != nil {
+		if err := compareFilesContentsWithContext(ctx, t, csvFile, "dmap4_query1.csv.expected"); err != nil {
 			t.Error(err)
 		}
 		
@@ -516,14 +536,17 @@ func testDMap4AppendWithServer(t *testing.T) {
 }
 
 func TestDMap5CSV(t *testing.T) {
+	cleanupTmpFiles(t)
+	testLogger := NewTestLogger("TestDMap5CSV")
+	defer testLogger.WriteLogFile()
 	runDualModeTest(t, DualModeTest{
 		Name:           "TestDMap5CSV",
-		ServerlessTest: testDMap5CSVServerless,
-		ServerTest:     testDMap5CSVWithServer,
+		ServerlessTest: func(t *testing.T) { testDMap5CSVServerless(t, testLogger) },
+		ServerTest:     func(t *testing.T) { testDMap5CSVWithServer(t, testLogger) },
 	})
 }
 
-func testDMap5CSVServerless(t *testing.T) {
+func testDMap5CSVServerless(t *testing.T, logger *TestLogger) {
 	inFile := "dmap5.csv.in"
 	csvFile := "dmap5_serverless.csv.tmp"
 	expectedCsvFile := "dmap5.csv.expected"
@@ -535,7 +558,8 @@ func testDMap5CSVServerless(t *testing.T) {
 		"group by $hostname set $timecount = `count($time)`, $time = `$time`, "+
 		"$min_goroutines = `min($goroutines)` logformat csv outfile %s", csvFile)
 
-	ctx, cancel := createTestContextWithTimeout(t)
+	ctxTimeout, cancel := createTestContextWithTimeout(t)
+	ctx := WithTestLogger(ctxTimeout, logger)
 	defer cancel()
 	_, err := runCommand(ctx, t, outFile,
 		"../dmap", "--query", query, "--cfg", "none", inFile)
@@ -544,7 +568,7 @@ func testDMap5CSVServerless(t *testing.T) {
 		return
 	}
 
-	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+	if err := compareFilesContentsWithContext(ctx, t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 	}
 	// Verify the query file contains the expected query
@@ -553,7 +577,8 @@ func testDMap5CSVServerless(t *testing.T) {
 	}
 }
 
-func testDMap5CSVWithServer(t *testing.T) {
+func testDMap5CSVWithServer(t *testing.T, logger *TestLogger) {
+	ctx := WithTestLogger(context.Background(), logger)
 	inFile := "dmap5.csv.in"
 	csvFile := "dmap5_server.csv.tmp"
 	expectedCsvFile := "dmap5.csv.expected"
@@ -585,7 +610,7 @@ func testDMap5CSVWithServer(t *testing.T) {
 		return
 	}
 
-	if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil {
+	if err := compareFilesContentsWithContext(ctx, t, csvFile, expectedCsvFile); err != nil {
 		t.Error(err)
 	}
 	// Verify the query file contains the expected query