diff options
| author | Paul Buetow <paul@buetow.org> | 2026-03-05 08:50:33 +0200 |
|---|---|---|
| committer | Paul Buetow <paul@buetow.org> | 2026-03-05 08:50:33 +0200 |
| commit | 5d1b9f1062d38c301c0995ec6da980bdf5e48332 (patch) | |
| tree | 81e1a8963ea66cf06164e89beb6cd2da0ee325f7 /benchmarks | |
| parent | bb46cfbccea301721fb93485ea7169f5841feda3 (diff) | |
Improve lint/vet reliability and refactor client runtime/bootstrap
Diffstat (limited to 'benchmarks')
| -rw-r--r-- | benchmarks/profile_runner.go | 58 | ||||
| -rw-r--r-- | benchmarks/testdata_generator.go | 48 |
2 files changed, 57 insertions, 49 deletions
diff --git a/benchmarks/profile_runner.go b/benchmarks/profile_runner.go index e7934dd..946c1ca 100644 --- a/benchmarks/profile_runner.go +++ b/benchmarks/profile_runner.go @@ -53,41 +53,41 @@ func RunProfiledCommand(b *testing.B, config ProfileConfig, tool string, args .. // Build command path cmdPath := filepath.Join("..", tool) - + // Add profiling flags profileArgs := []string{} if config.EnableCPU || config.EnableMem { profileArgs = append(profileArgs, "-profile") profileArgs = append(profileArgs, "-profiledir", config.ProfileDir) } - + // Combine all arguments allArgs := append(profileArgs, args...) - + // Create command cmd := exec.Command(cmdPath, allArgs...) - + // Set up output capture - outputFile := filepath.Join(config.ProfileDir, fmt.Sprintf("%s_output_%s.log", + outputFile := filepath.Join(config.ProfileDir, fmt.Sprintf("%s_output_%s.log", tool, time.Now().Format("20060102_150405"))) output, err := os.Create(outputFile) if err != nil { return nil, fmt.Errorf("creating output file: %w", err) } defer output.Close() - + cmd.Stdout = output cmd.Stderr = output - + // Record start time start := time.Now() - + // Run command err = cmd.Run() - + // Record duration duration := time.Since(start) - + result := &ProfileResult{ Tool: tool, Operation: strings.Join(args, "_"), @@ -95,12 +95,12 @@ func RunProfiledCommand(b *testing.B, config ProfileConfig, tool string, args .. ExitCode: cmd.ProcessState.ExitCode(), Error: err, } - + // Find generated profile files timestamp := time.Now().Format("20060102_1504") - profiles, _ := filepath.Glob(filepath.Join(config.ProfileDir, + profiles, _ := filepath.Glob(filepath.Join(config.ProfileDir, fmt.Sprintf("%s_*_%s*.prof", tool, timestamp))) - + for _, profile := range profiles { if strings.Contains(profile, "_cpu_") { result.CPUProfile = profile @@ -110,14 +110,14 @@ func RunProfiledCommand(b *testing.B, config ProfileConfig, tool string, args .. result.AllocProfile = profile } } - + return result, nil } // ProfileBenchmark runs a benchmark with profiling enabled func ProfileBenchmark(b *testing.B, name string, tool string, args ...string) { config := DefaultProfileConfig() - + b.Run(name+"_profiled", func(b *testing.B) { // Generate test data if needed testFile := "" @@ -130,7 +130,7 @@ func ProfileBenchmark(b *testing.B, name string, tool string, args ...string) { } testFile = GenerateTestFile(b, testConfig) defer os.Remove(testFile) - + // Replace placeholder in args for i, arg := range args { if arg == "__TESTFILE__" { @@ -138,13 +138,13 @@ func ProfileBenchmark(b *testing.B, name string, tool string, args ...string) { } } } - + // Run profiled command result, err := RunProfiledCommand(b, config, tool, args...) if err != nil && result.ExitCode != 0 { b.Fatalf("Command failed: %v", err) } - + // Report results b.Logf("Profile run completed in %v", result.Duration) if result.CPUProfile != "" { @@ -156,7 +156,7 @@ func ProfileBenchmark(b *testing.B, name string, tool string, args ...string) { if result.AllocProfile != "" { b.Logf("Allocation profile: %s", result.AllocProfile) } - + // Analyze profiles using dtail-tools dtailToolsPath := filepath.Join("..", "dtail-tools") if _, err := os.Stat(dtailToolsPath); err == nil { @@ -173,14 +173,14 @@ func ProfileBenchmark(b *testing.B, name string, tool string, args ...string) { // analyzeProfileWithTools runs dtail-tools profile analyze on a profile file func analyzeProfileWithTools(b *testing.B, dtailToolsPath, profilePath, profileType string) { b.Logf("\n%s Profile Analysis:", profileType) - + cmd := exec.Command(dtailToolsPath, "profile", "-mode", "analyze", profilePath) output, err := cmd.CombinedOutput() if err != nil { b.Logf("Failed to analyze profile: %v", err) return } - + // Print analysis output lines := strings.Split(string(output), "\n") // Print first 10 lines of analysis @@ -191,22 +191,24 @@ func analyzeProfileWithTools(b *testing.B, dtailToolsPath, profilePath, profileT } } -// Profiling benchmarks for each tool +// BenchmarkDCatWithProfiling runs dcat with profiling enabled. func BenchmarkDCatWithProfiling(b *testing.B) { ProfileBenchmark(b, "Simple", "dcat", "--plain", "--cfg", "none", "__TESTFILE__") } +// BenchmarkDGrepWithProfiling runs dgrep with profiling enabled. func BenchmarkDGrepWithProfiling(b *testing.B) { - ProfileBenchmark(b, "Regex", "dgrep", "--plain", "--cfg", "none", + ProfileBenchmark(b, "Regex", "dgrep", "--plain", "--cfg", "none", "-regex", "error|warning", "__TESTFILE__") } +// BenchmarkDMapWithProfiling runs dmap with profiling enabled. func BenchmarkDMapWithProfiling(b *testing.B) { // First generate a CSV file for dmap csvFile := filepath.Join(os.TempDir(), "dmap_test.csv") generateCSVTestData(b, csvFile, 10000) defer os.Remove(csvFile) - + ProfileBenchmark(b, "Count", "dmap", "--plain", "--cfg", "none", "-query", fmt.Sprintf("select count(*) from %s", csvFile)) } @@ -218,17 +220,17 @@ func generateCSVTestData(b *testing.B, filename string, rows int) { b.Fatalf("Failed to create CSV file: %v", err) } defer f.Close() - + // Write header fmt.Fprintln(f, "timestamp,user,action,duration") - + // Write data for i := 0; i < rows; i++ { timestamp := time.Now().Add(time.Duration(i) * time.Second).Format("2006-01-02 15:04:05") user := fmt.Sprintf("user%d", i%100) action := []string{"login", "query", "logout"}[i%3] duration := 100 + i%500 - + fmt.Fprintf(f, "%s,%s,%s,%d\n", timestamp, user, action, duration) } -}
\ No newline at end of file +} diff --git a/benchmarks/testdata_generator.go b/benchmarks/testdata_generator.go index 8ee4e29..74487aa 100644 --- a/benchmarks/testdata_generator.go +++ b/benchmarks/testdata_generator.go @@ -18,10 +18,12 @@ import ( // FileSize represents the size category of test files type FileSize int +// Supported benchmark file sizes. const ( - Small FileSize = 10 * 1024 * 1024 // 10MB - Medium FileSize = 100 * 1024 * 1024 // 100MB - Large FileSize = 1024 * 1024 * 1024 // 1GB + // Small represents a 10MB test file. + Small FileSize = 10 * 1024 * 1024 // 10MB + Medium FileSize = 100 * 1024 * 1024 // 100MB + Large FileSize = 1024 * 1024 * 1024 // 1GB ) func (fs FileSize) String() string { @@ -40,7 +42,9 @@ func (fs FileSize) String() string { // LogFormat represents different log format types type LogFormat int +// Supported synthetic log formats. const ( + // SimpleLogFormat contains plain text log lines. SimpleLogFormat LogFormat = iota MapReduceLogFormat MixedLogFormat @@ -49,7 +53,9 @@ const ( // CompressionType represents file compression options type CompressionType int +// Supported compression modes for generated test files. const ( + // NoCompression stores test data without compression. NoCompression CompressionType = iota GzipCompression ZstdCompression @@ -163,30 +169,30 @@ func generateCompressedFile(tmpFile, finalFile string, config TestDataConfig, cr // writeLogLines generates log content based on config func writeLogLines(w io.Writer, config TestDataConfig) error { rng := rand.New(rand.NewSource(time.Now().UnixNano())) - + // Calculate approximate lines needed avgLineSize := 150 // bytes totalLines := int(config.Size) / avgLineSize - + // Pre-generate some template lines for variation templateLines := generateTemplateLines(config.Format, config.LineVariation, config.Pattern, config.PatternRate, rng) - + bytesWritten := 0 for i := 0; i < totalLines && bytesWritten < int(config.Size); i++ { // Pick a random template line line := templateLines[rng.Intn(len(templateLines))] - + // Write with current timestamp timestampedLine := strings.Replace(line, "{TIMESTAMP}", generateTimestamp(i), 1) timestampedLine = strings.Replace(timestampedLine, "{COUNTER}", fmt.Sprintf("%d", i), 1) - + n, err := fmt.Fprintln(w, timestampedLine) if err != nil { return err } bytesWritten += n } - + return nil } @@ -194,10 +200,10 @@ func writeLogLines(w io.Writer, config TestDataConfig) error { func generateTemplateLines(format LogFormat, variation int, pattern string, patternRate int, rng *rand.Rand) []string { numTemplates := max(10, variation) // At least 10 templates templates := make([]string, 0, numTemplates) - + for i := 0; i < numTemplates; i++ { includePattern := pattern != "" && rng.Intn(100) < patternRate - + switch format { case SimpleLogFormat: templates = append(templates, generateSimpleLogLine(i, includePattern, pattern, rng)) @@ -211,7 +217,7 @@ func generateTemplateLines(format LogFormat, variation int, pattern string, patt } } } - + return templates } @@ -219,14 +225,14 @@ func generateTemplateLines(format LogFormat, variation int, pattern string, patt func generateSimpleLogLine(id int, includePattern bool, pattern string, rng *rand.Rand) string { levels := []string{"INFO", "WARN", "ERROR", "DEBUG"} level := levels[rng.Intn(len(levels))] - + message := fmt.Sprintf("Processing request %d", id) if includePattern && pattern != "" { message = fmt.Sprintf("%s %s", message, pattern) } - + // Format: LEVEL|TIMESTAMP|THREAD|FILE:LINE|MESSAGE - return fmt.Sprintf("%s|{TIMESTAMP}|thread-%d|app.go:%d|%s", + return fmt.Sprintf("%s|{TIMESTAMP}|thread-%d|app.go:%d|%s", level, rng.Intn(10)+1, rng.Intn(1000)+1, message) } @@ -235,12 +241,12 @@ func generateMapReduceLogLine(id int, includePattern bool, pattern string, rng * goroutines := rng.Intn(50) + 10 connections := rng.Intn(100) lifetime := rng.Intn(1000) + 100 - + message := "MAPREDUCE:STATS" if includePattern && pattern != "" { message = fmt.Sprintf("%s|%s", message, pattern) } - + // Format matching the integration test data return fmt.Sprintf("INFO|{TIMESTAMP}|1|stats.go:56|8|%d|7|0.%02d|471h%dm%ds|%s|currentConnections=%d|lifetimeConnections=%d", goroutines, rng.Intn(100), rng.Intn(60), rng.Intn(60), message, connections, lifetime) @@ -260,19 +266,19 @@ func CleanupBenchmarkFiles(pattern string) error { if pattern == "" { pattern = "dtail_bench_*.tmp*" } - + tempDir := os.TempDir() matches, err := filepath.Glob(filepath.Join(tempDir, pattern)) if err != nil { return err } - + for _, match := range matches { if err := os.Remove(match); err != nil && !os.IsNotExist(err) { return err } } - + return nil } @@ -282,4 +288,4 @@ func max(a, b int) int { return a } return b -}
\ No newline at end of file +} |
