diff options
| author | Paul Buetow <paul@buetow.org> | 2026-07-22 23:52:43 +0300 |
|---|---|---|
| committer | Paul Buetow <paul@buetow.org> | 2026-07-22 23:52:43 +0300 |
| commit | 17bf7e042496a4afcbf6ee7a583378adb3ec502d (patch) | |
| tree | 380d89c2ca5157a067244857aa4fd5ebf2387915 | |
| parent | bc2767c87c4090798c4c7d15e101ed066e947301 (diff) | |
build: DTail fork — benchmarks, PGO tooling, Makefile, docker
Squashed development of the build/benchmark/profiling tooling:
- Turbo-vs-normal and per-tool benchmark harnesses under benchmarks/ (with dated
baseline and result artifacts preserved), plus MapReduce/SSH/network suites.
- Profile-Guided Optimization workflow (internal/tools/pgo lives with the code):
Makefile targets (make pgo / pgo-quick / build-pgo), profile generation with
non-zero-sample and representativeness guards.
- Makefile targets for build, test (unit + integration), lint, vet, benchmarking
and profiling; docker config updates.
- .gitignore hardened against pgo profiles and accidental integration-test
debug/output dumps.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
38 files changed, 6356 insertions, 51 deletions
@@ -2,18 +2,55 @@ *.csv *.tmp *.out +.serena +Test*.log +*.query +append:* cache/ log/ tags +benchmark_results/ +benchmarks/baselines/current*.txt +benchmarks/baselines/comparison*.txt +benchmarks/profiles/ +benchmarks/testdata/ /cache/ /log/ +/profiles/ +/profiling/*.prof +/profiling/profiles/ +/profiling/testdata/ +/testdata/ +test_*.log /dtail /dgrep /dcat /dmap /dserver /dtailhealth +/dtail-tools known_hosts id_rsa id_rsa.pub ssh_host_key +pgo-profiles/ +pgo-build/ +.amp +.claude + +# Accidental integration-test debug/output dumps (never commit these) +integrationtests/*_output*.txt +integrationtests/test_output*.txt +integrationtests/turbo_test_output*.txt +integrationtests/manual_*.txt +integrationtests/debug_*.txt +integrationtests/strace_output.txt +integrationtests/test_trace_output.txt +integrationtests/output_*turbo*.txt +integrationtests/test_c.txt +integrationtests/test_colors.txt +integrationtests/test_plain.txt +integrationtests/test_config.json +integrationtests/test_server.json +integrationtests/test_server_config.json +/TOOD.md @@ -5,8 +5,12 @@ endif ifdef DTAIL_USE_PROPRIETARY GO_TAGS+=proprietary endif +ifdef DTAIL_NO_ZSTD +GO_TAGS+=nozstd +endif all: build -build: dserver dcat dgrep dmap dtail dtailhealth +build: dserver dcat dgrep dmap dtail dtailhealth dtail-tools +build-pgo: pgo-build-binaries dserver: ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dserver ./cmd/dserver/main.go dcat: @@ -19,6 +23,8 @@ dtail: ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dtail ./cmd/dtail/main.go dtailhealth: ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dtailhealth ./cmd/dtailhealth/main.go +dtail-tools: + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dtail-tools ./cmd/dtail-tools/main.go install: ${GO} install -tags '${GO_TAGS}' ./cmd/dserver/main.go ${GO} install -tags '${GO_TAGS}' ./cmd/dcat/main.go @@ -30,20 +36,193 @@ clean: ls ./cmd/ | while read cmd; do \ test -f $$cmd && rm $$cmd; \ done + @echo "Removing .tmp files..." + find . -name "*.tmp" -type f -delete + @echo "Removing .prof files..." + find . -name "*.prof" -type f -delete vet: - find . -type d | egrep -v '(./examples|./log|./doc)' | while read dir; do \ - echo ${GO} vet $$dir; \ - ${GO} vet $$dir; \ + @set -e; \ + packages=`${GO} list ./... | grep -v '^github.com/mimecast/dtail/benchmarks/cmd$$'`; \ + for pkg in $$packages; do \ + echo ${GO} vet $$pkg; \ + ${GO} vet $$pkg; \ done sh -c 'grep -R NEXT: .' sh -c 'grep -R TODO: .' lint: - ${GO} get golang.org/x/lint/golint - find . -type d | while read dir; do \ - echo golint $$dir; \ - golint $$dir; \ - done | grep -F .go: + @set -e; \ + ${GO} install golang.org/x/lint/golint@v0.0.0-20241112194109-818c5a804067; \ + gobin=`${GO} env GOBIN`; \ + if [ -z "$$gobin" ]; then \ + gobin=`${GO} env GOPATH`/bin; \ + fi; \ + golint_bin=$$gobin/golint; \ + packages=`${GO} list ./... | grep -v '^github.com/mimecast/dtail/benchmarks/cmd$$'`; \ + echo "Using $$golint_bin"; \ + output=`$$golint_bin $$packages || true`; \ + if [ -n "$$output" ]; then \ + echo "$$output"; \ + exit 1; \ + fi test: ${GO} clean -testcache set -e; find . -name '*_test.go' | while read file; do dirname $$file; done | \ - sort -u | while read dir; do ${GO} test -tags '${GO_TAGS}' --race -v $$dir || exit 2; done + sort -u | while read dir; do ${GO} test -tags '${GO_TAGS}' --race -v -failfast $$dir || exit 2; done +test-integration: clean build + ${GO} clean -testcache + DTAIL_INTEGRATION_TEST_RUN_MODE=yes ${GO} test -v -tags '${GO_TAGS}' --race -count=1 ./integrationtests +benchmark: build dtail-tools + ./dtail-tools benchmark -mode run +benchmark-quick: build dtail-tools + ./dtail-tools benchmark -mode run -quick +benchmark-full: build dtail-tools + ./dtail-tools benchmark -mode run -iterations 3x +benchmark-baseline: build dtail-tools + @read -p "Enter a descriptive name for this baseline (e.g. 'before-optimization', 'v1.0-release'): " tag; \ + if [ -z "$$tag" ]; then \ + echo "Error: Baseline name cannot be empty"; \ + exit 1; \ + fi; \ + ./dtail-tools benchmark -mode baseline -tag "$$tag" +benchmark-baseline-quick: build dtail-tools + @read -p "Enter a descriptive name for this baseline (e.g. 'before-optimization', 'v1.0-release'): " tag; \ + if [ -z "$$tag" ]; then \ + echo "Error: Baseline name cannot be empty"; \ + exit 1; \ + fi; \ + ./dtail-tools benchmark -mode baseline -tag "$$tag" -quick +benchmark-compare: build dtail-tools + @if [ -z "${BASELINE}" ]; then \ + echo "Usage: make benchmark-compare BASELINE=benchmarks/baselines/baseline_TIMESTAMP.txt"; \ + ./dtail-tools benchmark -mode list; \ + exit 1; \ + fi + ./dtail-tools benchmark -mode compare -baseline ${BASELINE} + +# Profiling targets +profile-all: build dtail-tools + ./dtail-tools profile -mode full +profile-quick: build dtail-tools + ./dtail-tools profile -mode quick +profile-dmap: build dtail-tools + ./dtail-tools profile -mode dmap +profile-list: dtail-tools + ./dtail-tools profile -mode list + +# Interactive profile analysis +profile-analyze: dtail-tools + @if [ -z "${PROFILE}" ]; then \ + echo "Usage: make profile-analyze PROFILE=profiles/dcat_cpu_*.prof"; \ + ./dtail-tools profile -mode list; \ + else \ + ./dtail-tools profile -mode analyze ${PROFILE}; \ + fi + +# Generate flame graph (web interface) +profile-web: dtail-tools + @if [ -z "${PROFILE}" ]; then \ + echo "Usage: make profile-web PROFILE=profiles/dcat_cpu_*.prof"; \ + ./dtail-tools profile -mode list; \ + else \ + ./dtail-tools profile -mode analyze ${PROFILE} -web; \ + fi + +# Clean profiles +profile-clean: + @echo "Cleaning profile directory..." + rm -rf profiles testdata + @echo "Profile directory cleaned" + +# Show profiling help +profile-help: + @echo "DTail Profiling Targets:" + @echo "" + @echo " make profile-quick - Quick profiling with small datasets" + @echo " make profile-all - Full profiling suite" + @echo " make profile-dmap - Profile dmap specifically" + @echo " make profile-list - List available profiles" + @echo "" + @echo " make profile-analyze PROFILE=<file> - Analyze a specific profile" + @echo " make profile-web PROFILE=<file> - Open web interface for profile" + @echo "" + @echo " make profile-clean - Clean all profiles" + @echo "" + @echo "Examples:" + @echo " make profile-quick # Fast profiling" + @echo " make profile-analyze PROFILE=profiles/dcat_cpu_*.prof" + @echo "" + +.PHONY: profile-all profile-quick profile-dmap profile-list profile-analyze profile-web profile-clean profile-help + +## Profile-Guided Optimization targets +pgo: build dtail-tools + @echo "Running Profile-Guided Optimization for all commands..." + ./dtail-tools pgo + +pgo-quick: build dtail-tools + @echo "Running quick PGO with smaller datasets..." + ./dtail-tools pgo -datasize 100000 -iterations 2 + +pgo-commands: build dtail-tools + @if [ -z "${COMMANDS}" ]; then \ + echo "Usage: make pgo-commands COMMANDS='dcat dgrep'"; \ + exit 1; \ + fi + ./dtail-tools pgo ${COMMANDS} + +pgo-clean: + @echo "Cleaning PGO artifacts..." + rm -rf pgo-profiles pgo-build + +pgo-help: + @echo "DTail PGO (Profile-Guided Optimization) Targets:" + @echo "" + @echo " make pgo - Run PGO for all commands (full optimization)" + @echo " make pgo-quick - Quick PGO with smaller datasets" + @echo " make pgo-commands - PGO for specific commands" + @echo " Example: make pgo-commands COMMANDS='dcat dgrep'" + @echo " make pgo-clean - Remove PGO artifacts" + @echo "" + @echo "After running PGO, optimized binaries will be in pgo-build/" + @echo "" + +# Build PGO-optimized binaries without running benchmarks +# This assumes PGO profiles already exist in pgo-profiles/ +pgo-build-binaries: dtail-tools + @if [ ! -d "pgo-profiles" ]; then \ + echo "Error: pgo-profiles directory not found."; \ + echo "Run 'make pgo' first to generate profiles, or 'make pgo-generate' to only generate profiles."; \ + exit 1; \ + fi + @echo "Building PGO-optimized binaries using existing profiles..." + @mkdir -p pgo-build + @for cmd in dcat dgrep dmap dtail dserver; do \ + profile="pgo-profiles/$$cmd.pprof"; \ + if [ -f "$$profile" ]; then \ + echo "Building $$cmd with PGO..."; \ + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -pgo=$$profile -o pgo-build/$$cmd ./cmd/$$cmd/main.go; \ + else \ + echo "Warning: Profile $$profile not found, building without PGO..."; \ + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o pgo-build/$$cmd ./cmd/$$cmd/main.go; \ + fi \ + done + @echo "PGO-optimized binaries built in pgo-build/" + +# Generate PGO profiles without building optimized binaries +pgo-generate: build dtail-tools + @echo "Generating PGO profiles..." + ./dtail-tools pgo -profileonly + @echo "PGO profiles generated in pgo-profiles/" + +# Install PGO-optimized binaries to system +install-pgo: pgo-build-binaries + @echo "Installing PGO-optimized binaries..." + @for cmd in dcat dgrep dmap dtail dserver; do \ + if [ -f "pgo-build/$$cmd" ]; then \ + echo "Installing $$cmd..."; \ + cp pgo-build/$$cmd ${GOPATH}/bin/$$cmd || sudo cp pgo-build/$$cmd /usr/local/bin/$$cmd; \ + fi \ + done + @echo "PGO-optimized binaries installed" + +.PHONY: pgo pgo-quick pgo-commands pgo-clean pgo-help pgo-build-binaries pgo-generate install-pgo diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..3494086 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,232 @@ +# DTail Benchmarks + +This directory contains comprehensive benchmarks for the DTail toolset (dcat, dgrep, dmap). + +## Overview + +The benchmarking framework tests performance across: +- Different file sizes (10MB, 100MB, 1GB) +- Various compression formats (none, gzip, zstd) +- Different query patterns and complexities +- Server mode vs serverless operation + +## Prerequisites + +Before running benchmarks, ensure all DTail binaries are built: + +```bash +cd .. +make build +``` + +## Running Benchmarks + +### Quick Benchmarks (Small Files Only) +```bash +go test -bench=BenchmarkQuick ./benchmarks +``` + +### All Benchmarks +```bash +go test -bench=. ./benchmarks +``` + +### Creating Baseline Snapshots +```bash +# Create a baseline before making changes (will prompt for name) +make benchmark-baseline + +# Create a quick baseline (small files only, will prompt for name) +make benchmark-baseline-quick + +# Create a baseline with a descriptive tag (no prompt) +./dtail-tools benchmark -mode baseline -tag "before-optimization" + +# Create a baseline interactively (will prompt if no tag provided) +make benchmark-baseline + +# Create a comprehensive baseline (3x iterations) +./dtail-tools benchmark -mode baseline -iterations 3x -tag "v1.0-release" +``` + +### Comparing Performance +```bash +# Compare with a specific baseline using make +make benchmark-compare BASELINE=benchmarks/baselines/baseline_20240125_143022.txt + +# Use the benchmark script for more options +./dtail-tools benchmark -mode compare -baseline benchmarks/baselines/baseline_20240125_143022.txt + +# List available baselines +./dtail-tools benchmark -mode list +``` + +### Specific Tool Benchmarks +```bash +# DCat benchmarks only +go test -bench=BenchmarkDCat ./benchmarks + +# DGrep benchmarks only +go test -bench=BenchmarkDGrep ./benchmarks + +# DMap benchmarks only +go test -bench=BenchmarkDMap ./benchmarks +``` + +### With Memory Profiling +```bash +go test -bench=. -benchmem ./benchmarks +``` + +### Custom Configuration +```bash +# Run with specific file sizes +DTAIL_BENCH_SIZES=small,medium go test -bench=. ./benchmarks + +# Keep temporary files for inspection +DTAIL_BENCH_KEEP_FILES=true go test -bench=. ./benchmarks + +# Set custom timeout +DTAIL_BENCH_TIMEOUT=30m go test -bench=. ./benchmarks +``` + +## Benchmark Categories + +### DCat Benchmarks +- **Simple**: Sequential file reading +- **Multiple Files**: Reading 10-100 files concurrently +- **Compressed**: Performance with gzip/zstd compression +- **Server Mode**: Client-server performance comparison + +### DGrep Benchmarks +- **Simple Pattern**: Basic string matching with varying hit rates +- **Regex Pattern**: Complex regex performance +- **Context Lines**: Impact of --before/--after flags +- **Inverted**: Performance of --invert grep +- **Compressed**: Grep on compressed files + +### DMap Benchmarks +- **Simple Aggregation**: Basic count, sum, avg operations +- **Group By Cardinality**: Performance with different group sizes +- **Complex Queries**: WHERE clauses and multiple conditions +- **Time Intervals**: Time-based grouping performance +- **Custom Functions**: Performance of maskdigits, md5sum, etc. + +## Output + +Benchmark results are saved in multiple formats: +- `benchmark_results/results_TIMESTAMP.json` - Machine-readable JSON +- `benchmark_results/results_TIMESTAMP.csv` - Spreadsheet-compatible CSV +- `benchmark_results/results_TIMESTAMP.md` - Human-readable Markdown report +- `benchmark_results/latest.json` - Most recent results for easy access + +## Interpreting Results + +Key metrics: +- **MB/sec**: Throughput in megabytes per second +- **lines/sec**: Lines processed per second +- **compression_ratio**: For compressed file benchmarks +- **matched_lines**: For grep benchmarks +- **approx_groups**: For MapReduce group by operations + +## Performance Tuning + +For accurate benchmarks: +1. Run on isolated hardware +2. Disable CPU frequency scaling +3. Close unnecessary applications +4. Run multiple times and average results + +## Continuous Integration + +The benchmarks can be integrated into CI/CD pipelines: + +```yaml +# Example GitHub Actions workflow +- name: Run Benchmarks + run: | + make build + go test -bench=BenchmarkQuick ./benchmarks +``` + +## Troubleshooting + +### "Command not found" errors +Ensure DTail binaries are built: `make build` + +### Disk space issues +Benchmarks create large temporary files. Ensure sufficient disk space (>2GB). + +### Timeout errors +Increase timeout: `DTAIL_BENCH_TIMEOUT=60m go test -bench=. ./benchmarks` + +## Baseline Management + +The benchmarking framework includes tools for creating and comparing performance baselines: + +### Creating Baselines +Baselines capture the complete benchmark output including: +- Git commit hash +- Timestamp +- All benchmark results with timing and memory allocation data +- Descriptive names for easy identification + +The system will prompt for a meaningful baseline name to ensure proper documentation: + +```bash +# Simple baseline (prompts for name) +make benchmark-baseline +> Enter a descriptive name for this baseline: before-cache-optimization + +# Quick baseline for rapid testing (prompts for name) +make benchmark-baseline-quick +> Enter a descriptive name for this baseline: initial-performance-check + +# Tagged baseline with description (no prompt) +./dtail-tools benchmark -mode baseline -tag "before-refactoring" + +# Full baseline with multiple iterations +./dtail-tools benchmark -mode baseline -iterations 3x -memory -tag "release-v2.0" +``` + +Baseline files are named with the pattern: +`baseline_YYYYMMDD_HHMMSS_descriptive-name.txt` + +### Comparing Performance +Compare current performance against a baseline to detect regressions or improvements: + +```bash +# Using make +make benchmark-compare BASELINE=benchmarks/baselines/baseline_20240125_143022.txt + +# Using benchmark script (provides benchstat analysis if available) +./dtail-tools benchmark -mode compare -baseline benchmarks/baselines/baseline_20240125_143022.txt +``` + +### Managing Baselines +```bash +# List all baselines +./dtail-tools benchmark -mode list + +# View a specific baseline +cat benchmarks/baselines/baseline_20240125_143022.txt + +# Clean old baselines (keeps last 10) +./dtail-tools benchmark -mode clean +``` + +### Best Practices for Baselines +1. Create a baseline before starting optimization work +2. Tag baselines with descriptive names (e.g., "before-cache-impl", "v1.0-release") +3. Use full baselines for release comparisons +4. Commit important baseline files to version control for team reference +5. Run benchmarks on consistent hardware for accurate comparisons + +## Contributing + +When adding new benchmarks: +1. Follow existing naming conventions +2. Include warmup runs +3. Report relevant metrics +4. Clean up temporary files +5. Document in this README
\ No newline at end of file diff --git a/benchmarks/baselines/baseline_20250626_103142_v4.3.0.txt b/benchmarks/baselines/baseline_20250626_103142_v4.3.0.txt new file mode 100644 index 0000000..4443e7f --- /dev/null +++ b/benchmarks/baselines/baseline_20250626_103142_v4.3.0.txt @@ -0,0 +1,424 @@ +Git commit: 41ec9cf +Date: Thu 26 Jun 10:31:42 EEST 2025 +Tag: v4.3.0 +---------------------------------------- +goos: linux +goarch: amd64 +pkg: github.com/mimecast/dtail/benchmarks +cpu: 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz +BenchmarkAll/DCat/Size=10MB-8 5 224532507 ns/op 17.77 MB/sec 313760 lines/sec +BenchmarkAll/DCat/Size=100MB-8 1 1925483552 ns/op 20.66 MB/sec 364791 lines/sec +BenchmarkAll/DCat/Size=1GB-8 1 25906967716 ns/op 15.66 MB/sec 277035 lines/sec +BenchmarkAll/DGrep/Size=10MB/HitRate=1%-8 10 130062984 ns/op 30.70 MB/sec 1.000 hit_rate_% 541364 lines/sec 20998 matched_lines +BenchmarkAll/DGrep/Size=10MB/HitRate=10%-8 10 110567400 ns/op 36.61 MB/sec 10.00 hit_rate_% 638102 lines/sec 22025 matched_lines +BenchmarkAll/DGrep/Size=10MB/HitRate=50%-8 7 166733549 ns/op 24.93 MB/sec 50.00 hit_rate_% 421688 lines/sec 41879 matched_lines +BenchmarkAll/DGrep/Size=10MB/HitRate=90%-8 5 252056920 ns/op 17.24 MB/sec 90.00 hit_rate_% 278744 lines/sec 65722 matched_lines +BenchmarkAll/DGrep/Size=100MB/HitRate=1%-8 1 1065585024 ns/op 37.71 MB/sec 1.000 hit_rate_% 663620 lines/sec 251213 matched_lines +BenchmarkAll/DGrep/Size=100MB/HitRate=10%-8 2 938644170 ns/op 42.75 MB/sec 10.00 hit_rate_% 753207 lines/sec 209112 matched_lines +BenchmarkAll/DGrep/Size=100MB/HitRate=50%-8 1 1798489060 ns/op 23.68 MB/sec 50.00 hit_rate_% 395058 lines/sec 461682 matched_lines +BenchmarkAll/DGrep/Size=100MB/HitRate=90%-8 1 2417024028 ns/op 18.08 MB/sec 90.00 hit_rate_% 293759 lines/sec 657133 matched_lines +BenchmarkAll/DGrep/Size=1GB/HitRate=1%-8 1 7364616736 ns/op 55.66 MB/sec 1.000 hit_rate_% 981824 lines/sec 1430313 matched_lines +BenchmarkAll/DGrep/Size=1GB/HitRate=10%-8 1 9692376704 ns/op 42.93 MB/sec 10.00 hit_rate_% 748544 lines/sec 2146443 matched_lines +BenchmarkAll/DGrep/Size=1GB/HitRate=50%-8 1 21529290491 ns/op 19.96 MB/sec 50.00 hit_rate_% 335888 lines/sec 4151061 matched_lines +BenchmarkAll/DGrep/Size=1GB/HitRate=90%-8 1 30476626794 ns/op 14.71 MB/sec 90.00 hit_rate_% 236933 lines/sec 7015053 matched_lines +BenchmarkAll/DMap/Size=10MB/Query=count-8 2 548543184 ns/op 14.12 MB/sec 127916 records/sec +BenchmarkAll/DMap/Size=10MB/Query=sum_avg-8 2 581903756 ns/op 13.30 MB/sec 120440 records/sec +BenchmarkAll/DMap/Size=10MB/Query=min_max-8 3 434457794 ns/op 17.77 MB/sec 161364 records/sec +BenchmarkAll/DMap/Size=10MB/Query=multi-8 3 466597339 ns/op 16.57 MB/sec 150203 records/sec +BenchmarkAll/DMap/Size=100MB/Query=count-8 1 2588103178 ns/op 29.86 MB/sec 270917 records/sec +BenchmarkAll/DMap/Size=100MB/Query=sum_avg-8 1 2886462985 ns/op 26.81 MB/sec 242874 records/sec +BenchmarkAll/DMap/Size=100MB/Query=min_max-8 1 2985275095 ns/op 25.92 MB/sec 234780 records/sec +BenchmarkAll/DMap/Size=100MB/Query=multi-8 1 2891554257 ns/op 26.72 MB/sec 242407 records/sec +BenchmarkAll/DMap/Size=1GB/Query=count-8 1 35728674597 ns/op 22.13 MB/sec 200635 records/sec +BenchmarkAll/DMap/Size=1GB/Query=sum_avg-8 1 32971434564 ns/op 23.98 MB/sec 217552 records/sec +BenchmarkAll/DMap/Size=1GB/Query=min_max-8 1 30821590005 ns/op 25.73 MB/sec 232805 records/sec +BenchmarkAll/DMap/Size=1GB/Query=multi-8 1 29579206903 ns/op 26.77 MB/sec 242608 records/sec +BenchmarkQuick/DCat/Size=10MB-8 3 426642922 ns/op 9.363 MB/sec 165106 lines/sec +BenchmarkQuick/DGrep/Size=10MB/HitRate=1%-8 8 157126961 ns/op 25.38 MB/sec 1.000 hit_rate_% 449067 lines/sec 21106 matched_lines +BenchmarkQuick/DGrep/Size=10MB/HitRate=10%-8 6 176532672 ns/op 22.81 MB/sec 10.00 hit_rate_% 400664 lines/sec 23897 matched_lines +BenchmarkQuick/DGrep/Size=10MB/HitRate=50%-8 4 258052402 ns/op 16.14 MB/sec 50.00 hit_rate_% 272832 lines/sec 40544 matched_lines +BenchmarkQuick/DGrep/Size=10MB/HitRate=90%-8 3 395698300 ns/op 10.99 MB/sec 90.00 hit_rate_% 178679 lines/sec 64304 matched_lines +BenchmarkQuick/DMap/Size=10MB/Query=count-8 3 452625097 ns/op 17.09 MB/sec 154851 records/sec +BenchmarkQuick/DMap/Size=10MB/Query=sum_avg-8 2 570774552 ns/op 13.54 MB/sec 122863 records/sec +BenchmarkQuick/DMap/Size=10MB/Query=min_max-8 3 442242744 ns/op 17.46 MB/sec 158383 records/sec +BenchmarkQuick/DMap/Size=10MB/Query=multi-8 3 353881726 ns/op 21.85 MB/sec 198048 records/sec +BenchmarkDCatSimple/Size=10MB-8 4 276786728 ns/op 14.37 MB/sec 254263 lines/sec +BenchmarkDCatSimple/Size=100MB-8 1 2529378974 ns/op 15.73 MB/sec 277897 lines/sec +BenchmarkDCatSimple/Size=1GB-8 1 38612836823 ns/op 10.51 MB/sec 185710 lines/sec +BenchmarkDCatMultipleFiles/Files=10-8 4 272343112 ns/op 14.59 MB/sec 10.00 files 258102 lines/sec +BenchmarkDCatMultipleFiles/Files=50-8 1 1305119765 ns/op 15.23 MB/sec 50.00 files 269256 lines/sec +BenchmarkDCatMultipleFiles/Files=100-8 1 3324677029 ns/op 11.93 MB/sec 100.0 files 210862 lines/sec +BenchmarkDCatCompressed/Size=10MB/Compression=none-8 3 406499383 ns/op 24.70 MB/sec 2.536 compression_ratio 172660 lines/sec +BenchmarkDCatCompressed/Size=10MB/Compression=gzip-8 4 257403608 ns/op 39.14 MB/sec 35.65 compression_ratio 273617 lines/sec +BenchmarkDCatCompressed/Size=10MB/Compression=zstd-8 4 259385565 ns/op 38.79 MB/sec 30.48 compression_ratio 271171 lines/sec +BenchmarkDCatCompressed/Size=100MB/Compression=none-8 1 2676607981 ns/op 37.50 MB/sec 2.539 compression_ratio 262109 lines/sec +SIGQUIT: quit +PC=0x47c881 m=0 sigcode=0 + +goroutine 0 gp=0x814d80 m=0 mp=0x815c60 [idle]: +runtime.futex(0x815da0, 0x80, 0x0, 0x0, 0x0, 0x0) + /usr/lib/golang/src/runtime/sys_linux_amd64.s:557 +0x21 fp=0x7ffec2268c48 sp=0x7ffec2268c40 pc=0x47c881 +runtime.futexsleep(0x7ffec2268cc0?, 0x413f91?, 0x1c2268cf0?) + /usr/lib/golang/src/runtime/os_linux.go:75 +0x30 fp=0x7ffec2268c98 sp=0x7ffec2268c48 pc=0x439490 +runtime.notesleep(0x815da0) + /usr/lib/golang/src/runtime/lock_futex.go:47 +0x87 fp=0x7ffec2268cd0 sp=0x7ffec2268c98 pc=0x4136a7 +runtime.mPark(...) + /usr/lib/golang/src/runtime/proc.go:1887 +runtime.stopm() + /usr/lib/golang/src/runtime/proc.go:2910 +0x8c fp=0x7ffec2268d00 sp=0x7ffec2268cd0 pc=0x444aec +runtime.findRunnable() + /usr/lib/golang/src/runtime/proc.go:3647 +0xd9c fp=0x7ffec2268e78 sp=0x7ffec2268d00 pc=0x4465bc +runtime.schedule() + /usr/lib/golang/src/runtime/proc.go:4020 +0xb1 fp=0x7ffec2268eb0 sp=0x7ffec2268e78 pc=0x4476b1 +runtime.park_m(0xc000003340) + /usr/lib/golang/src/runtime/proc.go:4144 +0x285 fp=0x7ffec2268f10 sp=0x7ffec2268eb0 pc=0x447b65 +runtime.mcall() + /usr/lib/golang/src/runtime/asm_amd64.s:459 +0x4e fp=0x7ffec2268f28 sp=0x7ffec2268f10 pc=0x478a2e + +goroutine 1 gp=0xc000002380 m=nil [chan receive]: +runtime.gopark(0x0?, 0x200000000000000?, 0x40?, 0x20?, 0x7f28e1b3e108?) + /usr/lib/golang/src/runtime/proc.go:435 +0xce fp=0xc00011d7c0 sp=0xc00011d7a0 pc=0x47330e +runtime.chanrecv(0xc0004d80e0, 0x0, 0x1) + /usr/lib/golang/src/runtime/chan.go:664 +0x445 fp=0xc00011d838 sp=0xc00011d7c0 pc=0x40e5c5 +runtime.chanrecv1(0x414fde?, 0x7f289987f3a8?) + /usr/lib/golang/src/runtime/chan.go:506 +0x12 fp=0xc00011d860 sp=0xc00011d838 pc=0x40e172 +testing.(*B).run1(0xc00022a2c8) + /usr/lib/golang/src/testing/benchmark.go:247 +0xa6 fp=0xc00011d950 sp=0xc00011d860 pc=0x4eb546 +testing.(*B).Run(0xc00022a008, {0x69616b?, 0x49a1f3?}, 0x6a0700) + /usr/lib/golang/src/testing/benchmark.go:847 +0x491 fp=0xc00011da50 sp=0xc00011d950 pc=0x4ee031 +testing.runBenchmarks.func1(0xc00022a008) + /usr/lib/golang/src/testing/benchmark.go:708 +0x37 fp=0xc00011da90 sp=0xc00011da50 pc=0x4ed137 +testing.(*B).runN(0xc00022a008, 0x1) + /usr/lib/golang/src/testing/benchmark.go:219 +0x190 fp=0xc00011db20 sp=0xc00011da90 pc=0x4eb330 +testing.runBenchmarks({0x69a424, 0x24}, 0x0?, {0x7fa8a0, 0x11, 0x8bb612e140?}) + /usr/lib/golang/src/testing/benchmark.go:717 +0x5f6 fp=0xc00011dc20 sp=0xc00011db20 pc=0x4ed076 +testing.(*M).Run(0xc000198140) + /usr/lib/golang/src/testing/testing.go:2158 +0x9b6 fp=0xc00011de50 sp=0xc00011dc20 pc=0x4fa116 +github.com/mimecast/dtail/benchmarks.TestMain(0xc000198140) + /home/paul/git/dtail/benchmarks/benchmark_test.go:17 +0x77 fp=0xc00011deb0 sp=0xc00011de50 pc=0x55fdb7 +main.main() + _testmain.go:79 +0xa8 fp=0xc00011df50 sp=0xc00011deb0 pc=0x56dc28 +runtime.main() + /usr/lib/golang/src/runtime/proc.go:283 +0x28b fp=0xc00011dfe0 sp=0xc00011df50 pc=0x43fc4b +runtime.goexit({}) + /usr/lib/golang/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc00011dfe8 sp=0xc00011dfe0 pc=0x47aa61 + +goroutine 2 gp=0xc000002e00 m=nil [force gc (idle), 10 minutes]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /usr/lib/golang/src/runtime/proc.go:435 +0xce fp=0xc000058fa8 sp=0xc000058f88 pc=0x47330e +runtime.goparkunlock(...) + /usr/lib/golang/src/runtime/proc.go:441 +runtime.forcegchelper() + /usr/lib/golang/src/runtime/proc.go:348 +0xb3 fp=0xc000058fe0 sp=0xc000058fa8 pc=0x43ff93 +runtime.goexit({}) + /usr/lib/golang/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000058fe8 sp=0xc000058fe0 pc=0x47aa61 +created by runtime.init.7 in goroutine 1 + /usr/lib/golang/src/runtime/proc.go:336 +0x1a + +goroutine 3 gp=0xc000003340 m=nil [GC sweep wait]: +runtime.gopark(0x814601?, 0x0?, 0x0?, 0x0?, 0x0?) + /usr/lib/golang/src/runtime/proc.go:435 +0xce fp=0xc000059780 sp=0xc000059760 pc=0x47330e +runtime.goparkunlock(...) + /usr/lib/golang/src/runtime/proc.go:441 +runtime.bgsweep(0xc00007c000) + /usr/lib/golang/src/runtime/mgcsweep.go:316 +0xdf fp=0xc0000597c8 sp=0xc000059780 pc=0x4288ff +runtime.gcenable.gowrap1() + /usr/lib/golang/src/runtime/mgc.go:204 +0x25 fp=0xc0000597e0 sp=0xc0000597c8 pc=0x41cde5 +runtime.goexit({}) + /usr/lib/golang/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0000597e8 sp=0xc0000597e0 pc=0x47aa61 +created by runtime.gcenable in goroutine 1 + /usr/lib/golang/src/runtime/mgc.go:204 +0x66 + +goroutine 4 gp=0xc000003500 m=nil [GC scaveng |