diff options
| author | Paul Buetow <paul@buetow.org> | 2025-06-28 19:52:43 +0300 |
|---|---|---|
| committer | Paul Buetow <paul@buetow.org> | 2025-06-28 19:52:43 +0300 |
| commit | 0aa3222cef46d527bb9437afa9ddd90f3a80a9d8 (patch) | |
| tree | aa56290ef19fa1a14b3281f2be38b04cd129db48 | |
| parent | c75b6595f6cb0c94f4ecc05ca7c27ec0e83de368 (diff) | |
refactor: consolidate optimization flags into DTAIL_TURBOBOOST_ENABLE
- Replace DTAIL_CHANNELLESS_GREP and DTAIL_OPTIMIZED_READER with single flag
- Rename documentation to TURBOBOOST_OPTIMIZATION.md
- Fix channel-less adapter to use blocking sends (prevent data loss)
- Update logging messages to reference "turbo boost" mode
The DTAIL_TURBOBOOST_ENABLE variable now controls all performance
optimizations and can be extended to other commands in the future.
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
| -rw-r--r-- | TURBOBOOST_OPTIMIZATION.md (renamed from CHANNELLESS_GREP_IMPLEMENTATION.md) | 21 | ||||
| -rw-r--r-- | internal/server/handlers/channelless_adapter.go | 14 | ||||
| -rw-r--r-- | internal/server/handlers/readcommand.go | 20 |
3 files changed, 31 insertions, 24 deletions
diff --git a/CHANNELLESS_GREP_IMPLEMENTATION.md b/TURBOBOOST_OPTIMIZATION.md index af79d9c..f13943f 100644 --- a/CHANNELLESS_GREP_IMPLEMENTATION.md +++ b/TURBOBOOST_OPTIMIZATION.md @@ -1,8 +1,8 @@ -# Channel-less dgrep Implementation +# DTail Turbo Boost Optimization ## Overview -This document describes the channel-less implementation of dgrep that was created to address performance bottlenecks caused by channel overhead in the original implementation. +This document describes the turbo boost optimization feature that provides significant performance improvements for DTail operations by using channel-less processing and optimized I/O. ## Problem Statement @@ -100,4 +100,19 @@ dgrep -regex "pattern" file.log 2. Add configurable buffer sizes 3. Implement zero-copy optimizations 4. Add performance metrics collection -5. Consider using io_uring on Linux for async I/O
\ No newline at end of file +5. Consider using io_uring on Linux for async I/O + +## Usage + +To enable turbo boost optimizations: + +```bash +export DTAIL_TURBOBOOST_ENABLE=yes +``` + +This enables: +- Channel-less implementation for grep and cat operations +- Optimized buffered I/O reader (256KB buffer) +- Buffer pooling to reduce memory allocations + +The turbo boost mode is designed to be extended to other DTail commands in the future. diff --git a/internal/server/handlers/channelless_adapter.go b/internal/server/handlers/channelless_adapter.go index 9e5bc9c..a950408 100644 --- a/internal/server/handlers/channelless_adapter.go +++ b/internal/server/handlers/channelless_adapter.go @@ -2,10 +2,8 @@ package handlers import ( "bytes" - "fmt" "github.com/mimecast/dtail/internal/io/line" - "github.com/mimecast/dtail/internal/io/pool" ) // ChannellessLineProcessor adapts the channel-less processor to work with the existing handler infrastructure @@ -30,15 +28,9 @@ func (p *ChannellessLineProcessor) ProcessLine(lineContent *bytes.Buffer, lineNu // Create a line object that matches what the original implementation expects l := line.New(lineContent, lineNum, 100, sourceID) - // Send through the channel - select { - case p.lines <- l: - return nil - default: - // Channel full, recycle the buffer - pool.RecycleBytesBuffer(lineContent) - return fmt.Errorf("lines channel full") - } + // Send through the channel (blocking to ensure no lines are lost) + p.lines <- l + return nil } // Flush does nothing for this implementation diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go index 7a351ba..abdbe9c 100644 --- a/internal/server/handlers/readcommand.go +++ b/internal/server/handlers/readcommand.go @@ -166,14 +166,14 @@ func (r *readCommand) read(ctx context.Context, ltx lcontext.LContext, } } - // Check if we should use the channel-less implementation - channellessEnabled := config.Env("DTAIL_CHANNELLESS_GREP") - dlog.Server.Info(r.server.user, "Channel-less check: enabled=", channellessEnabled, "mode=", r.mode) + // Check if we should use the turbo boost optimizations + turboBoostEnabled := config.Env("DTAIL_TURBOBOOST_ENABLE") + dlog.Server.Info(r.server.user, "Turbo boost check: enabled=", turboBoostEnabled, "mode=", r.mode) // Only enable channel-less for server mode, not serverless mode // Use the serverless field directly as it's more reliable - if channellessEnabled && (r.mode == omode.CatClient || r.mode == omode.GrepClient) && !r.server.serverless { + if turboBoostEnabled && (r.mode == omode.CatClient || r.mode == omode.GrepClient) && !r.server.serverless { // Log to stderr for testing verification - only in server mode - fmt.Fprintf(os.Stderr, "[DTAIL] Using channel-less implementation for %s\n", path) + fmt.Fprintf(os.Stderr, "[DTAIL] Turbo boost enabled: using channel-less implementation for %s\n", path) r.readWithProcessor(ctx, ltx, path, globID, re, reader) return } @@ -217,13 +217,13 @@ func (r *readCommand) readWithProcessor(ctx context.Context, ltx lcontext.LConte lines := r.server.lines aggregate := r.server.aggregate - // Use the optimized version if available - useOptimized := config.Env("DTAIL_OPTIMIZED_READER") + // Use the optimized version if turbo boost is enabled + turboBoostEnabled := config.Env("DTAIL_TURBOBOOST_ENABLE") // Log to stderr for testing verification - only in server mode if !r.server.serverless { - if useOptimized { - fmt.Fprintf(os.Stderr, "[DTAIL] Using optimized reader for %s\n", path) + if turboBoostEnabled { + fmt.Fprintf(os.Stderr, "[DTAIL] Turbo boost enabled: using optimized reader for %s\n", path) } else { fmt.Fprintf(os.Stderr, "[DTAIL] Using standard processor reader for %s\n", path) } @@ -240,7 +240,7 @@ func (r *readCommand) readWithProcessor(ctx context.Context, ltx lcontext.LConte defer processor.Close() var err error - if useOptimized { + if turboBoostEnabled { err = reader.StartWithProcessorOptimized(ctx, ltx, processor, re) } else { err = reader.StartWithProcessor(ctx, ltx, processor, re) |
