summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2025-06-28 19:52:43 +0300
committerPaul Buetow <paul@buetow.org>2025-06-28 19:52:43 +0300
commit0aa3222cef46d527bb9437afa9ddd90f3a80a9d8 (patch)
treeaa56290ef19fa1a14b3281f2be38b04cd129db48
parentc75b6595f6cb0c94f4ecc05ca7c27ec0e83de368 (diff)
refactor: consolidate optimization flags into DTAIL_TURBOBOOST_ENABLE
- Replace DTAIL_CHANNELLESS_GREP and DTAIL_OPTIMIZED_READER with single flag - Rename documentation to TURBOBOOST_OPTIMIZATION.md - Fix channel-less adapter to use blocking sends (prevent data loss) - Update logging messages to reference "turbo boost" mode The DTAIL_TURBOBOOST_ENABLE variable now controls all performance optimizations and can be extended to other commands in the future. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
-rw-r--r--TURBOBOOST_OPTIMIZATION.md (renamed from CHANNELLESS_GREP_IMPLEMENTATION.md)21
-rw-r--r--internal/server/handlers/channelless_adapter.go14
-rw-r--r--internal/server/handlers/readcommand.go20
3 files changed, 31 insertions, 24 deletions
diff --git a/CHANNELLESS_GREP_IMPLEMENTATION.md b/TURBOBOOST_OPTIMIZATION.md
index af79d9c..f13943f 100644
--- a/CHANNELLESS_GREP_IMPLEMENTATION.md
+++ b/TURBOBOOST_OPTIMIZATION.md
@@ -1,8 +1,8 @@
-# Channel-less dgrep Implementation
+# DTail Turbo Boost Optimization
## Overview
-This document describes the channel-less implementation of dgrep that was created to address performance bottlenecks caused by channel overhead in the original implementation.
+This document describes the turbo boost optimization feature that provides significant performance improvements for DTail operations by using channel-less processing and optimized I/O.
## Problem Statement
@@ -100,4 +100,19 @@ dgrep -regex "pattern" file.log
2. Add configurable buffer sizes
3. Implement zero-copy optimizations
4. Add performance metrics collection
-5. Consider using io_uring on Linux for async I/O \ No newline at end of file
+5. Consider using io_uring on Linux for async I/O
+
+## Usage
+
+To enable turbo boost optimizations:
+
+```bash
+export DTAIL_TURBOBOOST_ENABLE=yes
+```
+
+This enables:
+- Channel-less implementation for grep and cat operations
+- Optimized buffered I/O reader (256KB buffer)
+- Buffer pooling to reduce memory allocations
+
+The turbo boost mode is designed to be extended to other DTail commands in the future.
diff --git a/internal/server/handlers/channelless_adapter.go b/internal/server/handlers/channelless_adapter.go
index 9e5bc9c..a950408 100644
--- a/internal/server/handlers/channelless_adapter.go
+++ b/internal/server/handlers/channelless_adapter.go
@@ -2,10 +2,8 @@ package handlers
import (
"bytes"
- "fmt"
"github.com/mimecast/dtail/internal/io/line"
- "github.com/mimecast/dtail/internal/io/pool"
)
// ChannellessLineProcessor adapts the channel-less processor to work with the existing handler infrastructure
@@ -30,15 +28,9 @@ func (p *ChannellessLineProcessor) ProcessLine(lineContent *bytes.Buffer, lineNu
// Create a line object that matches what the original implementation expects
l := line.New(lineContent, lineNum, 100, sourceID)
- // Send through the channel
- select {
- case p.lines <- l:
- return nil
- default:
- // Channel full, recycle the buffer
- pool.RecycleBytesBuffer(lineContent)
- return fmt.Errorf("lines channel full")
- }
+ // Send through the channel (blocking to ensure no lines are lost)
+ p.lines <- l
+ return nil
}
// Flush does nothing for this implementation
diff --git a/internal/server/handlers/readcommand.go b/internal/server/handlers/readcommand.go
index 7a351ba..abdbe9c 100644
--- a/internal/server/handlers/readcommand.go
+++ b/internal/server/handlers/readcommand.go
@@ -166,14 +166,14 @@ func (r *readCommand) read(ctx context.Context, ltx lcontext.LContext,
}
}
- // Check if we should use the channel-less implementation
- channellessEnabled := config.Env("DTAIL_CHANNELLESS_GREP")
- dlog.Server.Info(r.server.user, "Channel-less check: enabled=", channellessEnabled, "mode=", r.mode)
+ // Check if we should use the turbo boost optimizations
+ turboBoostEnabled := config.Env("DTAIL_TURBOBOOST_ENABLE")
+ dlog.Server.Info(r.server.user, "Turbo boost check: enabled=", turboBoostEnabled, "mode=", r.mode)
// Only enable channel-less for server mode, not serverless mode
// Use the serverless field directly as it's more reliable
- if channellessEnabled && (r.mode == omode.CatClient || r.mode == omode.GrepClient) && !r.server.serverless {
+ if turboBoostEnabled && (r.mode == omode.CatClient || r.mode == omode.GrepClient) && !r.server.serverless {
// Log to stderr for testing verification - only in server mode
- fmt.Fprintf(os.Stderr, "[DTAIL] Using channel-less implementation for %s\n", path)
+ fmt.Fprintf(os.Stderr, "[DTAIL] Turbo boost enabled: using channel-less implementation for %s\n", path)
r.readWithProcessor(ctx, ltx, path, globID, re, reader)
return
}
@@ -217,13 +217,13 @@ func (r *readCommand) readWithProcessor(ctx context.Context, ltx lcontext.LConte
lines := r.server.lines
aggregate := r.server.aggregate
- // Use the optimized version if available
- useOptimized := config.Env("DTAIL_OPTIMIZED_READER")
+ // Use the optimized version if turbo boost is enabled
+ turboBoostEnabled := config.Env("DTAIL_TURBOBOOST_ENABLE")
// Log to stderr for testing verification - only in server mode
if !r.server.serverless {
- if useOptimized {
- fmt.Fprintf(os.Stderr, "[DTAIL] Using optimized reader for %s\n", path)
+ if turboBoostEnabled {
+ fmt.Fprintf(os.Stderr, "[DTAIL] Turbo boost enabled: using optimized reader for %s\n", path)
} else {
fmt.Fprintf(os.Stderr, "[DTAIL] Using standard processor reader for %s\n", path)
}
@@ -240,7 +240,7 @@ func (r *readCommand) readWithProcessor(ctx context.Context, ltx lcontext.LConte
defer processor.Close()
var err error
- if useOptimized {
+ if turboBoostEnabled {
err = reader.StartWithProcessorOptimized(ctx, ltx, processor, re)
} else {
err = reader.StartWithProcessor(ctx, ltx, processor, re)