From f7deedf50f6fa00c7dfe989ed381989e202b3057 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 8 Apr 2026 10:14:40 +0300 Subject: feat: add gobreaker circuit breakers for OpenAI and Gemini API calls Introduce internal/apicircuit with sony/gobreaker: trip after five consecutive failures, 45s open state, 2m count reset, half-open trial cap. context.Canceled is not counted as a failure for breaker stats. Wrap OpenAI TTS, Gemini TTS, OpenAI DALL-E/chat (image path), and Nano Banana Gemini GenerateContent calls. HTTP timeouts remain unchanged in httpctx. Made-with: Cursor --- go.mod | 1 + go.sum | 2 + internal/apicircuit/apicircuit.go | 93 ++++++++++++++++++++++++++++++++++ internal/apicircuit/apicircuit_test.go | 27 ++++++++++ internal/audio/gemini_provider.go | 9 ++-- internal/audio/openai_provider.go | 7 ++- internal/image/nanobanana.go | 33 +++++++----- internal/image/openai.go | 11 ++-- 8 files changed, 162 insertions(+), 21 deletions(-) create mode 100644 internal/apicircuit/apicircuit.go create mode 100644 internal/apicircuit/apicircuit_test.go diff --git a/go.mod b/go.mod index 8763e31..d8dfecd 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/magefile/mage v1.15.0 github.com/mattn/go-sqlite3 v1.14.28 github.com/sashabaranov/go-openai v1.40.5 + github.com/sony/gobreaker v1.0.0 github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 github.com/spf13/viper v1.20.1 diff --git a/go.sum b/go.sum index 03d16d3..730ac61 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,8 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ= +github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= diff --git a/internal/apicircuit/apicircuit.go b/internal/apicircuit/apicircuit.go new file mode 100644 index 0000000..22218f8 --- /dev/null +++ b/internal/apicircuit/apicircuit.go @@ -0,0 +1,93 @@ +// Package apicircuit wraps outbound API calls with sony/gobreaker circuit breakers +// so repeated failures against OpenAI or Gemini do not pile up unbounded work. +// +// HTTP deadlines remain in internal/httpctx; breakers add a separate open/half-open +// gate when the remote service is clearly unhealthy. +package apicircuit + +import ( + "context" + "errors" + "time" + + "github.com/sony/gobreaker" +) + +const ( + // breakerInterval clears rolling failure counts in the closed state so stale + // errors do not keep the breaker sensitive forever. + breakerInterval = 2 * time.Minute + // breakerOpenTimeout is how long the breaker stays open before trying half-open. + breakerOpenTimeout = 45 * time.Second + // breakerMaxHalfOpenRequests limits trial traffic while recovering. + breakerMaxHalfOpenRequests = 3 + // breakerTripAfterConsecutiveFailures opens the circuit after this many + // consecutive failed requests in the closed state. + breakerTripAfterConsecutiveFailures uint32 = 5 +) + +// isSuccessful counts only real API outcomes: nil is success; context.Canceled is +// treated as success so user abort does not trip the breaker. Timeouts and +// remote errors still count as failures. +func isSuccessful(err error) bool { + if err == nil { + return true + } + return errors.Is(err, context.Canceled) +} + +func readyToTrip(counts gobreaker.Counts) bool { + return counts.ConsecutiveFailures >= breakerTripAfterConsecutiveFailures +} + +func newBreaker(name string) *gobreaker.CircuitBreaker { + return gobreaker.NewCircuitBreaker(gobreaker.Settings{ + Name: name, + MaxRequests: breakerMaxHalfOpenRequests, + Interval: breakerInterval, + Timeout: breakerOpenTimeout, + ReadyToTrip: readyToTrip, + IsSuccessful: isSuccessful, + }) +} + +var ( + openAITTSBreaker = newBreaker("openai-tts") + geminiTTSBreaker = newBreaker("gemini-tts") + openAIImageBreaker = newBreaker("openai-image") + geminiNanoBananaBreaker = newBreaker("gemini-nanobanana") +) + +func runValue[T any](cb *gobreaker.CircuitBreaker, fn func() (T, error)) (T, error) { + var zero T + v, err := cb.Execute(func() (interface{}, error) { + return fn() + }) + if err != nil { + return zero, err + } + if v == nil { + return zero, nil + } + return v.(T), nil +} + +// OpenAITTS runs one OpenAI text-to-speech call through its circuit breaker. +func OpenAITTS[T any](fn func() (T, error)) (T, error) { + return runValue(openAITTSBreaker, fn) +} + +// GeminiTTS runs one Gemini TTS GenerateContent call through its circuit breaker. +func GeminiTTS[T any](fn func() (T, error)) (T, error) { + return runValue(geminiTTSBreaker, fn) +} + +// OpenAIImage runs one OpenAI image or chat call (DALL-E path) through its breaker. +func OpenAIImage[T any](fn func() (T, error)) (T, error) { + return runValue(openAIImageBreaker, fn) +} + +// GeminiNanoBanana runs one Gemini call from Nano Banana (scene text, image gen). +func GeminiNanoBanana[T any](fn func() (T, error)) (T, error) { + return runValue(geminiNanoBananaBreaker, fn) +} diff --git a/internal/apicircuit/apicircuit_test.go b/internal/apicircuit/apicircuit_test.go new file mode 100644 index 0000000..5f84c14 --- /dev/null +++ b/internal/apicircuit/apicircuit_test.go @@ -0,0 +1,27 @@ +package apicircuit + +import ( + "context" + "errors" + "testing" +) + +func TestOpenAITTS_Success(t *testing.T) { + t.Parallel() + v, err := OpenAITTS(func() (string, error) { + return "ok", nil + }) + if err != nil || v != "ok" { + t.Fatalf("OpenAITTS() = %q, %v; want ok, nil", v, err) + } +} + +func TestIsSuccessful_ContextCanceled(t *testing.T) { + t.Parallel() + if !isSuccessful(context.Canceled) { + t.Fatal("context.Canceled should not count as breaker failure") + } + if isSuccessful(errors.New("api error")) { + t.Fatal("arbitrary errors must count as failure") + } +} diff --git a/internal/audio/gemini_provider.go b/internal/audio/gemini_provider.go index b573a02..74478bd 100644 --- a/internal/audio/gemini_provider.go +++ b/internal/audio/gemini_provider.go @@ -13,6 +13,7 @@ import ( "google.golang.org/genai" + "codeberg.org/snonux/totalrecall/internal/apicircuit" "codeberg.org/snonux/totalrecall/internal/httpctx" ) @@ -78,9 +79,11 @@ func (p *GeminiProvider) GenerateAudio(ctx context.Context, text string, outputF SpeechConfig: p.speechConfig(), } - response, err := p.client.Models.GenerateContent(ctx, p.config.TTSModel, []*genai.Content{ - genai.NewContentFromText(prompt, genai.RoleUser), - }, req) + response, err := apicircuit.GeminiTTS(func() (*genai.GenerateContentResponse, error) { + return p.client.Models.GenerateContent(ctx, p.config.TTSModel, []*genai.Content{ + genai.NewContentFromText(prompt, genai.RoleUser), + }, req) + }) if err != nil { return fmt.Errorf("gemini API error: %w", err) } diff --git a/internal/audio/openai_provider.go b/internal/audio/openai_provider.go index 5b59a3e..58a9fb6 100644 --- a/internal/audio/openai_provider.go +++ b/internal/audio/openai_provider.go @@ -11,6 +11,7 @@ import ( "github.com/sashabaranov/go-openai" + "codeberg.org/snonux/totalrecall/internal/apicircuit" "codeberg.org/snonux/totalrecall/internal/httpctx" ) @@ -92,8 +93,10 @@ func (p *OpenAIProvider) GenerateAudio(ctx context.Context, text string, outputF } } - // Make the API call - response, err := p.client.CreateSpeech(ctx, req) + // Make the API call (circuit breaker limits load when OpenAI is unhealthy). + response, err := apicircuit.OpenAITTS(func() (openai.RawResponse, error) { + return p.client.CreateSpeech(ctx, req) + }) if err != nil { // Check if it's a model access error errStr := err.Error() diff --git a/internal/image/nanobanana.go b/internal/image/nanobanana.go index 7694069..4d12d08 100644 --- a/internal/image/nanobanana.go +++ b/internal/image/nanobanana.go @@ -17,6 +17,7 @@ import ( "google.golang.org/genai" + "codeberg.org/snonux/totalrecall/internal/apicircuit" "codeberg.org/snonux/totalrecall/internal/config" "codeberg.org/snonux/totalrecall/internal/httpctx" ) @@ -382,12 +383,14 @@ func (c *NanoBananaClient) generateSceneDescription(ctx context.Context, bulgari func (c *NanoBananaClient) generateText(ctx context.Context, model, systemPrompt, userPrompt string, temperature float32, maxOutputTokens int32) (string, error) { temp := temperature - resp, err := c.client.Models.GenerateContent(ctx, model, []*genai.Content{ - genai.NewContentFromText(userPrompt, genai.RoleUser), - }, &genai.GenerateContentConfig{ - SystemInstruction: genai.NewContentFromText(systemPrompt, genai.RoleUser), - Temperature: &temp, - MaxOutputTokens: maxOutputTokens, + resp, err := apicircuit.GeminiNanoBanana(func() (*genai.GenerateContentResponse, error) { + return c.client.Models.GenerateContent(ctx, model, []*genai.Content{ + genai.NewContentFromText(userPrompt, genai.RoleUser), + }, &genai.GenerateContentConfig{ + SystemInstruction: genai.NewContentFromText(systemPrompt, genai.RoleUser), + Temperature: &temp, + MaxOutputTokens: maxOutputTokens, + }) }) if err != nil { return "", fmt.Errorf("gemini API error: %w", err) @@ -412,9 +415,11 @@ func (c *NanoBananaClient) generateImage(ctx context.Context, prompt, aspectRati }, } - resp, err := c.client.Models.GenerateContent(ctx, c.modelName(), []*genai.Content{ - genai.NewContentFromText(prompt, genai.RoleUser), - }, cfg) + resp, err := apicircuit.GeminiNanoBanana(func() (*genai.GenerateContentResponse, error) { + return c.client.Models.GenerateContent(ctx, c.modelName(), []*genai.Content{ + genai.NewContentFromText(prompt, genai.RoleUser), + }, cfg) + }) if err != nil { return nil, "", &SearchError{ Provider: nanoBananaSource, @@ -466,10 +471,12 @@ func (c *NanoBananaClient) generateImageWithRefs(ctx context.Context, prompt, as ) parts = append(parts, &genai.Part{Text: refNote + prompt}) - resp, err := c.client.Models.GenerateContent(ctx, c.modelName(), - []*genai.Content{{Role: string(genai.RoleUser), Parts: parts}}, - cfg, - ) + resp, err := apicircuit.GeminiNanoBanana(func() (*genai.GenerateContentResponse, error) { + return c.client.Models.GenerateContent(ctx, c.modelName(), + []*genai.Content{{Role: string(genai.RoleUser), Parts: parts}}, + cfg, + ) + }) if err != nil { return nil, "", &SearchError{ Provider: nanoBananaSource, diff --git a/internal/image/openai.go b/internal/image/openai.go index 9fb8148..1bd5b2d 100644 --- a/internal/image/openai.go +++ b/internal/image/openai.go @@ -12,6 +12,7 @@ import ( "github.com/sashabaranov/go-openai" + "codeberg.org/snonux/totalrecall/internal/apicircuit" "codeberg.org/snonux/totalrecall/internal/httpctx" ) @@ -154,8 +155,10 @@ func (c *OpenAIClient) Search(ctx context.Context, opts *SearchOptions) ([]Searc req.Style = c.style } - // Generate the image - resp, err := c.client.CreateImage(ctx, req) + // Generate the image (circuit breaker limits load when OpenAI is unhealthy). + resp, err := apicircuit.OpenAIImage(func() (openai.ImageResponse, error) { + return c.client.CreateImage(ctx, req) + }) if err != nil { return nil, &SearchError{ Provider: "openai", @@ -295,7 +298,9 @@ func (c *OpenAIClient) generateSceneDescription(ctx context.Context, bulgarianWo MaxTokens: 100, } - resp, err := c.client.CreateChatCompletion(ctx, req) + resp, err := apicircuit.OpenAIImage(func() (openai.ChatCompletionResponse, error) { + return c.client.CreateChatCompletion(ctx, req) + }) if err != nil { return "", fmt.Errorf("scene generation failed: %w", err) } -- cgit v1.2.3