summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2026-04-08 10:14:40 +0300
committerPaul Buetow <paul@buetow.org>2026-04-08 10:14:40 +0300
commitf7deedf50f6fa00c7dfe989ed381989e202b3057 (patch)
tree72fe642a99101da79d2103af911c8dcf88b08484
parent14995ca7a76e2a72a2169def7f55f3b9e2433ee2 (diff)
feat: add gobreaker circuit breakers for OpenAI and Gemini API calls
Introduce internal/apicircuit with sony/gobreaker: trip after five consecutive failures, 45s open state, 2m count reset, half-open trial cap. context.Canceled is not counted as a failure for breaker stats. Wrap OpenAI TTS, Gemini TTS, OpenAI DALL-E/chat (image path), and Nano Banana Gemini GenerateContent calls. HTTP timeouts remain unchanged in httpctx. Made-with: Cursor
-rw-r--r--go.mod1
-rw-r--r--go.sum2
-rw-r--r--internal/apicircuit/apicircuit.go93
-rw-r--r--internal/apicircuit/apicircuit_test.go27
-rw-r--r--internal/audio/gemini_provider.go9
-rw-r--r--internal/audio/openai_provider.go7
-rw-r--r--internal/image/nanobanana.go33
-rw-r--r--internal/image/openai.go11
8 files changed, 162 insertions, 21 deletions
diff --git a/go.mod b/go.mod
index 8763e31..d8dfecd 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,7 @@ require (
github.com/magefile/mage v1.15.0
github.com/mattn/go-sqlite3 v1.14.28
github.com/sashabaranov/go-openai v1.40.5
+ github.com/sony/gobreaker v1.0.0
github.com/spf13/cobra v1.9.1
github.com/spf13/pflag v1.0.6
github.com/spf13/viper v1.20.1
diff --git a/go.sum b/go.sum
index 03d16d3..730ac61 100644
--- a/go.sum
+++ b/go.sum
@@ -104,6 +104,8 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY=
github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ=
+github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
diff --git a/internal/apicircuit/apicircuit.go b/internal/apicircuit/apicircuit.go
new file mode 100644
index 0000000..22218f8
--- /dev/null
+++ b/internal/apicircuit/apicircuit.go
@@ -0,0 +1,93 @@
+// Package apicircuit wraps outbound API calls with sony/gobreaker circuit breakers
+// so repeated failures against OpenAI or Gemini do not pile up unbounded work.
+//
+// HTTP deadlines remain in internal/httpctx; breakers add a separate open/half-open
+// gate when the remote service is clearly unhealthy.
+package apicircuit
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/sony/gobreaker"
+)
+
+const (
+ // breakerInterval clears rolling failure counts in the closed state so stale
+ // errors do not keep the breaker sensitive forever.
+ breakerInterval = 2 * time.Minute
+ // breakerOpenTimeout is how long the breaker stays open before trying half-open.
+ breakerOpenTimeout = 45 * time.Second
+ // breakerMaxHalfOpenRequests limits trial traffic while recovering.
+ breakerMaxHalfOpenRequests = 3
+ // breakerTripAfterConsecutiveFailures opens the circuit after this many
+ // consecutive failed requests in the closed state.
+ breakerTripAfterConsecutiveFailures uint32 = 5
+)
+
+// isSuccessful counts only real API outcomes: nil is success; context.Canceled is
+// treated as success so user abort does not trip the breaker. Timeouts and
+// remote errors still count as failures.
+func isSuccessful(err error) bool {
+ if err == nil {
+ return true
+ }
+ return errors.Is(err, context.Canceled)
+}
+
+func readyToTrip(counts gobreaker.Counts) bool {
+ return counts.ConsecutiveFailures >= breakerTripAfterConsecutiveFailures
+}
+
+func newBreaker(name string) *gobreaker.CircuitBreaker {
+ return gobreaker.NewCircuitBreaker(gobreaker.Settings{
+ Name: name,
+ MaxRequests: breakerMaxHalfOpenRequests,
+ Interval: breakerInterval,
+ Timeout: breakerOpenTimeout,
+ ReadyToTrip: readyToTrip,
+ IsSuccessful: isSuccessful,
+ })
+}
+
+var (
+ openAITTSBreaker = newBreaker("openai-tts")
+ geminiTTSBreaker = newBreaker("gemini-tts")
+ openAIImageBreaker = newBreaker("openai-image")
+ geminiNanoBananaBreaker = newBreaker("gemini-nanobanana")
+)
+
+func runValue[T any](cb *gobreaker.CircuitBreaker, fn func() (T, error)) (T, error) {
+ var zero T
+ v, err := cb.Execute(func() (interface{}, error) {
+ return fn()
+ })
+ if err != nil {
+ return zero, err
+ }
+ if v == nil {
+ return zero, nil
+ }
+ return v.(T), nil
+}
+
+// OpenAITTS runs one OpenAI text-to-speech call through its circuit breaker.
+func OpenAITTS[T any](fn func() (T, error)) (T, error) {
+ return runValue(openAITTSBreaker, fn)
+}
+
+// GeminiTTS runs one Gemini TTS GenerateContent call through its circuit breaker.
+func GeminiTTS[T any](fn func() (T, error)) (T, error) {
+ return runValue(geminiTTSBreaker, fn)
+}
+
+// OpenAIImage runs one OpenAI image or chat call (DALL-E path) through its breaker.
+func OpenAIImage[T any](fn func() (T, error)) (T, error) {
+ return runValue(openAIImageBreaker, fn)
+}
+
+// GeminiNanoBanana runs one Gemini call from Nano Banana (scene text, image gen).
+func GeminiNanoBanana[T any](fn func() (T, error)) (T, error) {
+ return runValue(geminiNanoBananaBreaker, fn)
+}
diff --git a/internal/apicircuit/apicircuit_test.go b/internal/apicircuit/apicircuit_test.go
new file mode 100644
index 0000000..5f84c14
--- /dev/null
+++ b/internal/apicircuit/apicircuit_test.go
@@ -0,0 +1,27 @@
+package apicircuit
+
+import (
+ "context"
+ "errors"
+ "testing"
+)
+
+func TestOpenAITTS_Success(t *testing.T) {
+ t.Parallel()
+ v, err := OpenAITTS(func() (string, error) {
+ return "ok", nil
+ })
+ if err != nil || v != "ok" {
+ t.Fatalf("OpenAITTS() = %q, %v; want ok, nil", v, err)
+ }
+}
+
+func TestIsSuccessful_ContextCanceled(t *testing.T) {
+ t.Parallel()
+ if !isSuccessful(context.Canceled) {
+ t.Fatal("context.Canceled should not count as breaker failure")
+ }
+ if isSuccessful(errors.New("api error")) {
+ t.Fatal("arbitrary errors must count as failure")
+ }
+}
diff --git a/internal/audio/gemini_provider.go b/internal/audio/gemini_provider.go
index b573a02..74478bd 100644
--- a/internal/audio/gemini_provider.go
+++ b/internal/audio/gemini_provider.go
@@ -13,6 +13,7 @@ import (
"google.golang.org/genai"
+ "codeberg.org/snonux/totalrecall/internal/apicircuit"
"codeberg.org/snonux/totalrecall/internal/httpctx"
)
@@ -78,9 +79,11 @@ func (p *GeminiProvider) GenerateAudio(ctx context.Context, text string, outputF
SpeechConfig: p.speechConfig(),
}
- response, err := p.client.Models.GenerateContent(ctx, p.config.TTSModel, []*genai.Content{
- genai.NewContentFromText(prompt, genai.RoleUser),
- }, req)
+ response, err := apicircuit.GeminiTTS(func() (*genai.GenerateContentResponse, error) {
+ return p.client.Models.GenerateContent(ctx, p.config.TTSModel, []*genai.Content{
+ genai.NewContentFromText(prompt, genai.RoleUser),
+ }, req)
+ })
if err != nil {
return fmt.Errorf("gemini API error: %w", err)
}
diff --git a/internal/audio/openai_provider.go b/internal/audio/openai_provider.go
index 5b59a3e..58a9fb6 100644
--- a/internal/audio/openai_provider.go
+++ b/internal/audio/openai_provider.go
@@ -11,6 +11,7 @@ import (
"github.com/sashabaranov/go-openai"
+ "codeberg.org/snonux/totalrecall/internal/apicircuit"
"codeberg.org/snonux/totalrecall/internal/httpctx"
)
@@ -92,8 +93,10 @@ func (p *OpenAIProvider) GenerateAudio(ctx context.Context, text string, outputF
}
}
- // Make the API call
- response, err := p.client.CreateSpeech(ctx, req)
+ // Make the API call (circuit breaker limits load when OpenAI is unhealthy).
+ response, err := apicircuit.OpenAITTS(func() (openai.RawResponse, error) {
+ return p.client.CreateSpeech(ctx, req)
+ })
if err != nil {
// Check if it's a model access error
errStr := err.Error()
diff --git a/internal/image/nanobanana.go b/internal/image/nanobanana.go
index 7694069..4d12d08 100644
--- a/internal/image/nanobanana.go
+++ b/internal/image/nanobanana.go
@@ -17,6 +17,7 @@ import (
"google.golang.org/genai"
+ "codeberg.org/snonux/totalrecall/internal/apicircuit"
"codeberg.org/snonux/totalrecall/internal/config"
"codeberg.org/snonux/totalrecall/internal/httpctx"
)
@@ -382,12 +383,14 @@ func (c *NanoBananaClient) generateSceneDescription(ctx context.Context, bulgari
func (c *NanoBananaClient) generateText(ctx context.Context, model, systemPrompt, userPrompt string, temperature float32, maxOutputTokens int32) (string, error) {
temp := temperature
- resp, err := c.client.Models.GenerateContent(ctx, model, []*genai.Content{
- genai.NewContentFromText(userPrompt, genai.RoleUser),
- }, &genai.GenerateContentConfig{
- SystemInstruction: genai.NewContentFromText(systemPrompt, genai.RoleUser),
- Temperature: &temp,
- MaxOutputTokens: maxOutputTokens,
+ resp, err := apicircuit.GeminiNanoBanana(func() (*genai.GenerateContentResponse, error) {
+ return c.client.Models.GenerateContent(ctx, model, []*genai.Content{
+ genai.NewContentFromText(userPrompt, genai.RoleUser),
+ }, &genai.GenerateContentConfig{
+ SystemInstruction: genai.NewContentFromText(systemPrompt, genai.RoleUser),
+ Temperature: &temp,
+ MaxOutputTokens: maxOutputTokens,
+ })
})
if err != nil {
return "", fmt.Errorf("gemini API error: %w", err)
@@ -412,9 +415,11 @@ func (c *NanoBananaClient) generateImage(ctx context.Context, prompt, aspectRati
},
}
- resp, err := c.client.Models.GenerateContent(ctx, c.modelName(), []*genai.Content{
- genai.NewContentFromText(prompt, genai.RoleUser),
- }, cfg)
+ resp, err := apicircuit.GeminiNanoBanana(func() (*genai.GenerateContentResponse, error) {
+ return c.client.Models.GenerateContent(ctx, c.modelName(), []*genai.Content{
+ genai.NewContentFromText(prompt, genai.RoleUser),
+ }, cfg)
+ })
if err != nil {
return nil, "", &SearchError{
Provider: nanoBananaSource,
@@ -466,10 +471,12 @@ func (c *NanoBananaClient) generateImageWithRefs(ctx context.Context, prompt, as
)
parts = append(parts, &genai.Part{Text: refNote + prompt})
- resp, err := c.client.Models.GenerateContent(ctx, c.modelName(),
- []*genai.Content{{Role: string(genai.RoleUser), Parts: parts}},
- cfg,
- )
+ resp, err := apicircuit.GeminiNanoBanana(func() (*genai.GenerateContentResponse, error) {
+ return c.client.Models.GenerateContent(ctx, c.modelName(),
+ []*genai.Content{{Role: string(genai.RoleUser), Parts: parts}},
+ cfg,
+ )
+ })
if err != nil {
return nil, "", &SearchError{
Provider: nanoBananaSource,
diff --git a/internal/image/openai.go b/internal/image/openai.go
index 9fb8148..1bd5b2d 100644
--- a/internal/image/openai.go
+++ b/internal/image/openai.go
@@ -12,6 +12,7 @@ import (
"github.com/sashabaranov/go-openai"
+ "codeberg.org/snonux/totalrecall/internal/apicircuit"
"codeberg.org/snonux/totalrecall/internal/httpctx"
)
@@ -154,8 +155,10 @@ func (c *OpenAIClient) Search(ctx context.Context, opts *SearchOptions) ([]Searc
req.Style = c.style
}
- // Generate the image
- resp, err := c.client.CreateImage(ctx, req)
+ // Generate the image (circuit breaker limits load when OpenAI is unhealthy).
+ resp, err := apicircuit.OpenAIImage(func() (openai.ImageResponse, error) {
+ return c.client.CreateImage(ctx, req)
+ })
if err != nil {
return nil, &SearchError{
Provider: "openai",
@@ -295,7 +298,9 @@ func (c *OpenAIClient) generateSceneDescription(ctx context.Context, bulgarianWo
MaxTokens: 100,
}
- resp, err := c.client.CreateChatCompletion(ctx, req)
+ resp, err := apicircuit.OpenAIImage(func() (openai.ChatCompletionResponse, error) {
+ return c.client.CreateChatCompletion(ctx, req)
+ })
if err != nil {
return "", fmt.Errorf("scene generation failed: %w", err)
}