summaryrefslogtreecommitdiff
path: root/internal/processor
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2026-04-10 23:11:30 +0300
committerPaul Buetow <paul@buetow.org>2026-04-10 23:11:30 +0300
commit65430e9876a32988839deb7a70dc1e42da21cf64 (patch)
treee0de040cb53d5b90587908f2d55e86e49664fc35 /internal/processor
parent4b4b3665230faafbd994f09894cb07c8f9d4b359 (diff)
Release 0.28.2v0.28.2
Diffstat (limited to 'internal/processor')
-rw-r--r--internal/processor/failed_asset_retry.go290
-rw-r--r--internal/processor/image_downloader.go10
-rw-r--r--internal/processor/processor_test.go148
3 files changed, 448 insertions, 0 deletions
diff --git a/internal/processor/failed_asset_retry.go b/internal/processor/failed_asset_retry.go
new file mode 100644
index 0000000..88d1abe
--- /dev/null
+++ b/internal/processor/failed_asset_retry.go
@@ -0,0 +1,290 @@
+package processor
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "codeberg.org/snonux/totalrecall/internal"
+ "codeberg.org/snonux/totalrecall/internal/anki"
+ "codeberg.org/snonux/totalrecall/internal/audio"
+ "codeberg.org/snonux/totalrecall/internal/store"
+)
+
+type failedAssetKind string
+
+const (
+ failedAssetAudio failedAssetKind = "audio"
+ failedAssetImage failedAssetKind = "image"
+ failedAssetBgBgAudioPair failedAssetKind = "front and back audio"
+ failedAssetBgBgFrontAudio failedAssetKind = "front audio"
+ failedAssetBgBgBackAudio failedAssetKind = "back audio"
+)
+
+type failedAssetPlan struct {
+ Card store.CardDirectory
+ CardType internal.CardType
+ Translation string
+ ImagePrompt string
+ Assets []failedAssetKind
+}
+
+// RetryFailedAssets scans the existing card output directory for incomplete or
+// failed asset generations and retries them in deterministic order. The retry
+// loop stops immediately on the first fresh error so users can rerun the same
+// command after an upstream rate limit clears.
+func (p *Processor) RetryFailedAssets() error {
+ if err := os.MkdirAll(p.Flags.OutputDir, 0755); err != nil {
+ return fmt.Errorf("failed to create output directory: %w", err)
+ }
+
+ plans, err := p.scanFailedAssetPlans()
+ if err != nil {
+ return err
+ }
+ if len(plans) == 0 {
+ fmt.Printf("No failed assets found in: %s\n", p.Flags.OutputDir)
+ return nil
+ }
+
+ totalAssets := 0
+ for _, plan := range plans {
+ totalAssets += len(plan.Assets)
+ }
+
+ fmt.Printf("Found %d failed asset(s) across %d card(s) in %s\n", totalAssets, len(plans), p.Flags.OutputDir)
+
+ regenerated := 0
+ for _, plan := range plans {
+ fmt.Printf("\nRetrying card: %s\n", plan.Card.Word)
+ for _, asset := range plan.Assets {
+ fmt.Printf(" Regenerating %s...\n", asset)
+
+ assetCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ err := p.regenerateFailedAsset(assetCtx, plan, asset)
+ cancel()
+ if err != nil {
+ return fmt.Errorf("stopped after %d successful regeneration(s); %s for %q failed: %w", regenerated, asset, plan.Card.Word, err)
+ }
+
+ regenerated++
+ }
+ }
+
+ fmt.Printf("\nRegenerated %d failed asset(s).\n", regenerated)
+ return nil
+}
+
+func (p *Processor) scanFailedAssetPlans() ([]failedAssetPlan, error) {
+ cards := p.cardStore.ListCardDirectories(nil)
+ plans := make([]failedAssetPlan, 0, len(cards))
+
+ for _, card := range cards {
+ plan := p.buildFailedAssetPlan(card)
+ if len(plan.Assets) == 0 {
+ continue
+ }
+ plans = append(plans, plan)
+ }
+
+ return plans, nil
+}
+
+func (p *Processor) buildFailedAssetPlan(card store.CardDirectory) failedAssetPlan {
+ plan := failedAssetPlan{
+ Card: card,
+ CardType: internal.LoadCardType(card.Path),
+ Translation: readStoredTranslation(card.Path),
+ ImagePrompt: readStoredImagePrompt(card.Path),
+ }
+
+ if !p.Flags.SkipAudio {
+ if plan.CardType.IsBgBg() {
+ frontReady := audioAssetReady(card.Path, "audio_front", p.EffectiveAudioFormat())
+ backReady := audioAssetReady(card.Path, "audio_back", p.EffectiveAudioFormat())
+
+ switch {
+ case !frontReady && !backReady:
+ plan.Assets = append(plan.Assets, failedAssetBgBgAudioPair)
+ case !frontReady:
+ plan.Assets = append(plan.Assets, failedAssetBgBgFrontAudio)
+ case !backReady:
+ plan.Assets = append(plan.Assets, failedAssetBgBgBackAudio)
+ }
+ } else if !audioAssetReady(card.Path, "audio", p.EffectiveAudioFormat()) {
+ plan.Assets = append(plan.Assets, failedAssetAudio)
+ }
+ }
+
+ if !p.Flags.SkipImages && !imageAssetReady(card.Path) {
+ plan.Assets = append(plan.Assets, failedAssetImage)
+ }
+
+ return plan
+}
+
+func (p *Processor) regenerateFailedAsset(ctx context.Context, plan failedAssetPlan, asset failedAssetKind) error {
+ switch asset {
+ case failedAssetAudio:
+ return p.generateAudio(ctx, plan.Card.Word)
+ case failedAssetImage:
+ return p.downloadImagesWithPrompt(ctx, plan.Card.Word, plan.Translation, plan.ImagePrompt)
+ case failedAssetBgBgAudioPair:
+ if strings.TrimSpace(plan.Translation) == "" {
+ return fmt.Errorf("missing back-side text in translation.txt")
+ }
+ return p.generateAudioBgBg(ctx, plan.Card.Word, plan.Translation)
+ case failedAssetBgBgFrontAudio:
+ return p.generateCardAudioSideInDir(ctx, plan.Card.Word, plan.Card.Path, "audio_front", "front audio")
+ case failedAssetBgBgBackAudio:
+ if strings.TrimSpace(plan.Translation) == "" {
+ return fmt.Errorf("missing back-side text in translation.txt")
+ }
+ return p.generateCardAudioSideInDir(ctx, plan.Translation, plan.Card.Path, "audio_back", "back audio")
+ default:
+ return fmt.Errorf("unknown failed asset kind %q", asset)
+ }
+}
+
+func (p *Processor) generateCardAudioSideInDir(ctx context.Context, text, wordDir, filenameBase, label string) error {
+ provider := p.AudioProviderName()
+ voice := p.audioVoiceForProvider()
+ p.logSelectedAudioVoice(provider, voice)
+
+ run := func(candidate string) error {
+ if candidate != voice {
+ fmt.Printf(" Retrying Gemini audio with voice: %s\n", candidate)
+ }
+ fmt.Printf(" Generating %s for '%s'...\n", label, text)
+ return p.generateAudioWithVoiceAndFilenameInDir(ctx, text, candidate, filenameBase, wordDir)
+ }
+
+ if provider == "gemini" && p.GeminiVoice() == "" {
+ _, err := audio.RunWithVoiceFallbacks(voice, run, func(candidate string) {
+ fmt.Printf(" Warning: Gemini returned no audio for voice %s\n", candidate)
+ })
+ return err
+ }
+
+ return run(voice)
+}
+
+func audioAssetReady(wordDir, baseName, preferredFormat string) bool {
+ paths := anki.ResolveAudioPaths(wordDir, baseName, preferredFormat)
+ if len(paths) == 0 {
+ return false
+ }
+
+ for _, path := range paths {
+ if !fileExistsAndNonEmpty(path) || !fileExistsAndNonEmpty(audio.AttributionPath(path)) {
+ return false
+ }
+ }
+
+ return fileExistsAndNonEmpty(filepath.Join(wordDir, "audio_metadata.txt"))
+}
+
+func imageAssetReady(wordDir string) bool {
+ if firstUsableImagePath(wordDir) == "" {
+ return false
+ }
+ if !fileExistsAndNonEmpty(filepath.Join(wordDir, "image_attribution.txt")) {
+ return false
+ }
+ return readStoredImagePrompt(wordDir) != ""
+}
+
+func firstUsableImagePath(wordDir string) string {
+ imagePatterns := []string{
+ "image_*.jpg",
+ "image_*.png",
+ "image_*.webp",
+ "image.jpg",
+ "image.png",
+ "image.webp",
+ }
+
+ for _, pattern := range imagePatterns {
+ if strings.Contains(pattern, "*") {
+ matches, _ := filepath.Glob(filepath.Join(wordDir, pattern))
+ for _, match := range matches {
+ if fileExistsAndNonEmpty(match) {
+ return match
+ }
+ }
+ continue
+ }
+
+ path := filepath.Join(wordDir, pattern)
+ if fileExistsAndNonEmpty(path) {
+ return path
+ }
+ }
+
+ return ""
+}
+
+func readStoredTranslation(wordDir string) string {
+ data, err := os.ReadFile(filepath.Join(wordDir, "translation.txt"))
+ if err != nil {
+ return ""
+ }
+
+ parts := strings.SplitN(string(data), "=", 2)
+ if len(parts) != 2 {
+ return strings.TrimSpace(string(data))
+ }
+
+ return strings.TrimSpace(parts[1])
+}
+
+func readStoredImagePrompt(wordDir string) string {
+ data, err := os.ReadFile(filepath.Join(wordDir, "image_prompt.txt"))
+ if err != nil {
+ return ""
+ }
+
+ prompt := strings.TrimSpace(string(data))
+ if prompt == "" || looksLikeFailedPrompt(prompt) {
+ return ""
+ }
+
+ return prompt
+}
+
+func looksLikeFailedPrompt(prompt string) bool {
+ lower := strings.ToLower(strings.TrimSpace(prompt))
+ if lower == "" {
+ return true
+ }
+
+ markers := []string{
+ "rate limit",
+ "too many requests",
+ "quota exceeded",
+ "resource exhausted",
+ "failed to generate",
+ "generation failed",
+ "temporarily unavailable",
+ "http 429",
+ }
+
+ for _, marker := range markers {
+ if strings.Contains(lower, marker) {
+ return true
+ }
+ }
+
+ return false
+}
+
+func fileExistsAndNonEmpty(path string) bool {
+ info, err := os.Stat(path)
+ if err != nil || info.IsDir() {
+ return false
+ }
+ return info.Size() > 0
+}
diff --git a/internal/processor/image_downloader.go b/internal/processor/image_downloader.go
index 1d6c002..fa0f6cf 100644
--- a/internal/processor/image_downloader.go
+++ b/internal/processor/image_downloader.go
@@ -24,6 +24,13 @@ import (
// AI image providers can generate more contextually accurate images.
// ctx is passed to the image downloader so the caller's deadline applies.
func (p *Processor) downloadImagesWithTranslation(ctx context.Context, word, translationText string) error {
+ return p.downloadImagesWithPrompt(ctx, word, translationText, "")
+}
+
+// downloadImagesWithPrompt downloads images for a word and optionally reuses a
+// previously-saved prompt. When customPrompt is empty the provider generates a
+// fresh prompt as usual.
+func (p *Processor) downloadImagesWithPrompt(ctx context.Context, word, translationText, customPrompt string) error {
searcher, err := p.newImageSearcher()
if err != nil {
return err
@@ -43,6 +50,9 @@ func (p *Processor) downloadImagesWithTranslation(ctx context.Context, word, tra
if translationText != "" {
searchOpts.Translation = translationText
}
+ if strings.TrimSpace(customPrompt) != "" {
+ searchOpts.CustomPrompt = strings.TrimSpace(customPrompt)
+ }
// Register a prompt callback so the AI-generated prompt is persisted
// to disk before the download completes (used by the GUI and for debugging).
diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go
index 649c092..64d6938 100644
--- a/internal/processor/processor_test.go
+++ b/internal/processor/processor_test.go
@@ -11,6 +11,7 @@ import (
"strings"
"testing"
+ "codeberg.org/snonux/totalrecall/internal"
"codeberg.org/snonux/totalrecall/internal/audio"
"codeberg.org/snonux/totalrecall/internal/cli"
"codeberg.org/snonux/totalrecall/internal/gui"
@@ -1333,6 +1334,153 @@ func TestProcessBatch_ValidFile(t *testing.T) {
}
}
+func TestRetryFailedAssets_RegeneratesMissingEnBgAssetsInOrder(t *testing.T) {
+ t.Setenv("GOOGLE_API_KEY", "test-google-key")
+
+ flags := cli.NewFlags()
+ flags.OutputDir = t.TempDir()
+ flags.ImageAPI = image.ImageProviderNanoBanana
+ flags.ImageAPISpecified = true
+ p := NewProcessor(flags, &Config{})
+
+ cardDir := p.findOrCreateWordDirectory("ябълка")
+ if err := os.WriteFile(filepath.Join(cardDir, "translation.txt"), []byte("ябълка = apple\n"), 0644); err != nil {
+ t.Fatalf("setup translation.txt: %v", err)
+ }
+
+ fakeProvider := &fakeAudioProvider{
+ generateFunc: func(_ string, outputFile string) error {
+ return os.WriteFile(outputFile, []byte("audio data"), 0644)
+ },
+ }
+ p.newAudioProvider = func(*audio.Config) (audio.Provider, error) {
+ return fakeProvider, nil
+ }
+ p.imageFactories.NewNanoBananaClient = func(*image.NanoBananaConfig) image.PromptAwareClient {
+ return &stubImageSearcher{}
+ }
+
+ output := captureStdout(t, func() {
+ if err := p.RetryFailedAssets(); err != nil {
+ t.Fatalf("RetryFailedAssets() unexpected error: %v", err)
+ }
+ })
+
+ if fakeProvider.generateCalls != 1 {
+ t.Fatalf("audio generate calls = %d, want 1", fakeProvider.generateCalls)
+ }
+ if !strings.Contains(output, "Regenerating audio") || !strings.Contains(output, "Regenerating image") {
+ t.Fatalf("stdout missing retry steps: %q", output)
+ }
+ if strings.Index(output, "Regenerating audio") > strings.Index(output, "Regenerating image") {
+ t.Fatalf("retry order is wrong, output = %q", output)
+ }
+ if _, err := os.Stat(filepath.Join(cardDir, "audio.mp3")); err != nil {
+ t.Fatalf("expected audio.mp3 to be created: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(cardDir, "image.png")); err != nil {
+ t.Fatalf("expected image.png to be created: %v", err)
+ }
+ promptData, err := os.ReadFile(filepath.Join(cardDir, "image_prompt.txt"))
+ if err != nil {
+ t.Fatalf("expected image_prompt.txt to be created: %v", err)
+ }
+ if strings.TrimSpace(string(promptData)) != "stub nanobanana prompt" {
+ t.Fatalf("image prompt = %q, want %q", strings.TrimSpace(string(promptData)), "stub nanobanana prompt")
+ }
+}
+
+func TestRetryFailedAssets_StopsOnFirstError(t *testing.T) {
+ flags := cli.NewFlags()
+ flags.OutputDir = t.TempDir()
+ flags.SkipImages = true
+ p := NewProcessor(flags, &Config{})
+
+ firstDir := p.findOrCreateWordDirectory("ябълка")
+ secondDir := p.findOrCreateWordDirectory("круша")
+ if err := os.WriteFile(filepath.Join(firstDir, "translation.txt"), []byte("ябълка = apple\n"), 0644); err != nil {
+ t.Fatalf("setup first translation.txt: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(secondDir, "translation.txt"), []byte("круша = pear\n"), 0644); err != nil {
+ t.Fatalf("setup second translation.txt: %v", err)
+ }
+
+ fakeProvider := &fakeAudioProvider{
+ generateFunc: func(_ string, _ string) error {
+ return errors.New("rate limit exceeded")
+ },
+ }
+ p.newAudioProvider = func(*audio.Config) (audio.Provider, error) {
+ return fakeProvider, nil
+ }
+
+ err := p.RetryFailedAssets()
+ if err == nil {
+ t.Fatal("expected RetryFailedAssets() to stop on first error")
+ }
+ if !strings.Contains(err.Error(), "rate limit exceeded") {
+ t.Fatalf("RetryFailedAssets() error = %v, want rate limit message", err)
+ }
+ if fakeProvider.generateCalls != 1 {
+ t.Fatalf("audio generate calls = %d, want 1", fakeProvider.generateCalls)
+ }
+ if _, statErr := os.Stat(filepath.Join(secondDir, "audio.mp3")); !os.IsNotExist(statErr) {
+ t.Fatalf("second card audio should not have been generated, stat err = %v", statErr)
+ }
+}
+
+func TestRetryFailedAssets_RegeneratesOnlyMissingBgBgBackAudio(t *testing.T) {
+ flags := cli.NewFlags()
+ flags.OutputDir = t.TempDir()
+ flags.SkipImages = true
+ p := NewProcessor(flags, &Config{})
+
+ cardDir := p.findOrCreateWordDirectory("ябълка")
+ if err := internal.SaveCardType(cardDir, internal.CardTypeBgBg); err != nil {
+ t.Fatalf("setup cardtype.txt: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(cardDir, "translation.txt"), []byte("ябълка = плод\n"), 0644); err != nil {
+ t.Fatalf("setup translation.txt: %v", err)
+ }
+
+ frontAudio := filepath.Join(cardDir, "audio_front.mp3")
+ if err := os.WriteFile(frontAudio, []byte("front audio"), 0644); err != nil {
+ t.Fatalf("setup audio_front.mp3: %v", err)
+ }
+ if err := os.WriteFile(audio.AttributionPath(frontAudio), []byte("front attribution"), 0644); err != nil {
+ t.Fatalf("setup front attribution: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(cardDir, "audio_metadata.txt"), []byte("format=mp3\naudio_file=audio_front.mp3\n"), 0644); err != nil {
+ t.Fatalf("setup audio_metadata.txt: %v", err)
+ }
+
+ fakeProvider := &fakeAudioProvider{
+ generateFunc: func(_ string, outputFile string) error {
+ return os.WriteFile(outputFile, []byte("back audio"), 0644)
+ },
+ }
+ p.newAudioProvider = func(*audio.Config) (audio.Provider, error) {
+ return fakeProvider, nil
+ }
+
+ if err := p.RetryFailedAssets(); err != nil {
+ t.Fatalf("RetryFailedAssets() unexpected error: %v", err)
+ }
+
+ if fakeProvider.generateCalls != 1 {
+ t.Fatalf("audio generate calls = %d, want 1", fakeProvider.generateCalls)
+ }
+ if !strings.HasSuffix(fakeProvider.lastOutputFile, "audio_back.mp3") {
+ t.Fatalf("last output file = %q, want audio_back.mp3", fakeProvider.lastOutputFile)
+ }
+ if _, err := os.Stat(filepath.Join(cardDir, "audio_back.mp3")); err != nil {
+ t.Fatalf("expected audio_back.mp3 to be created: %v", err)
+ }
+ if _, err := os.Stat(frontAudio); err != nil {
+ t.Fatalf("front audio should remain present: %v", err)
+ }
+}
+
func TestProcessWordWithTranslation_ProvidedTranslation(t *testing.T) {
flags := cli.NewFlags()
flags.OutputDir = t.TempDir()