From 65430e9876a32988839deb7a70dc1e42da21cf64 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 10 Apr 2026 23:11:30 +0300 Subject: Release 0.28.2 --- internal/processor/failed_asset_retry.go | 290 +++++++++++++++++++++++++++++++ internal/processor/image_downloader.go | 10 ++ internal/processor/processor_test.go | 148 ++++++++++++++++ 3 files changed, 448 insertions(+) create mode 100644 internal/processor/failed_asset_retry.go (limited to 'internal/processor') diff --git a/internal/processor/failed_asset_retry.go b/internal/processor/failed_asset_retry.go new file mode 100644 index 0000000..88d1abe --- /dev/null +++ b/internal/processor/failed_asset_retry.go @@ -0,0 +1,290 @@ +package processor + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "codeberg.org/snonux/totalrecall/internal" + "codeberg.org/snonux/totalrecall/internal/anki" + "codeberg.org/snonux/totalrecall/internal/audio" + "codeberg.org/snonux/totalrecall/internal/store" +) + +type failedAssetKind string + +const ( + failedAssetAudio failedAssetKind = "audio" + failedAssetImage failedAssetKind = "image" + failedAssetBgBgAudioPair failedAssetKind = "front and back audio" + failedAssetBgBgFrontAudio failedAssetKind = "front audio" + failedAssetBgBgBackAudio failedAssetKind = "back audio" +) + +type failedAssetPlan struct { + Card store.CardDirectory + CardType internal.CardType + Translation string + ImagePrompt string + Assets []failedAssetKind +} + +// RetryFailedAssets scans the existing card output directory for incomplete or +// failed asset generations and retries them in deterministic order. The retry +// loop stops immediately on the first fresh error so users can rerun the same +// command after an upstream rate limit clears. +func (p *Processor) RetryFailedAssets() error { + if err := os.MkdirAll(p.Flags.OutputDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + plans, err := p.scanFailedAssetPlans() + if err != nil { + return err + } + if len(plans) == 0 { + fmt.Printf("No failed assets found in: %s\n", p.Flags.OutputDir) + return nil + } + + totalAssets := 0 + for _, plan := range plans { + totalAssets += len(plan.Assets) + } + + fmt.Printf("Found %d failed asset(s) across %d card(s) in %s\n", totalAssets, len(plans), p.Flags.OutputDir) + + regenerated := 0 + for _, plan := range plans { + fmt.Printf("\nRetrying card: %s\n", plan.Card.Word) + for _, asset := range plan.Assets { + fmt.Printf(" Regenerating %s...\n", asset) + + assetCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + err := p.regenerateFailedAsset(assetCtx, plan, asset) + cancel() + if err != nil { + return fmt.Errorf("stopped after %d successful regeneration(s); %s for %q failed: %w", regenerated, asset, plan.Card.Word, err) + } + + regenerated++ + } + } + + fmt.Printf("\nRegenerated %d failed asset(s).\n", regenerated) + return nil +} + +func (p *Processor) scanFailedAssetPlans() ([]failedAssetPlan, error) { + cards := p.cardStore.ListCardDirectories(nil) + plans := make([]failedAssetPlan, 0, len(cards)) + + for _, card := range cards { + plan := p.buildFailedAssetPlan(card) + if len(plan.Assets) == 0 { + continue + } + plans = append(plans, plan) + } + + return plans, nil +} + +func (p *Processor) buildFailedAssetPlan(card store.CardDirectory) failedAssetPlan { + plan := failedAssetPlan{ + Card: card, + CardType: internal.LoadCardType(card.Path), + Translation: readStoredTranslation(card.Path), + ImagePrompt: readStoredImagePrompt(card.Path), + } + + if !p.Flags.SkipAudio { + if plan.CardType.IsBgBg() { + frontReady := audioAssetReady(card.Path, "audio_front", p.EffectiveAudioFormat()) + backReady := audioAssetReady(card.Path, "audio_back", p.EffectiveAudioFormat()) + + switch { + case !frontReady && !backReady: + plan.Assets = append(plan.Assets, failedAssetBgBgAudioPair) + case !frontReady: + plan.Assets = append(plan.Assets, failedAssetBgBgFrontAudio) + case !backReady: + plan.Assets = append(plan.Assets, failedAssetBgBgBackAudio) + } + } else if !audioAssetReady(card.Path, "audio", p.EffectiveAudioFormat()) { + plan.Assets = append(plan.Assets, failedAssetAudio) + } + } + + if !p.Flags.SkipImages && !imageAssetReady(card.Path) { + plan.Assets = append(plan.Assets, failedAssetImage) + } + + return plan +} + +func (p *Processor) regenerateFailedAsset(ctx context.Context, plan failedAssetPlan, asset failedAssetKind) error { + switch asset { + case failedAssetAudio: + return p.generateAudio(ctx, plan.Card.Word) + case failedAssetImage: + return p.downloadImagesWithPrompt(ctx, plan.Card.Word, plan.Translation, plan.ImagePrompt) + case failedAssetBgBgAudioPair: + if strings.TrimSpace(plan.Translation) == "" { + return fmt.Errorf("missing back-side text in translation.txt") + } + return p.generateAudioBgBg(ctx, plan.Card.Word, plan.Translation) + case failedAssetBgBgFrontAudio: + return p.generateCardAudioSideInDir(ctx, plan.Card.Word, plan.Card.Path, "audio_front", "front audio") + case failedAssetBgBgBackAudio: + if strings.TrimSpace(plan.Translation) == "" { + return fmt.Errorf("missing back-side text in translation.txt") + } + return p.generateCardAudioSideInDir(ctx, plan.Translation, plan.Card.Path, "audio_back", "back audio") + default: + return fmt.Errorf("unknown failed asset kind %q", asset) + } +} + +func (p *Processor) generateCardAudioSideInDir(ctx context.Context, text, wordDir, filenameBase, label string) error { + provider := p.AudioProviderName() + voice := p.audioVoiceForProvider() + p.logSelectedAudioVoice(provider, voice) + + run := func(candidate string) error { + if candidate != voice { + fmt.Printf(" Retrying Gemini audio with voice: %s\n", candidate) + } + fmt.Printf(" Generating %s for '%s'...\n", label, text) + return p.generateAudioWithVoiceAndFilenameInDir(ctx, text, candidate, filenameBase, wordDir) + } + + if provider == "gemini" && p.GeminiVoice() == "" { + _, err := audio.RunWithVoiceFallbacks(voice, run, func(candidate string) { + fmt.Printf(" Warning: Gemini returned no audio for voice %s\n", candidate) + }) + return err + } + + return run(voice) +} + +func audioAssetReady(wordDir, baseName, preferredFormat string) bool { + paths := anki.ResolveAudioPaths(wordDir, baseName, preferredFormat) + if len(paths) == 0 { + return false + } + + for _, path := range paths { + if !fileExistsAndNonEmpty(path) || !fileExistsAndNonEmpty(audio.AttributionPath(path)) { + return false + } + } + + return fileExistsAndNonEmpty(filepath.Join(wordDir, "audio_metadata.txt")) +} + +func imageAssetReady(wordDir string) bool { + if firstUsableImagePath(wordDir) == "" { + return false + } + if !fileExistsAndNonEmpty(filepath.Join(wordDir, "image_attribution.txt")) { + return false + } + return readStoredImagePrompt(wordDir) != "" +} + +func firstUsableImagePath(wordDir string) string { + imagePatterns := []string{ + "image_*.jpg", + "image_*.png", + "image_*.webp", + "image.jpg", + "image.png", + "image.webp", + } + + for _, pattern := range imagePatterns { + if strings.Contains(pattern, "*") { + matches, _ := filepath.Glob(filepath.Join(wordDir, pattern)) + for _, match := range matches { + if fileExistsAndNonEmpty(match) { + return match + } + } + continue + } + + path := filepath.Join(wordDir, pattern) + if fileExistsAndNonEmpty(path) { + return path + } + } + + return "" +} + +func readStoredTranslation(wordDir string) string { + data, err := os.ReadFile(filepath.Join(wordDir, "translation.txt")) + if err != nil { + return "" + } + + parts := strings.SplitN(string(data), "=", 2) + if len(parts) != 2 { + return strings.TrimSpace(string(data)) + } + + return strings.TrimSpace(parts[1]) +} + +func readStoredImagePrompt(wordDir string) string { + data, err := os.ReadFile(filepath.Join(wordDir, "image_prompt.txt")) + if err != nil { + return "" + } + + prompt := strings.TrimSpace(string(data)) + if prompt == "" || looksLikeFailedPrompt(prompt) { + return "" + } + + return prompt +} + +func looksLikeFailedPrompt(prompt string) bool { + lower := strings.ToLower(strings.TrimSpace(prompt)) + if lower == "" { + return true + } + + markers := []string{ + "rate limit", + "too many requests", + "quota exceeded", + "resource exhausted", + "failed to generate", + "generation failed", + "temporarily unavailable", + "http 429", + } + + for _, marker := range markers { + if strings.Contains(lower, marker) { + return true + } + } + + return false +} + +func fileExistsAndNonEmpty(path string) bool { + info, err := os.Stat(path) + if err != nil || info.IsDir() { + return false + } + return info.Size() > 0 +} diff --git a/internal/processor/image_downloader.go b/internal/processor/image_downloader.go index 1d6c002..fa0f6cf 100644 --- a/internal/processor/image_downloader.go +++ b/internal/processor/image_downloader.go @@ -24,6 +24,13 @@ import ( // AI image providers can generate more contextually accurate images. // ctx is passed to the image downloader so the caller's deadline applies. func (p *Processor) downloadImagesWithTranslation(ctx context.Context, word, translationText string) error { + return p.downloadImagesWithPrompt(ctx, word, translationText, "") +} + +// downloadImagesWithPrompt downloads images for a word and optionally reuses a +// previously-saved prompt. When customPrompt is empty the provider generates a +// fresh prompt as usual. +func (p *Processor) downloadImagesWithPrompt(ctx context.Context, word, translationText, customPrompt string) error { searcher, err := p.newImageSearcher() if err != nil { return err @@ -43,6 +50,9 @@ func (p *Processor) downloadImagesWithTranslation(ctx context.Context, word, tra if translationText != "" { searchOpts.Translation = translationText } + if strings.TrimSpace(customPrompt) != "" { + searchOpts.CustomPrompt = strings.TrimSpace(customPrompt) + } // Register a prompt callback so the AI-generated prompt is persisted // to disk before the download completes (used by the GUI and for debugging). diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index 649c092..64d6938 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "codeberg.org/snonux/totalrecall/internal" "codeberg.org/snonux/totalrecall/internal/audio" "codeberg.org/snonux/totalrecall/internal/cli" "codeberg.org/snonux/totalrecall/internal/gui" @@ -1333,6 +1334,153 @@ func TestProcessBatch_ValidFile(t *testing.T) { } } +func TestRetryFailedAssets_RegeneratesMissingEnBgAssetsInOrder(t *testing.T) { + t.Setenv("GOOGLE_API_KEY", "test-google-key") + + flags := cli.NewFlags() + flags.OutputDir = t.TempDir() + flags.ImageAPI = image.ImageProviderNanoBanana + flags.ImageAPISpecified = true + p := NewProcessor(flags, &Config{}) + + cardDir := p.findOrCreateWordDirectory("ябълка") + if err := os.WriteFile(filepath.Join(cardDir, "translation.txt"), []byte("ябълка = apple\n"), 0644); err != nil { + t.Fatalf("setup translation.txt: %v", err) + } + + fakeProvider := &fakeAudioProvider{ + generateFunc: func(_ string, outputFile string) error { + return os.WriteFile(outputFile, []byte("audio data"), 0644) + }, + } + p.newAudioProvider = func(*audio.Config) (audio.Provider, error) { + return fakeProvider, nil + } + p.imageFactories.NewNanoBananaClient = func(*image.NanoBananaConfig) image.PromptAwareClient { + return &stubImageSearcher{} + } + + output := captureStdout(t, func() { + if err := p.RetryFailedAssets(); err != nil { + t.Fatalf("RetryFailedAssets() unexpected error: %v", err) + } + }) + + if fakeProvider.generateCalls != 1 { + t.Fatalf("audio generate calls = %d, want 1", fakeProvider.generateCalls) + } + if !strings.Contains(output, "Regenerating audio") || !strings.Contains(output, "Regenerating image") { + t.Fatalf("stdout missing retry steps: %q", output) + } + if strings.Index(output, "Regenerating audio") > strings.Index(output, "Regenerating image") { + t.Fatalf("retry order is wrong, output = %q", output) + } + if _, err := os.Stat(filepath.Join(cardDir, "audio.mp3")); err != nil { + t.Fatalf("expected audio.mp3 to be created: %v", err) + } + if _, err := os.Stat(filepath.Join(cardDir, "image.png")); err != nil { + t.Fatalf("expected image.png to be created: %v", err) + } + promptData, err := os.ReadFile(filepath.Join(cardDir, "image_prompt.txt")) + if err != nil { + t.Fatalf("expected image_prompt.txt to be created: %v", err) + } + if strings.TrimSpace(string(promptData)) != "stub nanobanana prompt" { + t.Fatalf("image prompt = %q, want %q", strings.TrimSpace(string(promptData)), "stub nanobanana prompt") + } +} + +func TestRetryFailedAssets_StopsOnFirstError(t *testing.T) { + flags := cli.NewFlags() + flags.OutputDir = t.TempDir() + flags.SkipImages = true + p := NewProcessor(flags, &Config{}) + + firstDir := p.findOrCreateWordDirectory("ябълка") + secondDir := p.findOrCreateWordDirectory("круша") + if err := os.WriteFile(filepath.Join(firstDir, "translation.txt"), []byte("ябълка = apple\n"), 0644); err != nil { + t.Fatalf("setup first translation.txt: %v", err) + } + if err := os.WriteFile(filepath.Join(secondDir, "translation.txt"), []byte("круша = pear\n"), 0644); err != nil { + t.Fatalf("setup second translation.txt: %v", err) + } + + fakeProvider := &fakeAudioProvider{ + generateFunc: func(_ string, _ string) error { + return errors.New("rate limit exceeded") + }, + } + p.newAudioProvider = func(*audio.Config) (audio.Provider, error) { + return fakeProvider, nil + } + + err := p.RetryFailedAssets() + if err == nil { + t.Fatal("expected RetryFailedAssets() to stop on first error") + } + if !strings.Contains(err.Error(), "rate limit exceeded") { + t.Fatalf("RetryFailedAssets() error = %v, want rate limit message", err) + } + if fakeProvider.generateCalls != 1 { + t.Fatalf("audio generate calls = %d, want 1", fakeProvider.generateCalls) + } + if _, statErr := os.Stat(filepath.Join(secondDir, "audio.mp3")); !os.IsNotExist(statErr) { + t.Fatalf("second card audio should not have been generated, stat err = %v", statErr) + } +} + +func TestRetryFailedAssets_RegeneratesOnlyMissingBgBgBackAudio(t *testing.T) { + flags := cli.NewFlags() + flags.OutputDir = t.TempDir() + flags.SkipImages = true + p := NewProcessor(flags, &Config{}) + + cardDir := p.findOrCreateWordDirectory("ябълка") + if err := internal.SaveCardType(cardDir, internal.CardTypeBgBg); err != nil { + t.Fatalf("setup cardtype.txt: %v", err) + } + if err := os.WriteFile(filepath.Join(cardDir, "translation.txt"), []byte("ябълка = плод\n"), 0644); err != nil { + t.Fatalf("setup translation.txt: %v", err) + } + + frontAudio := filepath.Join(cardDir, "audio_front.mp3") + if err := os.WriteFile(frontAudio, []byte("front audio"), 0644); err != nil { + t.Fatalf("setup audio_front.mp3: %v", err) + } + if err := os.WriteFile(audio.AttributionPath(frontAudio), []byte("front attribution"), 0644); err != nil { + t.Fatalf("setup front attribution: %v", err) + } + if err := os.WriteFile(filepath.Join(cardDir, "audio_metadata.txt"), []byte("format=mp3\naudio_file=audio_front.mp3\n"), 0644); err != nil { + t.Fatalf("setup audio_metadata.txt: %v", err) + } + + fakeProvider := &fakeAudioProvider{ + generateFunc: func(_ string, outputFile string) error { + return os.WriteFile(outputFile, []byte("back audio"), 0644) + }, + } + p.newAudioProvider = func(*audio.Config) (audio.Provider, error) { + return fakeProvider, nil + } + + if err := p.RetryFailedAssets(); err != nil { + t.Fatalf("RetryFailedAssets() unexpected error: %v", err) + } + + if fakeProvider.generateCalls != 1 { + t.Fatalf("audio generate calls = %d, want 1", fakeProvider.generateCalls) + } + if !strings.HasSuffix(fakeProvider.lastOutputFile, "audio_back.mp3") { + t.Fatalf("last output file = %q, want audio_back.mp3", fakeProvider.lastOutputFile) + } + if _, err := os.Stat(filepath.Join(cardDir, "audio_back.mp3")); err != nil { + t.Fatalf("expected audio_back.mp3 to be created: %v", err) + } + if _, err := os.Stat(frontAudio); err != nil { + t.Fatalf("front audio should remain present: %v", err) + } +} + func TestProcessWordWithTranslation_ProvidedTranslation(t *testing.T) { flags := cli.NewFlags() flags.OutputDir = t.TempDir() -- cgit v1.2.3