summaryrefslogtreecommitdiff
path: root/internal
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2025-07-15 21:12:18 +0300
committerPaul Buetow <paul@buetow.org>2025-07-15 21:12:18 +0300
commit9c77f2a7bef485fa137f123cbf55b42cacb2b285 (patch)
treed4ee67a4d3de0c17fc86e465f24480b768e80d13 /internal
parentb2e1d035ab8d32f6c3351ae5f0a514e569294ef0 (diff)
feat: add OpenAI gpt-4o-mini-tts support with voice instructions
- Add support for OpenAI's new gpt-4o-mini-tts model with customizable voice instructions - Add OpenAIInstruction field to audio configuration for natural language voice control - Update CLI with --openai-instruction flag for runtime voice customization - Enhanced cache key generation to include voice instructions - Update default model to gpt-4o-mini-tts with Bulgarian-optimized instructions - Add support for new voices: ash, ballad, coral, sage, verse - Improve error handling for models requiring special API access - Update documentation with examples and model information - Create .totalrecall.yaml.example with comprehensive configuration options Note: The gpt-4o-mini-tts model requires special API access and may not be available to all accounts yet. πŸ€– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
Diffstat (limited to 'internal')
-rw-r--r--internal/audio/openai_provider.go41
-rw-r--r--internal/audio/provider.go16
-rw-r--r--internal/image/openai.go63
-rw-r--r--internal/image/translate.go9
4 files changed, 107 insertions, 22 deletions
diff --git a/internal/audio/openai_provider.go b/internal/audio/openai_provider.go
index 9efbcd2..a61957a 100644
--- a/internal/audio/openai_provider.go
+++ b/internal/audio/openai_provider.go
@@ -62,14 +62,29 @@ func (p *OpenAIProvider) GenerateAudio(ctx context.Context, text string, outputF
}
}
+ // Preprocess text for clearer Bulgarian pronunciation
+ processedText := p.preprocessBulgarianText(text)
+
// Prepare the TTS request
+ // OpenAI TTS will automatically detect and pronounce Bulgarian text
+ fmt.Printf("OpenAI TTS: Using model '%s' with voice '%s' at speed %.2f\n", p.config.OpenAIModel, p.config.OpenAIVoice, p.config.OpenAISpeed)
+ if p.config.OpenAIInstruction != "" && (p.config.OpenAIModel == "gpt-4o-mini-tts" || p.config.OpenAIModel == "gpt-4o-mini-audio-preview") {
+ fmt.Printf("OpenAI TTS Instruction: '%s'\n", p.config.OpenAIInstruction)
+ }
+ fmt.Printf("OpenAI TTS Input: '%s'\n", processedText)
+
req := openai.CreateSpeechRequest{
Model: openai.SpeechModel(p.config.OpenAIModel),
- Input: text,
+ Input: processedText,
Voice: openai.SpeechVoice(p.config.OpenAIVoice),
Speed: p.config.OpenAISpeed,
}
+ // Add instructions for gpt-4o-mini-tts model
+ if p.config.OpenAIInstruction != "" && (p.config.OpenAIModel == "gpt-4o-mini-tts" || p.config.OpenAIModel == "gpt-4o-mini-audio-preview") {
+ req.Instructions = p.config.OpenAIInstruction
+ }
+
// Determine response format based on output file extension
ext := strings.ToLower(filepath.Ext(outputFile))
switch ext {
@@ -93,6 +108,11 @@ func (p *OpenAIProvider) GenerateAudio(ctx context.Context, text string, outputF
// Make the API call
response, err := p.client.CreateSpeech(ctx, req)
if err != nil {
+ // Check if it's a model access error
+ errStr := err.Error()
+ if strings.Contains(errStr, "does not have access to model") && (p.config.OpenAIModel == "gpt-4o-mini-tts" || p.config.OpenAIModel == "gpt-4o-mini-audio-preview") {
+ return fmt.Errorf("OpenAI TTS API error: %w\nNote: The %s model requires access. Try using --openai-model tts-1-hd instead", err, p.config.OpenAIModel)
+ }
return fmt.Errorf("OpenAI TTS API error: %w", err)
}
defer response.Close()
@@ -147,6 +167,21 @@ func (p *OpenAIProvider) IsAvailable() error {
return nil
}
+// preprocessBulgarianText prepares Bulgarian text for clearer TTS pronunciation
+func (p *OpenAIProvider) preprocessBulgarianText(text string) string {
+ // For single words, we add subtle punctuation to create natural pauses
+ // without repeating the word
+
+ // First, clean the text
+ cleanedText := strings.TrimSpace(text)
+
+ // Add ellipsis after the word to create a natural pause and slow down
+ // This helps the TTS engine pronounce it more carefully
+ processedText := fmt.Sprintf("%s...", cleanedText)
+
+ return processedText
+}
+
// getCacheFilePath generates a cache file path for the given text
func (p *OpenAIProvider) getCacheFilePath(text string) string {
// Create a hash of the text and settings
@@ -155,6 +190,10 @@ func (p *OpenAIProvider) getCacheFilePath(text string) string {
h.Write([]byte(p.config.OpenAIModel))
h.Write([]byte(p.config.OpenAIVoice))
h.Write([]byte(fmt.Sprintf("%.2f", p.config.OpenAISpeed)))
+ // Include instruction in cache key for gpt-4o-mini-tts
+ if p.config.OpenAIModel == "gpt-4o-mini-tts" && p.config.OpenAIInstruction != "" {
+ h.Write([]byte(p.config.OpenAIInstruction))
+ }
hash := hex.EncodeToString(h.Sum(nil))
// Use first 2 chars as subdirectory for better file system performance
diff --git a/internal/audio/provider.go b/internal/audio/provider.go
index c803b61..94605b7 100644
--- a/internal/audio/provider.go
+++ b/internal/audio/provider.go
@@ -31,10 +31,11 @@ type Config struct {
ESpeakWordGap int
// OpenAI-specific settings
- OpenAIKey string
- OpenAIModel string // "tts-1" or "tts-1-hd"
- OpenAIVoice string // "alloy", "echo", "fable", "onyx", "nova", "shimmer"
- OpenAISpeed float64 // 0.25 to 4.0
+ OpenAIKey string
+ OpenAIModel string // "tts-1", "tts-1-hd", or "gpt-4o-mini-tts"
+ OpenAIVoice string // "alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "verse"
+ OpenAISpeed float64 // 0.25 to 4.0
+ OpenAIInstruction string // Voice instructions for gpt-4o-mini-tts model
// Caching settings
EnableCache bool
@@ -52,9 +53,10 @@ func DefaultProviderConfig() *Config {
ESpeakPitch: 50,
ESpeakAmplitude: 100,
ESpeakWordGap: 0,
- OpenAIModel: "tts-1",
- OpenAIVoice: "nova",
- OpenAISpeed: 1.0,
+ OpenAIModel: "gpt-4o-mini-tts", // New model with voice instructions support
+ OpenAIVoice: "nova",
+ OpenAISpeed: 0.8, // Slightly slower for clarity (note: may be ignored by gpt-4o-mini-tts)
+ OpenAIInstruction: "Speak slowly and clearly with natural Bulgarian pronunciation, emphasizing each syllable distinctly",
EnableCache: true,
CacheDir: "./.audio_cache",
}
diff --git a/internal/image/openai.go b/internal/image/openai.go
index a5a3e31..839c735 100644
--- a/internal/image/openai.go
+++ b/internal/image/openai.go
@@ -48,10 +48,10 @@ func NewOpenAIClient(config *OpenAIConfig) *OpenAIClient {
// Set defaults
if config.Model == "" {
- config.Model = "dall-e-2"
+ config.Model = "dall-e-3"
}
if config.Size == "" {
- config.Size = "512x512"
+ config.Size = "1024x1024"
}
if config.Quality == "" {
config.Quality = "standard"
@@ -97,6 +97,7 @@ func (c *OpenAIClient) Search(ctx context.Context, opts *SearchOptions) ([]Searc
cacheFile := c.getCacheFilePath(opts.Query)
if info, err := os.Stat(cacheFile); err == nil && info.Size() > 0 {
// Return cached result
+ fmt.Printf("Using cached image for '%s'\n", opts.Query)
result := SearchResult{
ID: c.generateImageID(opts.Query),
URL: cacheFile,
@@ -112,11 +113,20 @@ func (c *OpenAIClient) Search(ctx context.Context, opts *SearchOptions) ([]Searc
}
// Translate Bulgarian word to English for better results
- translatedWord := translateBulgarianToEnglish(opts.Query)
+ translatedWord, err := c.translateBulgarianToEnglish(ctx, opts.Query)
+ if err != nil {
+ // If translation fails, fall back to using the original word
+ fmt.Printf("Translation failed: %v, using original word\n", err)
+ translatedWord = opts.Query
+ }
// Create educational prompt
prompt := c.createEducationalPrompt(opts.Query, translatedWord)
+ // Log the prompt to stdout for debugging
+ fmt.Printf("OpenAI Image Generation Prompt: %s\n", prompt)
+ fmt.Printf("OpenAI Image Generation: Using model '%s' with size '%s'\n", c.model, c.size)
+
// Create the image generation request
req := openai.ImageRequest{
Prompt: prompt,
@@ -220,22 +230,47 @@ func (c *OpenAIClient) Name() string {
// createEducationalPrompt generates a prompt optimized for language learning
func (c *OpenAIClient) createEducationalPrompt(bulgarianWord, englishTranslation string) string {
- // Create a prompt that generates clear, educational images
- // suitable for language learning flashcards
+ // Create a simple, clear prompt for educational images
return fmt.Sprintf(
- "A simple, clear, photorealistic educational image showing %s, "+
- "suitable for language learning flashcards. "+
- "The image should be easily recognizable, with good lighting, "+
- "plain background, and focused on a single clear subject. "+
+ "Generate a simple, clear image of: %s. "+
+ "This is for the Bulgarian word '%s' which means %s. "+
+ "The image should be educational and suitable for language learning flashcards. "+
+ "Requirements: single main subject, plain background, clear and recognizable. "+
"No text, labels, or writing in the image.",
- englishTranslation,
+ englishTranslation, bulgarianWord, englishTranslation,
)
}
-// translateBulgarianToEnglish translates a Bulgarian word to English
-func translateBulgarianToEnglish(word string) string {
- // Use the existing translation function from translate.go
- return translateBulgarianQuery(word)
+// translateBulgarianToEnglish translates a Bulgarian word to English using OpenAI
+func (c *OpenAIClient) translateBulgarianToEnglish(ctx context.Context, word string) (string, error) {
+ // Use OpenAI chat completion to translate
+ fmt.Printf("OpenAI Translation: Using model 'gpt-4o-mini' to translate '%s'\n", word)
+
+ req := openai.ChatCompletionRequest{
+ Model: openai.GPT4oMini,
+ Messages: []openai.ChatCompletionMessage{
+ {
+ Role: openai.ChatMessageRoleUser,
+ Content: fmt.Sprintf("Translate the Bulgarian word '%s' to English. Respond with only the English translation, nothing else.", word),
+ },
+ },
+ Temperature: 0.3, // Lower temperature for more consistent translations
+ MaxTokens: 50,
+ }
+
+ resp, err := c.client.CreateChatCompletion(ctx, req)
+ if err != nil {
+ return "", fmt.Errorf("translation failed: %w", err)
+ }
+
+ if len(resp.Choices) == 0 || resp.Choices[0].Message.Content == "" {
+ return "", fmt.Errorf("no translation received")
+ }
+
+ translation := strings.TrimSpace(resp.Choices[0].Message.Content)
+ fmt.Printf("Translated '%s' to '%s'\n", word, translation)
+
+ return translation, nil
}
// getCacheFilePath generates a cache file path for the given word
diff --git a/internal/image/translate.go b/internal/image/translate.go
index 03d5875..38d16f9 100644
--- a/internal/image/translate.go
+++ b/internal/image/translate.go
@@ -8,6 +8,15 @@ func translateBulgarianQuery(query string) string {
// Common Bulgarian words for flashcard creation
translations := map[string]string{
"ябълка": "apple",
+ "ΠΌΠ°Π»ΠΈΠ½ΠΊΠ°": "raspberry",
+ "ягода": "strawberry",
+ "Ρ‡Π΅Ρ€Π΅ΡˆΠ°": "cherry",
+ "ΠΊΡ€ΡƒΡˆΠ°": "pear",
+ "праскова": "peach",
+ "Π³Ρ€ΠΎΠ·Π΄Π΅": "grapes",
+ "Π±Π°Π½Π°Π½": "banana",
+ "ΠΏΠΎΡ€Ρ‚ΠΎΠΊΠ°Π»": "orange",
+ "Π»ΠΈΠΌΠΎΠ½": "lemon",
"ΠΊΠΎΡ‚ΠΊΠ°": "cat",
"ΠΊΡƒΡ‡Π΅": "dog",
"хляб": "bread",