diff options
| author | Paul Buetow <paul@buetow.org> | 2025-08-17 09:05:45 +0300 |
|---|---|---|
| committer | Paul Buetow <paul@buetow.org> | 2025-08-17 09:05:45 +0300 |
| commit | d5fbb6ef5957894eb5be0854bdb328a6774abddb (patch) | |
| tree | 7558a3d1c70db85d54f3367a5c81c28e7260392b /cmd | |
| parent | 5d9e197a394089f66539320e77abb3f3689b3381 (diff) | |
cli: stream responses in hexai when supported (OpenAI, Ollama)
- Add llm.Streamer optional interface
- Implement ChatStream for OpenAI (SSE) and Ollama (JSON stream)
- CLI uses streaming; LSP unchanged (non-streaming)
- README: document streaming behavior for CLI
Diffstat (limited to 'cmd')
| -rw-r--r-- | cmd/hexai/main.go | 42 |
1 files changed, 29 insertions, 13 deletions
diff --git a/cmd/hexai/main.go b/cmd/hexai/main.go index c356800..6cbd288 100644 --- a/cmd/hexai/main.go +++ b/cmd/hexai/main.go @@ -83,18 +83,34 @@ func main() { {Role: "system", Content: system}, {Role: "user", Content: input}, } - out, err := client.Chat(context.Background(), msgs) - dur := time.Since(start) - if err != nil { - fmt.Fprintf(os.Stderr, logging.AnsiBase+"hexai: error: %v"+logging.AnsiReset+"\n", err) - os.Exit(1) + var out string + if s, ok := client.(llm.Streamer); ok { + var b strings.Builder + err := s.ChatStream(context.Background(), msgs, func(chunk string) { + b.WriteString(chunk) + fmt.Fprint(os.Stdout, chunk) + }) + dur := time.Since(start) + if err != nil { + fmt.Fprintf(os.Stderr, logging.AnsiBase+"hexai: error: %v"+logging.AnsiReset+"\n", err) + os.Exit(1) + } + out = b.String() + // Summary + inSize := len(input) + outSize := len(out) + fmt.Fprintf(os.Stderr, "\n"+logging.AnsiBase+"done provider=%s model=%s time=%s in_bytes=%d out_bytes=%d"+logging.AnsiReset+"\n", client.Name(), client.DefaultModel(), dur.Round(time.Millisecond), inSize, outSize) + } else { + outText, err := client.Chat(context.Background(), msgs) + dur := time.Since(start) + if err != nil { + fmt.Fprintf(os.Stderr, logging.AnsiBase+"hexai: error: %v"+logging.AnsiReset+"\n", err) + os.Exit(1) + } + out = outText + fmt.Fprint(os.Stdout, out) + inSize := len(input) + outSize := len(out) + fmt.Fprintf(os.Stderr, "\n"+logging.AnsiBase+"done provider=%s model=%s time=%s in_bytes=%d out_bytes=%d"+logging.AnsiReset+"\n", client.Name(), client.DefaultModel(), dur.Round(time.Millisecond), inSize, outSize) } - - // Write assistant output to stdout - fmt.Fprint(os.Stdout, out) - - // Summary to stderr (preceded by a blank line) - inSize := len(input) - outSize := len(out) - fmt.Fprintf(os.Stderr, "\n"+logging.AnsiBase+"done provider=%s model=%s time=%s in_bytes=%d out_bytes=%d"+logging.AnsiReset+"\n", client.Name(), client.DefaultModel(), dur.Round(time.Millisecond), inSize, outSize) } |
