Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

mlxrunner: add Phonon text-to-speech support (#19166)

Jeffrey Morgan6ef847b8fef4
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 2ac1b12f2..02f79be5e 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -903,23 +903,12 @@ func RunHandler(cmd *cobra.Command, args []string) error {
}
prompts := args[1:]
- // prepend stdin to the prompt if provided
- if !term.IsTerminal(int(os.Stdin.Fd())) {
- in, err := io.ReadAll(os.Stdin)
- if err != nil {
- return err
- }
-
- // Only prepend stdin content if it's not empty
- stdinContent := string(in)
- if len(stdinContent) > 0 {
- prompts = append([]string{stdinContent}, prompts...)
- }
+ stdinPiped := !term.IsTerminal(int(os.Stdin.Fd()))
+ if stdinPiped {
opts.ShowConnect = false
opts.WordWrap = false
interactive = false
}
- opts.Prompt = strings.Join(prompts, " ")
if len(prompts) > 0 {
interactive = false
}
@@ -966,12 +955,32 @@ func RunHandler(cmd *cobra.Command, args []string) error {
applyShowResponseToRunOptions(&opts, info)
+ // prepend stdin to the prompt if provided; speech models read it as it
+ // arrives instead
+ speech := slices.Contains(info.Capabilities, model.CapabilitySpeech)
+ if stdinPiped && !speech {
+ in, err := io.ReadAll(os.Stdin)
+ if err != nil {
+ return err
+ }
+
+ // Only prepend stdin content if it's not empty
+ stdinContent := string(in)
+ if len(stdinContent) > 0 {
+ prompts = append([]string{stdinContent}, prompts...)
+ }
+ }
+ opts.Prompt = strings.Join(prompts, " ")
+
if slices.Contains(info.Capabilities, model.CapabilityDecision) {
return runDecision(cmd, opts, interactive)
}
if cmd.Flags().Changed("ask") {
return fmt.Errorf("%s is not a decision model, so it can't answer --ask questions", name)
}
+ if cmd.Flags().Changed("voice") && !speech {
+ return fmt.Errorf("%s is not a speech model, so it can't use --voice", name)
+ }
// Check if this is an embedding model
isEmbeddingModel := slices.Contains(info.Capabilities, model.CapabilityEmbedding)
@@ -996,6 +1005,18 @@ func RunHandler(cmd *cobra.Command, args []string) error {
return generateEmbedding(cmd, name, opts.Prompt, opts.KeepAlive, truncate, dimensions)
}
+ if speech {
+ voice, err := cmd.Flags().GetString("voice")
+ if err != nil {
+ return err
+ }
+ var input io.Reader
+ if stdinPiped {
+ input = os.Stdin
+ }
+ return runSpeech(cmd.Context(), name, voice, opts.Prompt, input, interactive)
+ }
+
if slices.Contains(info.Capabilities, model.CapabilityImage) {
return errors.New("image generation models are not currently supported")
}
@@ -1261,7 +1282,10 @@ func ListRunningHandler(cmd *cobra.Command, args []string) error {
} else {
until = format.HumanTime(m.ExpiresAt, "Never")
}
- ctxStr := strconv.Itoa(m.ContextLength)
+ ctxStr := "-"
+ if m.ContextLength > 0 {
+ ctxStr = strconv.Itoa(m.ContextLength)
+ }
data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), procStr, ctxStr, m.Runner, until})
}
}
@@ -2597,6 +2621,7 @@ func NewCLI() *cobra.Command {
runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)")
runCmd.Flags().Bool("truncate", false, "For embedding models: truncate inputs exceeding context length (default: true). Set --truncate=false to error instead")
runCmd.Flags().Int("dimensions", 0, "Truncate output embeddings to specified dimension (embedding models only)")
+ runCmd.Flags().String("voice", "", "Voice to speak with (speech models only)")
var asked []askedQuestion
runCmd.Flags().Var(questionFlag{"ask", &asked}, "ask", "Ask a decision model a question, answered yes or no unless --choice or --level follow")
runCmd.Flags().Var(questionFlag{"choice", &asked}, "choice", `Add a label for the preceding --ask to choose from, e.g. billing="Payments and refunds"`)
diff --git a/cmd/speech.go b/cmd/speech.go
new file mode 100644
index 000000000..c4c1ddaf0
--- /dev/null
+++ b/cmd/speech.go
@@ -0,0 +1,324 @@
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "os/exec"
+ "os/signal"
+ "runtime"
+ "strings"
+ "unicode/utf8"
+
+ "golang.org/x/term"
+
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/readline"
+)
+
+// runSpeech reads text aloud: piped input as it arrives followed by any
+// text arguments, the text arguments alone, or each line typed at the prompt.
+func runSpeech(ctx context.Context, model, voice, text string, input io.Reader, interactive bool) error {
+ if input != nil {
+ return speak(ctx, model, voice, io.MultiReader(input, strings.NewReader(" "+text)))
+ }
+ if text != "" || !interactive {
+ return speak(ctx, model, voice, strings.NewReader(text))
+ }
+ if _, err := player(model, ""); err != nil {
+ return err
+ }
+
+ scanner, err := readline.New(readline.Prompt{
+ Prompt: ">>> ",
+ Placeholder: "Enter text to speak (/bye to exit)",
+ })
+ if err != nil {
+ return err
+ }
+ if envconfig.NoHistory() {
+ scanner.HistoryDisable()
+ }
+ for {
+ // Clear anything typed while audio played; readline redraws it.
+ fmt.Print(readline.CursorBOL + readline.ClearToEOL)
+ line, err := scanner.Readline()
+ switch {
+ case errors.Is(err, io.EOF):
+ fmt.Println()
+ return nil
+ case errors.Is(err, readline.ErrInterrupt):
+ if line == "" {
+ fmt.Println("\nUse Ctrl + d or /bye to exit.")
+ }
+ continue
+ case errors.Is(err, readline.ErrEditPrompt):
+ continue
+ case err != nil:
+ return err
+ }
+ text := strings.TrimSpace(line)
+ if text == "/bye" {
+ return nil
+ }
+ if text == "" {
+ continue
+ }
+ // Ctrl+C stops speaking and returns to the prompt.
+ speakCtx, stop := signal.NotifyContext(ctx, os.Interrupt)
+ if err := speak(speakCtx, model, voice, strings.NewReader(text)); err != nil && speakCtx.Err() == nil {
+ fmt.Fprintln(os.Stderr, "Error:", err)
+ }
+ stop()
+ }
+}
+
+// speak plays text from r a few sentences at a time as it arrives, so a chat
+// model piped into a speech model is heard while it is still writing. When
+// stdout is not a terminal it writes one WAV instead, for example
+// `ollama run model "text" > speech.wav` on a machine without speakers.
+func speak(ctx context.Context, model, voice string, r io.Reader) error {
+ toTerminal := term.IsTerminal(int(os.Stdout.Fd()))
+ if toTerminal {
+ if _, err := player(model, ""); err != nil {
+ return err
+ }
+ }
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ // Generate the next piece while the current one plays.
+ type result struct {
+ wav []byte
+ err error
+ }
+ audio := make(chan result, 1)
+ go func() {
+ defer close(audio)
+ for piece := range speechPieces(ctx, r) {
+ wav, err := synthesize(ctx, model, voice, piece)
+ select {
+ case audio <- result{wav, err}:
+ case <-ctx.Done():
+ return
+ }
+ if err != nil {
+ return
+ }
+ }
+ }()
+
+ var out []byte
+ spoken := false
+ for res := range audio {
+ if res.err != nil {
+ return res.err
+ }
+ spoken = true
+ switch {
+ case toTerminal:
+ if err := play(ctx, model, res.wav); err != nil {
+ return err
+ }
+ case out == nil:
+ out = res.wav
+ default:
+ out = append(out, res.wav[44:]...)
+ }
+ }
+ if !spoken {
+ return fmt.Errorf("speech models require input text. Usage: ollama run %s \"your text here\"", model)
+ }
+ if toTerminal {
+ return nil
+ }
+ setWAVSizes(out)
+ _, err := os.Stdout.Write(out)
+ return err
+}
+
+// minSpeechPiece is how much text speechPieces gathers before it sends a
+// sentence on, so speech starts quickly without many tiny requests.
+const minSpeechPiece = 100
+
+// speechPieces sends text from r on as soon as it ends at a sentence or
+// paragraph break, sized for /v1/audio/speech, and the rest when r ends.
+func speechPieces(ctx context.Context, r io.Reader) <-chan string {
+ pieces := make(chan string)
+ go func() {
+ defer close(pieces)
+ send := func(text string) bool {
+ for _, piece := range speechChunks(text, maxSpeechInput) {
+ select {
+ case pieces <- piece:
+ case <-ctx.Done():
+ return false
+ }
+ }
+ return true
+ }
+ var pending string
+ buf := make([]byte, 4096)
+ for {
+ n, err := r.Read(buf)
+ pending += string(buf[:n])
+ if cut := speechCut(pending); cut > 0 {
+ if !send(pending[:cut]) {
+ return
+ }
+ pending = pending[cut:]
+ }
+ if err != nil {
+ send(pending)
+ return
+ }
+ }
+ }()
+ return pieces
+}
+
+// speechCut returns where text can be cut so the part before it is spoken
+// now: after the last paragraph break or sentence end, once a paragraph or at
+// least minSpeechPiece characters have arrived. It returns 0 to wait.
+func speechCut(text string) int {
+ para := strings.LastIndex(text, "\n\n")
+ cut := max(para, lastSentenceEnd(text))
+ if para > 0 || (cut > 0 && utf8.RuneCountInString(text[:cut]) >= minSpeechPiece) {
+ return cut
+ }
+ return 0
+}
+
+// maxSpeechInput is the most characters /v1/audio/speech accepts at once.
+const maxSpeechInput = 4096
+
+// speechChunks splits text into one piece per paragraph, cutting paragraphs
+// longer than limit characters at the last sentence end or space that fits.
+func speechChunks(text string, limit int) []string {
+ var chunks []string
+ for para := range strings.SplitSeq(strings.ReplaceAll(text, "\r\n", "\n"), "\n\n") {
+ for p := strings.TrimSpace(para); p != ""; p = strings.TrimSpace(p) {
+ end := len(p)
+ if utf8.RuneCountInString(p) > limit {
+ end = 0
+ for range limit {
+ _, size := utf8.DecodeRuneInString(p[end:])
+ end += size
+ }
+ if i := lastSentenceEnd(p[:end]); i > 0 {
+ end = i
+ } else if i := strings.LastIndexAny(p[:end], " \t\n"); i > 0 {
+ end = i
+ }
+ }
+ chunks = append(chunks, strings.TrimSpace(p[:end]))
+ p = p[end:]
+ }
+ }
+ return chunks
+}
+
+// lastSentenceEnd returns the index just past the last '.', '!' or '?' in s
+// that is followed by whitespace, or -1.
+func lastSentenceEnd(s string) int {
+ for i := len(s) - 2; i > 0; i-- {
+ if strings.IndexByte(".!?", s[i]) >= 0 && (s[i+1] == ' ' || s[i+1] == '\n') {
+ return i + 1
+ }
+ }
+ return -1
+}
+
+func synthesize(ctx context.Context, model, voice, text string) ([]byte, error) {
+ body, err := json.Marshal(map[string]string{"model": model, "voice": voice, "input": text, "response_format": "wav"})
+ if err != nil {
+ return nil, err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, envconfig.Host().JoinPath("v1/audio/speech").String(), bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ data, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, err
+ }
+ if res.StatusCode != http.StatusOK {
+ var e struct{ Error struct{ Message string } }
+ if json.Unmarshal(data, &e) == nil && e.Error.Message != "" {
+ return nil, errors.New(e.Error.Message)
+ }
+ return nil, fmt.Errorf("speech request failed: %s", res.Status)
+ }
+ if len(data) <= 44 {
+ return nil, errors.New("speech response contained no audio")
+ }
+ setWAVSizes(data)
+ return data, nil
+}
+
+// setWAVSizes records the length of a WAV the server streamed as unknown,
+// for players that need it.
+func setWAVSizes(wav []byte) {
+ binary.LittleEndian.PutUint32(wav[4:], uint32(len(wav)-8))
+ binary.LittleEndian.PutUint32(wav[40:], uint32(len(wav)-44))
+}
+
+// play uses the system's built-in WAV player.
+func play(ctx context.Context, model string, wav []byte) error {
+ f, err := os.CreateTemp("", "ollama-*.wav")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(f.Name())
+ if _, err := f.Write(wav); err != nil {
+ f.Close()
+ return err
+ }
+ if err := f.Close(); err != nil {
+ return err
+ }
+
+ p, err := player(model, f.Name())
+ if err != nil {
+ return err
+ }
+ c := exec.CommandContext(ctx, p[0], p[1:]...)
+ c.Stderr = os.Stderr
+ if err := c.Run(); err != nil {
+ return fmt.Errorf("%s failed: %w. %s", p[0], err, saveHint(model))
+ }
+ return nil
+}
+
+// player returns the command that plays the WAV file at path.
+func player(model, path string) ([]string, error) {
+ players := [][]string{{"paplay", path}, {"pw-play", path}, {"aplay", "-q", path}, {"ffplay", "-nodisp", "-autoexit", "-loglevel", "error", path}}
+ switch runtime.GOOS {
+ case "darwin":
+ players = [][]string{{"afplay", path}}
+ case "windows":
+ players = [][]string{{"powershell", "-NoProfile", "-Command", "(New-Object Media.SoundPlayer '" + strings.ReplaceAll(path, "'", "''") + "').PlaySync()"}}
+ }
+ for _, p := range players {
+ if _, err := exec.LookPath(p[0]); err == nil {
+ return p, nil
+ }
+ }
+ return nil, errors.New("no audio player found. " + saveHint(model))
+}
+
+func saveHint(model string) string {
+ return fmt.Sprintf(`To save the audio instead, run: ollama run %s "text" > speech.wav`, model)
+}
diff --git a/cmd/speech_test.go b/cmd/speech_test.go
new file mode 100644
index 000000000..59377e90a
--- /dev/null
+++ b/cmd/speech_test.go
@@ -0,0 +1,102 @@
+package cmd
+
+import (
+ "encoding/binary"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestSpeechCut(t *testing.T) {
+ long := strings.Repeat("This sentence is long enough to say. ", 3)
+ for _, tc := range []struct {
+ text string
+ want int
+ }{
+ {"Too short to say yet. ", 0},
+ {"Heading\n\nBody", 7},
+ {long + "And then", len(long) - 1},
+ {"no sentence end yet " + strings.Repeat("words ", 20), 0},
+ } {
+ if got := speechCut(tc.text); got != tc.want {
+ t.Errorf("speechCut(%q) = %d, want %d", tc.text, got, tc.want)
+ }
+ }
+}
+
+// Speech starts once a few sentences arrive, before the input ends.
+func TestSpeechPiecesStream(t *testing.T) {
+ r, w := io.Pipe()
+ pieces := speechPieces(t.Context(), r)
+ go w.Write([]byte(strings.Repeat("This sentence is long enough to say. ", 3) + "Unfinished"))
+ select {
+ case p := <-pieces:
+ if !strings.HasSuffix(p, "to say.") {
+ t.Errorf("first piece = %q", p)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("speech waited for the end of the input")
+ }
+ go func() {
+ w.Write([]byte(" tail"))
+ w.Close()
+ }()
+ var rest []string
+ for p := range pieces {
+ rest = append(rest, p)
+ }
+ if !slices.Equal(rest, []string{"Unfinished tail"}) {
+ t.Errorf("rest = %q", rest)
+ }
+}
+
+func TestSpeechChunks(t *testing.T) {
+ for _, tc := range []struct {
+ text string
+ limit int
+ want []string
+ }{
+ {"Hello.", 100, []string{"Hello."}},
+ {"One.\n\n Two.\r\n\r\nThree.", 100, []string{"One.", "Two.", "Three."}},
+ {"One two. Three four. Five six.", 21, []string{"One two. Three four.", "Five six."}},
+ {"one two three four", 10, []string{"one two", "three four"}},
+ {"abcdefghijkl", 5, []string{"abcde", "fghij", "kl"}},
+ {"ééééé", 2, []string{"éé", "éé", "é"}},
+ {" \n\n ", 100, nil},
+ } {
+ if got := speechChunks(tc.text, tc.limit); !slices.Equal(got, tc.want) {
+ t.Errorf("speechChunks(%q, %d) = %q, want %q", tc.text, tc.limit, got, tc.want)
+ }
+ }
+}
+
+func TestSynthesize(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req struct{ Input, Voice string }
+ if r.URL.Path != "/v1/audio/speech" || json.NewDecoder(r.Body).Decode(&req) != nil || req.Input == "" || req.Voice != "Toby" {
+ w.WriteHeader(http.StatusBadRequest)
+ w.Write([]byte(`{"error":{"message":"input is required"}}`))
+ return
+ }
+ wav := make([]byte, 48)
+ binary.LittleEndian.PutUint32(wav[4:], 0xffffffff)
+ binary.LittleEndian.PutUint32(wav[40:], 0xffffffff)
+ copy(wav[44:], []byte{1, 0, 2, 0})
+ w.Write(wav)
+ }))
+ defer srv.Close()
+ t.Setenv("OLLAMA_HOST", srv.URL)
+
+ wav, err := synthesize(t.Context(), "phonon", "Toby", "hello")
+ if err != nil || binary.LittleEndian.Uint32(wav[4:]) != 40 || binary.LittleEndian.Uint32(wav[40:]) != 4 {
+ t.Fatalf("WAV lengths not recorded: %x, %v", wav, err)
+ }
+ if _, err := synthesize(t.Context(), "phonon", "Toby", ""); err == nil || err.Error() != "input is required" {
+ t.Fatalf("server error lost: %v", err)
+ }
+}
diff --git a/create/create.go b/create/create.go
index 4fa0f08ac..9ba9ecabe 100644
--- a/create/create.go
+++ b/create/create.go
@@ -23,6 +23,10 @@ import (
// safetensors-backed models.
const SafetensorsMinOllamaVersion = "0.19.0"
+// phononMinOllamaVersion is the first Ollama release that runs Phonon speech
+// models.
+const phononMinOllamaVersion = "0.40.9"
+
// IsSafetensorsLLMModel checks for a safetensors completion or decision model.
func IsSafetensorsLLMModel(modelName string) bool {
name := model.ParseName(modelName)
diff --git a/create/metadata.go b/create/metadata.go
index 2756be8d0..053968f92 100644
--- a/create/metadata.go
+++ b/create/metadata.go
@@ -79,6 +79,8 @@ func inferSafetensorsConfig(modelDir string, cfg sourceModelConfig, parserOverri
if decisionVision {
capabilities = append(capabilities, "vision")
}
+ } else if cfg.Architecture() == "Phonon" {
+ capabilities = []string{"speech"}
} else {
capabilities = inferCompletionCapabilitiesFromConfig(cfg, chatTemplate, parserName)
}
diff --git a/create/metadata_test.go b/create/metadata_test.go
index 18b648f88..31c8f786d 100644
--- a/create/metadata_test.go
+++ b/create/metadata_test.go
@@ -237,6 +237,11 @@ func TestInferSafetensorsConfigFamilies(t *testing.T) {
wantRenderer: "flint",
wantCaps: []string{"completion", "tools", "thinking"},
},
+ {
+ name: "phonon",
+ config: `{"architectures":["Phonon"],"model_type":"phonon"}`,
+ wantCaps: []string{"speech"},
+ },
{
name: "nested llm config",
config: `{"model_type":"nemotron_h_omni","llm_config":{"model_type":"nemotron_h"}}`,
diff --git a/create/pipeline.go b/create/pipeline.go
index f2a127201..af1ef6efe 100644
--- a/create/pipeline.go
+++ b/create/pipeline.go
@@ -10,6 +10,8 @@ import (
"path/filepath"
"strings"
+ "golang.org/x/mod/semver"
+
"github.com/ollama/ollama/types/model"
)
@@ -70,6 +72,10 @@ func Create(ctx context.Context, modelName, modelDir string, opts PipelineOption
return err
}
}
+ // Older releases can't run Phonon, whatever the Modelfile or client says.
+ if inv.Config.Architecture() == "Phonon" && semver.Compare("v"+modelConfig.Requires, "v"+phononMinOllamaVersion) < 0 {
+ modelConfig.Requires = phononMinOllamaVersion
+ }
if err := validateMLXSource(inv.Config, false, opts.Validation); err != nil {
return err
}
@@ -79,6 +85,9 @@ func Create(ctx context.Context, modelName, modelDir string, opts PipelineOption
if inv.Config.Architecture() == "LayaForDecision" && opts.Quantize != "" {
return fmt.Errorf("Laya currently requires unquantized weights")
}
+ if inv.Config.Architecture() == "Phonon" && opts.Quantize != "" {
+ return fmt.Errorf("Phonon currently requires unquantized weights")
+ }
class, err := Classify(inv, opts.Quantize)
if err != nil {
return err
diff --git a/create/pipeline_test.go b/create/pipeline_test.go
index 9215432c2..342c8b458 100644
--- a/create/pipeline_test.go
+++ b/create/pipeline_test.go
@@ -75,6 +75,27 @@ func TestCreatePipeline(t *testing.T) {
}
}
+func TestCreatePipelinePhononRequires(t *testing.T) {
+ dir := t.TempDir()
+ writeConfigJSON(t, dir, `{"architectures":["Phonon"],"model_type":"phonon"}`)
+ createTestSafetensors(t, filepath.Join(dir, "model.safetensors"), []*st.TensorData{
+ st.NewTensorDataFromBytes("flow_lm.bos_emb", "F32", []int32{4}, make([]byte, 4*4)),
+ })
+ // Remote creates send the generic safetensors minimum; it must not win.
+ for requires, want := range map[string]string{"": phononMinOllamaVersion, "0.19.0": phononMinOllamaVersion, "0.41.0": "0.41.0"} {
+ var got string
+ opts := testPipelineOptions()
+ opts.Requires = requires
+ err := Create(context.Background(), "phonon", dir, opts, newCaptureStore(), func(_ context.Context, _ string, info ManifestInfo) error {
+ got = info.ModelConfig.Requires
+ return nil
+ }, func(string) {})
+ if err != nil || got != want {
+ t.Errorf("requires %q: got %q, %v; want %q", requires, got, err, want)
+ }
+ }
+}
+
func TestCreatePipelineRejectsUnsupportedArchitectureBeforeWriting(t *testing.T) {
dir := t.TempDir()
writeConfigJSON(t, dir, `{"architectures":["UnsupportedForCausalLM"]}`)
diff --git a/docs/api/openai-compatibility.mdx b/docs/api/openai-compatibility.mdx
index 272f98ea7..e577ac345 100644
--- a/docs/api/openai-compatibility.mdx
+++ b/docs/api/openai-compatibility.mdx
@@ -315,6 +315,31 @@ curl -X POST http://localhost:11434/v1/chat/completions \
- [x] `dimensions`
- [ ] `user`
+### `/v1/audio/speech`
+
+> Note: Added in Ollama v0.40.8
+
+Generates audio with [speech models](/capabilities/speech). The response streams while audio is generated.
+
+#### Supported request fields
+
+- [x] `model`
+- [x] `input` (up to 4,096 characters)
+- [x] `voice` (one of the model's voice names, such as `Freya`; omit it for the model's default)
+- [x] `response_format`
+ - [x] `wav` (default)
+ - [x] `pcm`
+ - [ ] `mp3`, `opus`, `aac`, `flac`
+- [x] `speed` (`1` only)
+- [x] `stream_format` (`audio` only)
+- [ ] `instructions`
+
+#### Notes
+
+- Audio is 16-bit, mono, 24 kHz. `pcm` returns raw little-endian samples with no header.
+- The default `response_format` is `wav`, not `mp3`.
+- Unsupported values return a `400` error instead of being ignored.
+
### Responses API
`POST /v1/responses`
diff --git a/docs/capabilities/speech.mdx b/docs/capabilities/speech.mdx
new file mode 100644
index 000000000..1323221bc
--- /dev/null
+++ b/docs/capabilities/speech.mdx
@@ -0,0 +1,128 @@
+---
+title: Speech
+description: Turn text into spoken audio with text-to-speech models.
+---
+
+Speech models read text aloud. Ollama streams the audio as it is generated, so playback can start before the whole text has been spoken.
+
+## Setup
+
+Start [Ollama](/quickstart) v0.40.8 or later, then download [Phonon](https://ollama.com/library/phonon):
+
+```shell setup.sh
+ollama pull phonon
+```
+
+Phonon speaks English with four voices: `Freya` (the default), `Harper`, `Sterling`, and `Toby`. Voice names are case-sensitive.
+
+## Speak from the terminal
+
+`ollama run` plays the audio through your computer's speakers:
+
+```shell examples.sh
+# Speak some text
+ollama run phonon "Hello from Ollama."
+
+# Choose a voice
+ollama run phonon --voice Toby "Hi, this is Toby."
+
+# Speak text from another program
+echo "Text from another program works too." | ollama run phonon
+
+# Speak a chat model's answer
+ollama run gemma4 --hidethinking "Tell me a fun fact about owls in two sentences." | ollama run phonon
+```
+
+Piped text is spoken as it arrives, so a chat model's answer starts playing after its first sentence or two.
+
+Run `ollama run phonon` without any text to type lines at a `>>>` prompt. Each line is spoken when you press Enter. Press Ctrl+C to stop speaking, and type `/bye` or press Ctrl+D to exit.
+
+Audio plays with your system's built-in player: `afplay` on macOS, the first of `paplay`, `pw-play`, `aplay`, or `ffplay` that is installed on Linux, and PowerShell on Windows.
+
+### Save audio to a file
+
+When the output is redirected, `ollama run` writes a WAV file instead of playing it. This also works on servers without speakers:
+
+```shell
+ollama run phonon "Save this to a file." > speech.wav
+```
+
+To hear a remote server, run `ollama` on your own computer and point it at the server. The audio plays where the command runs:
+
+```shell
+OLLAMA_HOST=my-server:11434 ollama run phonon "Hello from the server."
+```
+
+## Generate speech with the API
+
+Use the OpenAI-compatible `/v1/audio/speech` endpoint:
+
+<Tabs>
+ <Tab title="cURL">
+ ```shell
+ curl http://localhost:11434/v1/audio/speech \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "phonon",
+ "input": "Hello from Ollama.",
+ "voice": "Freya"
+ }' --output speech.wav
+ ```
+ </Tab>
+ <Tab title="Python">
+ ```python
+ from openai import OpenAI
+
+ client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")
+
+ with client.audio.speech.with_streaming_response.create(
+ model="phonon",
+ voice="Freya",
+ input="Hello from Ollama.",
+ response_format="wav",
+ ) as response:
+ response.stream_to_file("speech.wav")
+ ```
+ </Tab>
+ <Tab title="JavaScript">
+ ```javascript
+ import fs from 'node:fs'
+ import OpenAI from 'openai'
+
+ const openai = new OpenAI({ baseURL: 'http://localhost:11434/v1', apiKey: 'ollama' })
+
+ const response = await openai.audio.speech.create({
+ model: 'phonon',
+ voice: 'Freya',
+ input: 'Hello from Ollama.',
+ response_format: 'wav',
+ })
+ fs.writeFileSync('speech.wav', Buffer.from(await response.arrayBuffer()))
+ ```
+ </Tab>
+</Tabs>
+
+The response streams while audio is generated, so you can start playing it right away:
+
+```shell
+curl -sN http://localhost:11434/v1/audio/speech \
+ -H "Content-Type: application/json" \
+ -d '{"model": "phonon", "input": "Playback starts while the rest is still being generated."}' \
+ | ffplay -nodisp -autoexit -loglevel error -
+```
+
+### Audio formats
+
+| `response_format` | Output |
+| --- | --- |
+| `wav` (default) | A WAV file: 16-bit, mono, 24 kHz. The header is written before the length is known, so its size fields hold the maximum value. |
+| `pcm` | Raw 16-bit little-endian samples, mono, 24 kHz, with no header. |
+
+### Limits
+
+- `input` can be up to 4,096 characters per request. `ollama run` splits longer text at paragraph and sentence breaks for you.
+- Text is spoken as written. Numbers, dates, and abbreviations are not expanded into words, so write them out (`twenty-one`, not `21`) for predictable results. Emoji, bullet characters, and `*` emphasis markers are skipped.
+- Options the model doesn't support, such as other formats, speeds, or `instructions`, return a `400` error instead of being ignored. An unknown voice returns a `400` error that lists the available voices.
+- Speech models only generate speech: chat, completion, and embedding requests to them return a `400` error.
+
+`ollama show phonon` and `/api/show` list the model's capabilities as `speech`. See [OpenAI compatibility](/api/openai-compatibility) for every supported field.
diff --git a/docs/cli.mdx b/docs/cli.mdx
index ae4215544..e93d3e5e2 100644
--- a/docs/cli.mdx
+++ b/docs/cli.mdx
@@ -96,6 +96,18 @@ ollama run nimble --ask "How urgent is this?" \
The state is piped in or given after the model name, as text or JSON. Each `--ask` question is answered yes or no, unless it's followed by labels to choose from (`--choice`) or levels to score it on (`--level`). See [Decision](/capabilities/decision#ask-from-the-terminal).
+### Speak with a speech model
+
+```
+ollama run phonon "Hello from Ollama."
+
+ollama run phonon --voice Toby "Hi, this is Toby."
+
+ollama run phonon "Save this to a file." > speech.wav
+```
+
+The audio plays through your speakers, or is written as a WAV file when the output is redirected. Run it without text to speak each line you type. See [Speech](/capabilities/speech#speak-from-the-terminal).
+
### Download a model
```
diff --git a/docs/docs.json b/docs/docs.json
index ed96b7167..7e567e5aa 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -131,6 +131,7 @@
"/capabilities/decision",
"/capabilities/vision",
"/capabilities/embeddings",
+ "/capabilities/speech",
"/capabilities/tool-calling",
"/capabilities/web-search"
]
diff --git a/llm/speech.go b/llm/speech.go
new file mode 100644
index 000000000..b08fdce85
--- /dev/null
+++ b/llm/speech.go
@@ -0,0 +1,15 @@
+package llm
+
+import "context"
+
+// SpeechRequest asks a speech model to read Input aloud.
+type SpeechRequest struct {
+ Input string `json:"input"`
+ Voice string `json:"voice,omitempty"`
+}
+
+// Speaker is implemented by runners that serve speech models. Speech streams
+// 24 kHz mono 16-bit little-endian PCM to fn as it is generated.
+type Speaker interface {
+ Speech(ctx context.Context, req SpeechRequest, fn func(pcm []byte) error) error
+}
diff --git a/mlxrunner/client.go b/mlxrunner/client.go
index 649473622..0d46c8321 100644
--- a/mlxrunner/client.go
+++ b/mlxrunner/client.go
@@ -604,6 +604,14 @@ func (c *Client) Tokenize(ctx context.Context, content string) ([]int, error) {
}
defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ return nil, api.StatusError{StatusCode: resp.StatusCode, ErrorMessage: strings.TrimSpace(string(body))}
+ }
+
var tokens []int
if err := json.NewDecoder(resp.Body).Decode(&tokens); err != nil {
return nil, err
diff --git a/mlxrunner/client_speech.go b/mlxrunner/client_speech.go
new file mode 100644
index 000000000..b29b6c17e
--- /dev/null
+++ b/mlxrunner/client_speech.go
@@ -0,0 +1,61 @@
+package mlxrunner
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+)
+
+var _ llm.Speaker = (*Client)(nil)
+
+func (c *Client) Speech(ctx context.Context, input llm.SpeechRequest, fn func([]byte) error) error {
+ data, err := json.Marshal(input)
+ if err != nil {
+ return err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/v1/audio/speech", c.port), bytes.NewReader(data))
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ res, err := c.client.Do(req)
+ if err != nil {
+ if c.status != nil {
+ if errMsg := c.status.LastError(); errMsg != "" {
+ return fmt.Errorf("mlx runner failed: %s", errMsg)
+ }
+ }
+ return err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ body, err := io.ReadAll(res.Body)
+ if err != nil {
+ return err
+ }
+ return api.StatusError{StatusCode: res.StatusCode, ErrorMessage: strings.TrimSpace(string(body))}
+ }
+
+ buf := make([]byte, 32<<10)
+ for {
+ n, err := res.Body.Read(buf)
+ if n > 0 {
+ if err := fn(buf[:n]); err != nil {
+ return err
+ }
+ }
+ if errors.Is(err, io.EOF) {
+ return nil
+ } else if err != nil {
+ return err
+ }
+ }
+}
diff --git a/mlxrunner/client_speech_test.go b/mlxrunner/client_speech_test.go
new file mode 100644
index 000000000..a9027de8c
--- /dev/null
+++ b/mlxrunner/client_speech_test.go
@@ -0,0 +1,195 @@
+package mlxrunner
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/mlx/mlxtest"
+ "github.com/ollama/ollama/mlxrunner/model"
+)
+
+func TestClientSpeech(t *testing.T) {
+ for _, status := range []int{200, 400} {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/audio/speech" || r.Method != "POST" {
+ t.Errorf("unexpected request: %s %s", r.Method, r.URL)
+ }
+ if status != 200 {
+ http.Error(w, "unknown voice", status)
+ return
+ }
+ for _, b := range [][]byte{{1}, {0, 2}, {0}} {
+ _, _ = w.Write(b)
+ w.(http.Flusher).Flush()
+ }
+ }))
+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}
+ var got bytes.Buffer
+ err := client.Speech(t.Context(), llm.SpeechRequest{Input: "hello", Voice: "Freya"}, func(b []byte) error { _, err := got.Write(b); return err })
+ srv.Close()
+ if status == 200 {
+ if err != nil || !bytes.Equal(got.Bytes(), []byte{1, 0, 2, 0}) {
+ t.Fatalf("audio: %x, %v", got.Bytes(), err)
+ }
+ } else {
+ var se api.StatusError
+ if !errors.As(err, &se) || se.StatusCode != 400 || se.ErrorMessage != "unknown voice" {
+ t.Fatalf("error lost: %v", err)
+ }
+ }
+ }
+}
+
+func TestClientSpeechCancellation(t *testing.T) {
+ started := make(chan struct{})
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ close(started)
+ <-r.Context().Done()
+ }))
+ defer srv.Close()
+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ done := make(chan error, 1)
+ go func() { done <- client.Speech(ctx, llm.SpeechRequest{}, func([]byte) error { return nil }) }()
+ <-started
+ cancel()
+ if err := <-done; !errors.Is(err, context.Canceled) {
+ t.Fatalf("cancellation lost: %v", err)
+ }
+}
+
+func TestPCM16(t *testing.T) {
+ got := pcm16([]float32{-2, -1, 0, 1, 2})
+ want := []byte{1, 128, 1, 128, 0, 0, 255, 127, 255, 127}
+ if !bytes.Equal(got, want) {
+ t.Fatalf("PCM: %x, want %x", got, want)
+ }
+}
+
+type speechTestModel struct {
+ model.Model
+ chunks int
+ generated chan struct{}
+}
+
+func (m speechTestModel) Speak(_ context.Context, _ llm.SpeechRequest, emit func([]float32) error) error {
+ defer close(m.generated)
+ samples := make([]float32, 1<<16)
+ for range m.chunks {
+ if err := emit(samples); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// A client reading at playback speed must not hold the MLX worker.
+func TestSpeechDoesNotWaitForClient(t *testing.T) {
+ m := speechTestModel{chunks: 256, generated: make(chan struct{})} // 32 MiB, more than socket buffers hold
+ r := &Runner{Model: m, mlxThread: mlxtest.Worker(t)}
+ srv := httptest.NewServer(http.HandlerFunc(r.speechHandler))
+ defer srv.Close()
+ res, err := srv.Client().Post(srv.URL, "application/json", strings.NewReader("{}"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+ select {
+ case <-m.generated:
+ case <-time.After(10 * time.Second):
+ t.Fatal("generation waited for the client to read")
+ }
+ if n, err := io.Copy(io.Discard, res.Body); err != nil || n != 256<<17 {
+ t.Fatalf("read %d bytes: %v", n, err)
+ }
+}
+
+func TestClientSpeechInterruptedStream(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte{1, 0})
+ w.(http.Flusher).Flush()
+ panic(http.ErrAbortHandler)
+ }))
+ defer srv.Close()
+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}
+ var got bytes.Buffer
+ err := client.Speech(t.Context(), llm.SpeechRequest{}, func(b []byte) error { _, err := got.Write(b); return err })
+ if !errors.Is(err, io.ErrUnexpectedEOF) || !bytes.Equal(got.Bytes(), []byte{1, 0}) {
+ t.Fatalf("interrupted audio: %x, %v", got.Bytes(), err)
+ }
+}
+
+func TestClientSpeechConsumerFailure(t *testing.T) {
+ canceled := make(chan struct{})
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ io.Copy(io.Discard, r.Body)
+ w.Write([]byte{1, 0})
+ w.(http.Flusher).Flush()
+ <-r.Context().Done()
+ close(canceled)
+ }))
+ defer srv.Close()
+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}
+ stop := errors.New("consumer stopped reading")
+ err := client.Speech(t.Context(), llm.SpeechRequest{}, func([]byte) error { return stop })
+ if !errors.Is(err, stop) {
+ t.Fatalf("consumer error lost: %v", err)
+ }
+ select {
+ case <-canceled:
+ case <-time.After(5 * time.Second):
+ t.Fatal("runner request was not canceled")
+ }
+}
+
+// The runner's logging wrapper must preserve HTTP controls for streaming.
+type controlledSpeechWriter struct {
+ *httptest.ResponseRecorder
+ deadline time.Time
+ flushErr error
+}
+
+func (w *controlledSpeechWriter) SetWriteDeadline(t time.Time) error {
+ w.deadline = t
+ return nil
+}
+
+func (w *controlledSpeechWriter) FlushError() error { return w.flushErr }
+
+func TestSpeechResponseControls(t *testing.T) {
+ want := errors.New("write failed")
+ w := &controlledSpeechWriter{ResponseRecorder: httptest.NewRecorder(), flushErr: want}
+ controller := http.NewResponseController(&statusRecorder{ResponseWriter: w})
+ deadline := time.Now().Add(time.Second)
+ if err := controller.SetWriteDeadline(deadline); err != nil || !w.deadline.Equal(deadline) {
+ t.Fatalf("write deadline lost: %v, %v", w.deadline, err)
+ }
+ if err := controller.Flush(); !errors.Is(err, want) {
+ t.Fatalf("flush error lost: %v", err)
+ }
+}
+
+func TestSpeechModelRejectsTextTokenization(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "this model does not support text tokenization", http.StatusBadRequest)
+ }))
+ defer srv.Close()
+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}
+ _, err := client.Tokenize(t.Context(), "hello")
+ var status api.StatusError
+ if !errors.As(err, &status) || status.StatusCode != 400 || status.ErrorMessage != "this model does not support text tokenization" {
+ t.Fatalf("tokenization error lost: %v", err)
+ }
+}
diff --git a/mlxrunner/model/architectures/architectures.go b/mlxrunner/model/architectures/architectures.go
index c5a374e97..ec5b1ed8c 100644
--- a/mlxrunner/model/architectures/architectures.go
+++ b/mlxrunner/model/architectures/architectures.go
@@ -15,6 +15,7 @@ import (
_ "github.com/ollama/ollama/mlxrunner/model/laya"
_ "github.com/ollama/ollama/mlxrunner/model/llama"
_ "github.com/ollama/ollama/mlxrunner/model/nemotron_h"
+ _ "github.com/ollama/ollama/mlxrunner/model/phonon"
_ "github.com/ollama/ollama/mlxrunner/model/qwen3"
_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5"
_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5_moe"
diff --git a/mlxrunner/model/phonon/flow.go b/mlxrunner/model/phonon/flow.go
new file mode 100644
index 000000000..8650caf6f
--- /dev/null
+++ b/mlxrunner/model/phonon/flow.go
@@ -0,0 +1,215 @@
+package phonon
+
+import (
+ "fmt"
+ "math"
+
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlxrunner/batch"
+ "github.com/ollama/ollama/mlxrunner/cache"
+ "github.com/ollama/ollama/mlxrunner/nn"
+)
+
+// Layer is a pre-norm transformer layer, shared by the flow LM and the Mimi
+// decoder. Scale1 and Scale2 are Mimi's layer scales.
+type Layer struct {
+ Norm1 *nn.LayerNorm
+ Norm2 *nn.LayerNorm
+ InProj *nn.Linear
+ OutProj *nn.Linear
+ Linear1 *nn.Linear
+ Linear2 *nn.Linear
+ Scale1 *mlx.Array
+ Scale2 *mlx.Array
+}
+
+type Transformer struct {
+ Layers []*Layer
+ NumHeads int32
+ MaxPeriod float32
+ GELU string
+}
+
+func loadTransformer(tensors map[string]*mlx.Array, prefix string, layers, heads int32, period float32, gelu string) (*Transformer, error) {
+ t := &Transformer{NumHeads: heads, MaxPeriod: period, GELU: gelu}
+ for i := range layers {
+ p := fmt.Sprintf("%s.layers.%d.", prefix, i)
+ l := &Layer{
+ Norm1: layerNorm(tensors, p+"norm1", 1e-5),
+ Norm2: layerNorm(tensors, p+"norm2", 1e-5),
+ InProj: linear(tensors, p+"self_attn.in_proj"),
+ OutProj: linear(tensors, p+"self_attn.out_proj"),
+ Linear1: linear(tensors, p+"linear1"),
+ Linear2: linear(tensors, p+"linear2"),
+ // The runner loads "<name>.scale" tensors as "<name>_scale".
+ Scale1: tensors[p+"layer_scale_1_scale"],
+ Scale2: tensors[p+"layer_scale_2_scale"],
+ }
+ if l.Norm1 == nil || l.Norm2 == nil || l.InProj == nil || l.OutProj == nil || l.Linear1 == nil || l.Linear2 == nil {
+ return nil, fmt.Errorf("missing weights for %s", p)
+ }
+ if heads <= 0 || l.Norm1.Weight.Dim(0)%int(heads) != 0 {
+ return nil, fmt.Errorf("invalid attention heads for %s", p)
+ }
+ t.Layers = append(t.Layers, l)
+ }
+ return t, nil
+}
+
+// forward runs x [1, L, D] at absolute position offset.
+func (t *Transformer) forward(x *mlx.Array, offset int, caches []cache.Attention) *mlx.Array {
+ L, D := int32(x.Dim(1)), int32(x.Dim(2))
+ // Attention uses the batch shape and positions for both text and audio frames.
+ b := &batch.Batch{
+ InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, int(L)),
+ SeqOffsets: []int32{int32(offset)},
+ SeqQueryLens: []int32{L},
+ }
+ H, hd := t.NumHeads, D/t.NumHeads
+ scale := float32(1 / math.Sqrt(float64(hd)))
+ pos := mlx.FromValues([]int32{int32(offset)}, 1)
+ split := func(qkv *mlx.Array, i int32) *mlx.Array {
+ s := qkv.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(int(i*D), int((i+1)*D)))
+ return mlx.Transpose(mlx.Reshape(s, 1, L, H, hd), 0, 2, 1, 3)
+ }
+ for i, l := range t.Layers {
+ qkv := l.InProj.Forward(l.Norm1.Forward(x))
+ q := mlx.RoPEWithBase(split(qkv, 0), int(hd), true, t.MaxPeriod, 1, pos)
+ k := mlx.RoPEWithBase(split(qkv, 1), int(hd), true, t.MaxPeriod, 1, pos)
+ history := caches[i].Update(b, k, split(qkv, 2))
+ a := nn.ScaledDotProductAttention(b, q, scale, nn.WithKVHistory(history), nn.WithMask(nn.CausalMask()))
+ a = mlx.Reshape(mlx.Transpose(a, 0, 2, 1, 3), 1, L, D)
+ x = residual(x, l.OutProj.Forward(a), l.Scale1)
+ h := l.Linear1.Forward(l.Norm2.Forward(x))
+ if t.GELU == "erf" {
+ h = mlx.GELU(h)
+ } else {
+ h = mlx.GELUApprox(h)
+ }
+ h = l.Linear2.Forward(h)
+ x = residual(x, h, l.Scale2)
+ }
+ return x
+}
+
+func residual(x, update, scale *mlx.Array) *mlx.Array {
+ if scale != nil {
+ update = mlx.Mul(update, scale)
+ }
+ return mlx.Add(x, update)
+}
+
+// FlowHead is the AdaLN MLP that turns a transformer state into a latent with
+// one Lagrangian self-distillation step from s=0 to t=1.
+type FlowHead struct {
+ CondEmbed *nn.Linear
+ InputProj *nn.Linear
+ Blocks []*FlowBlock
+ Final *nn.Linear
+ // Modulation stacks every block's adaLN projection and the final
+ // layer's so one matmul produces all shifts, scales and gates.
+ Modulation *nn.Linear
+ // Time is the averaged embedding of the constant s=0, t=1 times.
+ Time *mlx.Array
+}
+
+type FlowBlock struct {
+ Norm *nn.LayerNorm
+ MLP0 *nn.Linear
+ MLP2 *nn.Linear
+}
+
+func loadFlowHead(tensors map[string]*mlx.Array, prefix string, depth int32, norm string) (*FlowHead, error) {
+ f := &FlowHead{
+ CondEmbed: linear(tensors, prefix+".cond_embed"),
+ InputProj: linear(tensors, prefix+".input_proj"),
+ Final: linear(tensors, prefix+".final_layer.linear"),
+ }
+ var mw, mb []*mlx.Array
+ for i := range depth {
+ p := fmt.Sprintf("%s.res_blocks.%d.", prefix, i)
+ b := &FlowBlock{Norm: layerNorm(tensors, p+"in_ln", 1e-6), MLP0: linear(tensors, p+"mlp.0"), MLP2: linear(tensors, p+"mlp.2")}
+ ada := linear(tensors, p+"adaLN_modulation.1")
+ if b.Norm == nil || b.MLP0 == nil || b.MLP2 == nil || ada == nil {
+ return nil, fmt.Errorf("missing weights for %s", p)
+ }
+ f.Blocks = append(f.Blocks, b)
+ mw, mb = append(mw, ada.Weight), append(mb, ada.Bias)
+ }
+ ada := linear(tensors, prefix+".final_layer.adaLN_modulation.1")
+ if f.CondEmbed == nil || f.InputProj == nil || f.Final == nil || ada == nil {
+ return nil, fmt.Errorf("missing weights for %s", prefix)
+ }
+ f.Modulation = nn.NewLinear(mlx.Concatenate(append(mw, ada.Weight), 0), mlx.Concatenate(append(mb, ada.Bias), 0))
+
+ // LSD conditions on a start and a target time; with one step they are
+ // always 0 and 1.
+ t0, t1 := timeEmbed(tensors, prefix+".time_embed.0", 0, norm), timeEmbed(tensors, prefix+".time_embed.1", 1, norm)
+ if t0 == nil || t1 == nil {
+ return nil, fmt.Errorf("missing time embedding for %s", prefix)
+ }
+ f.Time = mlx.MulScalar(mlx.Add(t0, t1), 0.5)
+ return f, nil
+}
+
+// timeEmbed evaluates a TimestepEmbedder at t with the checkpoint's norm.
+func timeEmbed(tensors map[string]*mlx.Array, prefix string, t float32, norm string) *mlx.Array {
+ freqs := tensors[prefix+".freqs"]
+ l0, l2, alpha := linear(tensors, prefix+".mlp.0"), linear(tensors, prefix+".mlp.2"), tensors[prefix+".mlp.3.alpha"]
+ if freqs == nil || l0 == nil || l2 == nil || alpha == nil {
+ return nil
+ }
+ args := mlx.MulScalar(freqs, t)
+ x := mlx.Concatenate([]*mlx.Array{mlx.Cos(args), mlx.Sin(args)}, 0)
+ x = l2.Forward(mlx.SiLU(l0.Forward(x)))
+ return timeNorm(x, alpha, norm)
+}
+
+func timeNorm(x, alpha *mlx.Array, norm string) *mlx.Array {
+ if norm == "rms" {
+ return mlx.RMSNormFn(x, alpha, 1e-5)
+ }
+ // Legacy checkpoints divide by the unbiased standard deviation, without centering x.
+ n := float32(x.Dim(0))
+ centered := mlx.Sub(x, mlx.Mean(x, -1, true))
+ variance := mlx.DivScalar(mlx.Sum(mlx.Mul(centered, centered), -1, true), n-1)
+ return mlx.Mul(mlx.Mul(x, alpha), mlx.RSqrt(mlx.AddScalar(variance, 1e-5)))
+}
+
+// sample returns the latent for state c [1, D] from noise [1, latent].
+func (f *FlowHead) sample(c, noise *mlx.Array) *mlx.Array {
+ y := mlx.Add(f.CondEmbed.Forward(c), f.Time)
+ mod := f.Modulation.Forward(mlx.SiLU(y))
+ dim := y.Dim(1)
+ chunk := func(i int) *mlx.Array { return mod.Slice(mlx.Slice(), mlx.Slice(i*dim, (i+1)*dim)) }
+
+ x := f.InputProj.Forward(noise)
+ for i, b := range f.Blocks {
+ h := modulate(b.Norm.Forward(x), chunk(3*i), chunk(3*i+1))
+ h = b.MLP2.Forward(mlx.SiLU(b.MLP0.Forward(h)))
+ x = mlx.Add(x, mlx.Mul(chunk(3*i+2), h))
+ }
+ n := 3 * len(f.Blocks)
+ x = modulate(mlx.LayerNormFn(x, nil, nil, 1e-6), chunk(n), chunk(n+1))
+ return mlx.Add(noise, f.Final.Forward(x))
+}
+
+func modulate(x, shift, scale *mlx.Array) *mlx.Array {
+ return mlx.Add(mlx.Mul(x, mlx.AddScalar(scale, 1)), shift)
+}
+
+func linear(tensors map[string]*mlx.Array, prefix string) *nn.Linear {
+ w := tensors[prefix+".weight"]
+ if w == nil {
+ return nil
+ }
+ return nn.NewLinear(w, tensors[prefix+".bias"])
+}
+
+func layerNorm(tensors map[string]*mlx.Array, prefix string, eps float32) *nn.LayerNorm {
+ w := tensors[prefix+".weight"]
+ if w == nil {
+ return nil
+ }
+ return &nn.LayerNorm{Weight: w, Bias: tensors[prefix+".bias"], Eps: eps}
+}
diff --git a/mlxrunner/model/phonon/mimi.go b/mlxrunner/model/phonon/mimi.go
new file mode 100644
index 000000000..919ebb3bb
--- /dev/null
+++ b/mlxrunner/model/phonon/mimi.go
@@ -0,0 +1,204 @@
+package phonon
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlxrunner/cache"
+)
+
+// Mimi is the streaming decoder half of the Mimi codec: latent frames in,
+// 24 kHz samples out.
+type Mimi struct {
+ OutputProj *mlx.Array // [latent, dim]
+ Upsample *ConvTr
+ Transformer *Transformer
+ Context int
+ Decoder []*SeanetLayer
+}
+
+// SeanetLayer is one decoder stage; a layer with no weights is an ELU.
+type SeanetLayer struct {
+ Conv *Conv
+ ConvTr *ConvTr
+ Res []*Conv
+}
+
+// Conv is a causal stride-1 convolution. Weight is [out, kernel, in].
+type Conv struct {
+ Weight *mlx.Array
+ Bias *mlx.Array
+ Dilation int
+}
+
+// ConvTr is a transposed convolution whose kernel is twice its stride, as
+// every Mimi upsampler's is. Weight is [in, kernel*out], or [kernel, channels]
+// when depthwise.
+type ConvTr struct {
+ Weight *mlx.Array
+ Bias *mlx.Array
+ Stride int
+ Depthwise bool
+}
+
+func loadMimi(tensors map[string]*mlx.Array, cfg *config) (*Mimi, error) {
+ t := cfg.Mimi
+ m := &Mimi{Context: t.Context}
+ if w := tensors["mimi.quantizer.output_proj.weight"]; w != nil {
+ m.OutputProj = mlx.Transpose(mlx.Squeeze(w, 2), 1, 0)
+ }
+ if w := tensors["mimi.upsample.convtr.convtr.weight"]; w != nil {
+ m.Upsample = &ConvTr{Weight: mlx.Transpose(mlx.Squeeze(w, 1), 1, 0), Stride: w.Dim(2) / 2, Depthwise: true}
+ }
+ if m.OutputProj == nil || m.Upsample == nil {
+ return nil, fmt.Errorf("missing mimi projection weights")
+ }
+ var err error
+ if m.Transformer, err = loadTransformer(tensors, "mimi.decoder_transformer.transformer", t.NumLayers, t.NumHeads, t.MaxPeriod, cfg.gelu(t.GELU)); err != nil {
+ return nil, err
+ }
+
+ const prefix = "mimi.decoder.model."
+ last := -1
+ for name := range tensors {
+ if rest, ok := strings.CutPrefix(name, prefix); ok {
+ index, _, ok := strings.Cut(rest, ".")
+ if i, err := strconv.Atoi(index); ok && err == nil {
+ last = max(last, i)
+ }
+ }
+ }
+ dilation := 1
+ for i := 0; i <= last; i++ {
+ p := prefix + strconv.Itoa(i)
+ l := &SeanetLayer{}
+ switch {
+ case tensors[p+".conv.weight"] != nil:
+ l.Conv = conv(tensors, p+".conv", 1)
+ case tensors[p+".convtr.weight"] != nil:
+ w := tensors[p+".convtr.weight"]
+ in, out, k := w.Dim(0), w.Dim(1), w.Dim(2)
+ l.ConvTr = &ConvTr{
+ Weight: mlx.Reshape(mlx.Transpose(w, 0, 2, 1), int32(in), int32(k*out)),
+ Bias: tensors[p+".convtr.bias"],
+ Stride: k / 2,
+ }
+ dilation = 1
+ case tensors[p+".block.1.conv.weight"] != nil:
+ if tensors[p+".block.3.conv.weight"] == nil {
+ return nil, fmt.Errorf("missing residual weights for %s", p)
+ }
+ l.Res = []*Conv{conv(tensors, p+".block.1.conv", dilation), conv(tensors, p+".block.3.conv", 1)}
+ dilation *= cfg.Mimi.DilationBase
+ }
+ m.Decoder = append(m.Decoder, l)
+ }
+ if last < 0 {
+ return nil, fmt.Errorf("missing mimi decoder weights")
+ }
+ return m, nil
+}
+
+func conv(tensors map[string]*mlx.Array, prefix string, dilation int) *Conv {
+ return &Conv{Weight: mlx.Transpose(tensors[prefix+".weight"], 0, 2, 1), Bias: tensors[prefix+".bias"], Dilation: dilation}
+}
+
+// mimiState is one stream's decoder state: convolution context, transposed
+// convolution overlap, and the transformer's attention window.
+type mimiState struct {
+ scope *mlx.Scope
+ conv map[*Conv]*mlx.Array
+ transpose map[*ConvTr]*mlx.Array
+ attention []cache.Attention
+ pos int
+}
+
+func (m *Mimi) newState() *mimiState {
+ s := &mimiState{
+ scope: mlx.NewScope(),
+ conv: make(map[*Conv]*mlx.Array),
+ transpose: make(map[*ConvTr]*mlx.Array),
+ }
+ for range m.Transformer.Layers {
+ s.attention = append(s.attention, cache.NewRotatingKVCache(m.Context))
+ }
+ return s
+}
+
+func (s *mimiState) close() {
+ for _, c := range s.attention {
+ c.Free()
+ }
+ s.scope.Close()
+}
+
+// decode turns normalized latents [1, T, latent] into samples [1, T*frame, 1].
+func (m *Mimi) decode(s *mimiState, latents *mlx.Array) *mlx.Array {
+ x := latents.Matmul(m.OutputProj)
+ x = m.Upsample.forward(s, x)
+ steps := x.Dim(1)
+ x = m.Transformer.forward(x, s.pos, s.attention)
+ s.pos += steps
+ for _, l := range m.Decoder {
+ switch {
+ case l.Conv != nil:
+ x = l.Conv.forward(s, x)
+ case l.ConvTr != nil:
+ x = l.ConvTr.forward(s, x)
+ case l.Res != nil:
+ x = mlx.Add(x, l.Res[1].forward(s, elu(l.Res[0].forward(s, elu(x)))))
+ default:
+ x = elu(x)
+ }
+ }
+ return x
+}
+
+func (c *Conv) forward(s *mimiState, x *mlx.Array) *mlx.Array {
+ if pad := (c.Weight.Dim(1) - 1) * c.Dilation; pad > 0 {
+ prev := s.conv[c]
+ if prev == nil {
+ prev = mlx.Zeros(x.DType(), 1, pad, x.Dim(2))
+ s.scope.Attach(prev)
+ s.conv[c] = prev
+ }
+ x = prev.Concatenate(1, x)
+ prev.Set(x.Slice(mlx.Slice(), mlx.Slice(x.Dim(1)-pad, x.Dim(1)), mlx.Slice()))
+ }
+ return mlx.Conv1d(x, c.Weight, c.Bias, 1, 0, int32(c.Dilation), 1)
+}
+
+// forward computes every kernel tap with one matmul, then overlap-adds the
+// second half of each step onto the first half of the next.
+func (c *ConvTr) forward(s *mimiState, x *mlx.Array) *mlx.Array {
+ T, S := x.Dim(1), c.Stride
+ var z *mlx.Array
+ if c.Depthwise {
+ z = mlx.Mul(mlx.ExpandDims(x, 2), c.Weight)
+ } else {
+ z = mlx.Reshape(x.Matmul(c.Weight), 1, int32(T), int32(2*S), -1)
+ }
+ C := z.Dim(3)
+ head := z.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(0, S), mlx.Slice())
+ tail := z.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(S, 2*S), mlx.Slice())
+ prev := s.transpose[c]
+ if prev == nil {
+ prev = mlx.Zeros(x.DType(), 1, 1, S, C)
+ s.scope.Attach(prev)
+ s.transpose[c] = prev
+ }
+ shifted := prev.Concatenate(1, tail.Slice(mlx.Slice(), mlx.Slice(0, T-1), mlx.Slice(), mlx.Slice()))
+ prev.Set(tail.Slice(mlx.Slice(), mlx.Slice(T-1, T), mlx.Slice(), mlx.Slice()))
+ y := mlx.Reshape(mlx.Add(head, shifted), 1, int32(T*S), int32(C))
+ if c.Bias != nil {
+ y = mlx.Add(y, c.Bias)
+ }
+ return y
+}
+
+var elu = mlx.Compile1("ELU", func(x *mlx.Array) *mlx.Array {
+ zero := mlx.FromValue[float32](0).AsType(x.DType())
+ return mlx.Where(x.Greater(zero), x, mlx.AddScalar(mlx.Exp(mlx.Minimum(x, zero)), -1))
+}, mlx.Shapeless())
diff --git a/mlxrunner/model/phonon/model.go b/mlxrunner/model/phonon/model.go
new file mode 100644
index 000000000..020c1b91a
--- /dev/null
+++ b/mlxrunner/model/phonon/model.go
@@ -0,0 +1,349 @@
+package phonon
+
+import (
+ "context"
+ "fmt"
+ "maps"
+ "math"
+ "math/rand/v2"
+ "net/http"
+ "slices"
+ "strings"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlxrunner/batch"
+ "github.com/ollama/ollama/mlxrunner/cache"
+ "github.com/ollama/ollama/mlxrunner/model"
+ "github.com/ollama/ollama/mlxrunner/nn"
+ "github.com/ollama/ollama/mlxrunner/tokenizer"
+)
+
+func init() { model.Register("Phonon", newModel) }
+
+// config follows the upstream Phonon config.json.
+type config struct {
+ ModelID *struct {
+ Sig string `json:"sig"`
+ Epoch int `json:"epoch"`
+ } `json:"model_id"`
+ FlowLM struct {
+ NumHeads int32 `json:"num_heads"`
+ NumLayers int32 `json:"num_layers"`
+ MaxPeriod float32 `json:"max_period"`
+ FlowDepth int32 `json:"flow_depth"`
+ GELU string `json:"gelu"`
+ TimeRMSNorm string `json:"time_rms_norm"`
+ } `json:"flow_lm"`
+ Mimi struct {
+ SampleRate int `json:"sample_rate"`
+ FrameRate float64 `json:"frame_rate"`
+ DilationBase int `json:"dilation_base"`
+ NumHeads int32 `json:"transformer_num_heads"`
+ NumLayers int32 `json:"transformer_num_layers"`
+ Context int `json:"transformer_context"`
+ MaxPeriod float32 `json:"transformer_max_period"`
+ GELU string `json:"gelu"`
+ } `json:"mimi"`
+ Temperature float32 `json:"temp"`
+ DecodeSteps int `json:"lsd_decode_steps"`
+ EOSThreshold float32 `json:"eos_threshold"`
+}
+
+const (
+ sampleRate = 24000
+ // mimiBatch is how many frames the codec decodes per call after the first.
+ mimiBatch = 8
+)
+
+type Model struct {
+ Embed *nn.Embedding
+ InputLinear *nn.Linear
+ BOS *mlx.Array
+ NumSpeakers *mlx.Array
+ Flow *Transformer
+ OutNorm *nn.LayerNorm
+ EOS *nn.Linear
+ Head *FlowHead
+ EmbMean *mlx.Array
+ EmbStd *mlx.Array
+ Mimi *Mimi
+ Voices map[string]*Voice
+
+ cfg *config
+ tok *unigram
+
+ // sample maps a transformer state and noise to a latent and EOS logit.
+ sample mlx.CompileFunc
+}
+
+// Voice is a speaker prompt already run through the flow LM: per-layer keys
+// and values [1, heads, frames, head_dim].
+type Voice struct {
+ Keys []*mlx.Array
+ Values []*mlx.Array
+}
+
+func newModel(root *model.Root) (model.Model, error) {
+ cfg := config{Temperature: 0.3}
+ if err := root.Manifest.ReadConfigJSON("config.json", &cfg); err != nil {
+ return nil, fmt.Errorf("load config: %w", err)
+ }
+ if err := cfg.validate(); err != nil {
+ return nil, err
+ }
+ data, err := root.Manifest.ReadConfig("tokenizer.json")
+ if err != nil {
+ return nil, fmt.Errorf("load tokenizer: %w", err)
+ }
+ tok, err := newUnigram(data)
+ if err != nil {
+ return nil, fmt.Errorf("parse tokenizer: %w", err)
+ }
+ return &Model{cfg: &cfg, tok: tok}, nil
+}
+
+// gelu follows upstream's checkpoint defaults; an explicit activation wins.
+func (c *config) gelu(value string) string {
+ if value != "" {
+ return value
+ }
+ if c.ModelID != nil {
+ return "erf"
+ }
+ return "tanh"
+}
+
+func (c *config) validate() error {
+ for _, value := range []string{c.FlowLM.GELU, c.Mimi.GELU} {
+ if value != "" && value != "erf" && value != "tanh" {
+ return fmt.Errorf("unsupported phonon GELU %q", value)
+ }
+ }
+ if value := c.FlowLM.TimeRMSNorm; value != "" && value != "rms_var" && value != "rms" {
+ return fmt.Errorf("unsupported phonon time norm %q", value)
+ }
+ if c.Mimi.SampleRate != sampleRate || c.Mimi.FrameRate != 12.5 {
+ return fmt.Errorf("unsupported mimi rates: %d Hz, %g frames/s", c.Mimi.SampleRate, c.Mimi.FrameRate)
+ }
+ if c.FlowLM.NumLayers <= 0 || c.FlowLM.NumHeads <= 0 || c.FlowLM.MaxPeriod <= 0 || c.FlowLM.FlowDepth <= 0 ||
+ c.Mimi.NumLayers <= 0 || c.Mimi.NumHeads <= 0 || c.Mimi.Context <= 0 || c.Mimi.MaxPeriod <= 0 || c.Mimi.DilationBase <= 0 {
+ return fmt.Errorf("invalid phonon transformer configuration")
+ }
+ if c.DecodeSteps != 1 {
+ return fmt.Errorf("phonon requires one LSD decode step, got %d", c.DecodeSteps)
+ }
+ if c.Temperature < 0 || math.IsNaN(float64(c.Temperature)) || math.IsInf(float64(c.Temperature), 0) ||
+ math.IsNaN(float64(c.EOSThreshold)) || math.IsInf(float64(c.EOSThreshold), 0) {
+ return fmt.Errorf("invalid phonon sampling configuration")
+ }
+ return nil
+}
+
+func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {
+ // The flow LM transformer is bandwidth bound and runs in bfloat16; the
+ // flow head and the Mimi decoder stay in float32.
+ for name, t := range tensors {
+ if d := t.DType(); d != mlx.DTypeFloat32 && d != mlx.DTypeBFloat16 && d != mlx.DTypeFloat16 {
+ return fmt.Errorf("phonon requires unquantized floating-point weights: %s", name)
+ }
+ dtype := mlx.DTypeFloat32
+ if strings.HasPrefix(name, "flow_lm.transformer.") || strings.HasPrefix(name, "voices.") ||
+ name == "flow_lm.num_speakers" || name == "flow_lm.conditioner.embed.weight" || name == "flow_lm.input_linear.weight" {
+ dtype = mlx.DTypeBFloat16
+ }
+ if t.DType() != dtype {
+ tensors[name] = t.AsType(dtype)
+ }
+ }
+ f := m.cfg.FlowLM
+ var err error
+ if m.Flow, err = loadTransformer(tensors, "flow_lm.transformer", f.NumLayers, f.NumHeads, f.MaxPeriod, m.cfg.gelu(f.GELU)); err != nil {
+ return err
+ }
+ if m.Head, err = loadFlowHead(tensors, "flow_lm.flow_net", f.FlowDepth, f.TimeRMSNorm); err != nil {
+ return err
+ }
+ if m.Mimi, err = loadMimi(tensors, m.cfg); err != nil {
+ return err
+ }
+ if w := tensors["flow_lm.conditioner.embed.weight"]; w != nil {
+ m.Embed = nn.NewEmbedding(w)
+ }
+ m.InputLinear = linear(tensors, "flow_lm.input_linear")
+ m.OutNorm = layerNorm(tensors, "flow_lm.out_norm", 1e-5)
+ m.EOS = linear(tensors, "flow_lm.out_eos")
+ m.BOS, m.EmbMean, m.EmbStd = tensors["flow_lm.bos_emb"], tensors["flow_lm.emb_mean"], tensors["flow_lm.emb_std"]
+ if m.Embed == nil || m.InputLinear == nil || m.OutNorm == nil || m.EOS == nil || m.BOS == nil || m.EmbMean == nil || m.EmbStd == nil {
+ return fmt.Errorf("missing flow_lm weights")
+ }
+ m.NumSpeakers = tensors["flow_lm.num_speakers"]
+ if m.NumSpeakers == nil || !slices.Equal(m.NumSpeakers.Dims(), []int{1, 1, m.Embed.Weight.Dim(1)}) {
+ return fmt.Errorf("missing or invalid num_speakers conditioning")
+ }
+ m.BOS = mlx.Reshape(m.BOS, 1, 1, -1)
+ m.sample = mlx.Compile("phonon_sample", func(in ...*mlx.Array) []*mlx.Array {
+ h := m.OutNorm.Forward(mlx.Reshape(in[0].AsType(mlx.DTypeFloat32), 1, -1))
+ latent := m.Head.sample(h, in[1])
+ return []*mlx.Array{mlx.ExpandDims(latent, 0), m.EOS.Forward(h)}
+ })
+
+ // The packager projects each speaker prompt into the flow LM's embedding
+ // space. Prefill it once at load time; requests start from these cached keys
+ // and values, without sharing mutable generation state.
+ m.Voices = map[string]*Voice{}
+ for name, t := range tensors {
+ voice, ok := strings.CutPrefix(name, "voices.")
+ if !ok {
+ continue
+ }
+ if voice == "" || t.NumDims() != 3 || t.Dim(0) != 1 || t.Dim(1) == 0 || t.Dim(2) != m.Embed.Weight.Dim(1) {
+ return fmt.Errorf("voice %q has shape %v", voice, t.Dims())
+ }
+ m.Voices[voice] = m.prefillVoice(t)
+ }
+ if len(m.Voices) == 0 {
+ return fmt.Errorf("model has no voices")
+ }
+ return nil
+}
+
+func (m *Model) prefillVoice(prompt *mlx.Array) *Voice {
+ caches := make([]cache.Attention, len(m.Flow.Layers))
+ for i := range caches {
+ caches[i] = cache.NewKVCache()
+ defer caches[i].Free()
+ }
+ kv := mlx.ScopedEval(func() []*mlx.Array {
+ m.Flow.forward(prompt, 0, caches)
+ var kv []*mlx.Array
+ for _, c := range caches {
+ kv = append(kv, c.State()...)
+ }
+ return kv
+ })
+ v := &Voice{}
+ for i := 0; i < len(kv); i += 2 {
+ v.Keys = append(v.Keys, kv[i])
+ v.Values = append(v.Values, kv[i+1])
+ }
+ return v
+}
+
+// Speech models load through model.Model but never run the text pipeline.
+func (m *Model) NewCaches() []cache.Cache { return nil }
+
+func (m *Model) Forward(*batch.Batch, []cache.Cache) (*mlx.Array, *mlx.Array) {
+ panic("phonon: text forward is not supported")
+}
+func (m *Model) Unembed(x *mlx.Array) *mlx.Array { return x }
+func (m *Model) Tokenizer() *tokenizer.Tokenizer { return nil }
+func (m *Model) MaxContextLength() int { return 0 }
+
+func (m *Model) VoiceNames() []string { return slices.Sorted(maps.Keys(m.Voices)) }
+
+func (m *Model) Speak(ctx context.Context, req llm.SpeechRequest, emit func([]float32) error) error {
+ name := req.Voice
+ if name == "" {
+ name = m.VoiceNames()[0]
+ }
+ voice, ok := m.Voices[name]
+ if !ok {
+ return api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: fmt.Sprintf("voice %q not found; available voices: %s", req.Voice, strings.Join(m.VoiceNames(), ", "))}
+ }
+ chunks, err := m.tok.chunks(req.Input)
+ if err != nil {
+ return api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: err.Error()}
+ }
+ rng := rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64()))
+ for _, chunk := range chunks {
+ if err := m.speak(ctx, chunk, voice, rng, emit); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// speak generates one chunk from a fresh copy of the voice state.
+func (m *Model) speak(ctx context.Context, text string, voice *Voice, rng *rand.Rand, emit func([]float32) error) error {
+ text, framesAfterEOS, err := prepareText(text)
+ if err != nil {
+ return err
+ }
+ tokens := m.tok.encode(text)
+ maxFrames := int(math.Ceil((float64(len(tokens))/3 + 2) * m.cfg.Mimi.FrameRate))
+
+ caches := make([]cache.Attention, len(m.Flow.Layers))
+ for i := range caches {
+ caches[i] = cache.NewKVCache()
+ defer caches[i].Free()
+ }
+ mimi := m.Mimi.newState()
+ defer mimi.close()
+ held := mlx.NewScope()
+ defer held.Close()
+
+ prev := m.BOS.Clone()
+ held.Attach(prev)
+ offset := voice.Keys[0].Dim(2)
+ mlx.Scoped(func() {
+ for i, c := range caches {
+ c.Update(nil, voice.Keys[i], voice.Values[i])
+ }
+ ids := mlx.FromValues(tokens, 1, len(tokens))
+ m.Flow.forward(m.Embed.Forward(ids), offset, caches)
+ })
+ offset += len(tokens)
+
+ std := math.Sqrt(float64(m.cfg.Temperature))
+ noise := make([]float32, m.BOS.Dim(2))
+ for generated, stop := 0, maxFrames; generated < stop; {
+ // Decode the first frame alone for latency, then in batches. EOS
+ // is read at the same sync points; frames past it are dropped.
+ n := min(mimiBatch, stop-generated)
+ if generated == 0 {
+ n = 1
+ }
+ var latents, logits []*mlx.Array
+ for step := generated; step < generated+n; step++ {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ for i := range noise {
+ noise[i] = float32(rng.NormFloat64() * std)
+ }
+ out := mlx.ScopedArrays(func() []*mlx.Array {
+ input := mlx.Add(m.InputLinear.Forward(prev.AsType(mlx.DTypeBFloat16)), m.NumSpeakers)
+ h := m.Flow.forward(input, offset+step, caches)
+ out := m.sample(h, mlx.FromValues(noise, 1, len(noise)))
+ prev.Set(out[0])
+ return out
+ })
+ held.Attach(out...)
+ mlx.AsyncEval(out...)
+ latents, logits = append(latents, out[0]), append(logits, out[1])
+ }
+ var eos []float32
+ mlx.Scoped(func() { eos = mlx.Concatenate(logits, 0).Floats() })
+ for i, e := range eos {
+ if e > m.cfg.EOSThreshold {
+ stop = min(stop, generated+i+1+framesAfterEOS)
+ break
+ }
+ }
+ n = min(n, stop-generated)
+ var samples []float32
+ mlx.Scoped(func() {
+ z := mlx.Add(mlx.Mul(mlx.Concatenate(latents[:n], 1), m.EmbStd), m.EmbMean)
+ samples = m.Mimi.decode(mimi, z).Floats()
+ })
+ held.Discard(latents...)
+ held.Discard(logits...)
+ generated += n
+ if err := emit(samples); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/mlxrunner/model/phonon/model_test.go b/mlxrunner/model/phonon/model_test.go
new file mode 100644
index 000000000..9d42031b6
--- /dev/null
+++ b/mlxrunner/model/phonon/model_test.go
@@ -0,0 +1,91 @@
+package phonon
+
+import (
+ "encoding/json"
+ "math"
+ "testing"
+
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlx/mlxtest"
+)
+
+func TestConfig(t *testing.T) {
+ const checkpoint = `{"flow_lm":{"num_heads":12,"num_layers":6,"max_period":10000,"flow_depth":6},"mimi":{"sample_rate":24000,"frame_rate":12.5,"dilation_base":2,"transformer_num_heads":8,"transformer_num_layers":2,"transformer_context":250,"transformer_max_period":10000},"temp":0.8,"lsd_decode_steps":1,"eos_threshold":4.0}`
+ var cfg config
+ if err := json.Unmarshal([]byte(checkpoint), &cfg); err != nil {
+ t.Fatal(err)
+ }
+ if err := cfg.validate(); err != nil {
+ t.Fatal(err)
+ }
+ if cfg.EOSThreshold != 4 || cfg.Temperature != .8 {
+ t.Fatalf("lost checkpoint sampling settings: %+v", cfg)
+ }
+ for _, tc := range []struct {
+ name string
+ change func(*config)
+ }{
+ {"GELU", func(c *config) { c.FlowLM.GELU = "relu" }},
+ {"time norm", func(c *config) { c.FlowLM.TimeRMSNorm = "other" }},
+ {"sample rate", func(c *config) { c.Mimi.SampleRate = 48000 }},
+ {"layers", func(c *config) { c.FlowLM.NumLayers = 0 }},
+ {"multiple LSD steps", func(c *config) { c.DecodeSteps = 2 }},
+ {"negative temperature", func(c *config) { c.Temperature = -1 }},
+ {"NaN threshold", func(c *config) { c.EOSThreshold = float32(math.NaN()) }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ c := cfg
+ tc.change(&c)
+ if c.validate() == nil {
+ t.Fatal("accepted unsupported configuration")
+ }
+ })
+ }
+}
+
+func TestCheckpointDefaults(t *testing.T) {
+ for _, tc := range []struct {
+ json, gelu string
+ temperature float32
+ }{
+ {`{}`, "tanh", .3},
+ {`{"model_id":{"sig":"7e71a02d","epoch":200},"temp":0.8}`, "erf", .8},
+ {`{"model_id":{"sig":"test","epoch":1},"flow_lm":{"gelu":"tanh"},"temp":0}`, "tanh", 0},
+ {`{"flow_lm":{"gelu":"erf"}}`, "erf", .3},
+ } {
+ cfg := config{Temperature: .3}
+ if err := json.Unmarshal([]byte(tc.json), &cfg); err != nil {
+ t.Fatal(err)
+ }
+ if got := cfg.gelu(cfg.FlowLM.GELU); got != tc.gelu {
+ t.Errorf("GELU = %q, want %q", got, tc.gelu)
+ }
+ if cfg.Temperature != tc.temperature {
+ t.Errorf("temperature = %v, want %v", cfg.Temperature, tc.temperature)
+ }
+ }
+}
+
+func TestTimeNorm(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ values := []float32{1, 2, 3, 6}
+ x, alpha := mlx.FromValues(values, 4), mlx.FromValues([]float32{2, 2, 2, 2}, 4)
+ for _, tc := range []struct {
+ norm string
+ denominator float64
+ }{
+ {"", math.Sqrt(14.0/3 + 1e-5)},
+ {"rms_var", math.Sqrt(14.0/3 + 1e-5)},
+ {"rms", math.Sqrt(12.5 + 1e-5)},
+ } {
+ out := timeNorm(x, alpha, tc.norm)
+ mlx.Eval(out)
+ for i, got := range out.Floats() {
+ want := 2 * float64(values[i]) / tc.denominator
+ if math.Abs(float64(got)-want) > 1e-5 {
+ t.Errorf("%s[%d] = %v, want %v", tc.norm, i, got, want)
+ }
+ }
+ }
+ })
+}
diff --git a/mlxrunner/model/phonon/normalize.go b/mlxrunner/model/phonon/normalize.go
new file mode 100644
index 000000000..b66e17aa4
--- /dev/null
+++ b/mlxrunner/model/phonon/normalize.go
@@ -0,0 +1,75 @@
+package phonon
+
+import (
+ "strings"
+ "unicode"
+)
+
+// normalizeText applies the reference's language-independent punctuation rules.
+// Number, currency, address and other language-specific rewrites belong in a
+// separate pass once the model's language is known.
+func normalizeText(text string) string {
+ var out []rune
+ push := func(c rune) {
+ if c == '.' || c == ',' {
+ for len(out) > 0 {
+ last := out[len(out)-1]
+ keep := strings.ContainsRune("\"'@+%#&", last) ||
+ last == '$' && len(out) > 1 && out[len(out)-2] >= '0' && out[len(out)-2] <= '9'
+ punctuation := last < 128 && (unicode.IsPunct(last) || unicode.IsSymbol(last))
+ if !unicode.IsSpace(last) && (!punctuation || keep) {
+ break
+ }
+ out = out[:len(out)-1]
+ }
+ }
+ out = append(out, c)
+ }
+ space := func() {
+ if len(out) > 0 && out[len(out)-1] != ' ' {
+ out = append(out, ' ')
+ }
+ }
+ runes := []rune(text)
+ for i, c := range runes {
+ switch {
+ case strings.ContainsRune("»«‹›“”„‟″‶⹂❝❞❠〝〞〟"", c):
+ push('"')
+ case strings.ContainsRune("‘’‚‛′‵❛❜❟'ʼʻʹ´`", c):
+ push('\'')
+ case strings.ContainsRune("‐‑‒―", c):
+ push('-')
+ case strings.ContainsRune("–—[]{}•‣◦·→←↑↓➡➜", c):
+ space()
+ case c == '*':
+ // A leading * before a number is a spoken star in the reference.
+ // Preserve it for language-specific expansion; remove markup.
+ if (i == 0 || unicode.IsSpace(runes[i-1])) && i+1 < len(runes) && runes[i+1] >= '0' && runes[i+1] <= '9' {
+ push(c)
+ } else {
+ space()
+ }
+ case c == '…':
+ push('.')
+ case c == ':' && i > 0 && i+1 < len(runes) && !unicode.IsSpace(runes[i-1]) && !unicode.IsSpace(runes[i+1]):
+ push(c)
+ case strings.ContainsRune(";:()", c):
+ push(',')
+ space()
+ case strings.ContainsRune("★☆❤♥", c):
+ // These carry meaning ("stars", "love"), unlike decorative emoji.
+ push(c)
+ case c >= 0x1F000 && c <= 0x1FAFF || c >= 0x2600 && c <= 0x27BF ||
+ c >= 0x2B00 && c <= 0x2BFF || c == 0xFE0F || c == 0x200D ||
+ unicode.IsControl(c) || unicode.IsSpace(c):
+ space()
+ default:
+ push(c)
+ }
+ }
+ text = strings.TrimRight(strings.TrimSpace(string(out)), "\"")
+ if len(text) >= 2 && text[len(text)-1] == '\'' && strings.ContainsRune(".!?,;", rune(text[len(text)-2])) {
+ text = text[:len(text)-1]
+ }
+ return text
+}
diff --git a/mlxrunner/model/phonon/normalize_test.go b/mlxrunner/model/phonon/normalize_test.go
new file mode 100644
index 000000000..fbaaca24c
--- /dev/null
+++ b/mlxrunner/model/phonon/normalize_test.go
@@ -0,0 +1,59 @@
+package phonon
+
+import "testing"
+
+func TestNormalizeText(t *testing.T) {
+ // Expected strings come from xn-ptts Normalize::for_lang(Lang::En).
+ for _, tc := range []struct{ input, want string }{
+ {"You’re sweet! Is there anything you’d like to talk about?", "You're sweet! Is there anything you'd like to talk about?"},
+ {"Youʼre sweet!", "You're sweet!"},
+ {"Of course—I would love to help! What’s on your mind?", "Of course I would love to help! What's on your mind?"},
+ {"Well… I am not sure… maybe we could try again.", "Well. I am not sure. maybe we could try again."},
+ {"🎉 Congratulations! You did an amazing job! 😊", "Congratulations! You did an amazing job!"},
+ {"Hello👋there! It is nice to meet you.", "Hello there! It is nice to meet you."},
+ {"I can help (if you would like); just let me know.", "I can help, if you would like, just let me know."},
+ {"Here is the plan: • Take a walk → Get some rest.", "Here is the plan, Take a walk Get some rest."},
+ {"That is [really] **wonderful** news!", "That is really wonderful news!"},
+ {"She said, “It is time to go.”", "She said, \"It is time to go."},
+ {"Hello\x00 there. How are you?", "Hello there. How are you?"},
+ {"sun ☀️ dev 👩‍💻 done", "sun dev done"},
+ {"time: 10:30", "time, 10:30"},
+ {"foo ; bar : baz", "foo, bar, baz"},
+ {"a;b:c", "a, b:c"},
+ {"fish &. 12%. #.", "fish &. 12%. #."},
+ {"hello... goodbye", "hello. goodbye"},
+ {"'Hello!'", "'Hello!"},
+ {"a ‐ b ‑ c ‒ d ― e – f — g", "a - b - c - d - e f g"},
+ {"👋 \x00", ""},
+ } {
+ got := normalizeText(tc.input)
+ if got != tc.want {
+ t.Errorf("normalize(%q) = %q; want %q", tc.input, got, tc.want)
+ }
+ if again := normalizeText(got); again != got {
+ t.Errorf("normalize is not idempotent for %q: %q", got, again)
+ }
+ }
+}
+
+func TestNormalizeTextPreservesLanguageSpecificMeaning(t *testing.T) {
+ // Until a language-specific pass exists, do not silently erase these.
+ for _, text := range []string{"I ❤ you", "Rated ★★★☆☆", "*5", "2 + 2 = 4", "21°C", "user@host", "It costs $5.", "12.50", "café", "日本語"} {
+ if got := normalizeText(text); got != text {
+ t.Errorf("normalize(%q) = %q", text, got)
+ }
+ }
+}
+
+func TestUnigramDecodeSpecialTokens(t *testing.T) {
+ u, err := newUnigram([]byte(`{
+ "added_tokens":[{"id":1,"content":"\u0000<s>","special":true}],
+ "model":{"type":"Unigram","unk_id":0,"vocab":[["<unk>",-10],["\u0000<s>",0],["▁hello",-1],["▁world",-1]]}
+ }`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := u.decode([]int32{1, 2, 1, 3, 1}); got != "hello world" {
+ t.Fatalf("decode with skipped special tokens = %q", got)
+ }
+}
diff --git a/mlxrunner/model/phonon/text.go b/mlxrunner/model/phonon/text.go
new file mode 100644
index 000000000..df8fc5034
--- /dev/null
+++ b/mlxrunner/model/phonon/text.go
@@ -0,0 +1,344 @@
+package phonon
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "math"
+ "slices"
+ "strconv"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+)
+
+const metaspace = "▁"
+
+// unigram is the SentencePiece Unigram tokenizer from tokenizer.json, with the
+// Prepend("▁") normalizer, Metaspace pre-tokenizer and byte fallback the
+// released checkpoints use.
+type unigram struct {
+ pieces []string
+ scores []float64
+ byteIDs [256]int32 // token ID for each byte, or -1
+ bytes []int // byte value of each <0xNN> piece, or -1
+ ids map[string]int32
+ maxLen int
+ unkID int32
+ unkScore float64
+ specials []specialToken
+ specialIDs []bool
+}
+
+type specialToken struct {
+ ID int32 `json:"id"`
+ Content string `json:"content"`
+ Special bool `json:"special"`
+}
+
+func newUnigram(data []byte) (*unigram, error) {
+ var t struct {
+ AddedTokens []specialToken `json:"added_tokens"`
+ Model struct {
+ Type string `json:"type"`
+ UnkID int32 `json:"unk_id"`
+ Vocab [][]any `json:"vocab"`
+ } `json:"model"`
+ }
+ if err := json.Unmarshal(data, &t); err != nil {
+ return nil, err
+ }
+ if t.Model.Type != "Unigram" {
+ return nil, fmt.Errorf("unsupported tokenizer model %q", t.Model.Type)
+ }
+ u := &unigram{ids: make(map[string]int32, len(t.Model.Vocab)), unkID: t.Model.UnkID, specialIDs: make([]bool, len(t.Model.Vocab))}
+ for i := range u.byteIDs {
+ u.byteIDs[i] = -1
+ }
+ minScore := math.Inf(1)
+ if len(t.Model.Vocab) == 0 || t.Model.UnkID < 0 || int(t.Model.UnkID) >= len(t.Model.Vocab) {
+ return nil, fmt.Errorf("invalid unigram vocabulary or unknown token")
+ }
+ for i, v := range t.Model.Vocab {
+ if len(v) != 2 {
+ return nil, fmt.Errorf("invalid vocab entry %d", i)
+ }
+ piece, ok1 := v[0].(string)
+ score, ok2 := v[1].(float64)
+ if !ok1 || !ok2 || piece == "" {
+ return nil, fmt.Errorf("invalid vocab entry %d", i)
+ }
+ b := -1
+ if len(piece) == 6 && strings.HasPrefix(piece, "<0x") && piece[5] == '>' {
+ if v, err := strconv.ParseUint(piece[3:5], 16, 8); err == nil {
+ b = int(v)
+ u.byteIDs[b] = int32(i)
+ }
+ }
+ u.pieces = append(u.pieces, piece)
+ u.scores = append(u.scores, score)
+ u.bytes = append(u.bytes, b)
+ u.ids[piece] = int32(i)
+ u.maxLen = max(u.maxLen, len(piece))
+ minScore = min(minScore, score)
+ }
+ u.unkScore = minScore - 10
+ for _, token := range t.AddedTokens {
+ if token.Content == "" || token.ID < 0 || int(token.ID) >= len(u.pieces) {
+ return nil, fmt.Errorf("invalid added token %q", token.Content)
+ }
+ u.specials = append(u.specials, token)
+ u.specialIDs[token.ID] = token.Special
+ }
+ return u, nil
+}
+
+func (u *unigram) encode(text string) []int32 {
+ var ids []int32
+ for text != "" {
+ at, selected := len(text), -1
+ for i, token := range u.specials {
+ if pos := strings.Index(text, token.Content); pos >= 0 && (pos < at ||
+ (pos == at && selected >= 0 && len(token.Content) > len(u.specials[selected].Content))) {
+ at, selected = pos, i
+ }
+ }
+ if at > 0 {
+ ids = append(ids, u.encodeText(text[:at])...)
+ }
+ if selected < 0 {
+ break
+ }
+ token := u.specials[selected]
+ ids = append(ids, token.ID)
+ text = text[at+len(token.Content):]
+ }
+ return ids
+}
+
+func (u *unigram) encodeText(text string) []int32 {
+ var ids []int32
+ for word := range strings.SplitSeq(strings.ReplaceAll(text, " ", metaspace), metaspace) {
+ ids = u.encodeWord(metaspace+word, ids)
+ }
+ return ids
+}
+
+// encodeWord appends the Viterbi segmentation of s.
+func (u *unigram) encodeWord(s string, ids []int32) []int32 {
+ n := len(s)
+ best := make([]float64, n+1)
+ from := make([]int, n+1)
+ tok := make([]int32, n+1)
+ for i := range from {
+ from[i] = -1
+ }
+ for i := 0; i < n; {
+ _, size := utf8.DecodeRuneInString(s[i:])
+ single := false
+ for j := i + 1; j <= min(n, i+u.maxLen); j++ {
+ id, ok := u.ids[s[i:j]]
+ if !ok {
+ continue
+ }
+ if score := best[i] + u.scores[id]; from[j] < 0 || score > best[j] {
+ best[j], from[j], tok[j] = score, i, id
+ }
+ single = single || j-i == size
+ }
+ if j := i + size; !single {
+ if score := best[i] + u.unkScore; from[j] < 0 || score > best[j] {
+ best[j], from[j], tok[j] = score, i, u.unkID
+ }
+ }
+ i += size
+ }
+
+ start := len(ids)
+ for j := n; j > 0; j = from[j] {
+ if tok[j] != u.unkID {
+ ids = append(ids, tok[j])
+ continue
+ }
+ // Backtracking visits pieces and their fallback bytes in reverse.
+ mark := len(ids)
+ for k := j - 1; k >= from[j]; k-- {
+ if id := u.byteIDs[s[k]]; id >= 0 {
+ ids = append(ids, id)
+ } else {
+ ids = append(ids[:mark], u.unkID)
+ break
+ }
+ }
+ }
+ slices.Reverse(ids[start:])
+ return ids
+}
+
+func (u *unigram) decode(ids []int32) string {
+ var b []byte
+ first := true
+ for _, id := range ids {
+ if u.specialIDs[id] {
+ continue
+ }
+ if c := u.bytes[id]; c >= 0 {
+ b = append(b, byte(c))
+ first = false
+ continue
+ }
+ piece := u.pieces[id]
+ if first {
+ piece = strings.ReplaceAll(piece, metaspace, "")
+ } else {
+ piece = strings.ReplaceAll(piece, metaspace, " ")
+ }
+ b = append(b, piece...)
+ first = false
+ }
+ return strings.ToValidUTF8(string(b), "�")
+}
+
+const maxChunkTokens = 50
+
+var errEmptyText = errors.New("input has no text to speak")
+
+// prepareText follows the reference prepare_text_prompt, after normalization.
+func prepareText(text string) (string, int, error) {
+ words := strings.Fields(text)
+ text = strings.Join(words, " ")
+ if text == "" {
+ return "", 0, errEmptyText
+ }
+ framesAfterEOS := 1
+ if len(words) <= 4 {
+ framesAfterEOS = 3
+ }
+ _, n := utf8.DecodeRuneInString(text)
+ text = strings.ToUpper(text[:n]) + text[n:]
+ if last, _ := utf8.DecodeLastRuneInString(text); unicode.IsLetter(last) || unicode.IsNumber(last) {
+ text += "."
+ }
+ return text, framesAfterEOS, nil
+}
+
+// chunks splits text into sentence groups of at most maxChunkTokens tokens,
+// as Phonon split_into_best_sentences does, then fits oversized sentences
+// to the reference runtime's hard limit.
+func (u *unigram) chunks(text string) ([]string, error) {
+ text, _, err := prepareText(normalizeText(text))
+ if err != nil {
+ return nil, err
+ }
+ tokens := u.encode(strings.TrimSpace(text))
+
+ type segment struct {
+ n int
+ text string
+ }
+ boundary := map[int32]bool{}
+ for _, id := range u.encode(".!...?")[1:] {
+ boundary[id] = true
+ }
+ var segs []segment
+ start, after := 0, false
+ for i, id := range tokens {
+ if boundary[id] {
+ after = true
+ continue
+ }
+ if after {
+ segs = append(segs, segment{i - start, u.decode(tokens[start:i])})
+ start = i
+ }
+ after = false
+ }
+ segs = append(segs, segment{len(tokens) - start, u.decode(tokens[start:])})
+
+ var chunks []string
+ var cur string
+ var n int
+ for _, s := range segs {
+ switch {
+ case cur == "":
+ cur, n = s.text, s.n
+ case n+s.n > maxChunkTokens:
+ chunks = append(chunks, strings.TrimSpace(cur))
+ cur, n = s.text, s.n
+ default:
+ cur += " " + s.text
+ n += s.n
+ }
+ }
+ if cur != "" {
+ chunks = append(chunks, strings.TrimSpace(cur))
+ }
+ return u.fitChunks(chunks)
+}
+
+// fitChunks mirrors upstream plan::fit_or_error. Long prompts otherwise outlast
+// the model's training segments and can lose words even with enough KV space.
+func (u *unigram) fitChunks(chunks []string) ([]string, error) {
+ const limit = 200
+ slices.Reverse(chunks)
+ var fitted []string
+ for len(chunks) > 0 {
+ text := chunks[len(chunks)-1]
+ chunks = chunks[:len(chunks)-1]
+ prepared, _, err := prepareText(text)
+ if err != nil {
+ return nil, err
+ }
+ if len(u.encode(prepared)) <= limit {
+ fitted = append(fitted, text)
+ continue
+ }
+ words := strings.Fields(text)
+ var left, right string
+ if len(words) > 1 {
+ cut := u.cutWords(words, limit/4)
+ left, right = strings.Join(words[:cut], " "), strings.Join(words[cut:], " ")
+ } else {
+ runes := []rune(text)
+ if len(runes) < 2 {
+ return nil, fmt.Errorf("text cannot fit the %d-token limit", limit)
+ }
+ left, right = string(runes[:len(runes)/2]), string(runes[len(runes)/2:])
+ }
+ chunks = append(chunks, right, left)
+ }
+ return fitted, nil
+}
+
+// Prefer a sentence end, then a clause end, then the most balanced word split.
+func (u *unigram) cutWords(words []string, minSide int) int {
+ ends := make([]int, len(words))
+ total := 0
+ for i, word := range words {
+ total += len(u.encode(word))
+ ends[i] = total
+ }
+ for _, marks := range []string{".!?", ",;:", ""} {
+ best, distance := 0, total+1
+ for i, end := range ends[:len(ends)-1] {
+ if marks != "" {
+ word := strings.TrimRight(words[i], "\"')]”’»")
+ last, _ := utf8.DecodeLastRuneInString(word)
+ if !strings.ContainsRune(marks, last) || end < minSide || total-end < minSide {
+ continue
+ }
+ }
+ d := end - (total - end)
+ if d < 0 {
+ d = -d
+ }
+ if d < distance {
+ best, distance = i+1, d
+ }
+ }
+ if best != 0 {
+ return best
+ }
+ }
+ return 1
+}
diff --git a/mlxrunner/model/phonon/text_test.go b/mlxrunner/model/phonon/text_test.go
new file mode 100644
index 000000000..7a874d9de
--- /dev/null
+++ b/mlxrunner/model/phonon/text_test.go
@@ -0,0 +1,158 @@
+package phonon
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "slices"
+ "strings"
+ "testing"
+)
+
+func TestPrepareText(t *testing.T) {
+ for _, tc := range []struct {
+ input, want string
+ tail int
+ }{
+ {" hello\nworld ", "Hello world.", 3},
+ {"wait—", "Wait—", 3},
+ {"\"Hello?\"", "\"Hello?\"", 3},
+ {"it's ready", "It's ready.", 3},
+ {"it’s ready", "It’s ready.", 3},
+ {"hello; world", "Hello; world.", 3},
+ {"one two\tthree\rfour five", "One two three four five.", 1},
+ } {
+ got, tail, err := prepareText(tc.input)
+ if err != nil || got != tc.want || tail != tc.tail {
+ t.Errorf("prepare(%q) = %q, %d, %v; want %q, %d", tc.input, got, tail, err, tc.want, tc.tail)
+ }
+ }
+ if _, _, err := prepareText(" \n\t"); err == nil {
+ t.Fatal("accepted empty text")
+ }
+}
+
+func TestUnigramMalformedVocabulary(t *testing.T) {
+ for _, vocab := range []string{`[]`, `[[]]`, `[["x"]]`, `[["x","not a score"]]`, `[["",0]]`} {
+ if _, err := newUnigram([]byte(`{"model":{"type":"Unigram","unk_id":0,"vocab":` + vocab + `}}`)); err == nil {
+ t.Errorf("accepted malformed vocabulary %s", vocab)
+ }
+ }
+}
+
+func TestUnigramByteFallback(t *testing.T) {
+ vocab := [][]any{{"<unk>", -10.0}, {metaspace, -1.0}, {"hello", -1.0}}
+ for i := range 256 {
+ vocab = append(vocab, []any{fmt.Sprintf("<0x%02X>", i), -20.0})
+ }
+ data, _ := json.Marshal(map[string]any{"model": map[string]any{"type": "Unigram", "unk_id": 0, "vocab": vocab}})
+ u, err := newUnigram(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, text := range []string{"hello", "hello café", "hello 👋", "hello hello", "日本語"} {
+ if got := u.decode(u.encode(text)); got != text {
+ t.Errorf("round trip %q => %q", text, got)
+ }
+ }
+ text := strings.Repeat("hello ", 300)
+ prepared, _, _ := prepareText(text)
+ chunks, err := u.chunks(text)
+ if err != nil || len(chunks) < 2 || strings.Join(chunks, " ") != prepared {
+ t.Fatalf("long sentence was not split losslessly: %v", err)
+ }
+ for _, chunk := range chunks {
+ prepared, _, _ := prepareText(chunk)
+ if n := len(u.encode(prepared)); n > 200 {
+ t.Fatalf("long sentence chunk has %d tokens", n)
+ }
+ }
+}
+
+// Run against a real checkpoint and fixtures generated with Hugging Face
+// tokenizers: PHONON_TOKENIZER=... PHONON_TOKEN_CASES=... go test ...
+func TestCheckpointTokenizer(t *testing.T) {
+ path := os.Getenv("PHONON_TOKENIZER")
+ if path == "" {
+ t.Skip("set PHONON_TOKENIZER for checkpoint parity")
+ }
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ u, err := newUnigram(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err = os.ReadFile(os.Getenv("PHONON_TOKEN_CASES"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var cases []struct {
+ Text string
+ IDs []int32
+ }
+ if err := json.Unmarshal(data, &cases); err != nil {
+ t.Fatal(err)
+ }
+ for _, tc := range cases {
+ if got := u.encode(tc.Text); !slices.Equal(got, tc.IDs) {
+ t.Errorf("%q: got %v; want %v", tc.Text, got, tc.IDs)
+ }
+ }
+ chunks, err := u.chunks(strings.Repeat("Hello from the new Phonon model. ", 20))
+ if err != nil || len(chunks) < 2 {
+ t.Fatalf("chunking: %v %v", chunks, err)
+ }
+ for _, chunk := range chunks {
+ if !strings.HasSuffix(chunk, ".") {
+ t.Fatalf("split inside sentence: %q", chunk)
+ }
+ }
+}
+
+// Fixtures come from xn-ptts plan::chunks + plan::fit_or_error, using the same
+// checkpoint tokenizer. They cover oversized sentences, clauses and UTF-8 words.
+func TestCheckpointChunking(t *testing.T) {
+ path := os.Getenv("PHONON_CHUNK_CASES")
+ if path == "" {
+ t.Skip("set PHONON_CHUNK_CASES for upstream chunking parity")
+ }
+ data, err := os.ReadFile(os.Getenv("PHONON_TOKENIZER"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ u, err := newUnigram(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err = os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var cases []struct {
+ Text string
+ Chunks []string
+ }
+ if err := json.Unmarshal(data, &cases); err != nil {
+ t.Fatal(err)
+ }
+ for _, tc := range cases {
+ got, err := u.chunks(tc.Text)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(got, tc.Chunks) {
+ t.Errorf("chunks for %.40q differ: got %d chunks, want %d", tc.Text, len(got), len(tc.Chunks))
+ }
+ for _, chunk := range got {
+ prepared, _, err := prepareText(chunk)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n := len(u.encode(prepared)); n > 200 {
+ t.Errorf("chunk has %d tokens", n)
+ }
+ }
+ }
+}
diff --git a/mlxrunner/model/speech.go b/mlxrunner/model/speech.go
new file mode 100644
index 000000000..4f95a2a55
--- /dev/null
+++ b/mlxrunner/model/speech.go
@@ -0,0 +1,13 @@
+package model
+
+import (
+ "context"
+
+ "github.com/ollama/ollama/llm"
+)
+
+// SpeechModel generates speech instead of text. The runner loads it like any
+// model and serves it with Speak, which emits 24 kHz mono samples.
+type SpeechModel interface {
+ Speak(ctx context.Context, req llm.SpeechRequest, emit func(samples []float32) error) error
+}
diff --git a/mlxrunner/runner.go b/mlxrunner/runner.go
index 1af2856bf..93e0f9dbb 100644
--- a/mlxrunner/runner.go
+++ b/mlxrunner/runner.go
@@ -137,6 +137,9 @@ func (r *Runner) loadModel(modelName string) (weights []*mlx.Array, err error) {
}
r.Model = m
+ if _, ok := m.(model.SpeechModel); ok {
+ return w
+ }
r.Tokenizer = m.Tokenizer()
r.contextLength = m.MaxContextLength()
caches := m.NewCaches()
diff --git a/mlxrunner/server.go b/mlxrunner/server.go
index c8c349a4d..259f60c58 100644
--- a/mlxrunner/server.go
+++ b/mlxrunner/server.go
@@ -124,6 +124,10 @@ func Execute(args []string) error {
})
mux.HandleFunc("POST /v1/completions", func(w http.ResponseWriter, r *http.Request) {
+ if runner.Tokenizer == nil {
+ http.Error(w, "this model does not support text generation", http.StatusBadRequest)
+ return
+ }
request := Request{Responses: make(chan CompletionResponse)}
if err := json.NewDecoder(r.Body).Decode(&request.CompletionRequest); err != nil {
@@ -195,6 +199,10 @@ func Execute(args []string) error {
})
mux.HandleFunc("POST /v1/tokenize", func(w http.ResponseWriter, r *http.Request) {
+ if runner.Tokenizer == nil {
+ http.Error(w, "this model does not support text tokenization", http.StatusBadRequest)
+ return
+ }
var b bytes.Buffer
if _, err := io.Copy(&b, r.Body); err != nil {
slog.Error("Failed to read request body", "error", err)
@@ -212,6 +220,10 @@ func Execute(args []string) error {
})
mux.HandleFunc("POST /v1/detokenize", func(w http.ResponseWriter, r *http.Request) {
+ if runner.Tokenizer == nil {
+ http.Error(w, "this model does not support text tokenization", http.StatusBadRequest)
+ return
+ }
var tokens []int32
if err := json.NewDecoder(r.Body).Decode(&tokens); err != nil {
http.Error(w, "Bad Request", http.StatusBadRequest)
@@ -224,6 +236,7 @@ func Execute(args []string) error {
})
mux.HandleFunc("POST /v1/embeddings", runner.handleEmbed)
+ mux.HandleFunc("POST /v1/audio/speech", runner.speechHandler)
for source, target := range map[string]string{
"GET /health": "/v1/status",
@@ -267,8 +280,11 @@ func (w *statusRecorder) Status() string {
return strconv.Itoa(w.code) + " " + http.StatusText(w.code)
}
-func (w *statusRecorder) Flush() {
- if f, ok := w.ResponseWriter.(http.Flusher); ok {
- f.Flush()
- }
+// Unwrap lets net/http's ResponseController reach deadlines and other controls.
+func (w *statusRecorder) Unwrap() http.ResponseWriter { return w.ResponseWriter }
+
+func (w *statusRecorder) FlushError() error {
+ return http.NewResponseController(w.ResponseWriter).Flush()
}
+
+func (w *statusRecorder) Flush() { _ = w.FlushError() }
diff --git a/mlxrunner/speech.go b/mlxrunner/speech.go
new file mode 100644
index 000000000..e08554365
--- /dev/null
+++ b/mlxrunner/speech.go
@@ -0,0 +1,158 @@
+package mlxrunner
+
+import (
+ "context"
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "math"
+ "net/http"
+ "sync"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlxrunner/model"
+)
+
+// speechHandler streams 24 kHz 16-bit PCM as the model generates it.
+func (r *Runner) speechHandler(w http.ResponseWriter, req *http.Request) {
+ speaker, ok := r.Model.(model.SpeechModel)
+ if !ok {
+ http.Error(w, "this model does not support speech", http.StatusNotImplemented)
+ return
+ }
+ var input llm.SpeechRequest
+ if err := json.NewDecoder(req.Body).Decode(&input); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ // Generation queues audio for a separate writer instead of waiting on the
+ // client, so a client reading at playback speed does not hold the MLX worker.
+ ctx, cancel := context.WithCancel(req.Context())
+ defer cancel()
+ queue := &audioQueue{ready: make(chan struct{}, 1)}
+ var streaming bool
+ var writeErr error
+ written := make(chan struct{})
+ go func() {
+ defer close(written)
+ streaming, writeErr = writeAudio(w, queue)
+ if writeErr != nil {
+ cancel()
+ }
+ }()
+ // The writer must not outlive the handler, even if generation panics.
+ defer func() { queue.close(); <-written }()
+
+ err := r.mlxThread.Do(ctx, func() error {
+ defer mlx.ClearCache()
+ return speaker.Speak(ctx, input, func(samples []float32) error {
+ queue.push(pcm16(samples))
+ return nil
+ })
+ })
+ queue.close()
+ <-written
+ if err == nil {
+ err = writeErr
+ }
+ switch {
+ case err == nil:
+ case streaming:
+ // Truncate the stream so the client sees the failure.
+ panic(http.ErrAbortHandler)
+ default:
+ status := http.StatusInternalServerError
+ var statusErr api.StatusError
+ if errors.As(err, &statusErr) {
+ status = statusErr.StatusCode
+ }
+ http.Error(w, err.Error(), status)
+ }
+}
+
+// writeAudio streams queued audio until the queue is closed and reports
+// whether any audio was written.
+func writeAudio(w http.ResponseWriter, queue *audioQueue) (streaming bool, err error) {
+ controller := http.NewResponseController(w)
+ defer controller.SetWriteDeadline(time.Time{})
+ for {
+ chunks, ok := queue.next()
+ if !ok {
+ return streaming, nil
+ }
+ for _, pcm := range chunks {
+ // A stalled reader must not hold the connection forever.
+ if err := controller.SetWriteDeadline(time.Now().Add(30 * time.Second)); err != nil {
+ return streaming, err
+ }
+ if !streaming {
+ w.Header().Set("Content-Type", "audio/pcm")
+ streaming = true
+ }
+ if _, err := w.Write(pcm); err != nil {
+ return streaming, err
+ }
+ }
+ if err := controller.Flush(); err != nil {
+ return streaming, err
+ }
+ }
+}
+
+// audioQueue hands audio from generation to the writer without ever blocking
+// generation.
+type audioQueue struct {
+ mu sync.Mutex
+ pending [][]byte
+ closed bool
+ ready chan struct{}
+}
+
+func (q *audioQueue) push(pcm []byte) {
+ q.mu.Lock()
+ q.pending = append(q.pending, pcm)
+ q.mu.Unlock()
+ q.wake()
+}
+
+func (q *audioQueue) close() {
+ q.mu.Lock()
+ q.closed = true
+ q.mu.Unlock()
+ q.wake()
+}
+
+func (q *audioQueue) wake() {
+ select {
+ case q.ready <- struct{}{}:
+ default:
+ }
+}
+
+// next waits for queued audio. It returns false once the queue is closed and
+// empty.
+func (q *audioQueue) next() ([][]byte, bool) {
+ for {
+ q.mu.Lock()
+ pending, closed := q.pending, q.closed
+ q.pending = nil
+ q.mu.Unlock()
+ if len(pending) > 0 || closed {
+ return pending, len(pending) > 0
+ }
+ <-q.ready
+ }
+}
+
+func pcm16(samples []float32) []byte {
+ b := make([]byte, 2*len(samples))
+ for i, s := range samples {
+ v := math.Round(float64(max(-1, min(1, s))) * math.MaxInt16)
+ binary.LittleEndian.PutUint16(b[2*i:], uint16(int16(v)))
+ }
+ return b
+}
diff --git a/server/images.go b/server/images.go
index 83d594be8..87950edd5 100644
--- a/server/images.go
+++ b/server/images.go
@@ -585,6 +585,7 @@ func (m *Model) CheckCapabilities(want ...model.Capability) error {
model.CapabilityThinking: errCapabilityThinking,
model.CapabilityImage: errCapabilityImage,
model.CapabilityDecision: errors.New("decision"),
+ model.CapabilitySpeech: errors.New("speech"),
}
for _, cap := range want {
diff --git a/server/routes.go b/server/routes.go
index 9cd9c8b93..09e99afb2 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -1893,8 +1893,8 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
Runner: m.Runner,
}
- // For safetensors completion and decision models, populate details from config.json.
- isSafetensorsLLM := m.IsMLX() && (slices.Contains(m.Config.Capabilities, "completion") || slices.Contains(m.Config.Capabilities, "decision"))
+ // For safetensors completion, decision and speech models, populate details from config.json.
+ isSafetensorsLLM := m.IsMLX() && (slices.Contains(m.Config.Capabilities, "completion") || slices.Contains(m.Config.Capabilities, "decision") || slices.Contains(m.Config.Capabilities, "speech"))
if isSafetensorsLLM {
if info, err := getSafetensorsLLMInfoForRunner(name, req.Runner); err == nil {
if arch, ok := info["general.architecture"].(string); ok && arch != "" {
@@ -2418,8 +2418,9 @@ func (s *Server) GenerateRoutes() (http.Handler, error) {
r.GET("/v1/models/:model", cloudModelPathPassthroughMiddleware(cloudErrRemoteModelDetailsUnavailable), middleware.RetrieveMiddleware(), s.ShowHandler)
r.POST("/v1/responses", s.withInferenceRequestLogging("/v1/responses", s.responsesCompactionMiddleware(), cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.ResponsesMiddleware(lookupThinking), s.ChatHandler)...)
r.POST("/v1/responses/compact", s.ResponsesCompactHandler)
- // OpenAI-compatible audio endpoint
+ // OpenAI-compatible audio endpoints
r.POST("/v1/audio/transcriptions", middleware.TranscriptionMiddleware(), s.ChatHandler)
+ r.POST("/v1/audio/speech", s.SpeechHandler)
// Inference (Anthropic compatibility)
r.POST("/v1/messages", s.withInferenceRequestLogging("/v1/messages", cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.AnthropicMessagesMiddleware(lookupThinking), s.ChatHandler)...)
@@ -2816,6 +2817,12 @@ func (s *Server) PsHandler(c *gin.Context) {
QuantizationLevel: m.Config.FileType,
}
+ contextLength := v.contextLength
+ if slices.Contains(m.Config.Capabilities, "speech") {
+ // Speech models have no text context.
+ contextLength = 0
+ }
+
mr := api.ProcessModelResponse{
Model: displayName,
Name: displayName,
@@ -2824,7 +2831,7 @@ func (s *Server) PsHandler(c *gin.Context) {
Digest: digest,
Details: modelDetails,
ExpiresAt: v.expiresAt,
- ContextLength: v.contextLength,
+ ContextLength: contextLength,
Runner: runner,
}
@@ -3595,19 +3602,24 @@ func countChatImages(msgs []api.Message) int {
}
func handleScheduleError(c *gin.Context, name string, err error) {
+ status, message := scheduleError(name, err)
+ c.JSON(status, gin.H{"error": message})
+}
+
+// scheduleError keeps native and OpenAI-compatible endpoints consistent while
+// allowing each to use its own error response format.
+func scheduleError(name string, err error) (int, string) {
switch {
- case errors.Is(err, errCapabilities), errors.Is(err, errRequired):
- c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
- case errors.Is(err, manifest.ErrNoCompatibleManifest):
- c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ case errors.Is(err, errCapabilities), errors.Is(err, errRequired), errors.Is(err, manifest.ErrNoCompatibleManifest):
+ return http.StatusBadRequest, err.Error()
case errors.Is(err, context.Canceled):
- c.JSON(499, gin.H{"error": "request canceled"})
+ return 499, "request canceled"
case errors.Is(err, ErrMaxQueue):
- c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})
+ return http.StatusServiceUnavailable, err.Error()
case errors.Is(err, os.ErrNotExist):
- c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found, try pulling it first", name)})
+ return http.StatusNotFound, fmt.Sprintf("model %q not found, try pulling it first", name)
default:
- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return http.StatusInternalServerError, err.Error()
}
}
diff --git a/server/speech.go b/server/speech.go
new file mode 100644
index 000000000..f83ae8a71
--- /dev/null
+++ b/server/speech.go
@@ -0,0 +1,163 @@
+package server
+
+import (
+ "cmp"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/openai"
+ "github.com/ollama/ollama/types/model"
+)
+
+const speechSampleRate = 24000
+
+// SpeechHandler serves the OpenAI-compatible POST /v1/audio/speech, streaming
+// audio as the model generates it.
+func (s *Server) SpeechHandler(c *gin.Context) {
+ var req struct {
+ Model string `json:"model"`
+ Input string `json:"input"`
+ Voice string `json:"voice"`
+ ResponseFormat string `json:"response_format"`
+ Speed *float64 `json:"speed"`
+ Instructions string `json:"instructions"`
+ StreamFormat string `json:"stream_format"`
+ }
+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 1<<20)
+ if err := c.ShouldBindJSON(&req); err != nil {
+ speechError(c, http.StatusBadRequest, err.Error())
+ return
+ }
+ format := cmp.Or(req.ResponseFormat, "wav")
+ switch {
+ case strings.TrimSpace(req.Model) == "":
+ speechError(c, http.StatusBadRequest, "model is required")
+ return
+ case strings.TrimSpace(req.Input) == "":
+ speechError(c, http.StatusBadRequest, "input is required")
+ return
+ case utf8.RuneCountInString(req.Input) > 4096:
+ speechError(c, http.StatusBadRequest, "input must be at most 4096 characters")
+ return
+ case format != "wav" && format != "pcm":
+ speechError(c, http.StatusBadRequest, fmt.Sprintf("response_format %q is not supported, use wav or pcm", format))
+ return
+ case req.Speed != nil && *req.Speed != 1:
+ speechError(c, http.StatusBadRequest, "only speed 1 is supported")
+ return
+ case req.Instructions != "":
+ speechError(c, http.StatusBadRequest, "instructions are not supported")
+ return
+ case req.StreamFormat != "" && req.StreamFormat != "audio":
+ speechError(c, http.StatusBadRequest, "only stream_format audio is supported")
+ return
+ }
+
+ r, _, _, err := s.scheduleRunner(c.Request.Context(), req.Model, "", []model.Capability{model.CapabilitySpeech}, nil, nil, nil)
+ speaker, ok := r.(llm.Speaker)
+ switch {
+ case errors.Is(err, errCapabilities), err == nil && !ok:
+ speechError(c, http.StatusBadRequest, fmt.Sprintf("%q does not support speech", req.Model))
+ return
+ case err != nil:
+ status, message := scheduleError(req.Model, err)
+ speechError(c, status, message)
+ return
+ }
+
+ streamSpeech(c, speaker, llm.SpeechRequest{Input: req.Input, Voice: req.Voice}, format)
+}
+
+func streamSpeech(c *gin.Context, speaker llm.Speaker, req llm.SpeechRequest, format string) {
+ // Write through Gin for status/byte accounting, but bypass its Flush
+ // method so ResponseController can return the underlying flush error.
+ var writer http.ResponseWriter = c.Writer
+ if w, ok := writer.(interface{ Unwrap() http.ResponseWriter }); ok {
+ writer = w.Unwrap()
+ }
+ controller := http.NewResponseController(writer)
+ defer controller.SetWriteDeadline(time.Time{})
+ started, clientGone := false, false
+ err := speaker.Speech(c.Request.Context(), req, func(pcm []byte) (err error) {
+ // Every error here comes from writing to the client.
+ defer func() { clientGone = err != nil }()
+ if len(pcm) == 0 {
+ return nil
+ }
+ // Bound a stalled client's writes so it cannot hold the model and
+ // back up the runner indefinitely. Reset for each chunk, not request.
+ if err := controller.SetWriteDeadline(time.Now().Add(30 * time.Second)); err != nil {
+ return err
+ }
+ if !started {
+ started = true
+ c.Header("Content-Type", "audio/"+format)
+ c.Status(http.StatusOK)
+ if format == "wav" {
+ if _, err := c.Writer.Write(wavHeader()); err != nil {
+ return err
+ }
+ }
+ }
+ if _, err := c.Writer.Write(pcm); err != nil {
+ return err
+ }
+ return controller.Flush()
+ })
+ if err == nil && !started {
+ err = errors.New("speech model produced no audio")
+ }
+ if err == nil {
+ return
+ }
+ if started {
+ // Close the connection rather than end the stream, so clients don't
+ // mistake partial audio for a complete response. A client that hangs
+ // up is not a failure worth a warning.
+ if !clientGone && c.Request.Context().Err() == nil {
+ slog.Warn("speech stream failed", "error", err)
+ }
+ if conn, _, err := controller.Hijack(); err == nil {
+ conn.Close()
+ }
+ return
+ }
+ status, message := http.StatusInternalServerError, err.Error()
+ var statusErr api.StatusError
+ if errors.As(err, &statusErr) {
+ status, message = statusErr.StatusCode, statusErr.ErrorMessage
+ }
+ speechError(c, status, message)
+}
+
+func speechError(c *gin.Context, status int, message string) {
+ c.AbortWithStatusJSON(status, openai.NewError(status, message))
+}
+
+// wavHeader describes 16-bit mono PCM of unknown length, as streamed WAV is.
+func wavHeader() []byte {
+ h := make([]byte, 44)
+ copy(h[0:], "RIFF")
+ binary.LittleEndian.PutUint32(h[4:], 0xffffffff)
+ copy(h[8:], "WAVEfmt ")
+ binary.LittleEndian.PutUint32(h[16:], 16)
+ binary.LittleEndian.PutUint16(h[20:], 1)
+ binary.LittleEndian.PutUint16(h[22:], 1)
+ binary.LittleEndian.PutUint32(h[24:], speechSampleRate)
+ binary.LittleEndian.PutUint32(h[28:], speechSampleRate*2)
+ binary.LittleEndian.PutUint16(h[32:], 2)
+ binary.LittleEndian.PutUint16(h[34:], 16)
+ copy(h[36:], "data")
+ binary.LittleEndian.PutUint32(h[40:], 0xffffffff)
+ return h
+}
diff --git a/server/speech_test.go b/server/speech_test.go
new file mode 100644
index 000000000..de47767cb
--- /dev/null
+++ b/server/speech_test.go
@@ -0,0 +1,214 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/manifest"
+)
+
+type speechFunc func(context.Context, llm.SpeechRequest, func([]byte) error) error
+
+func (f speechFunc) Speech(ctx context.Context, req llm.SpeechRequest, emit func([]byte) error) error {
+ return f(ctx, req, emit)
+}
+
+func TestSpeechValidation(t *testing.T) {
+ for _, body := range []string{
+ `{`, `{}`, `{"model":"test","input":" \n"}`,
+ `{"input":"hello"}`,
+ `{"model":"test","input":"hello","response_format":"mp3"}`,
+ `{"model":"test","input":"hello","speed":0}`,
+ `{"model":"test","input":"hello","speed":2}`,
+ `{"model":"test","input":"hello","stream_format":"sse"}`,
+ `{"model":"test","input":"hello","instructions":"whisper"}`,
+ `{"model":"test","input":"` + strings.Repeat("é", 4097) + `"}`,
+ } {
+ t.Run(body[:min(len(body), 90)], func(t *testing.T) {
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = httptest.NewRequest(http.MethodPost, "/v1/audio/speech", strings.NewReader(body))
+ // Invalid requests must fail before touching the scheduler.
+ (&Server{}).SpeechHandler(c)
+ var got struct {
+ Error struct {
+ Message string `json:"message"`
+ } `json:"error"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || w.Code != 400 || got.Error.Message == "" {
+ t.Fatalf("invalid error response: %d %s", w.Code, w.Body)
+ }
+ })
+ }
+}
+
+func TestSpeechStreamsBeforeCompletion(t *testing.T) {
+ for _, format := range []string{"pcm", "wav"} {
+ t.Run(format, func(t *testing.T) {
+ release := make(chan struct{})
+ defer close(release)
+ pcm := []byte{1, 0, 2, 0}
+ speaker := speechFunc(func(ctx context.Context, req llm.SpeechRequest, emit func([]byte) error) error {
+ if err := emit(pcm); err != nil {
+ return err
+ }
+ select {
+ case <-release:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ return nil
+ })
+ router := gin.New()
+ router.Use(gin.Recovery())
+ router.POST("/", func(c *gin.Context) { streamSpeech(c, speaker, llm.SpeechRequest{}, format) })
+ srv := httptest.NewServer(router)
+ defer srv.Close()
+ req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, srv.URL, nil)
+ res, err := srv.Client().Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+ if res.Header.Get("Content-Type") != "audio/"+format {
+ t.Fatal(res.Header)
+ }
+ if format == "wav" {
+ h := make([]byte, 44)
+ if _, err := io.ReadFull(res.Body, h); err != nil {
+ t.Fatal(err)
+ }
+ if string(h[:4]) != "RIFF" || string(h[8:16]) != "WAVEfmt " || string(h[36:40]) != "data" ||
+ binary.LittleEndian.Uint32(h[24:]) != 24000 || binary.LittleEndian.Uint16(h[22:]) != 1 ||
+ binary.LittleEndian.Uint16(h[34:]) != 16 {
+ t.Fatalf("bad WAV header: %x", h)
+ }
+ }
+ got := make([]byte, len(pcm))
+ if _, err := io.ReadFull(res.Body, got); err != nil || !bytes.Equal(got, pcm) {
+ t.Fatalf("first audio: %x, %v", got, err)
+ }
+ })
+ }
+}
+
+func TestSpeechErrors(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ emitted bool
+ err error
+ status int
+ }{
+ {"invalid voice", false, api.StatusError{StatusCode: 400, ErrorMessage: "invalid voice"}, 400},
+ {"empty output", false, nil, 500},
+ {"generation failed", true, errors.New("generation failed"), 200},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ router := gin.New()
+ router.Use(gin.Recovery())
+ router.POST("/", func(c *gin.Context) {
+ streamSpeech(c, speechFunc(func(_ context.Context, _ llm.SpeechRequest, emit func([]byte) error) error {
+ if tc.emitted {
+ if err := emit([]byte{1, 0}); err != nil {
+ return err
+ }
+ }
+ return tc.err
+ }), llm.SpeechRequest{}, "pcm")
+ })
+ srv := httptest.NewServer(router)
+ defer srv.Close()
+ res, err := srv.Client().Post(srv.URL, "application/json", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+ body, err := io.ReadAll(res.Body)
+ if res.StatusCode != tc.status {
+ t.Fatalf("status %d: %s", res.StatusCode, body)
+ }
+ if tc.emitted {
+ if !errors.Is(err, io.ErrUnexpectedEOF) {
+ t.Fatalf("partial audio looked complete: %v", err)
+ }
+ } else if err != nil || !bytes.Contains(body, []byte(`"error"`)) {
+ t.Fatalf("error response: %s, %v", body, err)
+ }
+ })
+ }
+}
+
+func TestSpeechScheduleErrors(t *testing.T) {
+ for _, tc := range []struct {
+ err error
+ status int
+ message string
+ }{
+ {fmt.Errorf("model %w", errRequired), 400, "model is required"},
+ {errCapabilities, 400, errCapabilities.Error()},
+ {manifest.ErrNoCompatibleManifest, 400, manifest.ErrNoCompatibleManifest.Error()},
+ {context.Canceled, 499, "request canceled"},
+ {ErrMaxQueue, 503, ErrMaxQueue.Error()},
+ {os.ErrNotExist, 404, `model "phonon" not found, try pulling it first`},
+ {errors.New("load failed"), 500, "load failed"},
+ } {
+ t.Run(tc.err.Error(), func(t *testing.T) {
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ status, message := scheduleError("phonon", tc.err)
+ speechError(c, status, message)
+ var body struct {
+ Error struct {
+ Message string
+ Type string
+ }
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
+ t.Fatal(err)
+ }
+ if w.Code != tc.status || body.Error.Message != tc.message || body.Error.Type == "" {
+ t.Fatalf("got %d %s; want %d %q", w.Code, w.Body, tc.status, tc.message)
+ }
+ })
+ }
+}
+
+// Gin's Flush method drops errors; speech must reach the underlying writer's
+// FlushError so a failed write stops the producer immediately.
+type speechFlushErrorWriter struct {
+ *httptest.ResponseRecorder
+ err error
+}
+
+func (w *speechFlushErrorWriter) SetWriteDeadline(time.Time) error { return nil }
+func (w *speechFlushErrorWriter) FlushError() error { return w.err }
+func (w *speechFlushErrorWriter) Flush() { _ = w.FlushError() }
+
+func TestSpeechFlushErrorStopsProducer(t *testing.T) {
+ want := errors.New("connection write failed")
+ w := &speechFlushErrorWriter{ResponseRecorder: httptest.NewRecorder(), err: want}
+ c, _ := gin.CreateTestContext(w)
+ c.Request = httptest.NewRequest(http.MethodPost, "/v1/audio/speech", nil)
+ var got error
+ streamSpeech(c, speechFunc(func(_ context.Context, _ llm.SpeechRequest, emit func([]byte) error) error {
+ got = emit([]byte{1, 0})
+ return got
+ }), llm.SpeechRequest{}, "pcm")
+ if !errors.Is(got, want) {
+ t.Fatalf("flush error did not reach producer: got %v, want %v", got, want)
+ }
+}
diff --git a/types/model/capability.go b/types/model/capability.go
index c6ebc7c9d..27e0cd571 100644
--- a/types/model/capability.go
+++ b/types/model/capability.go
@@ -12,13 +12,15 @@ const (
CapabilityImage = Capability("image")
CapabilityAudio = Capability("audio")
CapabilityDecision = Capability("decision")
+ CapabilitySpeech = Capability("speech")
)
// IsValid reports whether c is recognized, such as "decision"; empty or unknown labels are invalid.
func (c Capability) IsValid() bool {
switch c {
case CapabilityCompletion, CapabilityTools, CapabilityInsert, CapabilityVision,
- CapabilityEmbedding, CapabilityThinking, CapabilityImage, CapabilityAudio, CapabilityDecision:
+ CapabilityEmbedding, CapabilityThinking, CapabilityImage, CapabilityAudio, CapabilityDecision,
+ CapabilitySpeech:
return true
default:
return false