Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlxrunner: add Phonon text-to-speech support (#19166)
Jeffrey Morgan6ef847b8fef4
diff --git a/cmd/cmd.go b/cmd/cmd.goindex 2ac1b12f2..02f79be5e 100644--- a/cmd/cmd.go+++ b/cmd/cmd.go@@ -903,23 +903,12 @@ func RunHandler(cmd *cobra.Command, args []string) error {}prompts := args[1:]- // prepend stdin to the prompt if provided- if !term.IsTerminal(int(os.Stdin.Fd())) {- in, err := io.ReadAll(os.Stdin)- if err != nil {- return err- }-- // Only prepend stdin content if it's not empty- stdinContent := string(in)- if len(stdinContent) > 0 {- prompts = append([]string{stdinContent}, prompts...)- }+ stdinPiped := !term.IsTerminal(int(os.Stdin.Fd()))+ if stdinPiped {opts.ShowConnect = falseopts.WordWrap = falseinteractive = false}- opts.Prompt = strings.Join(prompts, " ")if len(prompts) > 0 {interactive = false}@@ -966,12 +955,32 @@ func RunHandler(cmd *cobra.Command, args []string) error {applyShowResponseToRunOptions(&opts, info)+ // prepend stdin to the prompt if provided; speech models read it as it+ // arrives instead+ speech := slices.Contains(info.Capabilities, model.CapabilitySpeech)+ if stdinPiped && !speech {+ in, err := io.ReadAll(os.Stdin)+ if err != nil {+ return err+ }++ // Only prepend stdin content if it's not empty+ stdinContent := string(in)+ if len(stdinContent) > 0 {+ prompts = append([]string{stdinContent}, prompts...)+ }+ }+ opts.Prompt = strings.Join(prompts, " ")+if slices.Contains(info.Capabilities, model.CapabilityDecision) {return runDecision(cmd, opts, interactive)}if cmd.Flags().Changed("ask") {return fmt.Errorf("%s is not a decision model, so it can't answer --ask questions", name)}+ if cmd.Flags().Changed("voice") && !speech {+ return fmt.Errorf("%s is not a speech model, so it can't use --voice", name)+ }// Check if this is an embedding modelisEmbeddingModel := slices.Contains(info.Capabilities, model.CapabilityEmbedding)@@ -996,6 +1005,18 @@ func RunHandler(cmd *cobra.Command, args []string) error {return generateEmbedding(cmd, name, opts.Prompt, opts.KeepAlive, truncate, dimensions)}+ if speech {+ voice, err := cmd.Flags().GetString("voice")+ if err != nil {+ return err+ }+ var input io.Reader+ if stdinPiped {+ input = os.Stdin+ }+ return runSpeech(cmd.Context(), name, voice, opts.Prompt, input, interactive)+ }+if slices.Contains(info.Capabilities, model.CapabilityImage) {return errors.New("image generation models are not currently supported")}@@ -1261,7 +1282,10 @@ func ListRunningHandler(cmd *cobra.Command, args []string) error {} else {until = format.HumanTime(m.ExpiresAt, "Never")}- ctxStr := strconv.Itoa(m.ContextLength)+ ctxStr := "-"+ if m.ContextLength > 0 {+ ctxStr = strconv.Itoa(m.ContextLength)+ }data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), procStr, ctxStr, m.Runner, until})}}@@ -2597,6 +2621,7 @@ func NewCLI() *cobra.Command {runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)")runCmd.Flags().Bool("truncate", false, "For embedding models: truncate inputs exceeding context length (default: true). Set --truncate=false to error instead")runCmd.Flags().Int("dimensions", 0, "Truncate output embeddings to specified dimension (embedding models only)")+ runCmd.Flags().String("voice", "", "Voice to speak with (speech models only)")var asked []askedQuestionrunCmd.Flags().Var(questionFlag{"ask", &asked}, "ask", "Ask a decision model a question, answered yes or no unless --choice or --level follow")runCmd.Flags().Var(questionFlag{"choice", &asked}, "choice", `Add a label for the preceding --ask to choose from, e.g. billing="Payments and refunds"`)diff --git a/cmd/speech.go b/cmd/speech.gonew file mode 100644index 000000000..c4c1ddaf0--- /dev/null+++ b/cmd/speech.go@@ -0,0 +1,324 @@+package cmd++import (+ "bytes"+ "context"+ "encoding/binary"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "os"+ "os/exec"+ "os/signal"+ "runtime"+ "strings"+ "unicode/utf8"++ "golang.org/x/term"++ "github.com/ollama/ollama/envconfig"+ "github.com/ollama/ollama/readline"+)++// runSpeech reads text aloud: piped input as it arrives followed by any+// text arguments, the text arguments alone, or each line typed at the prompt.+func runSpeech(ctx context.Context, model, voice, text string, input io.Reader, interactive bool) error {+ if input != nil {+ return speak(ctx, model, voice, io.MultiReader(input, strings.NewReader(" "+text)))+ }+ if text != "" || !interactive {+ return speak(ctx, model, voice, strings.NewReader(text))+ }+ if _, err := player(model, ""); err != nil {+ return err+ }++ scanner, err := readline.New(readline.Prompt{+ Prompt: ">>> ",+ Placeholder: "Enter text to speak (/bye to exit)",+ })+ if err != nil {+ return err+ }+ if envconfig.NoHistory() {+ scanner.HistoryDisable()+ }+ for {+ // Clear anything typed while audio played; readline redraws it.+ fmt.Print(readline.CursorBOL + readline.ClearToEOL)+ line, err := scanner.Readline()+ switch {+ case errors.Is(err, io.EOF):+ fmt.Println()+ return nil+ case errors.Is(err, readline.ErrInterrupt):+ if line == "" {+ fmt.Println("\nUse Ctrl + d or /bye to exit.")+ }+ continue+ case errors.Is(err, readline.ErrEditPrompt):+ continue+ case err != nil:+ return err+ }+ text := strings.TrimSpace(line)+ if text == "/bye" {+ return nil+ }+ if text == "" {+ continue+ }+ // Ctrl+C stops speaking and returns to the prompt.+ speakCtx, stop := signal.NotifyContext(ctx, os.Interrupt)+ if err := speak(speakCtx, model, voice, strings.NewReader(text)); err != nil && speakCtx.Err() == nil {+ fmt.Fprintln(os.Stderr, "Error:", err)+ }+ stop()+ }+}++// speak plays text from r a few sentences at a time as it arrives, so a chat+// model piped into a speech model is heard while it is still writing. When+// stdout is not a terminal it writes one WAV instead, for example+// `ollama run model "text" > speech.wav` on a machine without speakers.+func speak(ctx context.Context, model, voice string, r io.Reader) error {+ toTerminal := term.IsTerminal(int(os.Stdout.Fd()))+ if toTerminal {+ if _, err := player(model, ""); err != nil {+ return err+ }+ }+ ctx, cancel := context.WithCancel(ctx)+ defer cancel()++ // Generate the next piece while the current one plays.+ type result struct {+ wav []byte+ err error+ }+ audio := make(chan result, 1)+ go func() {+ defer close(audio)+ for piece := range speechPieces(ctx, r) {+ wav, err := synthesize(ctx, model, voice, piece)+ select {+ case audio <- result{wav, err}:+ case <-ctx.Done():+ return+ }+ if err != nil {+ return+ }+ }+ }()++ var out []byte+ spoken := false+ for res := range audio {+ if res.err != nil {+ return res.err+ }+ spoken = true+ switch {+ case toTerminal:+ if err := play(ctx, model, res.wav); err != nil {+ return err+ }+ case out == nil:+ out = res.wav+ default:+ out = append(out, res.wav[44:]...)+ }+ }+ if !spoken {+ return fmt.Errorf("speech models require input text. Usage: ollama run %s \"your text here\"", model)+ }+ if toTerminal {+ return nil+ }+ setWAVSizes(out)+ _, err := os.Stdout.Write(out)+ return err+}++// minSpeechPiece is how much text speechPieces gathers before it sends a+// sentence on, so speech starts quickly without many tiny requests.+const minSpeechPiece = 100++// speechPieces sends text from r on as soon as it ends at a sentence or+// paragraph break, sized for /v1/audio/speech, and the rest when r ends.+func speechPieces(ctx context.Context, r io.Reader) <-chan string {+ pieces := make(chan string)+ go func() {+ defer close(pieces)+ send := func(text string) bool {+ for _, piece := range speechChunks(text, maxSpeechInput) {+ select {+ case pieces <- piece:+ case <-ctx.Done():+ return false+ }+ }+ return true+ }+ var pending string+ buf := make([]byte, 4096)+ for {+ n, err := r.Read(buf)+ pending += string(buf[:n])+ if cut := speechCut(pending); cut > 0 {+ if !send(pending[:cut]) {+ return+ }+ pending = pending[cut:]+ }+ if err != nil {+ send(pending)+ return+ }+ }+ }()+ return pieces+}++// speechCut returns where text can be cut so the part before it is spoken+// now: after the last paragraph break or sentence end, once a paragraph or at+// least minSpeechPiece characters have arrived. It returns 0 to wait.+func speechCut(text string) int {+ para := strings.LastIndex(text, "\n\n")+ cut := max(para, lastSentenceEnd(text))+ if para > 0 || (cut > 0 && utf8.RuneCountInString(text[:cut]) >= minSpeechPiece) {+ return cut+ }+ return 0+}++// maxSpeechInput is the most characters /v1/audio/speech accepts at once.+const maxSpeechInput = 4096++// speechChunks splits text into one piece per paragraph, cutting paragraphs+// longer than limit characters at the last sentence end or space that fits.+func speechChunks(text string, limit int) []string {+ var chunks []string+ for para := range strings.SplitSeq(strings.ReplaceAll(text, "\r\n", "\n"), "\n\n") {+ for p := strings.TrimSpace(para); p != ""; p = strings.TrimSpace(p) {+ end := len(p)+ if utf8.RuneCountInString(p) > limit {+ end = 0+ for range limit {+ _, size := utf8.DecodeRuneInString(p[end:])+ end += size+ }+ if i := lastSentenceEnd(p[:end]); i > 0 {+ end = i+ } else if i := strings.LastIndexAny(p[:end], " \t\n"); i > 0 {+ end = i+ }+ }+ chunks = append(chunks, strings.TrimSpace(p[:end]))+ p = p[end:]+ }+ }+ return chunks+}++// lastSentenceEnd returns the index just past the last '.', '!' or '?' in s+// that is followed by whitespace, or -1.+func lastSentenceEnd(s string) int {+ for i := len(s) - 2; i > 0; i-- {+ if strings.IndexByte(".!?", s[i]) >= 0 && (s[i+1] == ' ' || s[i+1] == '\n') {+ return i + 1+ }+ }+ return -1+}++func synthesize(ctx context.Context, model, voice, text string) ([]byte, error) {+ body, err := json.Marshal(map[string]string{"model": model, "voice": voice, "input": text, "response_format": "wav"})+ if err != nil {+ return nil, err+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, envconfig.Host().JoinPath("v1/audio/speech").String(), bytes.NewReader(body))+ if err != nil {+ return nil, err+ }+ req.Header.Set("Content-Type", "application/json")+ res, err := http.DefaultClient.Do(req)+ if err != nil {+ return nil, err+ }+ defer res.Body.Close()+ data, err := io.ReadAll(res.Body)+ if err != nil {+ return nil, err+ }+ if res.StatusCode != http.StatusOK {+ var e struct{ Error struct{ Message string } }+ if json.Unmarshal(data, &e) == nil && e.Error.Message != "" {+ return nil, errors.New(e.Error.Message)+ }+ return nil, fmt.Errorf("speech request failed: %s", res.Status)+ }+ if len(data) <= 44 {+ return nil, errors.New("speech response contained no audio")+ }+ setWAVSizes(data)+ return data, nil+}++// setWAVSizes records the length of a WAV the server streamed as unknown,+// for players that need it.+func setWAVSizes(wav []byte) {+ binary.LittleEndian.PutUint32(wav[4:], uint32(len(wav)-8))+ binary.LittleEndian.PutUint32(wav[40:], uint32(len(wav)-44))+}++// play uses the system's built-in WAV player.+func play(ctx context.Context, model string, wav []byte) error {+ f, err := os.CreateTemp("", "ollama-*.wav")+ if err != nil {+ return err+ }+ defer os.Remove(f.Name())+ if _, err := f.Write(wav); err != nil {+ f.Close()+ return err+ }+ if err := f.Close(); err != nil {+ return err+ }++ p, err := player(model, f.Name())+ if err != nil {+ return err+ }+ c := exec.CommandContext(ctx, p[0], p[1:]...)+ c.Stderr = os.Stderr+ if err := c.Run(); err != nil {+ return fmt.Errorf("%s failed: %w. %s", p[0], err, saveHint(model))+ }+ return nil+}++// player returns the command that plays the WAV file at path.+func player(model, path string) ([]string, error) {+ players := [][]string{{"paplay", path}, {"pw-play", path}, {"aplay", "-q", path}, {"ffplay", "-nodisp", "-autoexit", "-loglevel", "error", path}}+ switch runtime.GOOS {+ case "darwin":+ players = [][]string{{"afplay", path}}+ case "windows":+ players = [][]string{{"powershell", "-NoProfile", "-Command", "(New-Object Media.SoundPlayer '" + strings.ReplaceAll(path, "'", "''") + "').PlaySync()"}}+ }+ for _, p := range players {+ if _, err := exec.LookPath(p[0]); err == nil {+ return p, nil+ }+ }+ return nil, errors.New("no audio player found. " + saveHint(model))+}++func saveHint(model string) string {+ return fmt.Sprintf(`To save the audio instead, run: ollama run %s "text" > speech.wav`, model)+}diff --git a/cmd/speech_test.go b/cmd/speech_test.gonew file mode 100644index 000000000..59377e90a--- /dev/null+++ b/cmd/speech_test.go@@ -0,0 +1,102 @@+package cmd++import (+ "encoding/binary"+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "slices"+ "strings"+ "testing"+ "time"+)++func TestSpeechCut(t *testing.T) {+ long := strings.Repeat("This sentence is long enough to say. ", 3)+ for _, tc := range []struct {+ text string+ want int+ }{+ {"Too short to say yet. ", 0},+ {"Heading\n\nBody", 7},+ {long + "And then", len(long) - 1},+ {"no sentence end yet " + strings.Repeat("words ", 20), 0},+ } {+ if got := speechCut(tc.text); got != tc.want {+ t.Errorf("speechCut(%q) = %d, want %d", tc.text, got, tc.want)+ }+ }+}++// Speech starts once a few sentences arrive, before the input ends.+func TestSpeechPiecesStream(t *testing.T) {+ r, w := io.Pipe()+ pieces := speechPieces(t.Context(), r)+ go w.Write([]byte(strings.Repeat("This sentence is long enough to say. ", 3) + "Unfinished"))+ select {+ case p := <-pieces:+ if !strings.HasSuffix(p, "to say.") {+ t.Errorf("first piece = %q", p)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("speech waited for the end of the input")+ }+ go func() {+ w.Write([]byte(" tail"))+ w.Close()+ }()+ var rest []string+ for p := range pieces {+ rest = append(rest, p)+ }+ if !slices.Equal(rest, []string{"Unfinished tail"}) {+ t.Errorf("rest = %q", rest)+ }+}++func TestSpeechChunks(t *testing.T) {+ for _, tc := range []struct {+ text string+ limit int+ want []string+ }{+ {"Hello.", 100, []string{"Hello."}},+ {"One.\n\n Two.\r\n\r\nThree.", 100, []string{"One.", "Two.", "Three."}},+ {"One two. Three four. Five six.", 21, []string{"One two. Three four.", "Five six."}},+ {"one two three four", 10, []string{"one two", "three four"}},+ {"abcdefghijkl", 5, []string{"abcde", "fghij", "kl"}},+ {"ééééé", 2, []string{"éé", "éé", "é"}},+ {" \n\n ", 100, nil},+ } {+ if got := speechChunks(tc.text, tc.limit); !slices.Equal(got, tc.want) {+ t.Errorf("speechChunks(%q, %d) = %q, want %q", tc.text, tc.limit, got, tc.want)+ }+ }+}++func TestSynthesize(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var req struct{ Input, Voice string }+ if r.URL.Path != "/v1/audio/speech" || json.NewDecoder(r.Body).Decode(&req) != nil || req.Input == "" || req.Voice != "Toby" {+ w.WriteHeader(http.StatusBadRequest)+ w.Write([]byte(`{"error":{"message":"input is required"}}`))+ return+ }+ wav := make([]byte, 48)+ binary.LittleEndian.PutUint32(wav[4:], 0xffffffff)+ binary.LittleEndian.PutUint32(wav[40:], 0xffffffff)+ copy(wav[44:], []byte{1, 0, 2, 0})+ w.Write(wav)+ }))+ defer srv.Close()+ t.Setenv("OLLAMA_HOST", srv.URL)++ wav, err := synthesize(t.Context(), "phonon", "Toby", "hello")+ if err != nil || binary.LittleEndian.Uint32(wav[4:]) != 40 || binary.LittleEndian.Uint32(wav[40:]) != 4 {+ t.Fatalf("WAV lengths not recorded: %x, %v", wav, err)+ }+ if _, err := synthesize(t.Context(), "phonon", "Toby", ""); err == nil || err.Error() != "input is required" {+ t.Fatalf("server error lost: %v", err)+ }+}diff --git a/create/create.go b/create/create.goindex 4fa0f08ac..9ba9ecabe 100644--- a/create/create.go+++ b/create/create.go@@ -23,6 +23,10 @@ import (// safetensors-backed models.const SafetensorsMinOllamaVersion = "0.19.0"+// phononMinOllamaVersion is the first Ollama release that runs Phonon speech+// models.+const phononMinOllamaVersion = "0.40.9"+// IsSafetensorsLLMModel checks for a safetensors completion or decision model.func IsSafetensorsLLMModel(modelName string) bool {name := model.ParseName(modelName)diff --git a/create/metadata.go b/create/metadata.goindex 2756be8d0..053968f92 100644--- a/create/metadata.go+++ b/create/metadata.go@@ -79,6 +79,8 @@ func inferSafetensorsConfig(modelDir string, cfg sourceModelConfig, parserOverriif decisionVision {capabilities = append(capabilities, "vision")}+ } else if cfg.Architecture() == "Phonon" {+ capabilities = []string{"speech"}} else {capabilities = inferCompletionCapabilitiesFromConfig(cfg, chatTemplate, parserName)}diff --git a/create/metadata_test.go b/create/metadata_test.goindex 18b648f88..31c8f786d 100644--- a/create/metadata_test.go+++ b/create/metadata_test.go@@ -237,6 +237,11 @@ func TestInferSafetensorsConfigFamilies(t *testing.T) {wantRenderer: "flint",wantCaps: []string{"completion", "tools", "thinking"},},+ {+ name: "phonon",+ config: `{"architectures":["Phonon"],"model_type":"phonon"}`,+ wantCaps: []string{"speech"},+ },{name: "nested llm config",config: `{"model_type":"nemotron_h_omni","llm_config":{"model_type":"nemotron_h"}}`,diff --git a/create/pipeline.go b/create/pipeline.goindex f2a127201..af1ef6efe 100644--- a/create/pipeline.go+++ b/create/pipeline.go@@ -10,6 +10,8 @@ import ("path/filepath""strings"+ "golang.org/x/mod/semver"+"github.com/ollama/ollama/types/model")@@ -70,6 +72,10 @@ func Create(ctx context.Context, modelName, modelDir string, opts PipelineOptionreturn err}}+ // Older releases can't run Phonon, whatever the Modelfile or client says.+ if inv.Config.Architecture() == "Phonon" && semver.Compare("v"+modelConfig.Requires, "v"+phononMinOllamaVersion) < 0 {+ modelConfig.Requires = phononMinOllamaVersion+ }if err := validateMLXSource(inv.Config, false, opts.Validation); err != nil {return err}@@ -79,6 +85,9 @@ func Create(ctx context.Context, modelName, modelDir string, opts PipelineOptionif inv.Config.Architecture() == "LayaForDecision" && opts.Quantize != "" {return fmt.Errorf("Laya currently requires unquantized weights")}+ if inv.Config.Architecture() == "Phonon" && opts.Quantize != "" {+ return fmt.Errorf("Phonon currently requires unquantized weights")+ }class, err := Classify(inv, opts.Quantize)if err != nil {return errdiff --git a/create/pipeline_test.go b/create/pipeline_test.goindex 9215432c2..342c8b458 100644--- a/create/pipeline_test.go+++ b/create/pipeline_test.go@@ -75,6 +75,27 @@ func TestCreatePipeline(t *testing.T) {}}+func TestCreatePipelinePhononRequires(t *testing.T) {+ dir := t.TempDir()+ writeConfigJSON(t, dir, `{"architectures":["Phonon"],"model_type":"phonon"}`)+ createTestSafetensors(t, filepath.Join(dir, "model.safetensors"), []*st.TensorData{+ st.NewTensorDataFromBytes("flow_lm.bos_emb", "F32", []int32{4}, make([]byte, 4*4)),+ })+ // Remote creates send the generic safetensors minimum; it must not win.+ for requires, want := range map[string]string{"": phononMinOllamaVersion, "0.19.0": phononMinOllamaVersion, "0.41.0": "0.41.0"} {+ var got string+ opts := testPipelineOptions()+ opts.Requires = requires+ err := Create(context.Background(), "phonon", dir, opts, newCaptureStore(), func(_ context.Context, _ string, info ManifestInfo) error {+ got = info.ModelConfig.Requires+ return nil+ }, func(string) {})+ if err != nil || got != want {+ t.Errorf("requires %q: got %q, %v; want %q", requires, got, err, want)+ }+ }+}+func TestCreatePipelineRejectsUnsupportedArchitectureBeforeWriting(t *testing.T) {dir := t.TempDir()writeConfigJSON(t, dir, `{"architectures":["UnsupportedForCausalLM"]}`)diff --git a/docs/api/openai-compatibility.mdx b/docs/api/openai-compatibility.mdxindex 272f98ea7..e577ac345 100644--- a/docs/api/openai-compatibility.mdx+++ b/docs/api/openai-compatibility.mdx@@ -315,6 +315,31 @@ curl -X POST http://localhost:11434/v1/chat/completions \- [x] `dimensions`- [ ] `user`+### `/v1/audio/speech`++> Note: Added in Ollama v0.40.8++Generates audio with [speech models](/capabilities/speech). The response streams while audio is generated.++#### Supported request fields++- [x] `model`+- [x] `input` (up to 4,096 characters)+- [x] `voice` (one of the model's voice names, such as `Freya`; omit it for the model's default)+- [x] `response_format`+ - [x] `wav` (default)+ - [x] `pcm`+ - [ ] `mp3`, `opus`, `aac`, `flac`+- [x] `speed` (`1` only)+- [x] `stream_format` (`audio` only)+- [ ] `instructions`++#### Notes++- Audio is 16-bit, mono, 24 kHz. `pcm` returns raw little-endian samples with no header.+- The default `response_format` is `wav`, not `mp3`.+- Unsupported values return a `400` error instead of being ignored.+### Responses API`POST /v1/responses`diff --git a/docs/capabilities/speech.mdx b/docs/capabilities/speech.mdxnew file mode 100644index 000000000..1323221bc--- /dev/null+++ b/docs/capabilities/speech.mdx@@ -0,0 +1,128 @@+---+title: Speech+description: Turn text into spoken audio with text-to-speech models.+---++Speech models read text aloud. Ollama streams the audio as it is generated, so playback can start before the whole text has been spoken.++## Setup++Start [Ollama](/quickstart) v0.40.8 or later, then download [Phonon](https://ollama.com/library/phonon):++```shell setup.sh+ollama pull phonon+```++Phonon speaks English with four voices: `Freya` (the default), `Harper`, `Sterling`, and `Toby`. Voice names are case-sensitive.++## Speak from the terminal++`ollama run` plays the audio through your computer's speakers:++```shell examples.sh+# Speak some text+ollama run phonon "Hello from Ollama."++# Choose a voice+ollama run phonon --voice Toby "Hi, this is Toby."++# Speak text from another program+echo "Text from another program works too." | ollama run phonon++# Speak a chat model's answer+ollama run gemma4 --hidethinking "Tell me a fun fact about owls in two sentences." | ollama run phonon+```++Piped text is spoken as it arrives, so a chat model's answer starts playing after its first sentence or two.++Run `ollama run phonon` without any text to type lines at a `>>>` prompt. Each line is spoken when you press Enter. Press Ctrl+C to stop speaking, and type `/bye` or press Ctrl+D to exit.++Audio plays with your system's built-in player: `afplay` on macOS, the first of `paplay`, `pw-play`, `aplay`, or `ffplay` that is installed on Linux, and PowerShell on Windows.++### Save audio to a file++When the output is redirected, `ollama run` writes a WAV file instead of playing it. This also works on servers without speakers:++```shell+ollama run phonon "Save this to a file." > speech.wav+```++To hear a remote server, run `ollama` on your own computer and point it at the server. The audio plays where the command runs:++```shell+OLLAMA_HOST=my-server:11434 ollama run phonon "Hello from the server."+```++## Generate speech with the API++Use the OpenAI-compatible `/v1/audio/speech` endpoint:++<Tabs>+ <Tab title="cURL">+ ```shell+ curl http://localhost:11434/v1/audio/speech \+ -H "Content-Type: application/json" \+ -d '{+ "model": "phonon",+ "input": "Hello from Ollama.",+ "voice": "Freya"+ }' --output speech.wav+ ```+ </Tab>+ <Tab title="Python">+ ```python+ from openai import OpenAI++ client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")++ with client.audio.speech.with_streaming_response.create(+ model="phonon",+ voice="Freya",+ input="Hello from Ollama.",+ response_format="wav",+ ) as response:+ response.stream_to_file("speech.wav")+ ```+ </Tab>+ <Tab title="JavaScript">+ ```javascript+ import fs from 'node:fs'+ import OpenAI from 'openai'++ const openai = new OpenAI({ baseURL: 'http://localhost:11434/v1', apiKey: 'ollama' })++ const response = await openai.audio.speech.create({+ model: 'phonon',+ voice: 'Freya',+ input: 'Hello from Ollama.',+ response_format: 'wav',+ })+ fs.writeFileSync('speech.wav', Buffer.from(await response.arrayBuffer()))+ ```+ </Tab>+</Tabs>++The response streams while audio is generated, so you can start playing it right away:++```shell+curl -sN http://localhost:11434/v1/audio/speech \+ -H "Content-Type: application/json" \+ -d '{"model": "phonon", "input": "Playback starts while the rest is still being generated."}' \+ | ffplay -nodisp -autoexit -loglevel error -+```++### Audio formats++| `response_format` | Output |+| --- | --- |+| `wav` (default) | A WAV file: 16-bit, mono, 24 kHz. The header is written before the length is known, so its size fields hold the maximum value. |+| `pcm` | Raw 16-bit little-endian samples, mono, 24 kHz, with no header. |++### Limits++- `input` can be up to 4,096 characters per request. `ollama run` splits longer text at paragraph and sentence breaks for you.+- Text is spoken as written. Numbers, dates, and abbreviations are not expanded into words, so write them out (`twenty-one`, not `21`) for predictable results. Emoji, bullet characters, and `*` emphasis markers are skipped.+- Options the model doesn't support, such as other formats, speeds, or `instructions`, return a `400` error instead of being ignored. An unknown voice returns a `400` error that lists the available voices.+- Speech models only generate speech: chat, completion, and embedding requests to them return a `400` error.++`ollama show phonon` and `/api/show` list the model's capabilities as `speech`. See [OpenAI compatibility](/api/openai-compatibility) for every supported field.diff --git a/docs/cli.mdx b/docs/cli.mdxindex ae4215544..e93d3e5e2 100644--- a/docs/cli.mdx+++ b/docs/cli.mdx@@ -96,6 +96,18 @@ ollama run nimble --ask "How urgent is this?" \The state is piped in or given after the model name, as text or JSON. Each `--ask` question is answered yes or no, unless it's followed by labels to choose from (`--choice`) or levels to score it on (`--level`). See [Decision](/capabilities/decision#ask-from-the-terminal).+### Speak with a speech model++```+ollama run phonon "Hello from Ollama."++ollama run phonon --voice Toby "Hi, this is Toby."++ollama run phonon "Save this to a file." > speech.wav+```++The audio plays through your speakers, or is written as a WAV file when the output is redirected. Run it without text to speak each line you type. See [Speech](/capabilities/speech#speak-from-the-terminal).+### Download a model```diff --git a/docs/docs.json b/docs/docs.jsonindex ed96b7167..7e567e5aa 100644--- a/docs/docs.json+++ b/docs/docs.json@@ -131,6 +131,7 @@"/capabilities/decision","/capabilities/vision","/capabilities/embeddings",+ "/capabilities/speech","/capabilities/tool-calling","/capabilities/web-search"]diff --git a/llm/speech.go b/llm/speech.gonew file mode 100644index 000000000..b08fdce85--- /dev/null+++ b/llm/speech.go@@ -0,0 +1,15 @@+package llm++import "context"++// SpeechRequest asks a speech model to read Input aloud.+type SpeechRequest struct {+ Input string `json:"input"`+ Voice string `json:"voice,omitempty"`+}++// Speaker is implemented by runners that serve speech models. Speech streams+// 24 kHz mono 16-bit little-endian PCM to fn as it is generated.+type Speaker interface {+ Speech(ctx context.Context, req SpeechRequest, fn func(pcm []byte) error) error+}diff --git a/mlxrunner/client.go b/mlxrunner/client.goindex 649473622..0d46c8321 100644--- a/mlxrunner/client.go+++ b/mlxrunner/client.go@@ -604,6 +604,14 @@ func (c *Client) Tokenize(ctx context.Context, content string) ([]int, error) {}defer resp.Body.Close()+ if resp.StatusCode != http.StatusOK {+ body, err := io.ReadAll(resp.Body)+ if err != nil {+ return nil, err+ }+ return nil, api.StatusError{StatusCode: resp.StatusCode, ErrorMessage: strings.TrimSpace(string(body))}+ }+var tokens []intif err := json.NewDecoder(resp.Body).Decode(&tokens); err != nil {return nil, errdiff --git a/mlxrunner/client_speech.go b/mlxrunner/client_speech.gonew file mode 100644index 000000000..b29b6c17e--- /dev/null+++ b/mlxrunner/client_speech.go@@ -0,0 +1,61 @@+package mlxrunner++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "strings"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/llm"+)++var _ llm.Speaker = (*Client)(nil)++func (c *Client) Speech(ctx context.Context, input llm.SpeechRequest, fn func([]byte) error) error {+ data, err := json.Marshal(input)+ if err != nil {+ return err+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/v1/audio/speech", c.port), bytes.NewReader(data))+ if err != nil {+ return err+ }+ req.Header.Set("Content-Type", "application/json")+ res, err := c.client.Do(req)+ if err != nil {+ if c.status != nil {+ if errMsg := c.status.LastError(); errMsg != "" {+ return fmt.Errorf("mlx runner failed: %s", errMsg)+ }+ }+ return err+ }+ defer res.Body.Close()+ if res.StatusCode != http.StatusOK {+ body, err := io.ReadAll(res.Body)+ if err != nil {+ return err+ }+ return api.StatusError{StatusCode: res.StatusCode, ErrorMessage: strings.TrimSpace(string(body))}+ }++ buf := make([]byte, 32<<10)+ for {+ n, err := res.Body.Read(buf)+ if n > 0 {+ if err := fn(buf[:n]); err != nil {+ return err+ }+ }+ if errors.Is(err, io.EOF) {+ return nil+ } else if err != nil {+ return err+ }+ }+}diff --git a/mlxrunner/client_speech_test.go b/mlxrunner/client_speech_test.gonew file mode 100644index 000000000..a9027de8c--- /dev/null+++ b/mlxrunner/client_speech_test.go@@ -0,0 +1,195 @@+package mlxrunner++import (+ "bytes"+ "context"+ "errors"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/mlx/mlxtest"+ "github.com/ollama/ollama/mlxrunner/model"+)++func TestClientSpeech(t *testing.T) {+ for _, status := range []int{200, 400} {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1/audio/speech" || r.Method != "POST" {+ t.Errorf("unexpected request: %s %s", r.Method, r.URL)+ }+ if status != 200 {+ http.Error(w, "unknown voice", status)+ return+ }+ for _, b := range [][]byte{{1}, {0, 2}, {0}} {+ _, _ = w.Write(b)+ w.(http.Flusher).Flush()+ }+ }))+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}+ var got bytes.Buffer+ err := client.Speech(t.Context(), llm.SpeechRequest{Input: "hello", Voice: "Freya"}, func(b []byte) error { _, err := got.Write(b); return err })+ srv.Close()+ if status == 200 {+ if err != nil || !bytes.Equal(got.Bytes(), []byte{1, 0, 2, 0}) {+ t.Fatalf("audio: %x, %v", got.Bytes(), err)+ }+ } else {+ var se api.StatusError+ if !errors.As(err, &se) || se.StatusCode != 400 || se.ErrorMessage != "unknown voice" {+ t.Fatalf("error lost: %v", err)+ }+ }+ }+}++func TestClientSpeechCancellation(t *testing.T) {+ started := make(chan struct{})+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.Copy(io.Discard, r.Body)+ close(started)+ <-r.Context().Done()+ }))+ defer srv.Close()+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ done := make(chan error, 1)+ go func() { done <- client.Speech(ctx, llm.SpeechRequest{}, func([]byte) error { return nil }) }()+ <-started+ cancel()+ if err := <-done; !errors.Is(err, context.Canceled) {+ t.Fatalf("cancellation lost: %v", err)+ }+}++func TestPCM16(t *testing.T) {+ got := pcm16([]float32{-2, -1, 0, 1, 2})+ want := []byte{1, 128, 1, 128, 0, 0, 255, 127, 255, 127}+ if !bytes.Equal(got, want) {+ t.Fatalf("PCM: %x, want %x", got, want)+ }+}++type speechTestModel struct {+ model.Model+ chunks int+ generated chan struct{}+}++func (m speechTestModel) Speak(_ context.Context, _ llm.SpeechRequest, emit func([]float32) error) error {+ defer close(m.generated)+ samples := make([]float32, 1<<16)+ for range m.chunks {+ if err := emit(samples); err != nil {+ return err+ }+ }+ return nil+}++// A client reading at playback speed must not hold the MLX worker.+func TestSpeechDoesNotWaitForClient(t *testing.T) {+ m := speechTestModel{chunks: 256, generated: make(chan struct{})} // 32 MiB, more than socket buffers hold+ r := &Runner{Model: m, mlxThread: mlxtest.Worker(t)}+ srv := httptest.NewServer(http.HandlerFunc(r.speechHandler))+ defer srv.Close()+ res, err := srv.Client().Post(srv.URL, "application/json", strings.NewReader("{}"))+ if err != nil {+ t.Fatal(err)+ }+ defer res.Body.Close()+ select {+ case <-m.generated:+ case <-time.After(10 * time.Second):+ t.Fatal("generation waited for the client to read")+ }+ if n, err := io.Copy(io.Discard, res.Body); err != nil || n != 256<<17 {+ t.Fatalf("read %d bytes: %v", n, err)+ }+}++func TestClientSpeechInterruptedStream(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte{1, 0})+ w.(http.Flusher).Flush()+ panic(http.ErrAbortHandler)+ }))+ defer srv.Close()+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}+ var got bytes.Buffer+ err := client.Speech(t.Context(), llm.SpeechRequest{}, func(b []byte) error { _, err := got.Write(b); return err })+ if !errors.Is(err, io.ErrUnexpectedEOF) || !bytes.Equal(got.Bytes(), []byte{1, 0}) {+ t.Fatalf("interrupted audio: %x, %v", got.Bytes(), err)+ }+}++func TestClientSpeechConsumerFailure(t *testing.T) {+ canceled := make(chan struct{})+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ io.Copy(io.Discard, r.Body)+ w.Write([]byte{1, 0})+ w.(http.Flusher).Flush()+ <-r.Context().Done()+ close(canceled)+ }))+ defer srv.Close()+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}+ stop := errors.New("consumer stopped reading")+ err := client.Speech(t.Context(), llm.SpeechRequest{}, func([]byte) error { return stop })+ if !errors.Is(err, stop) {+ t.Fatalf("consumer error lost: %v", err)+ }+ select {+ case <-canceled:+ case <-time.After(5 * time.Second):+ t.Fatal("runner request was not canceled")+ }+}++// The runner's logging wrapper must preserve HTTP controls for streaming.+type controlledSpeechWriter struct {+ *httptest.ResponseRecorder+ deadline time.Time+ flushErr error+}++func (w *controlledSpeechWriter) SetWriteDeadline(t time.Time) error {+ w.deadline = t+ return nil+}++func (w *controlledSpeechWriter) FlushError() error { return w.flushErr }++func TestSpeechResponseControls(t *testing.T) {+ want := errors.New("write failed")+ w := &controlledSpeechWriter{ResponseRecorder: httptest.NewRecorder(), flushErr: want}+ controller := http.NewResponseController(&statusRecorder{ResponseWriter: w})+ deadline := time.Now().Add(time.Second)+ if err := controller.SetWriteDeadline(deadline); err != nil || !w.deadline.Equal(deadline) {+ t.Fatalf("write deadline lost: %v, %v", w.deadline, err)+ }+ if err := controller.Flush(); !errors.Is(err, want) {+ t.Fatalf("flush error lost: %v", err)+ }+}++func TestSpeechModelRejectsTextTokenization(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "this model does not support text tokenization", http.StatusBadRequest)+ }))+ defer srv.Close()+ client := &Client{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client()}+ _, err := client.Tokenize(t.Context(), "hello")+ var status api.StatusError+ if !errors.As(err, &status) || status.StatusCode != 400 || status.ErrorMessage != "this model does not support text tokenization" {+ t.Fatalf("tokenization error lost: %v", err)+ }+}diff --git a/mlxrunner/model/architectures/architectures.go b/mlxrunner/model/architectures/architectures.goindex c5a374e97..ec5b1ed8c 100644--- a/mlxrunner/model/architectures/architectures.go+++ b/mlxrunner/model/architectures/architectures.go@@ -15,6 +15,7 @@ import (_ "github.com/ollama/ollama/mlxrunner/model/laya"_ "github.com/ollama/ollama/mlxrunner/model/llama"_ "github.com/ollama/ollama/mlxrunner/model/nemotron_h"+ _ "github.com/ollama/ollama/mlxrunner/model/phonon"_ "github.com/ollama/ollama/mlxrunner/model/qwen3"_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5"_ "github.com/ollama/ollama/mlxrunner/model/qwen3_5_moe"diff --git a/mlxrunner/model/phonon/flow.go b/mlxrunner/model/phonon/flow.gonew file mode 100644index 000000000..8650caf6f--- /dev/null+++ b/mlxrunner/model/phonon/flow.go@@ -0,0 +1,215 @@+package phonon++import (+ "fmt"+ "math"++ "github.com/ollama/ollama/mlx"+ "github.com/ollama/ollama/mlxrunner/batch"+ "github.com/ollama/ollama/mlxrunner/cache"+ "github.com/ollama/ollama/mlxrunner/nn"+)++// Layer is a pre-norm transformer layer, shared by the flow LM and the Mimi+// decoder. Scale1 and Scale2 are Mimi's layer scales.+type Layer struct {+ Norm1 *nn.LayerNorm+ Norm2 *nn.LayerNorm+ InProj *nn.Linear+ OutProj *nn.Linear+ Linear1 *nn.Linear+ Linear2 *nn.Linear+ Scale1 *mlx.Array+ Scale2 *mlx.Array+}++type Transformer struct {+ Layers []*Layer+ NumHeads int32+ MaxPeriod float32+ GELU string+}++func loadTransformer(tensors map[string]*mlx.Array, prefix string, layers, heads int32, period float32, gelu string) (*Transformer, error) {+ t := &Transformer{NumHeads: heads, MaxPeriod: period, GELU: gelu}+ for i := range layers {+ p := fmt.Sprintf("%s.layers.%d.", prefix, i)+ l := &Layer{+ Norm1: layerNorm(tensors, p+"norm1", 1e-5),+ Norm2: layerNorm(tensors, p+"norm2", 1e-5),+ InProj: linear(tensors, p+"self_attn.in_proj"),+ OutProj: linear(tensors, p+"self_attn.out_proj"),+ Linear1: linear(tensors, p+"linear1"),+ Linear2: linear(tensors, p+"linear2"),+ // The runner loads "<name>.scale" tensors as "<name>_scale".+ Scale1: tensors[p+"layer_scale_1_scale"],+ Scale2: tensors[p+"layer_scale_2_scale"],+ }+ if l.Norm1 == nil || l.Norm2 == nil || l.InProj == nil || l.OutProj == nil || l.Linear1 == nil || l.Linear2 == nil {+ return nil, fmt.Errorf("missing weights for %s", p)+ }+ if heads <= 0 || l.Norm1.Weight.Dim(0)%int(heads) != 0 {+ return nil, fmt.Errorf("invalid attention heads for %s", p)+ }+ t.Layers = append(t.Layers, l)+ }+ return t, nil+}++// forward runs x [1, L, D] at absolute position offset.+func (t *Transformer) forward(x *mlx.Array, offset int, caches []cache.Attention) *mlx.Array {+ L, D := int32(x.Dim(1)), int32(x.Dim(2))+ // Attention uses the batch shape and positions for both text and audio frames.+ b := &batch.Batch{+ InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, int(L)),+ SeqOffsets: []int32{int32(offset)},+ SeqQueryLens: []int32{L},+ }+ H, hd := t.NumHeads, D/t.NumHeads+ scale := float32(1 / math.Sqrt(float64(hd)))+ pos := mlx.FromValues([]int32{int32(offset)}, 1)+ split := func(qkv *mlx.Array, i int32) *mlx.Array {+ s := qkv.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(int(i*D), int((i+1)*D)))+ return mlx.Transpose(mlx.Reshape(s, 1, L, H, hd), 0, 2, 1, 3)+ }+ for i, l := range t.Layers {+ qkv := l.InProj.Forward(l.Norm1.Forward(x))+ q := mlx.RoPEWithBase(split(qkv, 0), int(hd), true, t.MaxPeriod, 1, pos)+ k := mlx.RoPEWithBase(split(qkv, 1), int(hd), true, t.MaxPeriod, 1, pos)+ history := caches[i].Update(b, k, split(qkv, 2))+ a := nn.ScaledDotProductAttention(b, q, scale, nn.WithKVHistory(history), nn.WithMask(nn.CausalMask()))+ a = mlx.Reshape(mlx.Transpose(a, 0, 2, 1, 3), 1, L, D)+ x = residual(x, l.OutProj.Forward(a), l.Scale1)+ h := l.Linear1.Forward(l.Norm2.Forward(x))+ if t.GELU == "erf" {+ h = mlx.GELU(h)+ } else {+ h = mlx.GELUApprox(h)+ }+ h = l.Linear2.Forward(h)+ x = residual(x, h, l.Scale2)+ }+ return x+}++func residual(x, update, scale *mlx.Array) *mlx.Array {+ if scale != nil {+ update = mlx.Mul(update, scale)+ }+ return mlx.Add(x, update)+}++// FlowHead is the AdaLN MLP that turns a transformer state into a latent with+// one Lagrangian self-distillation step from s=0 to t=1.+type FlowHead struct {+ CondEmbed *nn.Linear+ InputProj *nn.Linear+ Blocks []*FlowBlock+ Final *nn.Linear+ // Modulation stacks every block's adaLN projection and the final+ // layer's so one matmul produces all shifts, scales and gates.+ Modulation *nn.Linear+ // Time is the averaged embedding of the constant s=0, t=1 times.+ Time *mlx.Array+}++type FlowBlock struct {+ Norm *nn.LayerNorm+ MLP0 *nn.Linear+ MLP2 *nn.Linear+}++func loadFlowHead(tensors map[string]*mlx.Array, prefix string, depth int32, norm string) (*FlowHead, error) {+ f := &FlowHead{+ CondEmbed: linear(tensors, prefix+".cond_embed"),+ InputProj: linear(tensors, prefix+".input_proj"),+ Final: linear(tensors, prefix+".final_layer.linear"),+ }+ var mw, mb []*mlx.Array+ for i := range depth {+ p := fmt.Sprintf("%s.res_blocks.%d.", prefix, i)+ b := &FlowBlock{Norm: layerNorm(tensors, p+"in_ln", 1e-6), MLP0: linear(tensors, p+"mlp.0"), MLP2: linear(tensors, p+"mlp.2")}+ ada := linear(tensors, p+"adaLN_modulation.1")+ if b.Norm == nil || b.MLP0 == nil || b.MLP2 == nil || ada == nil {+ return nil, fmt.Errorf("missing weights for %s", p)+ }+ f.Blocks = append(f.Blocks, b)+ mw, mb = append(mw, ada.Weight), append(mb, ada.Bias)+ }+ ada := linear(tensors, prefix+".final_layer.adaLN_modulation.1")+ if f.CondEmbed == nil || f.InputProj == nil || f.Final == nil || ada == nil {+ return nil, fmt.Errorf("missing weights for %s", prefix)+ }+ f.Modulation = nn.NewLinear(mlx.Concatenate(append(mw, ada.Weight), 0), mlx.Concatenate(append(mb, ada.Bias), 0))++ // LSD conditions on a start and a target time; with one step they are+ // always 0 and 1.+ t0, t1 := timeEmbed(tensors, prefix+".time_embed.0", 0, norm), timeEmbed(tensors, prefix+".time_embed.1", 1, norm)+ if t0 == nil || t1 == nil {+ return nil, fmt.Errorf("missing time embedding for %s", prefix)+ }+ f.Time = mlx.MulScalar(mlx.Add(t0, t1), 0.5)+ return f, nil+}++// timeEmbed evaluates a TimestepEmbedder at t with the checkpoint's norm.+func timeEmbed(tensors map[string]*mlx.Array, prefix string, t float32, norm string) *mlx.Array {+ freqs := tensors[prefix+".freqs"]+ l0, l2, alpha := linear(tensors, prefix+".mlp.0"), linear(tensors, prefix+".mlp.2"), tensors[prefix+".mlp.3.alpha"]+ if freqs == nil || l0 == nil || l2 == nil || alpha == nil {+ return nil+ }+ args := mlx.MulScalar(freqs, t)+ x := mlx.Concatenate([]*mlx.Array{mlx.Cos(args), mlx.Sin(args)}, 0)+ x = l2.Forward(mlx.SiLU(l0.Forward(x)))+ return timeNorm(x, alpha, norm)+}++func timeNorm(x, alpha *mlx.Array, norm string) *mlx.Array {+ if norm == "rms" {+ return mlx.RMSNormFn(x, alpha, 1e-5)+ }+ // Legacy checkpoints divide by the unbiased standard deviation, without centering x.+ n := float32(x.Dim(0))+ centered := mlx.Sub(x, mlx.Mean(x, -1, true))+ variance := mlx.DivScalar(mlx.Sum(mlx.Mul(centered, centered), -1, true), n-1)+ return mlx.Mul(mlx.Mul(x, alpha), mlx.RSqrt(mlx.AddScalar(variance, 1e-5)))+}++// sample returns the latent for state c [1, D] from noise [1, latent].+func (f *FlowHead) sample(c, noise *mlx.Array) *mlx.Array {+ y := mlx.Add(f.CondEmbed.Forward(c), f.Time)+ mod := f.Modulation.Forward(mlx.SiLU(y))+ dim := y.Dim(1)+ chunk := func(i int) *mlx.Array { return mod.Slice(mlx.Slice(), mlx.Slice(i*dim, (i+1)*dim)) }++ x := f.InputProj.Forward(noise)+ for i, b := range f.Blocks {+ h := modulate(b.Norm.Forward(x), chunk(3*i), chunk(3*i+1))+ h = b.MLP2.Forward(mlx.SiLU(b.MLP0.Forward(h)))+ x = mlx.Add(x, mlx.Mul(chunk(3*i+2), h))+ }+ n := 3 * len(f.Blocks)+ x = modulate(mlx.LayerNormFn(x, nil, nil, 1e-6), chunk(n), chunk(n+1))+ return mlx.Add(noise, f.Final.Forward(x))+}++func modulate(x, shift, scale *mlx.Array) *mlx.Array {+ return mlx.Add(mlx.Mul(x, mlx.AddScalar(scale, 1)), shift)+}++func linear(tensors map[string]*mlx.Array, prefix string) *nn.Linear {+ w := tensors[prefix+".weight"]+ if w == nil {+ return nil+ }+ return nn.NewLinear(w, tensors[prefix+".bias"])+}++func layerNorm(tensors map[string]*mlx.Array, prefix string, eps float32) *nn.LayerNorm {+ w := tensors[prefix+".weight"]+ if w == nil {+ return nil+ }+ return &nn.LayerNorm{Weight: w, Bias: tensors[prefix+".bias"], Eps: eps}+}diff --git a/mlxrunner/model/phonon/mimi.go b/mlxrunner/model/phonon/mimi.gonew file mode 100644index 000000000..919ebb3bb--- /dev/null+++ b/mlxrunner/model/phonon/mimi.go@@ -0,0 +1,204 @@+package phonon++import (+ "fmt"+ "strconv"+ "strings"++ "github.com/ollama/ollama/mlx"+ "github.com/ollama/ollama/mlxrunner/cache"+)++// Mimi is the streaming decoder half of the Mimi codec: latent frames in,+// 24 kHz samples out.+type Mimi struct {+ OutputProj *mlx.Array // [latent, dim]+ Upsample *ConvTr+ Transformer *Transformer+ Context int+ Decoder []*SeanetLayer+}++// SeanetLayer is one decoder stage; a layer with no weights is an ELU.+type SeanetLayer struct {+ Conv *Conv+ ConvTr *ConvTr+ Res []*Conv+}++// Conv is a causal stride-1 convolution. Weight is [out, kernel, in].+type Conv struct {+ Weight *mlx.Array+ Bias *mlx.Array+ Dilation int+}++// ConvTr is a transposed convolution whose kernel is twice its stride, as+// every Mimi upsampler's is. Weight is [in, kernel*out], or [kernel, channels]+// when depthwise.+type ConvTr struct {+ Weight *mlx.Array+ Bias *mlx.Array+ Stride int+ Depthwise bool+}++func loadMimi(tensors map[string]*mlx.Array, cfg *config) (*Mimi, error) {+ t := cfg.Mimi+ m := &Mimi{Context: t.Context}+ if w := tensors["mimi.quantizer.output_proj.weight"]; w != nil {+ m.OutputProj = mlx.Transpose(mlx.Squeeze(w, 2), 1, 0)+ }+ if w := tensors["mimi.upsample.convtr.convtr.weight"]; w != nil {+ m.Upsample = &ConvTr{Weight: mlx.Transpose(mlx.Squeeze(w, 1), 1, 0), Stride: w.Dim(2) / 2, Depthwise: true}+ }+ if m.OutputProj == nil || m.Upsample == nil {+ return nil, fmt.Errorf("missing mimi projection weights")+ }+ var err error+ if m.Transformer, err = loadTransformer(tensors, "mimi.decoder_transformer.transformer", t.NumLayers, t.NumHeads, t.MaxPeriod, cfg.gelu(t.GELU)); err != nil {+ return nil, err+ }++ const prefix = "mimi.decoder.model."+ last := -1+ for name := range tensors {+ if rest, ok := strings.CutPrefix(name, prefix); ok {+ index, _, ok := strings.Cut(rest, ".")+ if i, err := strconv.Atoi(index); ok && err == nil {+ last = max(last, i)+ }+ }+ }+ dilation := 1+ for i := 0; i <= last; i++ {+ p := prefix + strconv.Itoa(i)+ l := &SeanetLayer{}+ switch {+ case tensors[p+".conv.weight"] != nil:+ l.Conv = conv(tensors, p+".conv", 1)+ case tensors[p+".convtr.weight"] != nil:+ w := tensors[p+".convtr.weight"]+ in, out, k := w.Dim(0), w.Dim(1), w.Dim(2)+ l.ConvTr = &ConvTr{+ Weight: mlx.Reshape(mlx.Transpose(w, 0, 2, 1), int32(in), int32(k*out)),+ Bias: tensors[p+".convtr.bias"],+ Stride: k / 2,+ }+ dilation = 1+ case tensors[p+".block.1.conv.weight"] != nil:+ if tensors[p+".block.3.conv.weight"] == nil {+ return nil, fmt.Errorf("missing residual weights for %s", p)+ }+ l.Res = []*Conv{conv(tensors, p+".block.1.conv", dilation), conv(tensors, p+".block.3.conv", 1)}+ dilation *= cfg.Mimi.DilationBase+ }+ m.Decoder = append(m.Decoder, l)+ }+ if last < 0 {+ return nil, fmt.Errorf("missing mimi decoder weights")+ }+ return m, nil+}++func conv(tensors map[string]*mlx.Array, prefix string, dilation int) *Conv {+ return &Conv{Weight: mlx.Transpose(tensors[prefix+".weight"], 0, 2, 1), Bias: tensors[prefix+".bias"], Dilation: dilation}+}++// mimiState is one stream's decoder state: convolution context, transposed+// convolution overlap, and the transformer's attention window.+type mimiState struct {+ scope *mlx.Scope+ conv map[*Conv]*mlx.Array+ transpose map[*ConvTr]*mlx.Array+ attention []cache.Attention+ pos int+}++func (m *Mimi) newState() *mimiState {+ s := &mimiState{+ scope: mlx.NewScope(),+ conv: make(map[*Conv]*mlx.Array),+ transpose: make(map[*ConvTr]*mlx.Array),+ }+ for range m.Transformer.Layers {+ s.attention = append(s.attention, cache.NewRotatingKVCache(m.Context))+ }+ return s+}++func (s *mimiState) close() {+ for _, c := range s.attention {+ c.Free()+ }+ s.scope.Close()+}++// decode turns normalized latents [1, T, latent] into samples [1, T*frame, 1].+func (m *Mimi) decode(s *mimiState, latents *mlx.Array) *mlx.Array {+ x := latents.Matmul(m.OutputProj)+ x = m.Upsample.forward(s, x)+ steps := x.Dim(1)+ x = m.Transformer.forward(x, s.pos, s.attention)+ s.pos += steps+ for _, l := range m.Decoder {+ switch {+ case l.Conv != nil:+ x = l.Conv.forward(s, x)+ case l.ConvTr != nil:+ x = l.ConvTr.forward(s, x)+ case l.Res != nil:+ x = mlx.Add(x, l.Res[1].forward(s, elu(l.Res[0].forward(s, elu(x)))))+ default:+ x = elu(x)+ }+ }+ return x+}++func (c *Conv) forward(s *mimiState, x *mlx.Array) *mlx.Array {+ if pad := (c.Weight.Dim(1) - 1) * c.Dilation; pad > 0 {+ prev := s.conv[c]+ if prev == nil {+ prev = mlx.Zeros(x.DType(), 1, pad, x.Dim(2))+ s.scope.Attach(prev)+ s.conv[c] = prev+ }+ x = prev.Concatenate(1, x)+ prev.Set(x.Slice(mlx.Slice(), mlx.Slice(x.Dim(1)-pad, x.Dim(1)), mlx.Slice()))+ }+ return mlx.Conv1d(x, c.Weight, c.Bias, 1, 0, int32(c.Dilation), 1)+}++// forward computes every kernel tap with one matmul, then overlap-adds the+// second half of each step onto the first half of the next.+func (c *ConvTr) forward(s *mimiState, x *mlx.Array) *mlx.Array {+ T, S := x.Dim(1), c.Stride+ var z *mlx.Array+ if c.Depthwise {+ z = mlx.Mul(mlx.ExpandDims(x, 2), c.Weight)+ } else {+ z = mlx.Reshape(x.Matmul(c.Weight), 1, int32(T), int32(2*S), -1)+ }+ C := z.Dim(3)+ head := z.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(0, S), mlx.Slice())+ tail := z.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(S, 2*S), mlx.Slice())+ prev := s.transpose[c]+ if prev == nil {+ prev = mlx.Zeros(x.DType(), 1, 1, S, C)+ s.scope.Attach(prev)+ s.transpose[c] = prev+ }+ shifted := prev.Concatenate(1, tail.Slice(mlx.Slice(), mlx.Slice(0, T-1), mlx.Slice(), mlx.Slice()))+ prev.Set(tail.Slice(mlx.Slice(), mlx.Slice(T-1, T), mlx.Slice(), mlx.Slice()))+ y := mlx.Reshape(mlx.Add(head, shifted), 1, int32(T*S), int32(C))+ if c.Bias != nil {+ y = mlx.Add(y, c.Bias)+ }+ return y+}++var elu = mlx.Compile1("ELU", func(x *mlx.Array) *mlx.Array {+ zero := mlx.FromValue[float32](0).AsType(x.DType())+ return mlx.Where(x.Greater(zero), x, mlx.AddScalar(mlx.Exp(mlx.Minimum(x, zero)), -1))+}, mlx.Shapeless())diff --git a/mlxrunner/model/phonon/model.go b/mlxrunner/model/phonon/model.gonew file mode 100644index 000000000..020c1b91a--- /dev/null+++ b/mlxrunner/model/phonon/model.go@@ -0,0 +1,349 @@+package phonon++import (+ "context"+ "fmt"+ "maps"+ "math"+ "math/rand/v2"+ "net/http"+ "slices"+ "strings"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/mlx"+ "github.com/ollama/ollama/mlxrunner/batch"+ "github.com/ollama/ollama/mlxrunner/cache"+ "github.com/ollama/ollama/mlxrunner/model"+ "github.com/ollama/ollama/mlxrunner/nn"+ "github.com/ollama/ollama/mlxrunner/tokenizer"+)++func init() { model.Register("Phonon", newModel) }++// config follows the upstream Phonon config.json.+type config struct {+ ModelID *struct {+ Sig string `json:"sig"`+ Epoch int `json:"epoch"`+ } `json:"model_id"`+ FlowLM struct {+ NumHeads int32 `json:"num_heads"`+ NumLayers int32 `json:"num_layers"`+ MaxPeriod float32 `json:"max_period"`+ FlowDepth int32 `json:"flow_depth"`+ GELU string `json:"gelu"`+ TimeRMSNorm string `json:"time_rms_norm"`+ } `json:"flow_lm"`+ Mimi struct {+ SampleRate int `json:"sample_rate"`+ FrameRate float64 `json:"frame_rate"`+ DilationBase int `json:"dilation_base"`+ NumHeads int32 `json:"transformer_num_heads"`+ NumLayers int32 `json:"transformer_num_layers"`+ Context int `json:"transformer_context"`+ MaxPeriod float32 `json:"transformer_max_period"`+ GELU string `json:"gelu"`+ } `json:"mimi"`+ Temperature float32 `json:"temp"`+ DecodeSteps int `json:"lsd_decode_steps"`+ EOSThreshold float32 `json:"eos_threshold"`+}++const (+ sampleRate = 24000+ // mimiBatch is how many frames the codec decodes per call after the first.+ mimiBatch = 8+)++type Model struct {+ Embed *nn.Embedding+ InputLinear *nn.Linear+ BOS *mlx.Array+ NumSpeakers *mlx.Array+ Flow *Transformer+ OutNorm *nn.LayerNorm+ EOS *nn.Linear+ Head *FlowHead+ EmbMean *mlx.Array+ EmbStd *mlx.Array+ Mimi *Mimi+ Voices map[string]*Voice++ cfg *config+ tok *unigram++ // sample maps a transformer state and noise to a latent and EOS logit.+ sample mlx.CompileFunc+}++// Voice is a speaker prompt already run through the flow LM: per-layer keys+// and values [1, heads, frames, head_dim].+type Voice struct {+ Keys []*mlx.Array+ Values []*mlx.Array+}++func newModel(root *model.Root) (model.Model, error) {+ cfg := config{Temperature: 0.3}+ if err := root.Manifest.ReadConfigJSON("config.json", &cfg); err != nil {+ return nil, fmt.Errorf("load config: %w", err)+ }+ if err := cfg.validate(); err != nil {+ return nil, err+ }+ data, err := root.Manifest.ReadConfig("tokenizer.json")+ if err != nil {+ return nil, fmt.Errorf("load tokenizer: %w", err)+ }+ tok, err := newUnigram(data)+ if err != nil {+ return nil, fmt.Errorf("parse tokenizer: %w", err)+ }+ return &Model{cfg: &cfg, tok: tok}, nil+}++// gelu follows upstream's checkpoint defaults; an explicit activation wins.+func (c *config) gelu(value string) string {+ if value != "" {+ return value+ }+ if c.ModelID != nil {+ return "erf"+ }+ return "tanh"+}++func (c *config) validate() error {+ for _, value := range []string{c.FlowLM.GELU, c.Mimi.GELU} {+ if value != "" && value != "erf" && value != "tanh" {+ return fmt.Errorf("unsupported phonon GELU %q", value)+ }+ }+ if value := c.FlowLM.TimeRMSNorm; value != "" && value != "rms_var" && value != "rms" {+ return fmt.Errorf("unsupported phonon time norm %q", value)+ }+ if c.Mimi.SampleRate != sampleRate || c.Mimi.FrameRate != 12.5 {+ return fmt.Errorf("unsupported mimi rates: %d Hz, %g frames/s", c.Mimi.SampleRate, c.Mimi.FrameRate)+ }+ if c.FlowLM.NumLayers <= 0 || c.FlowLM.NumHeads <= 0 || c.FlowLM.MaxPeriod <= 0 || c.FlowLM.FlowDepth <= 0 ||+ c.Mimi.NumLayers <= 0 || c.Mimi.NumHeads <= 0 || c.Mimi.Context <= 0 || c.Mimi.MaxPeriod <= 0 || c.Mimi.DilationBase <= 0 {+ return fmt.Errorf("invalid phonon transformer configuration")+ }+ if c.DecodeSteps != 1 {+ return fmt.Errorf("phonon requires one LSD decode step, got %d", c.DecodeSteps)+ }+ if c.Temperature < 0 || math.IsNaN(float64(c.Temperature)) || math.IsInf(float64(c.Temperature), 0) ||+ math.IsNaN(float64(c.EOSThreshold)) || math.IsInf(float64(c.EOSThreshold), 0) {+ return fmt.Errorf("invalid phonon sampling configuration")+ }+ return nil+}++func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {+ // The flow LM transformer is bandwidth bound and runs in bfloat16; the+ // flow head and the Mimi decoder stay in float32.+ for name, t := range tensors {+ if d := t.DType(); d != mlx.DTypeFloat32 && d != mlx.DTypeBFloat16 && d != mlx.DTypeFloat16 {+ return fmt.Errorf("phonon requires unquantized floating-point weights: %s", name)+ }+ dtype := mlx.DTypeFloat32+ if strings.HasPrefix(name, "flow_lm.transformer.") || strings.HasPrefix(name, "voices.") ||+ name == "flow_lm.num_speakers" || name == "flow_lm.conditioner.embed.weight" || name == "flow_lm.input_linear.weight" {+ dtype = mlx.DTypeBFloat16+ }+ if t.DType() != dtype {+ tensors[name] = t.AsType(dtype)+ }+ }+ f := m.cfg.FlowLM+ var err error+ if m.Flow, err = loadTransformer(tensors, "flow_lm.transformer", f.NumLayers, f.NumHeads, f.MaxPeriod, m.cfg.gelu(f.GELU)); err != nil {+ return err+ }+ if m.Head, err = loadFlowHead(tensors, "flow_lm.flow_net", f.FlowDepth, f.TimeRMSNorm); err != nil {+ return err+ }+ if m.Mimi, err = loadMimi(tensors, m.cfg); err != nil {+ return err+ }+ if w := tensors["flow_lm.conditioner.embed.weight"]; w != nil {+ m.Embed = nn.NewEmbedding(w)+ }+ m.InputLinear = linear(tensors, "flow_lm.input_linear")+ m.OutNorm = layerNorm(tensors, "flow_lm.out_norm", 1e-5)+ m.EOS = linear(tensors, "flow_lm.out_eos")+ m.BOS, m.EmbMean, m.EmbStd = tensors["flow_lm.bos_emb"], tensors["flow_lm.emb_mean"], tensors["flow_lm.emb_std"]+ if m.Embed == nil || m.InputLinear == nil || m.OutNorm == nil || m.EOS == nil || m.BOS == nil || m.EmbMean == nil || m.EmbStd == nil {+ return fmt.Errorf("missing flow_lm weights")+ }+ m.NumSpeakers = tensors["flow_lm.num_speakers"]+ if m.NumSpeakers == nil || !slices.Equal(m.NumSpeakers.Dims(), []int{1, 1, m.Embed.Weight.Dim(1)}) {+ return fmt.Errorf("missing or invalid num_speakers conditioning")+ }+ m.BOS = mlx.Reshape(m.BOS, 1, 1, -1)+ m.sample = mlx.Compile("phonon_sample", func(in ...*mlx.Array) []*mlx.Array {+ h := m.OutNorm.Forward(mlx.Reshape(in[0].AsType(mlx.DTypeFloat32), 1, -1))+ latent := m.Head.sample(h, in[1])+ return []*mlx.Array{mlx.ExpandDims(latent, 0), m.EOS.Forward(h)}+ })++ // The packager projects each speaker prompt into the flow LM's embedding+ // space. Prefill it once at load time; requests start from these cached keys+ // and values, without sharing mutable generation state.+ m.Voices = map[string]*Voice{}+ for name, t := range tensors {+ voice, ok := strings.CutPrefix(name, "voices.")+ if !ok {+ continue+ }+ if voice == "" || t.NumDims() != 3 || t.Dim(0) != 1 || t.Dim(1) == 0 || t.Dim(2) != m.Embed.Weight.Dim(1) {+ return fmt.Errorf("voice %q has shape %v", voice, t.Dims())+ }+ m.Voices[voice] = m.prefillVoice(t)+ }+ if len(m.Voices) == 0 {+ return fmt.Errorf("model has no voices")+ }+ return nil+}++func (m *Model) prefillVoice(prompt *mlx.Array) *Voice {+ caches := make([]cache.Attention, len(m.Flow.Layers))+ for i := range caches {+ caches[i] = cache.NewKVCache()+ defer caches[i].Free()+ }+ kv := mlx.ScopedEval(func() []*mlx.Array {+ m.Flow.forward(prompt, 0, caches)+ var kv []*mlx.Array+ for _, c := range caches {+ kv = append(kv, c.State()...)+ }+ return kv+ })+ v := &Voice{}+ for i := 0; i < len(kv); i += 2 {+ v.Keys = append(v.Keys, kv[i])+ v.Values = append(v.Values, kv[i+1])+ }+ return v+}++// Speech models load through model.Model but never run the text pipeline.+func (m *Model) NewCaches() []cache.Cache { return nil }++func (m *Model) Forward(*batch.Batch, []cache.Cache) (*mlx.Array, *mlx.Array) {+ panic("phonon: text forward is not supported")+}+func (m *Model) Unembed(x *mlx.Array) *mlx.Array { return x }+func (m *Model) Tokenizer() *tokenizer.Tokenizer { return nil }+func (m *Model) MaxContextLength() int { return 0 }++func (m *Model) VoiceNames() []string { return slices.Sorted(maps.Keys(m.Voices)) }++func (m *Model) Speak(ctx context.Context, req llm.SpeechRequest, emit func([]float32) error) error {+ name := req.Voice+ if name == "" {+ name = m.VoiceNames()[0]+ }+ voice, ok := m.Voices[name]+ if !ok {+ return api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: fmt.Sprintf("voice %q not found; available voices: %s", req.Voice, strings.Join(m.VoiceNames(), ", "))}+ }+ chunks, err := m.tok.chunks(req.Input)+ if err != nil {+ return api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: err.Error()}+ }+ rng := rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64()))+ for _, chunk := range chunks {+ if err := m.speak(ctx, chunk, voice, rng, emit); err != nil {+ return err+ }+ }+ return nil+}++// speak generates one chunk from a fresh copy of the voice state.+func (m *Model) speak(ctx context.Context, text string, voice *Voice, rng *rand.Rand, emit func([]float32) error) error {+ text, framesAfterEOS, err := prepareText(text)+ if err != nil {+ return err+ }+ tokens := m.tok.encode(text)+ maxFrames := int(math.Ceil((float64(len(tokens))/3 + 2) * m.cfg.Mimi.FrameRate))++ caches := make([]cache.Attention, len(m.Flow.Layers))+ for i := range caches {+ caches[i] = cache.NewKVCache()+ defer caches[i].Free()+ }+ mimi := m.Mimi.newState()+ defer mimi.close()+ held := mlx.NewScope()+ defer held.Close()++ prev := m.BOS.Clone()+ held.Attach(prev)+ offset := voice.Keys[0].Dim(2)+ mlx.Scoped(func() {+ for i, c := range caches {+ c.Update(nil, voice.Keys[i], voice.Values[i])+ }+ ids := mlx.FromValues(tokens, 1, len(tokens))+ m.Flow.forward(m.Embed.Forward(ids), offset, caches)+ })+ offset += len(tokens)++ std := math.Sqrt(float64(m.cfg.Temperature))+ noise := make([]float32, m.BOS.Dim(2))+ for generated, stop := 0, maxFrames; generated < stop; {+ // Decode the first frame alone for latency, then in batches. EOS+ // is read at the same sync points; frames past it are dropped.+ n := min(mimiBatch, stop-generated)+ if generated == 0 {+ n = 1+ }+ var latents, logits []*mlx.Array+ for step := generated; step < generated+n; step++ {+ if err := ctx.Err(); err != nil {+ return err+ }+ for i := range noise {+ noise[i] = float32(rng.NormFloat64() * std)+ }+ out := mlx.ScopedArrays(func() []*mlx.Array {+ input := mlx.Add(m.InputLinear.Forward(prev.AsType(mlx.DTypeBFloat16)), m.NumSpeakers)+ h := m.Flow.forward(input, offset+step, caches)+ out := m.sample(h, mlx.FromValues(noise, 1, len(noise)))+ prev.Set(out[0])+ return out+ })+ held.Attach(out...)+ mlx.AsyncEval(out...)+ latents, logits = append(latents, out[0]), append(logits, out[1])+ }+ var eos []float32+ mlx.Scoped(func() { eos = mlx.Concatenate(logits, 0).Floats() })+ for i, e := range eos {+ if e > m.cfg.EOSThreshold {+ stop = min(stop, generated+i+1+framesAfterEOS)+ break+ }+ }+ n = min(n, stop-generated)+ var samples []float32+ mlx.Scoped(func() {+ z := mlx.Add(mlx.Mul(mlx.Concatenate(latents[:n], 1), m.EmbStd), m.EmbMean)+ samples = m.Mimi.decode(mimi, z).Floats()+ })+ held.Discard(latents...)+ held.Discard(logits...)+ generated += n+ if err := emit(samples); err != nil {+ return err+ }+ }+ return nil+}diff --git a/mlxrunner/model/phonon/model_test.go b/mlxrunner/model/phonon/model_test.gonew file mode 100644index 000000000..9d42031b6--- /dev/null+++ b/mlxrunner/model/phonon/model_test.go@@ -0,0 +1,91 @@+package phonon++import (+ "encoding/json"+ "math"+ "testing"++ "github.com/ollama/ollama/mlx"+ "github.com/ollama/ollama/mlx/mlxtest"+)++func TestConfig(t *testing.T) {+ const checkpoint = `{"flow_lm":{"num_heads":12,"num_layers":6,"max_period":10000,"flow_depth":6},"mimi":{"sample_rate":24000,"frame_rate":12.5,"dilation_base":2,"transformer_num_heads":8,"transformer_num_layers":2,"transformer_context":250,"transformer_max_period":10000},"temp":0.8,"lsd_decode_steps":1,"eos_threshold":4.0}`+ var cfg config+ if err := json.Unmarshal([]byte(checkpoint), &cfg); err != nil {+ t.Fatal(err)+ }+ if err := cfg.validate(); err != nil {+ t.Fatal(err)+ }+ if cfg.EOSThreshold != 4 || cfg.Temperature != .8 {+ t.Fatalf("lost checkpoint sampling settings: %+v", cfg)+ }+ for _, tc := range []struct {+ name string+ change func(*config)+ }{+ {"GELU", func(c *config) { c.FlowLM.GELU = "relu" }},+ {"time norm", func(c *config) { c.FlowLM.TimeRMSNorm = "other" }},+ {"sample rate", func(c *config) { c.Mimi.SampleRate = 48000 }},+ {"layers", func(c *config) { c.FlowLM.NumLayers = 0 }},+ {"multiple LSD steps", func(c *config) { c.DecodeSteps = 2 }},+ {"negative temperature", func(c *config) { c.Temperature = -1 }},+ {"NaN threshold", func(c *config) { c.EOSThreshold = float32(math.NaN()) }},+ } {+ t.Run(tc.name, func(t *testing.T) {+ c := cfg+ tc.change(&c)+ if c.validate() == nil {+ t.Fatal("accepted unsupported configuration")+ }+ })+ }+}++func TestCheckpointDefaults(t *testing.T) {+ for _, tc := range []struct {+ json, gelu string+ temperature float32+ }{+ {`{}`, "tanh", .3},+ {`{"model_id":{"sig":"7e71a02d","epoch":200},"temp":0.8}`, "erf", .8},+ {`{"model_id":{"sig":"test","epoch":1},"flow_lm":{"gelu":"tanh"},"temp":0}`, "tanh", 0},+ {`{"flow_lm":{"gelu":"erf"}}`, "erf", .3},+ } {+ cfg := config{Temperature: .3}+ if err := json.Unmarshal([]byte(tc.json), &cfg); err != nil {+ t.Fatal(err)+ }+ if got := cfg.gelu(cfg.FlowLM.GELU); got != tc.gelu {+ t.Errorf("GELU = %q, want %q", got, tc.gelu)+ }+ if cfg.Temperature != tc.temperature {+ t.Errorf("temperature = %v, want %v", cfg.Temperature, tc.temperature)+ }+ }+}++func TestTimeNorm(t *testing.T) {+ mlxtest.Run(t, func(t *mlxtest.T) {+ values := []float32{1, 2, 3, 6}+ x, alpha := mlx.FromValues(values, 4), mlx.FromValues([]float32{2, 2, 2, 2}, 4)+ for _, tc := range []struct {+ norm string+ denominator float64+ }{+ {"", math.Sqrt(14.0/3 + 1e-5)},+ {"rms_var", math.Sqrt(14.0/3 + 1e-5)},+ {"rms", math.Sqrt(12.5 + 1e-5)},+ } {+ out := timeNorm(x, alpha, tc.norm)+ mlx.Eval(out)+ for i, got := range out.Floats() {+ want := 2 * float64(values[i]) / tc.denominator+ if math.Abs(float64(got)-want) > 1e-5 {+ t.Errorf("%s[%d] = %v, want %v", tc.norm, i, got, want)+ }+ }+ }+ })+}diff --git a/mlxrunner/model/phonon/normalize.go b/mlxrunner/model/phonon/normalize.gonew file mode 100644index 000000000..b66e17aa4--- /dev/null+++ b/mlxrunner/model/phonon/normalize.go@@ -0,0 +1,75 @@+package phonon++import (+ "strings"+ "unicode"+)++// normalizeText applies the reference's language-independent punctuation rules.+// Number, currency, address and other language-specific rewrites belong in a+// separate pass once the model's language is known.+func normalizeText(text string) string {+ var out []rune+ push := func(c rune) {+ if c == '.' || c == ',' {+ for len(out) > 0 {+ last := out[len(out)-1]+ keep := strings.ContainsRune("\"'@+%#&", last) ||+ last == '$' && len(out) > 1 && out[len(out)-2] >= '0' && out[len(out)-2] <= '9'+ punctuation := last < 128 && (unicode.IsPunct(last) || unicode.IsSymbol(last))+ if !unicode.IsSpace(last) && (!punctuation || keep) {+ break+ }+ out = out[:len(out)-1]+ }+ }+ out = append(out, c)+ }+ space := func() {+ if len(out) > 0 && out[len(out)-1] != ' ' {+ out = append(out, ' ')+ }+ }+ runes := []rune(text)+ for i, c := range runes {+ switch {+ case strings.ContainsRune("»«‹›“”„‟″‶⹂❝❞❠〝〞〟"", c):+ push('"')+ case strings.ContainsRune("‘’‚‛′‵❛❜❟'ʼʻʹ´`", c):+ push('\'')+ case strings.ContainsRune("‐‑‒―", c):+ push('-')+ case strings.ContainsRune("–—[]{}•‣◦·→←↑↓➡➜", c):+ space()+ case c == '*':+ // A leading * before a number is a spoken star in the reference.+ // Preserve it for language-specific expansion; remove markup.+ if (i == 0 || unicode.IsSpace(runes[i-1])) && i+1 < len(runes) && runes[i+1] >= '0' && runes[i+1] <= '9' {+ push(c)+ } else {+ space()+ }+ case c == '…':+ push('.')+ case c == ':' && i > 0 && i+1 < len(runes) && !unicode.IsSpace(runes[i-1]) && !unicode.IsSpace(runes[i+1]):+ push(c)+ case strings.ContainsRune(";:()", c):+ push(',')+ space()+ case strings.ContainsRune("★☆❤♥", c):+ // These carry meaning ("stars", "love"), unlike decorative emoji.+ push(c)+ case c >= 0x1F000 && c <= 0x1FAFF || c >= 0x2600 && c <= 0x27BF ||+ c >= 0x2B00 && c <= 0x2BFF || c == 0xFE0F || c == 0x200D ||+ unicode.IsControl(c) || unicode.IsSpace(c):+ space()+ default:+ push(c)+ }+ }+ text = strings.TrimRight(strings.TrimSpace(string(out)), "\"")+ if len(text) >= 2 && text[len(text)-1] == '\'' && strings.ContainsRune(".!?,;", rune(text[len(text)-2])) {+ text = text[:len(text)-1]+ }+ return text+}diff --git a/mlxrunner/model/phonon/normalize_test.go b/mlxrunner/model/phonon/normalize_test.gonew file mode 100644index 000000000..fbaaca24c--- /dev/null+++ b/mlxrunner/model/phonon/normalize_test.go@@ -0,0 +1,59 @@+package phonon++import "testing"++func TestNormalizeText(t *testing.T) {+ // Expected strings come from xn-ptts Normalize::for_lang(Lang::En).+ for _, tc := range []struct{ input, want string }{+ {"You’re sweet! Is there anything you’d like to talk about?", "You're sweet! Is there anything you'd like to talk about?"},+ {"Youʼre sweet!", "You're sweet!"},+ {"Of course—I would love to help! What’s on your mind?", "Of course I would love to help! What's on your mind?"},+ {"Well… I am not sure… maybe we could try again.", "Well. I am not sure. maybe we could try again."},+ {"🎉 Congratulations! You did an amazing job! 😊", "Congratulations! You did an amazing job!"},+ {"Hello👋there! It is nice to meet you.", "Hello there! It is nice to meet you."},+ {"I can help (if you would like); just let me know.", "I can help, if you would like, just let me know."},+ {"Here is the plan: • Take a walk → Get some rest.", "Here is the plan, Take a walk Get some rest."},+ {"That is [really] **wonderful** news!", "That is really wonderful news!"},+ {"She said, “It is time to go.”", "She said, \"It is time to go."},+ {"Hello\x00 there. How are you?", "Hello there. How are you?"},+ {"sun ☀️ dev 👩💻 done", "sun dev done"},+ {"time: 10:30", "time, 10:30"},+ {"foo ; bar : baz", "foo, bar, baz"},+ {"a;b:c", "a, b:c"},+ {"fish &. 12%. #.", "fish &. 12%. #."},+ {"hello... goodbye", "hello. goodbye"},+ {"'Hello!'", "'Hello!"},+ {"a ‐ b ‑ c ‒ d ― e – f — g", "a - b - c - d - e f g"},+ {"👋 \x00", ""},+ } {+ got := normalizeText(tc.input)+ if got != tc.want {+ t.Errorf("normalize(%q) = %q; want %q", tc.input, got, tc.want)+ }+ if again := normalizeText(got); again != got {+ t.Errorf("normalize is not idempotent for %q: %q", got, again)+ }+ }+}++func TestNormalizeTextPreservesLanguageSpecificMeaning(t *testing.T) {+ // Until a language-specific pass exists, do not silently erase these.+ for _, text := range []string{"I ❤ you", "Rated ★★★☆☆", "*5", "2 + 2 = 4", "21°C", "user@host", "It costs $5.", "12.50", "café", "日本語"} {+ if got := normalizeText(text); got != text {+ t.Errorf("normalize(%q) = %q", text, got)+ }+ }+}++func TestUnigramDecodeSpecialTokens(t *testing.T) {+ u, err := newUnigram([]byte(`{+ "added_tokens":[{"id":1,"content":"\u0000<s>","special":true}],+ "model":{"type":"Unigram","unk_id":0,"vocab":[["<unk>",-10],["\u0000<s>",0],["▁hello",-1],["▁world",-1]]}+ }`))+ if err != nil {+ t.Fatal(err)+ }+ if got := u.decode([]int32{1, 2, 1, 3, 1}); got != "hello world" {+ t.Fatalf("decode with skipped special tokens = %q", got)+ }+}diff --git a/mlxrunner/model/phonon/text.go b/mlxrunner/model/phonon/text.gonew file mode 100644index 000000000..df8fc5034--- /dev/null+++ b/mlxrunner/model/phonon/text.go@@ -0,0 +1,344 @@+package phonon++import (+ "encoding/json"+ "errors"+ "fmt"+ "math"+ "slices"+ "strconv"+ "strings"+ "unicode"+ "unicode/utf8"+)++const metaspace = "▁"++// unigram is the SentencePiece Unigram tokenizer from tokenizer.json, with the+// Prepend("▁") normalizer, Metaspace pre-tokenizer and byte fallback the+// released checkpoints use.+type unigram struct {+ pieces []string+ scores []float64+ byteIDs [256]int32 // token ID for each byte, or -1+ bytes []int // byte value of each <0xNN> piece, or -1+ ids map[string]int32+ maxLen int+ unkID int32+ unkScore float64+ specials []specialToken+ specialIDs []bool+}++type specialToken struct {+ ID int32 `json:"id"`+ Content string `json:"content"`+ Special bool `json:"special"`+}++func newUnigram(data []byte) (*unigram, error) {+ var t struct {+ AddedTokens []specialToken `json:"added_tokens"`+ Model struct {+ Type string `json:"type"`+ UnkID int32 `json:"unk_id"`+ Vocab [][]any `json:"vocab"`+ } `json:"model"`+ }+ if err := json.Unmarshal(data, &t); err != nil {+ return nil, err+ }+ if t.Model.Type != "Unigram" {+ return nil, fmt.Errorf("unsupported tokenizer model %q", t.Model.Type)+ }+ u := &unigram{ids: make(map[string]int32, len(t.Model.Vocab)), unkID: t.Model.UnkID, specialIDs: make([]bool, len(t.Model.Vocab))}+ for i := range u.byteIDs {+ u.byteIDs[i] = -1+ }+ minScore := math.Inf(1)+ if len(t.Model.Vocab) == 0 || t.Model.UnkID < 0 || int(t.Model.UnkID) >= len(t.Model.Vocab) {+ return nil, fmt.Errorf("invalid unigram vocabulary or unknown token")+ }+ for i, v := range t.Model.Vocab {+ if len(v) != 2 {+ return nil, fmt.Errorf("invalid vocab entry %d", i)+ }+ piece, ok1 := v[0].(string)+ score, ok2 := v[1].(float64)+ if !ok1 || !ok2 || piece == "" {+ return nil, fmt.Errorf("invalid vocab entry %d", i)+ }+ b := -1+ if len(piece) == 6 && strings.HasPrefix(piece, "<0x") && piece[5] == '>' {+ if v, err := strconv.ParseUint(piece[3:5], 16, 8); err == nil {+ b = int(v)+ u.byteIDs[b] = int32(i)+ }+ }+ u.pieces = append(u.pieces, piece)+ u.scores = append(u.scores, score)+ u.bytes = append(u.bytes, b)+ u.ids[piece] = int32(i)+ u.maxLen = max(u.maxLen, len(piece))+ minScore = min(minScore, score)+ }+ u.unkScore = minScore - 10+ for _, token := range t.AddedTokens {+ if token.Content == "" || token.ID < 0 || int(token.ID) >= len(u.pieces) {+ return nil, fmt.Errorf("invalid added token %q", token.Content)+ }+ u.specials = append(u.specials, token)+ u.specialIDs[token.ID] = token.Special+ }+ return u, nil+}++func (u *unigram) encode(text string) []int32 {+ var ids []int32+ for text != "" {+ at, selected := len(text), -1+ for i, token := range u.specials {+ if pos := strings.Index(text, token.Content); pos >= 0 && (pos < at ||+ (pos == at && selected >= 0 && len(token.Content) > len(u.specials[selected].Content))) {+ at, selected = pos, i+ }+ }+ if at > 0 {+ ids = append(ids, u.encodeText(text[:at])...)+ }+ if selected < 0 {+ break+ }+ token := u.specials[selected]+ ids = append(ids, token.ID)+ text = text[at+len(token.Content):]+ }+ return ids+}++func (u *unigram) encodeText(text string) []int32 {+ var ids []int32+ for word := range strings.SplitSeq(strings.ReplaceAll(text, " ", metaspace), metaspace) {+ ids = u.encodeWord(metaspace+word, ids)+ }+ return ids+}++// encodeWord appends the Viterbi segmentation of s.+func (u *unigram) encodeWord(s string, ids []int32) []int32 {+ n := len(s)+ best := make([]float64, n+1)+ from := make([]int, n+1)+ tok := make([]int32, n+1)+ for i := range from {+ from[i] = -1+ }+ for i := 0; i < n; {+ _, size := utf8.DecodeRuneInString(s[i:])+ single := false+ for j := i + 1; j <= min(n, i+u.maxLen); j++ {+ id, ok := u.ids[s[i:j]]+ if !ok {+ continue+ }+ if score := best[i] + u.scores[id]; from[j] < 0 || score > best[j] {+ best[j], from[j], tok[j] = score, i, id+ }+ single = single || j-i == size+ }+ if j := i + size; !single {+ if score := best[i] + u.unkScore; from[j] < 0 || score > best[j] {+ best[j], from[j], tok[j] = score, i, u.unkID+ }+ }+ i += size+ }++ start := len(ids)+ for j := n; j > 0; j = from[j] {+ if tok[j] != u.unkID {+ ids = append(ids, tok[j])+ continue+ }+ // Backtracking visits pieces and their fallback bytes in reverse.+ mark := len(ids)+ for k := j - 1; k >= from[j]; k-- {+ if id := u.byteIDs[s[k]]; id >= 0 {+ ids = append(ids, id)+ } else {+ ids = append(ids[:mark], u.unkID)+ break+ }+ }+ }+ slices.Reverse(ids[start:])+ return ids+}++func (u *unigram) decode(ids []int32) string {+ var b []byte+ first := true+ for _, id := range ids {+ if u.specialIDs[id] {+ continue+ }+ if c := u.bytes[id]; c >= 0 {+ b = append(b, byte(c))+ first = false+ continue+ }+ piece := u.pieces[id]+ if first {+ piece = strings.ReplaceAll(piece, metaspace, "")+ } else {+ piece = strings.ReplaceAll(piece, metaspace, " ")+ }+ b = append(b, piece...)+ first = false+ }+ return strings.ToValidUTF8(string(b), "�")+}++const maxChunkTokens = 50++var errEmptyText = errors.New("input has no text to speak")++// prepareText follows the reference prepare_text_prompt, after normalization.+func prepareText(text string) (string, int, error) {+ words := strings.Fields(text)+ text = strings.Join(words, " ")+ if text == "" {+ return "", 0, errEmptyText+ }+ framesAfterEOS := 1+ if len(words) <= 4 {+ framesAfterEOS = 3+ }+ _, n := utf8.DecodeRuneInString(text)+ text = strings.ToUpper(text[:n]) + text[n:]+ if last, _ := utf8.DecodeLastRuneInString(text); unicode.IsLetter(last) || unicode.IsNumber(last) {+ text += "."+ }+ return text, framesAfterEOS, nil+}++// chunks splits text into sentence groups of at most maxChunkTokens tokens,+// as Phonon split_into_best_sentences does, then fits oversized sentences+// to the reference runtime's hard limit.+func (u *unigram) chunks(text string) ([]string, error) {+ text, _, err := prepareText(normalizeText(text))+ if err != nil {+ return nil, err+ }+ tokens := u.encode(strings.TrimSpace(text))++ type segment struct {+ n int+ text string+ }+ boundary := map[int32]bool{}+ for _, id := range u.encode(".!...?")[1:] {+ boundary[id] = true+ }+ var segs []segment+ start, after := 0, false+ for i, id := range tokens {+ if boundary[id] {+ after = true+ continue+ }+ if after {+ segs = append(segs, segment{i - start, u.decode(tokens[start:i])})+ start = i+ }+ after = false+ }+ segs = append(segs, segment{len(tokens) - start, u.decode(tokens[start:])})++ var chunks []string+ var cur string+ var n int+ for _, s := range segs {+ switch {+ case cur == "":+ cur, n = s.text, s.n+ case n+s.n > maxChunkTokens:+ chunks = append(chunks, strings.TrimSpace(cur))+ cur, n = s.text, s.n+ default:+ cur += " " + s.text+ n += s.n+ }+ }+ if cur != "" {+ chunks = append(chunks, strings.TrimSpace(cur))+ }+ return u.fitChunks(chunks)+}++// fitChunks mirrors upstream plan::fit_or_error. Long prompts otherwise outlast+// the model's training segments and can lose words even with enough KV space.+func (u *unigram) fitChunks(chunks []string) ([]string, error) {+ const limit = 200+ slices.Reverse(chunks)+ var fitted []string+ for len(chunks) > 0 {+ text := chunks[len(chunks)-1]+ chunks = chunks[:len(chunks)-1]+ prepared, _, err := prepareText(text)+ if err != nil {+ return nil, err+ }+ if len(u.encode(prepared)) <= limit {+ fitted = append(fitted, text)+ continue+ }+ words := strings.Fields(text)+ var left, right string+ if len(words) > 1 {+ cut := u.cutWords(words, limit/4)+ left, right = strings.Join(words[:cut], " "), strings.Join(words[cut:], " ")+ } else {+ runes := []rune(text)+ if len(runes) < 2 {+ return nil, fmt.Errorf("text cannot fit the %d-token limit", limit)+ }+ left, right = string(runes[:len(runes)/2]), string(runes[len(runes)/2:])+ }+ chunks = append(chunks, right, left)+ }+ return fitted, nil+}++// Prefer a sentence end, then a clause end, then the most balanced word split.+func (u *unigram) cutWords(words []string, minSide int) int {+ ends := make([]int, len(words))+ total := 0+ for i, word := range words {+ total += len(u.encode(word))+ ends[i] = total+ }+ for _, marks := range []string{".!?", ",;:", ""} {+ best, distance := 0, total+1+ for i, end := range ends[:len(ends)-1] {+ if marks != "" {+ word := strings.TrimRight(words[i], "\"')]”’»")+ last, _ := utf8.DecodeLastRuneInString(word)+ if !strings.ContainsRune(marks, last) || end < minSide || total-end < minSide {+ continue+ }+ }+ d := end - (total - end)+ if d < 0 {+ d = -d+ }+ if d < distance {+ best, distance = i+1, d+ }+ }+ if best != 0 {+ return best+ }+ }+ return 1+}diff --git a/mlxrunner/model/phonon/text_test.go b/mlxrunner/model/phonon/text_test.gonew file mode 100644index 000000000..7a874d9de--- /dev/null+++ b/mlxrunner/model/phonon/text_test.go@@ -0,0 +1,158 @@+package phonon++import (+ "encoding/json"+ "fmt"+ "os"+ "slices"+ "strings"+ "testing"+)++func TestPrepareText(t *testing.T) {+ for _, tc := range []struct {+ input, want string+ tail int+ }{+ {" hello\nworld ", "Hello world.", 3},+ {"wait—", "Wait—", 3},+ {"\"Hello?\"", "\"Hello?\"", 3},+ {"it's ready", "It's ready.", 3},+ {"it’s ready", "It’s ready.", 3},+ {"hello; world", "Hello; world.", 3},+ {"one two\tthree\rfour five", "One two three four five.", 1},+ } {+ got, tail, err := prepareText(tc.input)+ if err != nil || got != tc.want || tail != tc.tail {+ t.Errorf("prepare(%q) = %q, %d, %v; want %q, %d", tc.input, got, tail, err, tc.want, tc.tail)+ }+ }+ if _, _, err := prepareText(" \n\t"); err == nil {+ t.Fatal("accepted empty text")+ }+}++func TestUnigramMalformedVocabulary(t *testing.T) {+ for _, vocab := range []string{`[]`, `[[]]`, `[["x"]]`, `[["x","not a score"]]`, `[["",0]]`} {+ if _, err := newUnigram([]byte(`{"model":{"type":"Unigram","unk_id":0,"vocab":` + vocab + `}}`)); err == nil {+ t.Errorf("accepted malformed vocabulary %s", vocab)+ }+ }+}++func TestUnigramByteFallback(t *testing.T) {+ vocab := [][]any{{"<unk>", -10.0}, {metaspace, -1.0}, {"hello", -1.0}}+ for i := range 256 {+ vocab = append(vocab, []any{fmt.Sprintf("<0x%02X>", i), -20.0})+ }+ data, _ := json.Marshal(map[string]any{"model": map[string]any{"type": "Unigram", "unk_id": 0, "vocab": vocab}})+ u, err := newUnigram(data)+ if err != nil {+ t.Fatal(err)+ }+ for _, text := range []string{"hello", "hello café", "hello 👋", "hello hello", "日本語"} {+ if got := u.decode(u.encode(text)); got != text {+ t.Errorf("round trip %q => %q", text, got)+ }+ }+ text := strings.Repeat("hello ", 300)+ prepared, _, _ := prepareText(text)+ chunks, err := u.chunks(text)+ if err != nil || len(chunks) < 2 || strings.Join(chunks, " ") != prepared {+ t.Fatalf("long sentence was not split losslessly: %v", err)+ }+ for _, chunk := range chunks {+ prepared, _, _ := prepareText(chunk)+ if n := len(u.encode(prepared)); n > 200 {+ t.Fatalf("long sentence chunk has %d tokens", n)+ }+ }+}++// Run against a real checkpoint and fixtures generated with Hugging Face+// tokenizers: PHONON_TOKENIZER=... PHONON_TOKEN_CASES=... go test ...+func TestCheckpointTokenizer(t *testing.T) {+ path := os.Getenv("PHONON_TOKENIZER")+ if path == "" {+ t.Skip("set PHONON_TOKENIZER for checkpoint parity")+ }+ data, err := os.ReadFile(path)+ if err != nil {+ t.Fatal(err)+ }+ u, err := newUnigram(data)+ if err != nil {+ t.Fatal(err)+ }+ data, err = os.ReadFile(os.Getenv("PHONON_TOKEN_CASES"))+ if err != nil {+ t.Fatal(err)+ }+ var cases []struct {+ Text string+ IDs []int32+ }+ if err := json.Unmarshal(data, &cases); err != nil {+ t.Fatal(err)+ }+ for _, tc := range cases {+ if got := u.encode(tc.Text); !slices.Equal(got, tc.IDs) {+ t.Errorf("%q: got %v; want %v", tc.Text, got, tc.IDs)+ }+ }+ chunks, err := u.chunks(strings.Repeat("Hello from the new Phonon model. ", 20))+ if err != nil || len(chunks) < 2 {+ t.Fatalf("chunking: %v %v", chunks, err)+ }+ for _, chunk := range chunks {+ if !strings.HasSuffix(chunk, ".") {+ t.Fatalf("split inside sentence: %q", chunk)+ }+ }+}++// Fixtures come from xn-ptts plan::chunks + plan::fit_or_error, using the same+// checkpoint tokenizer. They cover oversized sentences, clauses and UTF-8 words.+func TestCheckpointChunking(t *testing.T) {+ path := os.Getenv("PHONON_CHUNK_CASES")+ if path == "" {+ t.Skip("set PHONON_CHUNK_CASES for upstream chunking parity")+ }+ data, err := os.ReadFile(os.Getenv("PHONON_TOKENIZER"))+ if err != nil {+ t.Fatal(err)+ }+ u, err := newUnigram(data)+ if err != nil {+ t.Fatal(err)+ }+ data, err = os.ReadFile(path)+ if err != nil {+ t.Fatal(err)+ }+ var cases []struct {+ Text string+ Chunks []string+ }+ if err := json.Unmarshal(data, &cases); err != nil {+ t.Fatal(err)+ }+ for _, tc := range cases {+ got, err := u.chunks(tc.Text)+ if err != nil {+ t.Fatal(err)+ }+ if !slices.Equal(got, tc.Chunks) {+ t.Errorf("chunks for %.40q differ: got %d chunks, want %d", tc.Text, len(got), len(tc.Chunks))+ }+ for _, chunk := range got {+ prepared, _, err := prepareText(chunk)+ if err != nil {+ t.Fatal(err)+ }+ if n := len(u.encode(prepared)); n > 200 {+ t.Errorf("chunk has %d tokens", n)+ }+ }+ }+}diff --git a/mlxrunner/model/speech.go b/mlxrunner/model/speech.gonew file mode 100644index 000000000..4f95a2a55--- /dev/null+++ b/mlxrunner/model/speech.go@@ -0,0 +1,13 @@+package model++import (+ "context"++ "github.com/ollama/ollama/llm"+)++// SpeechModel generates speech instead of text. The runner loads it like any+// model and serves it with Speak, which emits 24 kHz mono samples.+type SpeechModel interface {+ Speak(ctx context.Context, req llm.SpeechRequest, emit func(samples []float32) error) error+}diff --git a/mlxrunner/runner.go b/mlxrunner/runner.goindex 1af2856bf..93e0f9dbb 100644--- a/mlxrunner/runner.go+++ b/mlxrunner/runner.go@@ -137,6 +137,9 @@ func (r *Runner) loadModel(modelName string) (weights []*mlx.Array, err error) {}r.Model = m+ if _, ok := m.(model.SpeechModel); ok {+ return w+ }r.Tokenizer = m.Tokenizer()r.contextLength = m.MaxContextLength()caches := m.NewCaches()diff --git a/mlxrunner/server.go b/mlxrunner/server.goindex c8c349a4d..259f60c58 100644--- a/mlxrunner/server.go+++ b/mlxrunner/server.go@@ -124,6 +124,10 @@ func Execute(args []string) error {})mux.HandleFunc("POST /v1/completions", func(w http.ResponseWriter, r *http.Request) {+ if runner.Tokenizer == nil {+ http.Error(w, "this model does not support text generation", http.StatusBadRequest)+ return+ }request := Request{Responses: make(chan CompletionResponse)}if err := json.NewDecoder(r.Body).Decode(&request.CompletionRequest); err != nil {@@ -195,6 +199,10 @@ func Execute(args []string) error {})mux.HandleFunc("POST /v1/tokenize", func(w http.ResponseWriter, r *http.Request) {+ if runner.Tokenizer == nil {+ http.Error(w, "this model does not support text tokenization", http.StatusBadRequest)+ return+ }var b bytes.Bufferif _, err := io.Copy(&b, r.Body); err != nil {slog.Error("Failed to read request body", "error", err)@@ -212,6 +220,10 @@ func Execute(args []string) error {})mux.HandleFunc("POST /v1/detokenize", func(w http.ResponseWriter, r *http.Request) {+ if runner.Tokenizer == nil {+ http.Error(w, "this model does not support text tokenization", http.StatusBadRequest)+ return+ }var tokens []int32if err := json.NewDecoder(r.Body).Decode(&tokens); err != nil {http.Error(w, "Bad Request", http.StatusBadRequest)@@ -224,6 +236,7 @@ func Execute(args []string) error {})mux.HandleFunc("POST /v1/embeddings", runner.handleEmbed)+ mux.HandleFunc("POST /v1/audio/speech", runner.speechHandler)for source, target := range map[string]string{"GET /health": "/v1/status",@@ -267,8 +280,11 @@ func (w *statusRecorder) Status() string {return strconv.Itoa(w.code) + " " + http.StatusText(w.code)}-func (w *statusRecorder) Flush() {- if f, ok := w.ResponseWriter.(http.Flusher); ok {- f.Flush()- }+// Unwrap lets net/http's ResponseController reach deadlines and other controls.+func (w *statusRecorder) Unwrap() http.ResponseWriter { return w.ResponseWriter }++func (w *statusRecorder) FlushError() error {+ return http.NewResponseController(w.ResponseWriter).Flush()}++func (w *statusRecorder) Flush() { _ = w.FlushError() }diff --git a/mlxrunner/speech.go b/mlxrunner/speech.gonew file mode 100644index 000000000..e08554365--- /dev/null+++ b/mlxrunner/speech.go@@ -0,0 +1,158 @@+package mlxrunner++import (+ "context"+ "encoding/binary"+ "encoding/json"+ "errors"+ "math"+ "net/http"+ "sync"+ "time"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/mlx"+ "github.com/ollama/ollama/mlxrunner/model"+)++// speechHandler streams 24 kHz 16-bit PCM as the model generates it.+func (r *Runner) speechHandler(w http.ResponseWriter, req *http.Request) {+ speaker, ok := r.Model.(model.SpeechModel)+ if !ok {+ http.Error(w, "this model does not support speech", http.StatusNotImplemented)+ return+ }+ var input llm.SpeechRequest+ if err := json.NewDecoder(req.Body).Decode(&input); err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }++ // Generation queues audio for a separate writer instead of waiting on the+ // client, so a client reading at playback speed does not hold the MLX worker.+ ctx, cancel := context.WithCancel(req.Context())+ defer cancel()+ queue := &audioQueue{ready: make(chan struct{}, 1)}+ var streaming bool+ var writeErr error+ written := make(chan struct{})+ go func() {+ defer close(written)+ streaming, writeErr = writeAudio(w, queue)+ if writeErr != nil {+ cancel()+ }+ }()+ // The writer must not outlive the handler, even if generation panics.+ defer func() { queue.close(); <-written }()++ err := r.mlxThread.Do(ctx, func() error {+ defer mlx.ClearCache()+ return speaker.Speak(ctx, input, func(samples []float32) error {+ queue.push(pcm16(samples))+ return nil+ })+ })+ queue.close()+ <-written+ if err == nil {+ err = writeErr+ }+ switch {+ case err == nil:+ case streaming:+ // Truncate the stream so the client sees the failure.+ panic(http.ErrAbortHandler)+ default:+ status := http.StatusInternalServerError+ var statusErr api.StatusError+ if errors.As(err, &statusErr) {+ status = statusErr.StatusCode+ }+ http.Error(w, err.Error(), status)+ }+}++// writeAudio streams queued audio until the queue is closed and reports+// whether any audio was written.+func writeAudio(w http.ResponseWriter, queue *audioQueue) (streaming bool, err error) {+ controller := http.NewResponseController(w)+ defer controller.SetWriteDeadline(time.Time{})+ for {+ chunks, ok := queue.next()+ if !ok {+ return streaming, nil+ }+ for _, pcm := range chunks {+ // A stalled reader must not hold the connection forever.+ if err := controller.SetWriteDeadline(time.Now().Add(30 * time.Second)); err != nil {+ return streaming, err+ }+ if !streaming {+ w.Header().Set("Content-Type", "audio/pcm")+ streaming = true+ }+ if _, err := w.Write(pcm); err != nil {+ return streaming, err+ }+ }+ if err := controller.Flush(); err != nil {+ return streaming, err+ }+ }+}++// audioQueue hands audio from generation to the writer without ever blocking+// generation.+type audioQueue struct {+ mu sync.Mutex+ pending [][]byte+ closed bool+ ready chan struct{}+}++func (q *audioQueue) push(pcm []byte) {+ q.mu.Lock()+ q.pending = append(q.pending, pcm)+ q.mu.Unlock()+ q.wake()+}++func (q *audioQueue) close() {+ q.mu.Lock()+ q.closed = true+ q.mu.Unlock()+ q.wake()+}++func (q *audioQueue) wake() {+ select {+ case q.ready <- struct{}{}:+ default:+ }+}++// next waits for queued audio. It returns false once the queue is closed and+// empty.+func (q *audioQueue) next() ([][]byte, bool) {+ for {+ q.mu.Lock()+ pending, closed := q.pending, q.closed+ q.pending = nil+ q.mu.Unlock()+ if len(pending) > 0 || closed {+ return pending, len(pending) > 0+ }+ <-q.ready+ }+}++func pcm16(samples []float32) []byte {+ b := make([]byte, 2*len(samples))+ for i, s := range samples {+ v := math.Round(float64(max(-1, min(1, s))) * math.MaxInt16)+ binary.LittleEndian.PutUint16(b[2*i:], uint16(int16(v)))+ }+ return b+}diff --git a/server/images.go b/server/images.goindex 83d594be8..87950edd5 100644--- a/server/images.go+++ b/server/images.go@@ -585,6 +585,7 @@ func (m *Model) CheckCapabilities(want ...model.Capability) error {model.CapabilityThinking: errCapabilityThinking,model.CapabilityImage: errCapabilityImage,model.CapabilityDecision: errors.New("decision"),+ model.CapabilitySpeech: errors.New("speech"),}for _, cap := range want {diff --git a/server/routes.go b/server/routes.goindex 9cd9c8b93..09e99afb2 100644--- a/server/routes.go+++ b/server/routes.go@@ -1893,8 +1893,8 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {Runner: m.Runner,}- // For safetensors completion and decision models, populate details from config.json.- isSafetensorsLLM := m.IsMLX() && (slices.Contains(m.Config.Capabilities, "completion") || slices.Contains(m.Config.Capabilities, "decision"))+ // For safetensors completion, decision and speech models, populate details from config.json.+ isSafetensorsLLM := m.IsMLX() && (slices.Contains(m.Config.Capabilities, "completion") || slices.Contains(m.Config.Capabilities, "decision") || slices.Contains(m.Config.Capabilities, "speech"))if isSafetensorsLLM {if info, err := getSafetensorsLLMInfoForRunner(name, req.Runner); err == nil {if arch, ok := info["general.architecture"].(string); ok && arch != "" {@@ -2418,8 +2418,9 @@ func (s *Server) GenerateRoutes() (http.Handler, error) {r.GET("/v1/models/:model", cloudModelPathPassthroughMiddleware(cloudErrRemoteModelDetailsUnavailable), middleware.RetrieveMiddleware(), s.ShowHandler)r.POST("/v1/responses", s.withInferenceRequestLogging("/v1/responses", s.responsesCompactionMiddleware(), cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.ResponsesMiddleware(lookupThinking), s.ChatHandler)...)r.POST("/v1/responses/compact", s.ResponsesCompactHandler)- // OpenAI-compatible audio endpoint+ // OpenAI-compatible audio endpointsr.POST("/v1/audio/transcriptions", middleware.TranscriptionMiddleware(), s.ChatHandler)+ r.POST("/v1/audio/speech", s.SpeechHandler)// Inference (Anthropic compatibility)r.POST("/v1/messages", s.withInferenceRequestLogging("/v1/messages", cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.AnthropicMessagesMiddleware(lookupThinking), s.ChatHandler)...)@@ -2816,6 +2817,12 @@ func (s *Server) PsHandler(c *gin.Context) {QuantizationLevel: m.Config.FileType,}+ contextLength := v.contextLength+ if slices.Contains(m.Config.Capabilities, "speech") {+ // Speech models have no text context.+ contextLength = 0+ }+mr := api.ProcessModelResponse{Model: displayName,Name: displayName,@@ -2824,7 +2831,7 @@ func (s *Server) PsHandler(c *gin.Context) {Digest: digest,Details: modelDetails,ExpiresAt: v.expiresAt,- ContextLength: v.contextLength,+ ContextLength: contextLength,Runner: runner,}@@ -3595,19 +3602,24 @@ func countChatImages(msgs []api.Message) int {}func handleScheduleError(c *gin.Context, name string, err error) {+ status, message := scheduleError(name, err)+ c.JSON(status, gin.H{"error": message})+}++// scheduleError keeps native and OpenAI-compatible endpoints consistent while+// allowing each to use its own error response format.+func scheduleError(name string, err error) (int, string) {switch {- case errors.Is(err, errCapabilities), errors.Is(err, errRequired):- c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})- case errors.Is(err, manifest.ErrNoCompatibleManifest):- c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})+ case errors.Is(err, errCapabilities), errors.Is(err, errRequired), errors.Is(err, manifest.ErrNoCompatibleManifest):+ return http.StatusBadRequest, err.Error()case errors.Is(err, context.Canceled):- c.JSON(499, gin.H{"error": "request canceled"})+ return 499, "request canceled"case errors.Is(err, ErrMaxQueue):- c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})+ return http.StatusServiceUnavailable, err.Error()case errors.Is(err, os.ErrNotExist):- c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found, try pulling it first", name)})+ return http.StatusNotFound, fmt.Sprintf("model %q not found, try pulling it first", name)default:- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})+ return http.StatusInternalServerError, err.Error()}}diff --git a/server/speech.go b/server/speech.gonew file mode 100644index 000000000..f83ae8a71--- /dev/null+++ b/server/speech.go@@ -0,0 +1,163 @@+package server++import (+ "cmp"+ "encoding/binary"+ "errors"+ "fmt"+ "log/slog"+ "net/http"+ "strings"+ "time"+ "unicode/utf8"++ "github.com/gin-gonic/gin"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/openai"+ "github.com/ollama/ollama/types/model"+)++const speechSampleRate = 24000++// SpeechHandler serves the OpenAI-compatible POST /v1/audio/speech, streaming+// audio as the model generates it.+func (s *Server) SpeechHandler(c *gin.Context) {+ var req struct {+ Model string `json:"model"`+ Input string `json:"input"`+ Voice string `json:"voice"`+ ResponseFormat string `json:"response_format"`+ Speed *float64 `json:"speed"`+ Instructions string `json:"instructions"`+ StreamFormat string `json:"stream_format"`+ }+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 1<<20)+ if err := c.ShouldBindJSON(&req); err != nil {+ speechError(c, http.StatusBadRequest, err.Error())+ return+ }+ format := cmp.Or(req.ResponseFormat, "wav")+ switch {+ case strings.TrimSpace(req.Model) == "":+ speechError(c, http.StatusBadRequest, "model is required")+ return+ case strings.TrimSpace(req.Input) == "":+ speechError(c, http.StatusBadRequest, "input is required")+ return+ case utf8.RuneCountInString(req.Input) > 4096:+ speechError(c, http.StatusBadRequest, "input must be at most 4096 characters")+ return+ case format != "wav" && format != "pcm":+ speechError(c, http.StatusBadRequest, fmt.Sprintf("response_format %q is not supported, use wav or pcm", format))+ return+ case req.Speed != nil && *req.Speed != 1:+ speechError(c, http.StatusBadRequest, "only speed 1 is supported")+ return+ case req.Instructions != "":+ speechError(c, http.StatusBadRequest, "instructions are not supported")+ return+ case req.StreamFormat != "" && req.StreamFormat != "audio":+ speechError(c, http.StatusBadRequest, "only stream_format audio is supported")+ return+ }++ r, _, _, err := s.scheduleRunner(c.Request.Context(), req.Model, "", []model.Capability{model.CapabilitySpeech}, nil, nil, nil)+ speaker, ok := r.(llm.Speaker)+ switch {+ case errors.Is(err, errCapabilities), err == nil && !ok:+ speechError(c, http.StatusBadRequest, fmt.Sprintf("%q does not support speech", req.Model))+ return+ case err != nil:+ status, message := scheduleError(req.Model, err)+ speechError(c, status, message)+ return+ }++ streamSpeech(c, speaker, llm.SpeechRequest{Input: req.Input, Voice: req.Voice}, format)+}++func streamSpeech(c *gin.Context, speaker llm.Speaker, req llm.SpeechRequest, format string) {+ // Write through Gin for status/byte accounting, but bypass its Flush+ // method so ResponseController can return the underlying flush error.+ var writer http.ResponseWriter = c.Writer+ if w, ok := writer.(interface{ Unwrap() http.ResponseWriter }); ok {+ writer = w.Unwrap()+ }+ controller := http.NewResponseController(writer)+ defer controller.SetWriteDeadline(time.Time{})+ started, clientGone := false, false+ err := speaker.Speech(c.Request.Context(), req, func(pcm []byte) (err error) {+ // Every error here comes from writing to the client.+ defer func() { clientGone = err != nil }()+ if len(pcm) == 0 {+ return nil+ }+ // Bound a stalled client's writes so it cannot hold the model and+ // back up the runner indefinitely. Reset for each chunk, not request.+ if err := controller.SetWriteDeadline(time.Now().Add(30 * time.Second)); err != nil {+ return err+ }+ if !started {+ started = true+ c.Header("Content-Type", "audio/"+format)+ c.Status(http.StatusOK)+ if format == "wav" {+ if _, err := c.Writer.Write(wavHeader()); err != nil {+ return err+ }+ }+ }+ if _, err := c.Writer.Write(pcm); err != nil {+ return err+ }+ return controller.Flush()+ })+ if err == nil && !started {+ err = errors.New("speech model produced no audio")+ }+ if err == nil {+ return+ }+ if started {+ // Close the connection rather than end the stream, so clients don't+ // mistake partial audio for a complete response. A client that hangs+ // up is not a failure worth a warning.+ if !clientGone && c.Request.Context().Err() == nil {+ slog.Warn("speech stream failed", "error", err)+ }+ if conn, _, err := controller.Hijack(); err == nil {+ conn.Close()+ }+ return+ }+ status, message := http.StatusInternalServerError, err.Error()+ var statusErr api.StatusError+ if errors.As(err, &statusErr) {+ status, message = statusErr.StatusCode, statusErr.ErrorMessage+ }+ speechError(c, status, message)+}++func speechError(c *gin.Context, status int, message string) {+ c.AbortWithStatusJSON(status, openai.NewError(status, message))+}++// wavHeader describes 16-bit mono PCM of unknown length, as streamed WAV is.+func wavHeader() []byte {+ h := make([]byte, 44)+ copy(h[0:], "RIFF")+ binary.LittleEndian.PutUint32(h[4:], 0xffffffff)+ copy(h[8:], "WAVEfmt ")+ binary.LittleEndian.PutUint32(h[16:], 16)+ binary.LittleEndian.PutUint16(h[20:], 1)+ binary.LittleEndian.PutUint16(h[22:], 1)+ binary.LittleEndian.PutUint32(h[24:], speechSampleRate)+ binary.LittleEndian.PutUint32(h[28:], speechSampleRate*2)+ binary.LittleEndian.PutUint16(h[32:], 2)+ binary.LittleEndian.PutUint16(h[34:], 16)+ copy(h[36:], "data")+ binary.LittleEndian.PutUint32(h[40:], 0xffffffff)+ return h+}diff --git a/server/speech_test.go b/server/speech_test.gonew file mode 100644index 000000000..de47767cb--- /dev/null+++ b/server/speech_test.go@@ -0,0 +1,214 @@+package server++import (+ "bytes"+ "context"+ "encoding/binary"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "os"+ "strings"+ "testing"+ "time"++ "github.com/gin-gonic/gin"+ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/manifest"+)++type speechFunc func(context.Context, llm.SpeechRequest, func([]byte) error) error++func (f speechFunc) Speech(ctx context.Context, req llm.SpeechRequest, emit func([]byte) error) error {+ return f(ctx, req, emit)+}++func TestSpeechValidation(t *testing.T) {+ for _, body := range []string{+ `{`, `{}`, `{"model":"test","input":" \n"}`,+ `{"input":"hello"}`,+ `{"model":"test","input":"hello","response_format":"mp3"}`,+ `{"model":"test","input":"hello","speed":0}`,+ `{"model":"test","input":"hello","speed":2}`,+ `{"model":"test","input":"hello","stream_format":"sse"}`,+ `{"model":"test","input":"hello","instructions":"whisper"}`,+ `{"model":"test","input":"` + strings.Repeat("é", 4097) + `"}`,+ } {+ t.Run(body[:min(len(body), 90)], func(t *testing.T) {+ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ c.Request = httptest.NewRequest(http.MethodPost, "/v1/audio/speech", strings.NewReader(body))+ // Invalid requests must fail before touching the scheduler.+ (&Server{}).SpeechHandler(c)+ var got struct {+ Error struct {+ Message string `json:"message"`+ } `json:"error"`+ }+ if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || w.Code != 400 || got.Error.Message == "" {+ t.Fatalf("invalid error response: %d %s", w.Code, w.Body)+ }+ })+ }+}++func TestSpeechStreamsBeforeCompletion(t *testing.T) {+ for _, format := range []string{"pcm", "wav"} {+ t.Run(format, func(t *testing.T) {+ release := make(chan struct{})+ defer close(release)+ pcm := []byte{1, 0, 2, 0}+ speaker := speechFunc(func(ctx context.Context, req llm.SpeechRequest, emit func([]byte) error) error {+ if err := emit(pcm); err != nil {+ return err+ }+ select {+ case <-release:+ case <-ctx.Done():+ return ctx.Err()+ }+ return nil+ })+ router := gin.New()+ router.Use(gin.Recovery())+ router.POST("/", func(c *gin.Context) { streamSpeech(c, speaker, llm.SpeechRequest{}, format) })+ srv := httptest.NewServer(router)+ defer srv.Close()+ req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, srv.URL, nil)+ res, err := srv.Client().Do(req)+ if err != nil {+ t.Fatal(err)+ }+ defer res.Body.Close()+ if res.Header.Get("Content-Type") != "audio/"+format {+ t.Fatal(res.Header)+ }+ if format == "wav" {+ h := make([]byte, 44)+ if _, err := io.ReadFull(res.Body, h); err != nil {+ t.Fatal(err)+ }+ if string(h[:4]) != "RIFF" || string(h[8:16]) != "WAVEfmt " || string(h[36:40]) != "data" ||+ binary.LittleEndian.Uint32(h[24:]) != 24000 || binary.LittleEndian.Uint16(h[22:]) != 1 ||+ binary.LittleEndian.Uint16(h[34:]) != 16 {+ t.Fatalf("bad WAV header: %x", h)+ }+ }+ got := make([]byte, len(pcm))+ if _, err := io.ReadFull(res.Body, got); err != nil || !bytes.Equal(got, pcm) {+ t.Fatalf("first audio: %x, %v", got, err)+ }+ })+ }+}++func TestSpeechErrors(t *testing.T) {+ for _, tc := range []struct {+ name string+ emitted bool+ err error+ status int+ }{+ {"invalid voice", false, api.StatusError{StatusCode: 400, ErrorMessage: "invalid voice"}, 400},+ {"empty output", false, nil, 500},+ {"generation failed", true, errors.New("generation failed"), 200},+ } {+ t.Run(tc.name, func(t *testing.T) {+ router := gin.New()+ router.Use(gin.Recovery())+ router.POST("/", func(c *gin.Context) {+ streamSpeech(c, speechFunc(func(_ context.Context, _ llm.SpeechRequest, emit func([]byte) error) error {+ if tc.emitted {+ if err := emit([]byte{1, 0}); err != nil {+ return err+ }+ }+ return tc.err+ }), llm.SpeechRequest{}, "pcm")+ })+ srv := httptest.NewServer(router)+ defer srv.Close()+ res, err := srv.Client().Post(srv.URL, "application/json", nil)+ if err != nil {+ t.Fatal(err)+ }+ defer res.Body.Close()+ body, err := io.ReadAll(res.Body)+ if res.StatusCode != tc.status {+ t.Fatalf("status %d: %s", res.StatusCode, body)+ }+ if tc.emitted {+ if !errors.Is(err, io.ErrUnexpectedEOF) {+ t.Fatalf("partial audio looked complete: %v", err)+ }+ } else if err != nil || !bytes.Contains(body, []byte(`"error"`)) {+ t.Fatalf("error response: %s, %v", body, err)+ }+ })+ }+}++func TestSpeechScheduleErrors(t *testing.T) {+ for _, tc := range []struct {+ err error+ status int+ message string+ }{+ {fmt.Errorf("model %w", errRequired), 400, "model is required"},+ {errCapabilities, 400, errCapabilities.Error()},+ {manifest.ErrNoCompatibleManifest, 400, manifest.ErrNoCompatibleManifest.Error()},+ {context.Canceled, 499, "request canceled"},+ {ErrMaxQueue, 503, ErrMaxQueue.Error()},+ {os.ErrNotExist, 404, `model "phonon" not found, try pulling it first`},+ {errors.New("load failed"), 500, "load failed"},+ } {+ t.Run(tc.err.Error(), func(t *testing.T) {+ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ status, message := scheduleError("phonon", tc.err)+ speechError(c, status, message)+ var body struct {+ Error struct {+ Message string+ Type string+ }+ }+ if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {+ t.Fatal(err)+ }+ if w.Code != tc.status || body.Error.Message != tc.message || body.Error.Type == "" {+ t.Fatalf("got %d %s; want %d %q", w.Code, w.Body, tc.status, tc.message)+ }+ })+ }+}++// Gin's Flush method drops errors; speech must reach the underlying writer's+// FlushError so a failed write stops the producer immediately.+type speechFlushErrorWriter struct {+ *httptest.ResponseRecorder+ err error+}++func (w *speechFlushErrorWriter) SetWriteDeadline(time.Time) error { return nil }+func (w *speechFlushErrorWriter) FlushError() error { return w.err }+func (w *speechFlushErrorWriter) Flush() { _ = w.FlushError() }++func TestSpeechFlushErrorStopsProducer(t *testing.T) {+ want := errors.New("connection write failed")+ w := &speechFlushErrorWriter{ResponseRecorder: httptest.NewRecorder(), err: want}+ c, _ := gin.CreateTestContext(w)+ c.Request = httptest.NewRequest(http.MethodPost, "/v1/audio/speech", nil)+ var got error+ streamSpeech(c, speechFunc(func(_ context.Context, _ llm.SpeechRequest, emit func([]byte) error) error {+ got = emit([]byte{1, 0})+ return got+ }), llm.SpeechRequest{}, "pcm")+ if !errors.Is(got, want) {+ t.Fatalf("flush error did not reach producer: got %v, want %v", got, want)+ }+}diff --git a/types/model/capability.go b/types/model/capability.goindex c6ebc7c9d..27e0cd571 100644--- a/types/model/capability.go+++ b/types/model/capability.go@@ -12,13 +12,15 @@ const (CapabilityImage = Capability("image")CapabilityAudio = Capability("audio")CapabilityDecision = Capability("decision")+ CapabilitySpeech = Capability("speech"))// IsValid reports whether c is recognized, such as "decision"; empty or unknown labels are invalid.func (c Capability) IsValid() bool {switch c {case CapabilityCompletion, CapabilityTools, CapabilityInsert, CapabilityVision,- CapabilityEmbedding, CapabilityThinking, CapabilityImage, CapabilityAudio, CapabilityDecision:+ CapabilityEmbedding, CapabilityThinking, CapabilityImage, CapabilityAudio, CapabilityDecision,+ CapabilitySpeech:return truedefault:return false