LLM inference in C/C++
C++ ggml
server : add POST /tts endpoint for TTS models (#26603)
Xuan-Son Nguyen5c0b49a23b46
diff --git a/common/common.cpp b/common/common.cppindex b0ddfcb915..6ef3b785e0 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1235,6 +1235,9 @@ common_gguf_info common_get_gguf_info(const std::string & fname) {info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id);}+ // same key as llama_model_chat_template(model, nullptr)+ info.has_chat_tmpl = gguf_find_key(gguf_ctx.get(), "tokenizer.chat_template") >= 0;+return info;}diff --git a/common/common.h b/common/common.hindex b089abd7e6..70d6a671f5 100644--- a/common/common.h+++ b/common/common.h@@ -985,6 +985,7 @@ common_decision_type common_get_decision_type(const struct llama_model * model);struct common_gguf_info {common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invaliduint32_t n_ctx_train = 0; // 0 if unknown+ bool has_chat_tmpl = false; // has the default tokenizer.chat_template};common_gguf_info common_get_gguf_info(const std::string & fname);diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex df61f82f00..e4750e52ec 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -4278,8 +4278,9 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_paramsstruct clip_cap clip_get_cap(const char * fname) {clip_cap res;clip_model_loader loader(fname, /* skip_tensors= */ true);- res.has_vision = loader.has_vision;- res.has_audio = loader.has_audio;+ res.has_vision = loader.has_vision;+ res.has_audio = loader.has_audio;+ res.has_gen_audio = loader.has_gen_audio;return res;}diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.hindex 19db476082..cdbaeae0d5 100644--- a/tools/mtmd/clip.h+++ b/tools/mtmd/clip.h@@ -148,5 +148,6 @@ std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx *struct clip_cap {bool has_vision;bool has_audio;+ bool has_gen_audio;};struct clip_cap clip_get_cap(const char * fname);diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cppindex 5fb7ea9a0d..47493d9069 100644--- a/tools/mtmd/mtmd-helper-gen.cpp+++ b/tools/mtmd/mtmd-helper-gen.cpp@@ -50,29 +50,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::strireturn LLAMA_TOKEN_NULL;}+static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {+ const char * c = (const char *) p;+ buf.insert(buf.end(), c, c + n);+}++// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe+static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate) {+ const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;+ const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;+ const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;+ const uint32_t rate32 = (uint32_t) rate;+ put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);+ put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);+ put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4);+ put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2);+ put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4);+}++static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {+ for (size_t i = 0; i < n; i++) {+ int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);+ put_bytes(buf, &s, 2);+ }+}+static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate) {// RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated headerif (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {return false;}- const uint32_t data_sz = (uint32_t) (pcm.size() * 2);- const uint32_t riff_sz = 36 + data_sz;- const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;- const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;- const uint32_t rate32 = (uint32_t) rate;- auto put = [&](const void * p, size_t n) {- const char * c = (const char *) p;- buf.insert(buf.end(), c, c + n);- };- put("RIFF", 4); put(&riff_sz, 4); put("WAVE", 4);- put("fmt ", 4); put(&fmt_sz, 4);- put(&fmt, 2); put(&ch, 2); put(&rate32, 4);- put(&byte_rate, 4); put(&align, 2); put(&bits, 2);- put("data", 4); put(&data_sz, 4);- for (float v : pcm) {- int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f);- put(&s, 2);- }+ write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate);+ append_wav16_pcm(buf, pcm.data(), pcm.size());return true;}@@ -92,6 +101,8 @@ public:// set out_stop on end-of-speech, h_state_out must be null if no frame is generatedvirtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0;virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;+ // forces any buffered codes through code2wav now, regardless of window_frames+ virtual int32_t flush() = 0;protected:llama_context * lctx;@@ -121,6 +132,9 @@ public:prompt_batch.reset();n_prompt = 0;prompt_pos = 0;+ stream = false;+ pcm_sent = 0;+ wav_header_sent = false;}int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {@@ -208,6 +222,7 @@ public:top_p = inp->top_p > 0 ? inp->top_p : def.top_p;seed = inp->seed;out_type = inp->out_type;+ stream = inp->stream;// the prompt above holds the whole text stream up to tts_eos, so every generated// frame adds tts_pad on top of the codes embedding@@ -301,31 +316,60 @@ public:}int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {- if (!flush_gen_wav()) {- return 1;+ *out_sample_rate = info.sample_rate;++ if (!stream) {+ // one-shot call: force out whatever's left, regardless of window_frames+ if (!flush_gen_wav()) {+ return 1;+ }+ if (out_n_samples) {+ *out_n_samples = (int64_t) audio_pcm.size();+ }+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {+ *out_data = (const char *) audio_pcm.data();+ *out_data_len = audio_pcm.size() * sizeof(float);+ return 0;+ }+ out_buf.clear();+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");+ return 1;+ }+ *out_data = out_buf.data();+ *out_data_len = out_buf.size();+ return 0;}- *out_sample_rate = info.sample_rate;+ // streaming: only return audio produced since the previous call+ const size_t n_new = audio_pcm.size() - pcm_sent;if (out_n_samples) {- *out_n_samples = (int64_t) audio_pcm.size();+ *out_n_samples = (int64_t) n_new;}if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {- *out_data = (const char *) audio_pcm.data();- *out_data_len = audio_pcm.size() * sizeof(float);+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);+ *out_data_len = n_new * sizeof(float);+ pcm_sent = audio_pcm.size();return 0;}out_buf.clear();- if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {- LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");- return 1;+ if (!wav_header_sent) {+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);+ wav_header_sent = true;}+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);+ pcm_sent = audio_pcm.size();*out_data = out_buf.data();*out_data_len = out_buf.size();return 0;}+ int32_t flush() override {+ return flush_gen_wav() ? 0 : 1;+ }+private:bool ensure_cache() {if (specials_ok) {@@ -369,7 +413,7 @@ private:LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");return false;}- const std::string marker = mtmd_default_marker();+ const std::string marker = mtmd_get_marker(mctx);mtmd_input_text text{ marker.c_str(), marker.size(), false, true };mtmd_input_chunks * chunks = mtmd_input_chunks_init();const mtmd_bitmap * bptr = bitmap;@@ -455,6 +499,9 @@ private:std::vector<float> h_state_buf;mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;std::vector<char> out_buf;+ bool stream = false;+ size_t pcm_sent = 0; // samples already returned by get_output()+ bool wav_header_sent = false;};// settings that only live in the reference's per-pack yaml, not in the checkpoint@@ -506,6 +553,9 @@ public:chunk_idx = 0;n_voice_pos = 0;chunk_budget = 0;+ stream = false;+ pcm_sent = 0;+ wav_header_sent = false;}int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {@@ -575,6 +625,7 @@ public:seed = inp->seed;out_type = inp->out_type;+ stream = inp->stream;return 0;}@@ -662,31 +713,60 @@ public:}int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {- if (!flush_gen_wav()) {- return 1;+ *out_sample_rate = info.sample_rate;++ if (!stream) {+ // one-shot call: force out whatever's left, regardless of window_frames+ if (!flush_gen_wav()) {+ return 1;+ }+ if (out_n_samples) {+ *out_n_samples = (int64_t) audio_pcm.size();+ }+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {+ *out_data = (const char *) audio_pcm.data();+ *out_data_len = audio_pcm.size() * sizeof(float);+ return 0;+ }+ out_buf.clear();+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");+ return 1;+ }+ *out_data = out_buf.data();+ *out_data_len = out_buf.size();+ return 0;}- *out_sample_rate = info.sample_rate;+ // streaming: only return audio produced since the previous call+ const size_t n_new = audio_pcm.size() - pcm_sent;if (out_n_samples) {- *out_n_samples = (int64_t) audio_pcm.size();+ *out_n_samples = (int64_t) n_new;}if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {- *out_data = (const char *) audio_pcm.data();- *out_data_len = audio_pcm.size() * sizeof(float);+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);+ *out_data_len = n_new * sizeof(float);+ pcm_sent = audio_pcm.size();return 0;}out_buf.clear();- if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {- LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");- return 1;+ if (!wav_header_sent) {+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);+ wav_header_sent = true;}+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);+ pcm_sent = audio_pcm.size();*out_data = out_buf.data();*out_data_len = out_buf.size();return 0;}+ int32_t flush() override {+ return flush_gen_wav() ? 0 : 1;+ }+private:bool ensure_cache() {if (specials_ok) {@@ -907,7 +987,7 @@ private:LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n");return false;}- const std::string marker = mtmd_default_marker();+ const std::string marker = mtmd_get_marker(mctx);mtmd_input_text text{ marker.c_str(), marker.size(), false, true };mtmd_input_chunks * chunks = mtmd_input_chunks_init();const mtmd_bitmap * bptr = bitmap;@@ -989,6 +1069,9 @@ private:std::vector<float> h_state_buf;mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;std::vector<char> out_buf;+ bool stream = false;+ size_t pcm_sent = 0; // samples already returned by get_output()+ bool wav_header_sent = false;};static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {@@ -1017,11 +1100,20 @@ void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) {}void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {- if (ctx->pipeline) {+ if (ctx && ctx->pipeline) {ctx->pipeline->reset();}}+struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {+ mtmd_helper_gen_audio_inp inp{};+ inp.top_k = 50;+ inp.top_p = 1.0f;+ inp.seed = UINT32_MAX; // random+ inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;+ return inp;+}+int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {if (!ctx->pipeline) {LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");@@ -1058,3 +1150,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t *}return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);}++int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) {+ if (!ctx->pipeline) {+ return 1;+ }+ return ctx->pipeline->flush();+}diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.hindex 7436230f0d..acbfefcbae 100644--- a/tools/mtmd/mtmd-helper.h+++ b/tools/mtmd/mtmd-helper.h@@ -204,6 +204,7 @@ enum mtmd_helper_gen_audio_outtype {MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono};struct mtmd_helper_gen_audio_inp {+ bool stream; // if true, output() must be called after each step_gen()llama_seq_id seq_id;const char * prompt;@@ -219,6 +220,8 @@ struct mtmd_helper_gen_audio_inp {enum mtmd_helper_gen_audio_outtype out_type;};+MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void);+MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx,struct mtmd_context * mctx);@@ -250,6 +253,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen(// out_data valid until next get_output() or reset() call// out_n_samples (optional, can be NULL) receives the number of generated PCM samples+// if inp->stream is true: returns only audio produced since the previous call, and+// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yetMTMD_API int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx,int32_t * out_sample_rate,@@ -257,6 +262,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output(size_t * out_data_len,int64_t * out_n_samples);+// forces any buffered codes through code2wav now, regardless of window_frames;+// call once when generation has ended, before the last get_output() in stream mode+MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);+#ifdef __cplusplus} // extern "C"#endif@@ -283,8 +292,41 @@ struct mtmd_helper_gen_audio_deleter {};using gen_audio_ptr = std::unique_ptr<mtmd_helper_gen_audio, mtmd_helper_gen_audio_deleter>;struct gen_audio {++ // sub-struct, RAII wrapper for mtmd_helper_gen_audio_inp+ struct inp {+ mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();+ std::string prompt_str;+ std::string lang_str;+ mtmd::bitmap_ptr speaker_ref_ptr;++ inp() = default;+ inp(inp &&) = default;+ inp & operator=(inp &&) = default;+ inp(const inp &) = delete;+ inp & operator=(const inp &) = delete;++ void set_prompt (std::string p) { prompt_str = std::move(p); }+ void set_lang (std::string l) { lang_str = std::move(l); }+ void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }++ // pointers are only valid as long as *this is alive+ const mtmd_helper_gen_audio_inp * get() {+ data.prompt = prompt_str.c_str();+ data.prompt_len = prompt_str.size();+ data.lang = lang_str.empty() ? nullptr : lang_str.c_str();+ data.speaker_ref = speaker_ref_ptr.get();+ return &data;+ }+ };+gen_audio_ptr ctx;- gen_audio(struct llama_context * lctx, struct mtmd_context * mctx) : ctx(mtmd_helper_gen_audio_init(lctx, mctx)) {}+ void init(struct llama_context * lctx, struct mtmd_context * mctx) {+ ctx.reset(mtmd_helper_gen_audio_init(lctx, mctx));+ }+ bool valid() const {+ return ctx.get() != nullptr;+ }void reset() {mtmd_helper_gen_audio_reset(ctx.get());}@@ -300,6 +342,9 @@ struct gen_audio {int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) {return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples);}+ int32_t flush() {+ return mtmd_helper_gen_audio_flush(ctx.get());+ }};} // namespace mtmd_helperdiff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex b3ac1979ec..ffd35b2bb3 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -2846,10 +2846,11 @@ struct mtmd_caps mtmd_get_cap_from_file(const char * fname) {mtmd_caps cap;cap.inp_audio = tmp.has_audio;cap.inp_vision = tmp.has_vision;+ cap.gen_audio = tmp.has_gen_audio;return cap;} catch (const std::exception & e) {LOG_ERR("%s: failed to get capabilities from file '%s': %s\n", __func__, fname, e.what());- return mtmd_caps{ false, false };+ return mtmd_caps{ false, false, false };}}diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.hindex df24c56568..c740f1e9fd 100644--- a/tools/mtmd/mtmd.h+++ b/tools/mtmd/mtmd.h@@ -360,6 +360,7 @@ MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);struct mtmd_caps {bool inp_vision;bool inp_audio;+ bool gen_audio;};MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname);diff --git a/tools/server/README.md b/tools/server/README.mdindex 2977fcd4d9..2886bca158 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -802,6 +802,50 @@ curl http://127.0.0.1:8012/v1/rerank \}' | jq```+### POST `/tts`: Generate speech audio from text++Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info, see [tts/README.md](../tts/README.md)++*Options:*++`input`: The text to speak (alias: `prompt`).++`lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional.++`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence.++`top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply.++`repeat_penalty`: Repetition penalty applied to the backbone sampler over the whole generation (default `1.05`). Without this, the backbone can loop and re-generate the same utterance.++`seed`: RNG seed for both the backbone sampler and the codec/vocoder (`-1` = random, the default). Set to a fixed value for reproducible output.++`n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token.++`response_format`: `wav` (default) or `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`).++`stream`: If `true`, the response is streamed as audio becomes available instead of waiting for the full generation to finish. WAV streaming writes an RFC-noncompliant header with an unknown (`0xFFFFFFFF`) size field, since the final length isn't known up front; most players and decoders (ffmpeg, VLC, ...) handle this by reading until EOF.++Note: it's highly recommended to always provide a speaker reference voice; otherwise, the model's performance may be degraded.++*Examples:*++```shell+curl -X POST http://127.0.0.1:9931/tts \+ -H "Content-Type: application/json" \+ -d '{"input": "Hello, this is a test."}' \+ -o output.wav+```++With a speaker reference uploaded as a file (`multipart/form-data`), instead of base64-encoding it into the JSON body:++```shell+curl -X POST http://127.0.0.1:9931/tts \+ -F "input=Hello, this is a test." \+ -F "speaker_ref=@/path/to/speaker-reference.wav;type=audio/wav" \+ -o output.wav+```+### POST `/infill`: For code infilling.Takes a prefix and a suffix and returns the predicted completion as stream.@@ -1262,11 +1306,12 @@ Each object in `data` has an `architecture` object. It has two string arrays:- `input_modalities` lists what the model can read. It always has `text`, plus each media type that the model supports.- `output_modalities` lists what the model can produce.-One output value is special:+Some output values are special:| Value | Meaning ||---|---|| `decisions` | The model is a native decision model. Serve it with [`/v1/systemone`](#post-v1systemone-typesafe-compatible-system-one-api). |+| `audio` | The model can generate speech. Serve it with [`/tts`](#post-tts-generate-speech-audio-from-text). A TTS-only model has `["audio"]` without `text`. |A language model that classifies with prompts does not get `decisions`. Only native decision models do.@@ -2054,7 +2099,7 @@ Note:- If a model is not running, it will be added or updated according to the source2. When the model is loaded, the info from `/v1/models` is forwarded to router's `/v1/models`. This includes metadata about the model and the runtime instance.-Each object in `data` has the same `architecture` object as [`GET /v1/models`](#get-v1models-openai-compatible-model-info-api) of a direct server. The server computes both arrays offline. It does not load the model, download files, or run inference. `output_modalities` comes from the GGUF metadata. `input_modalities` comes from the projector file. A native decision model shows `decisions` before its first load, after unload, and while it sleeps:+Each object in `data` has the same `architecture` object as [`GET /v1/models`](#get-v1models-openai-compatible-model-info-api) of a direct server. The server computes both arrays offline. It does not load the model, download files, or run inference. `output_modalities` comes from the GGUF metadata, plus `audio` when the projector file can generate speech. `input_modalities` comes from the projector file. A native decision model shows `decisions` before its first load, after unload, and while it sleeps:```json{diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex 0756876027..baaca58eb3 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -202,7 +202,8 @@ const char * get_media_marker() {// model output modalities//-std::vector<std::string> server_model_output_modalities(common_decision_type decision_type) {+std::vector<std::string> server_model_output_modalities(common_decision_type decision_type, bool has_cap_chat, bool has_cap_tts) {+ std::vector<std::string> out;switch (decision_type) {case COMMON_DECISION_TYPE_OPENJEV:case COMMON_DECISION_TYPE_LEV:@@ -216,13 +217,27 @@ std::vector<std::string> server_model_output_modalities(common_decision_type deccase COMMON_DECISION_TYPE_LFM2_D1:case COMMON_DECISION_TYPE_LFM2_D1_OMNI:case COMMON_DECISION_TYPE_H2O_LIGHTNING:- return {"decisions"};+ out = {"decisions"};+ break;case COMMON_DECISION_TYPE_GRANITE_SWITCH:- return {"text", "decisions"};+ out = {"text", "decisions"};+ break;default:- // fallback when there is no decision type or the metadata is bad- return {"text"};+ // a tts-only model cannot generate text+ if (has_cap_chat || !has_cap_tts) {+ // fallback when there is no decision type or the metadata is bad+ out = {"text"};+ }+ break;}+ if (has_cap_tts) {+ out.push_back("audio");+ }+ return out;+}++std::vector<std::string> server_model_output_modalities(const llama_model * model, bool has_cap_chat, bool has_cap_tts) {+ return server_model_output_modalities(common_get_decision_type(model), has_cap_chat, has_cap_tts);}json server_model_architecture_json(diff --git a/tools/server/server-common.h b/tools/server/server-common.hindex 2777521221..538e008e94 100644--- a/tools/server/server-common.h+++ b/tools/server/server-common.h@@ -114,7 +114,10 @@ const char * get_media_marker();//// output modalities for architecture.output_modalities in GET /models-std::vector<std::string> server_model_output_modalities(common_decision_type decision_type);+// has_cap_chat: false for a tts-only model, has_cap_tts: the model can generate audio+std::vector<std::string> server_model_output_modalities(common_decision_type decision_type, bool has_cap_chat, bool has_cap_tts);+// same, with the decision type read from a loaded model+std::vector<std::string> server_model_output_modalities(const llama_model * model, bool has_cap_chat, bool has_cap_tts);// architecture object of GET /models; shared by the direct server and the routerjson server_model_architecture_json(diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 6d8f8e7bee..b8e132d7bf 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -20,6 +20,7 @@#include "speculative.h"#include "mtmd.h"#include "mtmd-helper.h"+#include "base64.hpp"#include <algorithm>#include <cstddef>@@ -43,6 +44,14 @@constexpr int HTTP_POLLING_SECONDS = 1;static common_speculative_output_limits server_output_limits(const common_params & params) {+ if (!params.mmproj.path.empty()) {+ const auto mcaps = mtmd_get_cap_from_file(params.mmproj.path.c_str());+ if (mcaps.gen_audio) {+ // some TTS models decode with embeddings output for the whole batch+ return { params.n_batch, params.n_batch };+ }+ }+if (params.embedding ||(params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {return { params.n_batch, 1 };@@ -257,6 +266,27 @@ struct server_slot {mtmd_context * mctx = nullptr;mtmd::batch_ptr mbatch = nullptr;+ struct tts_ctx {+ mtmd_helper::gen_audio ctx;+ std::vector<float> h_state_prompt; // the first h_state after step_prompt done+ const float * h_state;+ llama_token sampled;+ int32_t n_decoded;+ bool stop;+ bool is_supported() const {+ return ctx.valid();+ }+ void reset() {+ ctx.reset();+ h_state_prompt.clear();+ h_state = nullptr;+ sampled = LLAMA_TOKEN_NULL;+ n_decoded = 0;+ stop = false;+ }+ };+ tts_ctx tts;+// speculative decodingcommon_speculative * spec;@@ -421,6 +451,9 @@ struct server_slot {// clear multimodal statembatch.reset();++ tts.reset();+audio_stream.reset();audio_wait = false;inp_embd.clear();@@ -622,6 +655,11 @@ struct server_slot {prompt_clear();}+ // mtmd_helper always re-eval the whole prompt+ if (task->type == SERVER_TASK_TYPE_TTS) {+ prompt_clear();+ }+callback_on_reset(*this);reset();@@ -975,6 +1013,14 @@ public:mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();const llama_vocab * vocab = nullptr;+ bool has_cap_tts() const {+ return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE;+ }++ bool has_cap_chat() const {+ return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx);+ }+server_decision_context decision;server_queue queue_tasks;@@ -1463,6 +1509,10 @@ private:slot.mctx = mctx;slot.prompt.tokens.has_mtmd = mctx != nullptr;+ if (has_cap_tts()) {+ slot.tts.ctx.init(ctx_tgt, mctx);+ }+SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);slot.callback_on_release = [this](int id_slot) {@@ -1944,6 +1994,20 @@ private:SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str());+ if (task.type == SERVER_TASK_TYPE_TTS) {+ GGML_ASSERT(has_cap_tts()); // should already checked in route handler+ if (!slot.tts.is_supported()) {+ slot.tts.ctx.init(ctx_tgt, slot.mctx);+ }+ // mtmd_helper always re-eval the whole prompt+ slot.prompt_clear();+ task.tts_inp.data.seq_id = slot.id;+ if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) {+ send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER);+ return false;+ }+ }+// initialize samplersif (task.need_sampling()) {try {@@ -1961,6 +2025,9 @@ private:// TODO: getting pre sampling logits is not yet supported with backend samplinguse_backend_sampling &= !need_pre_sample_logits;+ // TODO: check verify if this actually works with TTS+ use_backend_sampling &= task.type != SERVER_TASK_TYPE_TTS;+// TODO: tmp until backend sampling is fully implementedif (use_backend_sampling) {llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get()));@@ -1986,9 +2053,13 @@ private:slot.task = std::make_unique<const server_task>(std::move(task));- slot.state = slot.task->is_child()- ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt- : SLOT_STATE_STARTED;+ if (slot.task->type == SERVER_TASK_TYPE_TTS) {+ slot.state = SLOT_STATE_PROCESSING_PROMPT;+ } else {+ slot.state = slot.task->is_child()+ ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt+ : SLOT_STATE_STARTED;+ }// no prompt, the rows come from the audio stream, which audio controls reach from now onif (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {@@ -2268,6 +2339,18 @@ private:queue_results.send(std::move(res));}+ void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) {+ auto res = std::make_unique<server_task_result_tts>();++ res->id = slot.task->id;+ res->index = slot.task->index;+ res->sample_rate = sample_rate;+ res->audio.assign(data, data_len);+ res->final = final;++ queue_results.send(std::move(res));+ }+void send_final_response(server_slot & slot) {auto res = std::make_unique<server_task_result_cmpl_final>();@@ -2872,6 +2955,7 @@ private:case SERVER_TASK_TYPE_TRANSCRIBE_RT:case SERVER_TASK_TYPE_EMBEDDING:case SERVER_TASK_TYPE_RERANK:+ case SERVER_TASK_TYPE_TTS:case SERVER_TASK_TYPE_DECISION:{// special case: if input is provided via CLI, tokenize it first@@ -3397,6 +3481,14 @@ private:return;}+ // note: TTS slots bypass the shared batch entirely+ try {+ process_tts_slots();+ } catch (const std::exception & e) {+ SRV_ERR("process_tts_slots() failed: %s\n", e.what());+ abort_all_slots("process_tts_slots() failed: " + std::string(e.what()));+ }+GGML_ASSERT(batch.slot_batched || batch.size() == 0);if (batch.slot_batched) {@@ -3459,10 +3551,89 @@ private:}}+ void process_tts_slots() {+ iterate(slots, [&](server_slot & slot) {+ if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {+ return;+ }++ llama_set_embeddings(ctx_tgt, true);++ if (slot.state == SLOT_STATE_PROCESSING_PROMPT) {+ const int32_t ret = slot.tts.ctx.step_prompt(llama_n_batch(ctx_tgt));+ if (ret < 0) {+ send_error(slot, "TTS prompt processing failed", ERROR_TYPE_SERVER);+ slot.release();+ } else if (ret == 0) {+ // done prompt, do sample and save h_state_prompt+ slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);+ common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);+ const float * h_embd = llama_get_embeddings_ith(ctx_tgt, -1);+ slot.tts.h_state_prompt.assign(h_embd, h_embd + llama_model_n_embd(model_tgt));+ slot.tts.h_state = slot.tts.h_state_prompt.data();+ slot.state = SLOT_STATE_GENERATING;+ }+ return;+ }++ const int32_t n_predict = slot.n_predict_max > 0 ? slot.n_predict_max : 512;+ if (slot.tts.stop || slot.tts.n_decoded >= n_predict) {+ int32_t sample_rate = 0;+ const char * data = nullptr;+ size_t data_len = 0;+ // generation truly ends here: force out any sub-window remainder still buffered+ if (slot.tts.ctx.flush() != 0 || slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {+ send_error(slot, "failed to finalize TTS output", ERROR_TYPE_SERVER);+ } else {+ send_tts_result(slot, sample_rate, data, data_len, true);+ }+ slot.release();+ return;+ }++ const float * h_state_next = nullptr;+ bool stop = false;+ if (slot.tts.ctx.step_gen(slot.tts.sampled, slot.tts.h_state, &h_state_next, &stop) != 0) {+ send_error(slot, "TTS generation failed", ERROR_TYPE_SERVER);+ slot.release();+ return;+ }+ if (h_state_next == nullptr) {+ // end-of-speech without a new frame (e.g. pocket-tts eos head)+ slot.tts.stop = true;+ return;+ }+ slot.tts.h_state = h_state_next;+ slot.tts.n_decoded++;+ slot.tts.stop = stop;++ if (!stop) {+ slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);+ common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);+ }++ if (slot.task->params.stream) {+ int32_t sample_rate = 0;+ const char * data = nullptr;+ size_t data_len = 0;+ if (slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {+ send_error(slot, "TTS streaming output failed", ERROR_TYPE_SERVER);+ slot.release();+ } else if (data_len > 0) {+ send_tts_result(slot, sample_rate, data, data_len, false);+ }+ }+ });+ }+void pre_decode() {// apply context-shift if needed// TODO: simplify and improveiterate(slots, [&](server_slot & slot) {+ if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) {+ // TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch+ return;+ }if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {if (!params_base.ctx_shift) {// this check is redundant (for good)@@ -3535,7 +3706,7 @@ private:// determine which slots are generating and draftingiterate(slots, [&](server_slot & slot) {- if (slot.state != SLOT_STATE_GENERATING) {+ if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {return;}@@ -3681,7 +3852,7 @@ private:return; // batch is full, skip remaining slots}- if (!slot.is_processing()) {+ if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {return;}@@ -4926,11 +5097,13 @@ server_context_meta server_context::get_meta() const {/* model_aliases */ impl->model_aliases,/* model_tags */ impl->model_tags,/* model_path */ impl->params_base.model.path,- /* model_output_modalities */ server_model_output_modalities(common_get_decision_type(impl->model_tgt)),+ /* model_output_modalities */ server_model_output_modalities(impl->model_tgt, impl->has_cap_chat(), impl->has_cap_tts()),/* has_mtmd */ impl->mctx != nullptr,/* has_inp_image */ impl->chat_params.allow_image,/* has_inp_audio */ impl->chat_params.allow_audio,/* has_inp_video */ impl->chat_params.allow_video,+ /* has_cap_chat */ impl->has_cap_chat(),+ /* has_cap_tts */ impl->has_cap_tts(),/* has_audio_stream */ impl->mctx != nullptr && mtmd_support_audio_stream(impl->mctx),/* json_ui_settings */ impl->json_ui_settings,/* slot_n_ctx */ impl->n_ctx_slot(),@@ -5006,6 +5179,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(res->set_req(&req); // will also set spipe if needed+ if (!ctx_server.has_cap_chat()) {+ res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED));+ return res;+ }+int32_t sse_ping_interval = params.sse_ping_interval;try {@@ -6175,6 +6353,171 @@ void server_routes::init_routes() {return res;};+ this->post_tts = [this](const server_http_req & req) {+ auto res = create_response();+ res->set_req(&req); // will also set spipe if needed++ if (!ctx_server.has_cap_tts()) {+ res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));+ return res;+ }++ const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);++ json body = json::parse(req.body);++ // multipart form fields arrive as strings, correct the types of the numeric and boolean params+ for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {+ if (body.contains(k) && body[k].is_string()) {+ body[k] = std::stoll(body[k].get<std::string>());+ }+ }+ for (const char * k : { "top_p", "repeat_penalty" }) {+ if (body.contains(k) && body[k].is_string()) {+ body[k] = std::stof(body[k].get<std::string>());+ }+ }+ if (body.contains("stream") && body["stream"].is_string()) {+ body["stream"] = body["stream"] == "true";+ }++ std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));+ if (prompt.empty()) {+ res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));+ return res;+ }++ const std::string response_format = json_value(body, "response_format", std::string("wav"));++ server_task task(SERVER_TASK_TYPE_TTS);+ server_schema::eval_tts_schema(params, task, body);+ const bool stream = task.params.stream;+ task.tts_inp.set_prompt(prompt);+ task.tts_inp.set_lang(json_value(body, "lang", std::string()));+ task.tts_inp.data.stream = stream;+ task.tts_inp.data.out_type = response_format == "pcm"+ ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM+ : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;+ // note: -1 is clamped to 0 by the sampler, it will disable the penalty+ task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;+ task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG+ if (task.tts_inp.data.top_k > 0) {+ task.params.sampling.top_k = task.tts_inp.data.top_k;+ }+ if (task.tts_inp.data.top_p > 0) {+ task.params.sampling.top_p = task.tts_inp.data.top_p;+ }++ // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")+ const unsigned char * speaker_ref_data = nullptr;+ size_t speaker_ref_len = 0;+ std::string speaker_ref_b64_decoded;++ auto speaker_ref_file = req.files.find("speaker_ref");+ if (speaker_ref_file != req.files.end()) {+ speaker_ref_data = speaker_ref_file->second.data.data();+ speaker_ref_len = speaker_ref_file->second.data.size();+ } else {+ std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());+ if (!speaker_ref_b64.empty()) {+ try {+ speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);+ } catch (const std::exception &) {+ res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));+ return res;+ }+ speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();+ speaker_ref_len = speaker_ref_b64_decoded.size();+ }+ }++ if (speaker_ref_len > 0) {+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);+ if (!wrapper.bitmap) {+ res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));+ return res;+ }+ task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));+ } else {+ SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");+ }++ auto & rd = res->rd;+ task.id = rd.get_new_id();+ rd.post_task(std::move(task));++ // raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)+ const std::string content_type = response_format == "pcm"+ ? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32"+ : "audio/wav";++ if (!stream) {+ auto result = rd.next(req.should_stop);+ if (!result) {+ GGML_ASSERT(req.should_stop());+ return res; // connection is closed+ }+ if (result->is_error()) {+ res->error(result->to_json());+ return res;+ }+ auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());+ GGML_ASSERT(tts_res != nullptr);+ res->status = 200;+ res->content_type = content_type;+ res->data = std::move(tts_res->audio);+ return res;+ } else {+ auto first_result = rd.next(req.should_stop);+ if (!first_result) {+ GGML_ASSERT(req.should_stop());+ return res; // connection is closed+ }+ if (first_result->is_error()) {+ res->error(first_result->to_json());+ return res;+ }+ auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());+ GGML_ASSERT(first_tts_res != nullptr);++ res->status = 200;+ res->content_type = content_type;+ res->data = std::move(first_tts_res->audio);+ bool is_done = first_tts_res->final;++ res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool {+ // flush buffered audio before is_done - the first result can already be final+ if (!res_this->data.empty()) {+ output = std::move(res_this->data);+ res_this->data.clear();+ return true;+ }+ if (is_done) {+ return false;+ }+ if (res_this->should_stop()) {+ return false;+ }++ server_response_reader & rd = res_this->rd;+ if (!rd.has_next()) {+ return false;+ }+ auto result = rd.next([&res_this]() { return res_this->should_stop(); });+ if (!result || result->is_error()) {+ return false;+ }+ auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());+ GGML_ASSERT(tts_res != nullptr);+ output = std::move(tts_res->audio);+ is_done = tts_res->final;+ return true;+ });+ }++ return res;+ };+this->post_systemone = [this](const server_http_req & req) {auto res = create_response();const auto & decision = ctx_server.decision;diff --git a/tools/server/server-context.h b/tools/server/server-context.hindex da75e53271..a579b0ecad 100644--- a/tools/server/server-context.h+++ b/tools/server/server-context.h@@ -24,6 +24,8 @@ struct server_context_meta {bool has_inp_image;bool has_inp_audio;bool has_inp_video;+ bool has_cap_chat;+ bool has_cap_tts;bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream()json json_ui_settings;int slot_n_ctx;@@ -155,6 +157,7 @@ struct server_routes {server_http_context::handler_t post_embeddings;server_http_context::handler_t post_embeddings_oai;server_http_context::handler_t post_rerank;+ server_http_context::handler_t post_tts;server_http_context::handler_t post_systemone;server_http_context::handler_t get_lora_adapters;server_http_context::handler_t post_lora_adapters;diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex c8e235129d..d38549db5f 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -521,27 +521,31 @@ void server_model_meta::update_caps(const common_params & base) {return;}- // read the output modalities and the trained context from the GGUF metadata- std::vector<std::string> output_modalities = {"text"};- if (!params.model.path.empty()) {- const common_gguf_info info = common_get_gguf_info(params.model.path);- output_modalities = server_model_output_modalities(info.decision_type);- n_ctx_train = info.n_ctx_train;- }-bool inp_image = false;bool inp_audio = false;+ bool gen_audio = false;try {if (!params.no_mmproj && !params.mmproj.path.empty()) {mtmd_caps caps = mtmd_get_cap_from_file(params.mmproj.path.c_str());inp_image = caps.inp_vision;inp_audio = caps.inp_audio;+ gen_audio = caps.gen_audio;}} catch (const std::exception & e) {LOG_WRN("failed to read the multimodal capabilities of '%s': %s\n", name.c_str(), e.what());// keep the output modalities from the GGUF metadata}+ // read the output modalities and the trained context from the GGUF metadata+ std::vector<std::string> output_modalities = {"text"};+ if (!params.model.path.empty()) {+ const common_gguf_info info = common_get_gguf_info(params.model.path);+ // same rule as mtmd_helper_model_can_chat()+ const bool can_chat = !gen_audio || info.has_chat_tmpl;+ output_modalities = server_model_output_modalities(info.decision_type, can_chat, gen_audio);+ n_ctx_train = info.n_ctx_train;+ }+// offline discovery cannot see video; a loaded model reports itarchitecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities);}diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cppindex 54e126aa0c..1e2dce8886 100644--- a/tools/server/server-schema.cpp+++ b/tools/server/server-schema.cpp@@ -511,6 +511,58 @@ task_params eval_llama_cmpl_schema(return params;}+//+// TTS schema+//++std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task) {+ std::vector<std::unique_ptr<field>> fields;+ auto add = [&](field * f) {+ fields.emplace_back(f);+ };++ add((new field_num("top_k", task.tts_inp.data.top_k))+ ->set_limits(0, INT32_MAX)+ ->set_desc("Top-k for the acoustic code predictor, 0 to use the model default"));++ add((new field_num("top_p", task.tts_inp.data.top_p))+ ->set_limits(0.0f, 1.0f)+ ->set_desc("Top-p for the acoustic code predictor, 0.0 to use the model default"));++ add((new field_num("seed", task.tts_inp.data.seed))+ ->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)"));++ add((new field_num("n_predict", task.params.n_predict))+ ->set_hard_limits(-1, INT32_MAX)+ ->add_alias("max_tokens")+ ->set_desc("Max number of audio frames to generate"));++ add((new field_num("repeat_penalty", task.params.sampling.penalty_repeat))+ ->set_desc("Repetition penalty applied to the backbone sampler over the whole generation"));++ add((new field_bool("stream", task.params.stream))+ ->set_desc("Stream the audio as it becomes available instead of waiting for the full generation"));++ return fields;+}++void eval_tts_schema(const common_params & params_base, server_task & task, const json & data) {+ // baseline defaults, individual requests can override them+ // TODO @ngxson : change the default values based on model+ task.params.sampling = params_base.sampling;+ task.params.sampling.penalty_repeat = 1.05f;+ task.params.n_predict = -1;+ task.tts_inp.data.top_k = 0;+ task.tts_inp.data.top_p = 0.0f;+ task.tts_inp.data.seed = params_base.sampling.seed;++ field_eval_context ctx(task.params);++ for (const auto & f : make_tts_schema(task)) {+ f->eval(ctx, data);+ }+}+//// eval() implementations//diff --git a/tools/server/server-schema.h b/tools/server/server-schema.hindex d0a81431bc..542136a975 100644--- a/tools/server/server-schema.h+++ b/tools/server/server-schema.h@@ -101,4 +101,12 @@ task_params eval_llama_cmpl_schema(const std::vector<llama_logit_bias> & logit_bias_eog,const json & data);+std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task);++// evaluates the /tts request params into task.params and task.tts_inp.data+void eval_tts_schema(+ const common_params & params_base,+ server_task & task,+ const json & data);+} // namespace server_schemadiff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex 2e513a33ec..d7ac2a1dc3 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -1508,6 +1508,17 @@ json server_task_result_rerank::to_json() {};}+//+// server_task_result_tts+//+json server_task_result_tts::to_json() {+ return json {+ {"sample_rate", sample_rate},+ {"n_bytes", audio.size()},+ {"final", final},+ };+}+//// server_task_result_decision//diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex de39ce3b83..810cefbfe9 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -10,6 +10,7 @@// TODO: prevent including the whole server-common.h as we only use server_tokens#include "server-common.h"+#include "mtmd-helper.h"enum server_task_type {@@ -29,6 +30,7 @@ enum server_task_type {SERVER_TASK_TYPE_SLOT_ERASE,SERVER_TASK_TYPE_GET_LORA,SERVER_TASK_TYPE_SET_LORA,+ SERVER_TASK_TYPE_TTS,};// TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common@@ -219,6 +221,9 @@ struct server_task {// used by SERVER_TASK_TYPE_SET_LORAstd::map<int, float> set_lora; // mapping adapter ID -> scale+ // used by SERVER_TASK_TYPE_TTS+ mtmd_helper::gen_audio::inp tts_inp;+server_task() = default;server_task(server_task_type type) : type(type) {}@@ -256,6 +261,7 @@ struct server_task {switch (type) {case SERVER_TASK_TYPE_COMPLETION:case SERVER_TASK_TYPE_INFILL:+ case SERVER_TASK_TYPE_TTS:case SERVER_TASK_TYPE_TRANSCRIBE_RT:return true;default:@@ -526,6 +532,16 @@ struct server_task_result_embd : server_task_result {json to_json_oaicompat();};+struct server_task_result_tts : server_task_result {+ std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type)+ int32_t sample_rate = 0;+ bool final = false; // true for the last chunk of a request++ virtual bool is_stop() override { return final; }++ virtual json to_json() override;+};+struct server_task_result_rerank : server_task_result {float score = -1e6;diff --git a/tools/server/server.cpp b/tools/server/server.cppindex 41f29ded5d..ce44405b86 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -247,6 +247,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &routes.post_control = models_routes->proxy_post;routes.post_responses_oai = models_routes->proxy_post;routes.post_transcriptions_oai = models_routes->proxy_post;+ routes.post_tts = models_routes->proxy_post;routes.ws_realtime = models_routes->proxy_ws;routes.post_anthropic_messages = models_routes->proxy_post;routes.post_anthropic_count_tokens = models_routes->proxy_post;@@ -293,6 +294,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));+ ctx_http.post("/tts", ex_wrapper(routes.post_tts));ctx_http.ws ("/v1/realtime", routes.ws_realtime);ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages APIctx_http.post("/infill", ex_wrapper(routes.post_infill));diff --git a/tools/tts/README.md b/tools/tts/README.mdindex 1b08d5ef32..55b99932c6 100644--- a/tools/tts/README.md+++ b/tools/tts/README.md@@ -12,6 +12,8 @@ Simple usage:llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output out.wav```+Note: it's highly recommended to always provide a speaker reference voice (via `--tts-speaker-file`); otherwise, the model's performance may be degraded.+Common params:- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc.- `-n <number_of_frames>` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by modeldiff --git a/tools/tts/tts.cpp b/tools/tts/tts.cppindex 6fd1936324..aea794e65d 100644--- a/tools/tts/tts.cpp+++ b/tools/tts/tts.cpp@@ -111,8 +111,9 @@ int main(int argc, char ** argv) {speaker_bitmap.reset(wrapper.bitmap);}- mtmd_helper::gen_audio gen(lctx, mctx.get());- mtmd_helper_gen_audio_inp inp{};+ mtmd_helper::gen_audio gen;+ gen.init(lctx, mctx.get());+ mtmd_helper_gen_audio_inp inp = mtmd_helper_gen_audio_inp_default();inp.seq_id = 0;inp.prompt = params.prompt.c_str();inp.prompt_len = params.prompt.size();@@ -121,7 +122,6 @@ int main(int argc, char ** argv) {inp.top_k = params.sampling.top_k;inp.top_p = params.sampling.top_p;inp.seed = params.sampling.seed;- inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;//// stage 1: process prompt via backbone model, generate semantic representation