LLM inference in C/C++
C++ ggml
server: add realtime transcription API (#30636)
Xuan-Son Nguyen1242be66f65d
diff --git a/tools/server/README.md b/tools/server/README.mdindex 8d9f535695..2977fcd4d9 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1491,6 +1491,31 @@ Acts on an in-flight completion identified by its `id` (the `id` field streamedReturns a JSON object with a boolean `success` field, and an optional `message` field describing the reason when `success` is `false`.+### WebSocket `/v1/realtime`: Realtime transcription++Transcribes streamed audio, for models with a streaming audio encoder. Compatible with OAI `/v1/realtime` schema.++Audio must be PCM16, mono, at the model sample rate. The sample rate is returned in `session.created` as `audio.input.format.rate`. Each `input_audio_buffer.append` carries a base64-encoded chunk of PCM16 samples.++*Client events:*++- `session.update`: accepted for compatibility, the server replies with `session.updated`+- `input_audio_buffer.append`: `audio` holds base64 PCM16 samples+- `input_audio_buffer.commit`: flushes the audio received so far, so the words still held back by the model's delay are produced. The server replies with `input_audio_buffer.committed` and starts a new item. With `"final": true`, the input ends instead and the last item is completed once the stream drains.+- `input_audio_buffer.clear`: accepted for compatibility, audio already sent cannot be discarded++Closing the websocket also ends the input.++*Server events:*++- `session.created` and `session.updated`: carry the `session` object+- `conversation.item.input_audio_transcription.delta`: `delta` holds the new text for `item_id`+- `conversation.item.input_audio_transcription.completed`: `transcript` holds the full text of `item_id`+- `input_audio_buffer.committed`: `item_id` of the committed item, `previous_item_id` of the item committed before it (`null` for the first one)+- `error`: `error.message` describes the problem++Note that words held back by the model's delay at the time of a commit are attributed to the next item.+### POST `/v1/responses`: OpenAI-compatible Responses API*Options:*@@ -1992,6 +2017,8 @@ For **GET** endpoints (`/props`, `/metrics`, etc.) The router uses the `model` qGET /props?model=ggml-org%2Fgemma-3-4b-it-GGUF%3AQ4_K_M```+For the **WebSocket** endpoint (`/v1/realtime`) The router also uses the `model` query parameter.+By default, the model will be loaded automatically if it's not loaded. To disable this, add `--no-models-autoload` when starting the server. Additionally, you can include `?autoload=true|false` in the query param to control this behavior per-request.### GET `/models`: List available modelsdiff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex d66aae426e..6d8f8e7bee 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -1,4 +1,7 @@#include "server-context.h"+#include <atomic>+#include <thread>+#include "base64.hpp"#include "server-chat.h"#include "server-common.h"#include "server-decision.h"@@ -360,8 +363,13 @@ struct server_slot {// for TTS models, this is the embd generated from prev step, decode this to generate next hidden state// corresponding to one token position (size = n_embd)+ // for realtime transcription, these are the rows from the last audio stream step (size = n_rows * n_embd)std::vector<float> inp_embd;+ // realtime transcription, see SERVER_TASK_TYPE_TRANSCRIBE_RT+ mtmd::audio_stream_ptr audio_stream;+ bool audio_wait = false; // the stream needs more audio before it can produce rows+server_slot_stats stats;// accepted tokens per draft position@@ -413,6 +421,9 @@ struct server_slot {// clear multimodal statembatch.reset();+ audio_stream.reset();+ audio_wait = false;+ inp_embd.clear();}void init_sampler() const {@@ -542,16 +553,28 @@ struct server_slot {// add sampled token of this slot to the batch, optionally add the speculative draft tokens if anyvoid handle_last_sampled_token(server_batch & batch) {bool add_ok = true;+ // the audio stream has no token before the first step, track the position with a dummy token+ const llama_token tok_track = sampled == LLAMA_TOKEN_NULL ? 0 : sampled;if (spec_draft.empty()) {// no speculative decodingi_batch = batch.size();if (!inp_embd.empty()) {- // text embeddings broadcast the same position across the M-RoPE sections- const llama_pos p = prompt.tokens.pos_next();- const llama_pos pos[GGML_MROPE_SECTIONS] = { p, p, p, 0 };- GGML_ASSERT((int32_t) inp_embd.size() == batch.n_embd);- add_ok &= batch.add_embd(id, inp_embd.data(), pos, true, false);+ const int32_t n_rows = (int32_t) inp_embd.size() / batch.n_embd;+ GGML_ASSERT((int32_t) inp_embd.size() == n_rows * batch.n_embd);+ i_batch = batch.size() + n_rows - 1;+ for (int32_t i = 0; i < n_rows; i++) {+ // text embeddings broadcast the same position across the M-RoPE sections+ const llama_pos p = prompt.tokens.pos_next();+ const llama_pos pos[GGML_MROPE_SECTIONS] = { p, p, p, 0 };+ add_ok &= batch.add_embd(id, inp_embd.data() + (size_t) i * batch.n_embd, pos, i == n_rows - 1, false);+ if (i < n_rows - 1) {+ prompt.tokens.push_back(tok_track);+ }+ }+ if (audio_stream) {+ inp_embd.clear(); // rows are consumed, the next step produces new ones+ }} else {add_ok &= batch.add(id, sampled, prompt.tokens.pos_next(), true, false);}@@ -579,7 +602,7 @@ struct server_slot {GGML_ASSERT(add_ok && "batch must be large enough to hold the sampled and draft tokens");- prompt.tokens.push_back(sampled);+ prompt.tokens.push_back(tok_track);prompt.tokens.insert(spec_draft);}@@ -595,7 +618,7 @@ struct server_slot {// do not keep context of the child slots - the parent's context is enough// nor of the stateless slots - no task ever reuses it- if (task->is_child() || task->is_stateless()) {+ if (task->is_child() || task->is_stateless() || audio_stream) {prompt_clear();}@@ -1967,6 +1990,23 @@ private:? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt: SLOT_STATE_STARTED;+ // no prompt, the rows come from the audio stream, which audio controls reach from now on+ if (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {+ slot.audio_stream.reset(mtmd_audio_stream_init(mctx, mtmd_audio_stream_params_default()));+ if (!slot.audio_stream) {+ send_error(slot, "failed to initialize the audio stream", ERROR_TYPE_SERVER);+ slot.release();+ return false;+ }+ slot.prompt_clear();+ slot.init_sampler();+ slot.sampled = LLAMA_TOKEN_NULL;+ slot.i_batch = -1;+ slot.stats.update_prompt_start();+ slot.state = SLOT_STATE_GENERATING;+ send_partial_response(slot, {}, false, true);+ }+// reset server kill-switch countern_empty_consecutive = 0;@@ -2829,6 +2869,7 @@ private:switch (task.type) {case SERVER_TASK_TYPE_COMPLETION:case SERVER_TASK_TYPE_INFILL:+ case SERVER_TASK_TYPE_TRANSCRIBE_RT:case SERVER_TASK_TYPE_EMBEDDING:case SERVER_TASK_TYPE_RERANK:case SERVER_TASK_TYPE_DECISION:@@ -2916,15 +2957,15 @@ private:server_slot * slot = get_slot_by_cmpl_id(task.params.control_cmpl_id);if (slot == nullptr) {- SRV_WRN("control %s on unknown completion id=%s, no live slot\n",- task.params.control_action.c_str(), task.params.control_cmpl_id.c_str());+ SRV_WRN("control %d on unknown completion id=%s, no live slot\n",+ (int) task.params.control_action, task.params.control_cmpl_id.c_str());res->success = false;res->message = "no active completion for this id";queue_results.send(std::move(res));break;}- if (task.params.control_action == "reasoning_end") {+ if (task.params.control_action == SERVER_CONTROL_ACTION_REASONING_END) {// the budget sampler only exists when reasoning control was armedif (!slot->task->params.sampling.reasoning_control) {res->success = false;@@ -2935,6 +2976,22 @@ private:// act on the live slot mid generation, never defercommon_sampler_reasoning_budget_force(slot->smpl.get());res->success = true;+ } else if (task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_APPEND+ || task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_COMMIT+ || task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_END) {+ if (!slot->audio_stream) {+ res->success = false;+ res->message = "not a realtime transcription slot";+ } else {+ auto * stream = slot->audio_stream.get();+ if (task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_APPEND) {+ mtmd_audio_stream_push(stream, task.params.control_audio.data(), task.params.control_audio.size());+ } else {+ mtmd_audio_stream_flush(stream, task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_END);+ }+ slot->audio_wait = false;+ res->success = true;+ }} else {res->success = false;res->message = "unknown control action";@@ -3254,6 +3311,39 @@ private:};#endif+ // fill slot.inp_embd with the next rows of the audio stream+ // returns false if the slot has nothing to decode now+ bool slot_audio_step(server_slot & slot) {+ if (!slot.inp_embd.empty()) {+ return true; // rows from a previous step, not decoded yet+ }++ const int32_t n = mtmd_audio_stream_step(slot.audio_stream.get(), slot.sampled);+ if (n > 0) {+ const float * rows = mtmd_audio_stream_get_output_embd(slot.audio_stream.get());+ slot.inp_embd.assign(rows, rows + (size_t) n * batch.n_embd);+ slot.audio_wait = false;+ return true;+ }+ if (n == 0) {+ slot.audio_wait = true;+ return false;+ }+ if (n == -2) {+ send_error(slot, "audio stream error", ERROR_TYPE_SERVER);+ slot.release();+ return false;+ }++ // end of stream+ slot.stop = STOP_TYPE_EOS;+ slot.has_next_token = false;+ slot.print_timings();+ send_final_response(slot);+ slot.release();+ return false;+ }+void update_slots() {#ifdef DEBUG_TIMINGSstatic int64_t t_prev = 0;@@ -3269,11 +3359,12 @@ private:#endif// check if all slots are idle+ // a realtime slot waiting for audio has no work, the next audio control task wakes the loop{bool all_idle = true;for (auto & slot : slots) {- if (slot.is_processing()) {+ if (slot.is_processing() && !slot.audio_wait) {all_idle = false;break;}@@ -3448,6 +3539,10 @@ private:return;}+ if (slot.audio_stream && !slot_audio_step(slot)) {+ return; // waiting for audio, or the stream ended+ }+// check if we can batch this slot with the previous oneif (!slot_batched) {slot_batched = &slot;@@ -3601,6 +3696,11 @@ private:return;}+ // no prompt, the rows come from the audio stream+ if (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {+ return;+ }+// this slot still has a prompt to be processedif (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_STARTED) {const auto & input_tokens = slot.task->tokens;@@ -4831,6 +4931,7 @@ server_context_meta server_context::get_meta() const {/* has_inp_image */ impl->chat_params.allow_image,/* has_inp_audio */ impl->chat_params.allow_audio,/* has_inp_video */ impl->chat_params.allow_video,+ /* has_audio_stream */ impl->mctx != nullptr && mtmd_support_audio_stream(impl->mctx),/* json_ui_settings */ impl->json_ui_settings,/* slot_n_ctx */ impl->n_ctx_slot(),/* pooling_type */ llama_pooling_type(impl->ctx_tgt),@@ -5611,7 +5712,7 @@ void server_routes::init_routes() {server_task task(SERVER_TASK_TYPE_CONTROL);task.id = rd.get_new_id();task.params.control_cmpl_id = cmpl_id;- task.params.control_action = action;+ task.params.control_action = SERVER_CONTROL_ACTION_REASONING_END;rd.post_task(std::move(task));}@@ -5686,6 +5787,197 @@ void server_routes::init_routes() {session);};+ // realtime transcription over websocket, modeled after the OpenAI realtime API+ // audio is PCM16 mono at the model sample rate, sent as base64 in input_audio_buffer.append events+ // input_audio_buffer.commit with "final": true ends the input, closing the socket also ends it+ this->ws_realtime = [this](const server_http_req &, server_http_context::ws_conn & conn) {+ auto send_event = [&conn](json ev) {+ ev["event_id"] = "event_" + random_string();+ conn.send(safe_json_to_str(ev));+ };+ auto send_error_event = [&send_event](const std::string & message, const std::string & type) {+ send_event(json {+ {"type", "error"},+ {"error", {+ {"type", type},+ {"code", nullptr},+ {"message", message},+ {"param", nullptr},+ }},+ });+ };++ if (!meta->has_audio_stream) {+ send_error_event("The current model does not support realtime transcription.", "invalid_request_error");+ conn.close();+ return;+ }++ const std::string session_id = "sess_" + random_string();+ const json session_config = json {+ {"id", session_id},+ {"object", "realtime.transcription_session"},+ {"type", "transcription"},+ {"audio", {{"input", {+ {"format", {{"type", "audio/pcm"}, {"rate", mtmd_get_audio_sample_rate(ctx_server.mctx)}}},+ {"transcription", {{"model", meta->model_name}}},+ {"turn_detection", nullptr},+ {"noise_reduction", nullptr},+ }}}},+ };++ auto res = create_response();+ auto & rd = res->rd;+ {+ server_task task(SERVER_TASK_TYPE_TRANSCRIBE_RT);+ task.id = rd.get_new_id();+ task.params.sampling = params.sampling;+ task.params.stream = true;+ task.params.oaicompat_cmpl_id = session_id;+ task.params.oaicompat_model = meta->model_name;+ rd.post_task(std::move(task));+ }++ // the slot answers once its audio stream is live, the session starts there+ {+ auto result = rd.next([&conn]() { return !conn.is_open(); });+ if (!result || result->is_error()) {+ if (result) {+ send_error_event(json_value(result->to_json(), "message", std::string("server error")), "server_error");+ }+ conn.close();+ return;+ }+ }+ send_event(json {{"type", "session.created"}, {"session", session_config}});++ // one item per committed audio buffer, deltas are attributed to the current item+ std::mutex mutex_item;+ std::string item_id = "item_" + random_string();+ std::string item_prev; // the committed item before item_id, empty for the first one+ std::string item_text;++ // called with mutex_item held+ auto send_completed = [&]() {+ send_event(json {+ {"type", "conversation.item.input_audio_transcription.completed"},+ {"item_id", item_id},+ {"content_index", 0},+ {"transcript", item_text},+ });+ item_text.clear();+ };++ auto post_control = [&](server_control_action action, std::vector<float> && audio) {+ server_response_reader ctl(queue_tasks, queue_results, HTTP_POLLING_SECONDS);+ server_task task(SERVER_TASK_TYPE_CONTROL);+ task.id = ctl.get_new_id();+ task.params.control_action = action;+ task.params.control_cmpl_id = session_id;+ task.params.control_audio = std::move(audio);+ ctl.post_task(std::move(task));+ auto result = ctl.next([&conn]() { return !conn.is_open(); });+ auto * control = result ? dynamic_cast<server_task_result_control *>(result.get()) : nullptr;+ return control && control->success;+ };++ // client events are read on this thread, the transcript is sent from the thread below+ std::atomic<bool> input_ended{false};+ std::thread reader([&]() {+ std::string msg;+ while (conn.read(msg) != 0) {+ json ev;+ try {+ ev = json::parse(msg);+ } catch (const std::exception &) {+ send_error_event("invalid JSON", "invalid_request_error");+ continue;+ }+ const std::string type = json_value(ev, "type", std::string());+ if (type == "session.update" || type == "transcription_session.update") {+ send_event(json {{"type", "session.updated"}, {"session", session_config}});+ } else if (type == "input_audio_buffer.append") {+ std::string pcm;+ try {+ pcm = base64::decode(json_value(ev, "audio", std::string()));+ } catch (const std::exception &) {+ send_error_event("invalid base64 audio", "invalid_request_error");+ continue;+ }+ std::vector<float> samples(pcm.size() / 2);+ for (size_t i = 0; i < samples.size(); i++) {+ int16_t v;+ memcpy(&v, pcm.data() + 2 * i, sizeof(v));+ samples[i] = v / 32768.0f;+ }+ if (!post_control(SERVER_CONTROL_ACTION_AUDIO_APPEND, std::move(samples))) {+ break;+ }+ } else if (type == "input_audio_buffer.commit") {+ // "final": true ends the input, the last item is completed when the stream drains+ if (json_value(ev, "final", false)) {+ input_ended = true;+ post_control(SERVER_CONTROL_ACTION_AUDIO_END, {});+ continue;+ }+ if (!post_control(SERVER_CONTROL_ACTION_AUDIO_COMMIT, {})) {+ break;+ }+ // the current item is the committed one, its transcript completes it and the next item opens+ std::lock_guard<std::mutex> lock(mutex_item);+ send_event(json {+ {"type", "input_audio_buffer.committed"},+ {"previous_item_id", item_prev.empty() ? json(nullptr) : json(item_prev)},+ {"item_id", item_id},+ });+ send_completed();+ item_prev = item_id;+ item_id = "item_" + random_string();+ } else if (type == "input_audio_buffer.clear") {+ send_event(json {{"type", "input_audio_buffer.cleared"}});+ } else {+ send_error_event("unknown event type: " + type, "invalid_request_error");+ }+ }+ if (!input_ended.exchange(true)) {+ post_control(SERVER_CONTROL_ACTION_AUDIO_END, {});+ }+ // the result loop waits on the socket, nobody reads it from here on+ conn.close();+ });++ while (true) {+ auto result = rd.next([&conn]() { return !conn.is_open(); });+ if (!result) {+ break; // connection closed+ }+ if (result->is_error()) {+ send_error_event(json_value(result->to_json(), "message", std::string("server error")), "server_error");+ break;+ }+ if (auto * partial = dynamic_cast<server_task_result_cmpl_partial *>(result.get())) {+ if (!partial->content.text.empty()) {+ std::lock_guard<std::mutex> lock(mutex_item);+ item_text += partial->content.text;+ send_event(json {+ {"type", "conversation.item.input_audio_transcription.delta"},+ {"item_id", item_id},+ {"content_index", 0},+ {"delta", partial->content.text},+ });+ }+ } else if (result->is_stop()) {+ std::lock_guard<std::mutex> lock(mutex_item);+ send_completed();+ break;+ }+ }++ input_ended = true;+ conn.close();+ reader.join();+ };+this->post_anthropic_messages = [this](const server_http_req & req) {auto res = create_response();std::vector<raw_buffer> files;diff --git a/tools/server/server-context.h b/tools/server/server-context.hindex ed23f7d4e6..da75e53271 100644--- a/tools/server/server-context.h+++ b/tools/server/server-context.h@@ -24,6 +24,7 @@ struct server_context_meta {bool has_inp_image;bool has_inp_audio;bool has_inp_video;+ bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream()json json_ui_settings;int slot_n_ctx;enum llama_pooling_type pooling_type;@@ -144,6 +145,7 @@ struct server_routes {server_http_context::handler_t post_responses_oai;server_http_context::handler_t post_responses_tok_oai;server_http_context::handler_t post_transcriptions_oai;+ server_http_context::ws_handler_t ws_realtime;server_http_context::handler_t post_anthropic_messages;server_http_context::handler_t post_anthropic_count_tokens;server_http_context::handler_t post_apply_template;diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cppindex 33cbb156c5..cf147c7e70 100644--- a/tools/server/server-http.cpp+++ b/tools/server/server-http.cpp@@ -11,6 +11,7 @@#include <memory>#include <string>#include <thread>+#include <mutex>//// HTTP implementation using cpp-httplib@@ -744,6 +745,56 @@ void server_http_context::post(const std::string & path, const server_http_conte}}+void server_http_context::ws(const std::string & path, const server_http_context::ws_handler_t & handler) const {+ const std::string full_path = path_prefix + path;++ auto callback = [handler](const httplib::Request & req, httplib::ws::WebSocket & sock) {+ struct conn_impl : server_http_context::ws_conn {+ httplib::ws::WebSocket & sock;+ std::mutex mutex_send;+ conn_impl(httplib::ws::WebSocket & sock) : sock(sock) {}+ int read(std::string & msg) override {+ switch (sock.read(msg)) {+ case httplib::ws::Text: return 1;+ case httplib::ws::Binary: return 2;+ default: return 0;+ }+ }+ bool send(const std::string & msg) override {+ std::lock_guard<std::mutex> lock(mutex_send);+ return sock.send(msg);+ }+ bool send_binary(const std::string & msg) override {+ std::lock_guard<std::mutex> lock(mutex_send);+ return sock.send(msg.data(), msg.size());+ }+ bool is_open() const override {+ return sock.is_open();+ }+ void close() override {+ sock.close();+ }+ };+ conn_impl conn(sock);+ const std::function<bool()> should_stop = [&sock]() {+ return !sock.is_open();+ };+ server_http_req request{+ get_params(req),+ get_headers(req),+ req.path,+ build_query_string(req),+ req.body,+ {},+ should_stop+ };+ handler(request, conn);+ };+ for (const auto & srv : pimpl->servers) {+ srv->WebSocket(full_path, callback);+ }+}+void server_http_context::del(const std::string & path, const server_http_context::handler_t & handler) const {handlers.emplace(path, handler);auto callback = [handler](const httplib::Request & req, httplib::Response & res) {diff --git a/tools/server/server-http.h b/tools/server/server-http.hindex 63fd2c186b..d123535e87 100644--- a/tools/server/server-http.h+++ b/tools/server/server-http.h@@ -74,6 +74,20 @@ struct server_http_context {using handler_t = std::function<server_http_res_ptr(const server_http_req & req)>;mutable std::unordered_map<std::string, handler_t> handlers;+ // websocket connection, see ws()+ struct ws_conn {+ // blocking read, returns 1 for a text message, 2 for a binary message, 0 when the connection is closed+ virtual int read(std::string & msg) = 0;+ // thread-safe, can be called while another thread is in read()+ virtual bool send(const std::string & msg) = 0;+ virtual bool send_binary(const std::string & msg) = 0;+ virtual bool is_open() const = 0;+ virtual void close() = 0;+ virtual ~ws_conn() = default;+ };+ // the handler runs on its own HTTP thread for the whole lifetime of the connection+ using ws_handler_t = std::function<void(const server_http_req & req, ws_conn & conn)>;+std::string path_prefix;int port = 9931;bool is_ssl = false;@@ -89,6 +103,7 @@ struct server_http_context {void get(const std::string & path, const handler_t & handler) const;void post(const std::string & path, const handler_t & handler) const;void del(const std::string & path, const handler_t & handler) const;+ void ws(const std::string & path, const ws_handler_t & handler) const;// Register the Google Cloud Platform (Vertex AI) compat (AIP_PREDICT_ROUTE env var, or /predict)// Must be called AFTER all other API routes are registereddiff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 95cbdd77a8..c8e235129d 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -1899,6 +1899,81 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, coreturn proxy;}+void server_models::proxy_ws(const server_http_req & req, server_http_context::ws_conn & conn, const std::string & name) {+ auto send_error = [&conn](const std::string & message) {+ conn.send(safe_json_to_str(json {+ {"type", "error"},+ {"error", {{"type", "server_error"}, {"code", nullptr}, {"message", message}, {"param", nullptr}}},+ }));+ conn.close();+ };++ auto meta = get_meta(name);+ if (!meta.has_value() || !meta->is_running()) {+ send_error("model name=" + name + " is not running");+ return;+ }+ {+ std::unique_lock<std::mutex> lk(mutex);+ mapping[name].meta.last_used = ggml_time_ms();+ mapping[name].req_count++;+ }+ SRV_INF("proxying websocket to model %s on port %d\n", name.c_str(), meta->port);++ std::string url = std::string("ws://") + CHILD_ADDR + ":" + std::to_string(meta->port) + req.path;+ if (!req.query_string.empty()) {+ url += '?' + req.query_string;+ }+ // the client library adds its own upgrade headers+ httplib::Headers headers;+ for (const auto & [key, value] : req.headers) {+ std::string lower = key;+ std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);+ if (lower == "host" || lower == "connection" || lower == "upgrade" || lower.compare(0, 14, "sec-websocket-") == 0) {+ continue;+ }+ headers.emplace(key, value);+ }++ {+ httplib::ws::WebSocketClient upstream(url, headers);+ if (!upstream.connect()) {+ send_error("failed to connect to model name=" + name);+ } else {+ // client to child+ std::thread pump([&]() {+ std::string msg;+ while (true) {+ const int res = conn.read(msg);+ if (res == 0 || !(res == 1 ? upstream.send(msg) : upstream.send(msg.data(), msg.size()))) {+ break;+ }+ }+ upstream.close();+ });+ // child to client+ std::string msg;+ while (true) {+ const auto res = upstream.read(msg);+ if (res == httplib::ws::Fail || !(res == httplib::ws::Text ? conn.send(msg) : conn.send_binary(msg))) {+ break;+ }+ }+ conn.close();+ pump.join();+ }+ }++ std::unique_lock<std::mutex> lk(mutex);+ auto it = mapping.find(name);+ if (it != mapping.end() && it->second.req_count > 0) {+ it->second.req_count--;+ if (it->second.req_count == 0) {+ sched->tick(lk);+ }+ }+}+void server_models::handle_child_state(const std::string & name, const std::string & raw_input) {server_state state;json payload;@@ -2277,6 +2352,24 @@ void server_models_routes::init_routes() {return models.proxy_request(req, method, name, false);};+ this->proxy_ws = [this](const server_http_req & req, server_http_context::ws_conn & conn) {+ std::string name = req.get_param("model");+ bool autoload = is_autoload(params, req);+ auto error_res = std::make_unique<server_http_res>();+ if (!router_validate_model(name, models, autoload, error_res)) {+ // error_res holds {"error": {...}}, send it as a websocket error event+ json err = json::parse(error_res->data);+ err["type"] = "error";+ conn.send(safe_json_to_str(err));+ conn.close();+ return;+ }+ if (autoload) {+ models.ensure_model_ready(name, req.should_stop);+ }+ models.proxy_ws(req, conn, name);+ };+this->proxy_post = [this](const server_http_req & req) {std::string method = "POST";json body = json::parse(req.body);diff --git a/tools/server/server-models.h b/tools/server/server-models.hindex b04fc7384d..500ce9fb32 100644--- a/tools/server/server-models.h+++ b/tools/server/server-models.h@@ -366,6 +366,8 @@ public:// proxy an HTTP request to the model instanceserver_http_res_ptr proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used, bool detached = false);+ // forward a websocket connection to the child, returns when either side closes+ void proxy_ws(const server_http_req & req, server_http_context::ws_conn & conn, const std::string & name);// handle message sent from server_child::notify_to_router()// raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string@@ -433,6 +435,8 @@ struct server_models_routes {server_http_context::handler_t get_router_props;server_http_context::handler_t proxy_get;server_http_context::handler_t proxy_post;+ server_http_context::ws_handler_t proxy_ws;+server_http_context::handler_t get_router_models;server_http_context::handler_t post_router_models_load;server_http_context::handler_t post_router_models_unload;diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex e7bb043988..de39ce3b83 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -18,6 +18,7 @@ enum server_task_type {SERVER_TASK_TYPE_RERANK,SERVER_TASK_TYPE_DECISION,SERVER_TASK_TYPE_INFILL,+ SERVER_TASK_TYPE_TRANSCRIBE_RT,SERVER_TASK_TYPE_CANCEL,SERVER_TASK_TYPE_CONTROL,SERVER_TASK_TYPE_NEXT_RESPONSE,@@ -48,6 +49,14 @@ enum stop_type {STOP_TYPE_LIMIT,};+enum server_control_action {+ SERVER_CONTROL_ACTION_NONE,+ SERVER_CONTROL_ACTION_REASONING_END,+ SERVER_CONTROL_ACTION_AUDIO_APPEND, // push control_audio into the slot's audio stream+ SERVER_CONTROL_ACTION_AUDIO_COMMIT, // flush the audio stream, more audio can follow+ SERVER_CONTROL_ACTION_AUDIO_END, // flush the audio stream and end it+};+struct task_params {bool stream = false;bool include_usage = false;@@ -86,8 +95,9 @@ struct task_params {std::string oaicompat_cmpl_id;// realtime control (SERVER_TASK_TYPE_CONTROL)- std::string control_action;- std::string control_cmpl_id;+ server_control_action control_action = SERVER_CONTROL_ACTION_NONE;+ std::string control_cmpl_id;+ std::vector<float> control_audio; // PCM F32 samples for SERVER_CONTROL_ACTION_AUDIO_APPEND// reported in generation_settings, parsing itself is owned by the chat sessioncommon_chat_format chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;@@ -233,6 +243,7 @@ struct server_task {switch (type) {case SERVER_TASK_TYPE_COMPLETION:case SERVER_TASK_TYPE_INFILL:+ case SERVER_TASK_TYPE_TRANSCRIBE_RT:return true;case SERVER_TASK_TYPE_DECISION:return !decision.labels.empty();@@ -245,6 +256,7 @@ struct server_task {switch (type) {case SERVER_TASK_TYPE_COMPLETION:case SERVER_TASK_TYPE_INFILL:+ case SERVER_TASK_TYPE_TRANSCRIBE_RT:return true;default:return false;diff --git a/tools/server/server.cpp b/tools/server/server.cppindex b168241076..41f29ded5d 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -247,6 +247,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &routes.post_control = models_routes->proxy_post;routes.post_responses_oai = models_routes->proxy_post;routes.post_transcriptions_oai = models_routes->proxy_post;+ routes.ws_realtime = models_routes->proxy_ws;routes.post_anthropic_messages = models_routes->proxy_post;routes.post_anthropic_count_tokens = models_routes->proxy_post;routes.post_infill = models_routes->proxy_post;@@ -292,6 +293,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));+ ctx_http.ws ("/v1/realtime", routes.ws_realtime);ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages APIctx_http.post("/infill", ex_wrapper(routes.post_infill));ctx_http.post("/embedding", ex_wrapper(routes.post_embeddings)); // legacy