Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio: LFM3-Audio / LFM3-Omni on MLX, with spoken replies (#15889)
Long Yixingd7aa02beccdd
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 5dc4b7e8793..3d5c7d5aad0 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -2384,8 +2384,11 @@ class InferenceBackend:use_adapter: Optional[Union[bool, str]] = None,cancel_event = None,extra_audio_arrays: Optional[list] = None,+ tools: Optional[list] = None,) -> Generator[str, None, None]:- """Audio goes into the chat messages via processor.apply_chat_template (Gemma 3n pattern)."""+ """Audio goes into the chat messages via processor.apply_chat_template (Gemma 3n pattern).+ ``tools`` (a list, empty for tool history alone) renders the catalogue and keeps+ ``messages`` as sent."""import threadingimport numpy as np@@ -2402,7 +2405,7 @@ class InferenceBackend:# Gemma 3n format: audio goes INTO apply_chat_template, one item per clip in order.audio_messages = messages_with_attached_image(- alternating_turns(messages),+ alternating_turns(messages) if tools is None else messages,system_prompt = system_prompt,fallback_user_text = "Please transcribe this audio.",structured_content = True,@@ -2412,11 +2415,9 @@ class InferenceBackend:)# Direct processor render, so neutralize here with this processor's own profile (#7066).- from core.inference.chat_template_helpers import markup_for_tokenizer+ from core.inference.chat_template_helpers import neutralize_for_render- audio_messages = neutralize_control_markup_in_messages(- audio_messages, None, markup_for_tokenizer(processor)- )+ audio_messages, tools, _ = neutralize_for_render(processor, audio_messages, tools)inputs = processor.apply_chat_template(audio_messages,@@ -2425,16 +2426,18 @@ class InferenceBackend:return_dict = True,return_tensors = "pt",truncation = False,+ **({"tools": tools} if tools else {}),).to(model.device)try:from transformers import TextIteratorStreamerfrom queue import Empty+ # As the text path: a rendered catalogue keeps native tool-call tokens for the healer.streamer = TextIteratorStreamer(- raw_tokenizer,+ NativeToolTokenDecoder(raw_tokenizer) if tools else raw_tokenizer,skip_prompt = True,- skip_special_tokens = True,+ skip_special_tokens = not tools,timeout = 0.2,)diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 498af28c413..3124bd9ff05 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -38,10 +38,9 @@ from core.inference.chat_template_helpers import (count_structured_videos as _count_vlm_videos,detect_reasoning_channel_markers,make_reasoning_normalizer,- markup_for_tokenizer,messages_have_tool_history as _vlm_messages_have_tool_history,messages_with_attached_image,- neutralize_control_markup_in_messages,+ neutralize_for_render,normalize_reasoning_snapshots,prompt_opens_reasoning_channel,strip_open_reasoning_prefill,@@ -1490,6 +1489,7 @@ def _render_registered_vlm_prompt(num_images,num_audios = 0,continue_final_message = False,+ tools = None,):"""Resumes a partial from the swept messages, since a raw partial could close the turn."""from mlx_vlm import prompt_utils@@ -1522,7 +1522,7 @@ def _render_registered_vlm_prompt(if _resumes_thought(messages, continue_final_message):return None# Recovery path: sweeps the caller's original list rather than reusing a copy (#7066).- swept = neutralize_control_markup_in_messages(messages, None, markup_for_tokenizer(processor))+ swept, tools, _ = neutralize_for_render(processor, messages, tools)partial = trailing_assistant_text(swept) if continue_final_message else Nonerendered = prompt_utils.apply_chat_template(processor,@@ -1531,6 +1531,7 @@ def _render_registered_vlm_prompt(add_generation_prompt = True,num_images = num_images,num_audios = num_audios,+ **({"tools": tools} if tools else {}),)if isinstance(rendered, str) and rendered.strip():# A prefilled open "<think>" would resume the answer inside the reasoning block.@@ -1734,6 +1735,35 @@ def _discard_video_clip(path: str) -> None:pass+_SPEECH_REPLY_SYSTEM_PROMPT = "Respond with interleaved text and audio."+++def _mlx_lm_serves(model_type) -> bool:+ from importlib.util import find_spec++ from mlx_lm.utils import MODEL_REMAPPING+ return find_spec(f"mlx_lm.models.{MODEL_REMAPPING.get(model_type, model_type)}") is not None+++def _zoo_declares_multimodal(model_name, hf_token = None) -> bool:+ """unsloth_zoo's verdict from mlx-vlm's own configs, for mlx-vlm-only families the transformers+ registry does not know (an audio-only speech model, a tower under another name)."""+ try:+ from utils.models.model_config import _read_raw_config_json++ config = _read_raw_config_json(model_name, hf_token)+ model_type = (config or {}).get("model_type")+ # A type mlx-lm serves keeps its text path, and skips zoo's scan of every mlx-vlm family.+ if not isinstance(model_type, str) or not model_type or _mlx_lm_serves(model_type):+ return False+ from unsloth_zoo.mlx.loader import _is_vlm++ return _is_vlm(config)+ except Exception as exc: # noqa: BLE001 -- detection must never fail a load+ logger.debug("unsloth_zoo multimodal detection skipped for %s (%s)", model_name, exc)+ return False++def _classify_mlx_audio_type(model,processor,@@ -3572,6 +3602,27 @@ def _stream_until_cancelled(stream_generate, cancel_event, *args, **kwargs):return+@contextmanager+def _slots_until_cancelled(model, cancel_event):+ """mlx-vlm's ``generate_audio`` takes no stop hook; LFM3 samples through ``stream_slots``, so end that loop."""+ slots = getattr(model, "stream_slots", None)+ if cancel_event is None or slots is None:+ yield+ return++ def until_cancelled(*args, **kwargs):+ for slot in slots(*args, **kwargs):+ if cancel_event.is_set():+ return+ yield slot++ model.stream_slots = until_cancelled+ try:+ yield+ finally:+ del model.stream_slots++class LogprobsUnsupported(ValueError):"""``n_probs`` asked of a model that does not sample one token per step."""@@ -5260,6 +5311,10 @@ class MLXInferenceBackend:self._resume_load_args = (bool(load_in_4bit), repr(dtype), repr(tuple(offload or ())))model_name = config.identifier if hasattr(config, "identifier") else str(config)is_vision = getattr(config, "is_vision", False)+ is_lora = getattr(config, "is_lora", False)+ zoo_multimodal = not is_vision and _zoo_declares_multimodal(+ getattr(config, "base_model", None) if is_lora else model_name, hf_token+ )kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)self._turboquant = turboquantif self._turboquant:@@ -5271,7 +5326,6 @@ class MLXInferenceBackend:) from excdistributed_rank, distributed_size = _mlx_distributed_rank_size(distributed_group)is_distributed = distributed_group is not None and distributed_size > 1- is_lora = getattr(config, "is_lora", False)self._turboquant_refusal = (_turboquant_refusal(is_vision = is_vision, is_distributed = is_distributed, is_lora = is_lora)if self._turboquant@@ -5303,6 +5357,7 @@ class MLXInferenceBackend:# Speculation runs on mlx-vlm models and caches.use_vlm = (is_vision+ or zoo_multimodalor speculatesor (self._turboquant and not self._turboquant_refusal)or batches_quantized_text@@ -5476,6 +5531,9 @@ class MLXInferenceBackend:FastMLXModel.from_pretrained, load_source, load_kwargs, offload)+ zoo_loaded_multimodal = zoo_multimodal and not served_by_mlx_lm+ if zoo_loaded_multimodal:+ is_vision = getattr(model, "_unsloth_supports_images", None) is Trueself._reads_vision = bool(is_vision)if use_vlm:processor = tokenizer_or_processor@@ -5501,7 +5559,7 @@ class MLXInferenceBackend:_audio_type = _classify_mlx_audio_type(model,self._processor,- is_vision,+ is_vision or zoo_loaded_multimodal,config_audio_type = _config_audio_type,)_served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(@@ -5675,6 +5733,8 @@ class MLXInferenceBackend:"is_audio": _audio_type is not None and _audio_type != "audio_vlm","audio_type": _audio_type,"has_audio_input": is_audio_input_type(_audio_type),+ "speaks": _audio_type == "audio_vlm"+ and getattr(self._model, "joint_audio_text_generation", False) is True,"has_video_input": is_vision and _mlx_reads_video(self._processor),"context_length": _served_ctx,# Matches llama.cpp requested_n_ctx; served window alone cannot tell if a size was asked for.@@ -7703,11 +7763,15 @@ class MLXInferenceBackend:cancel_event = None,stop = None,extra_audio_arrays = None,+ speak = False,+ tools = None,**_sampler,):"""Audio-input chat (omni models): waveform in, incremental text deltas out (the audio routeforwards deltas, unlike the snapshot-diffing text/vision paths). Greedy, so the worker's- sampler kwargs go unused."""+ sampler kwargs go unused. ``speak`` answers in the model's own voice instead: the reply+ text, then one ``{"speech": <base64 wav>, "sample_rate": ...}`` item. ``tools`` (a list,+ empty for tool history alone) renders the catalogue and keeps ``messages`` as sent."""entry = self.models.get(self.active_model_name) or {}if entry.get("audio_type") != "audio_vlm":raise RuntimeError(@@ -7719,13 +7783,19 @@ class MLXInferenceBackend:if extra_audio_arrays:raise RuntimeError("This MLX model takes one audio file per message.")+ if speak:+ yield from self._speak_audio_reply(+ messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event+ )+ return+from mlx_vlm import stream_generate as vlm_streamif not system_prompt:system_prompt = "You are an assistant that transcribes speech accurately."audio_messages = messages_with_attached_image(- alternating_turns(messages),+ alternating_turns(messages) if tools is None else messages,system_prompt = system_prompt,fallback_user_text = "Please transcribe this audio.",image = 0,@@ -7737,6 +7807,7 @@ class MLXInferenceBackend:audio_messages,num_images = 0,num_audios = 1,+ **({"tools": tools} if tools else {}),)if prompt is None:raise RuntimeError(@@ -7836,6 +7907,76 @@ class MLXInferenceBackend:if stopped:self._mark_stopped()+ def _speak_audio_reply(+ self,+ messages,+ system_prompt,+ audio_array,+ max_new_tokens,+ use_adapter,+ cancel_event = None,+ ):+ """One spoken question answered in interleaved text and speech, not streamed."""+ import base64++ from mlx_vlm.generate import generate_audio++ if not (self.models.get(self.active_model_name) or {}).get("speaks"):+ raise RuntimeError("The loaded model cannot answer in speech.")+ # The model card's prompt for speech chat; a user prompt still applies after it.+ system = "\n\n".join(p for p in (_SPEECH_REPLY_SYSTEM_PROMPT, system_prompt) if p)+ prompt = _render_registered_vlm_prompt(+ self._processor,+ self._model,+ messages_with_attached_image(+ alternating_turns(messages),+ system_prompt = system,+ image = 0,+ audio = audio_array,+ ),+ num_images = 0,+ num_audios = 1,+ )+ max_new_tokens = self._generation_limit(+ prompt, max_new_tokens, audio = [audio_array], cap = UNSET_GENERATION_BUDGET+ )+ with (+ self._generation_lock,+ _temporary_mlx_adapter_state(self._model, use_adapter),+ ExitStack() as generation_scope,+ ):+ self._release_vlm_snapshots()+ generation_scope.enter_context(self._generation_scopes("vision", "audio"))+ generation_scope.enter_context(_slots_until_cancelled(self._model, cancel_event))+ result = generate_audio(+ self._model,+ self._processor,+ prompt,+ audio = [audio_array],+ generation_mode = "interleaved",+ max_tokens = max_new_tokens,+ )+ cancelled = cancel_event is not None and cancel_event.is_set()+ # Nothing to encode once stopped, nor for a reply that ended before its first speech frame.+ spoke = not cancelled and result.audio is not None and result.audio.size > 0+ wav = result.to_wav_bytes() if spoke else None+ self.last_generation_stats = _build_generation_stats(+ result.prompt_tokens,+ result.prompt_tps,+ result.generation_tokens,+ result.generation_tps,+ finish_reason = result.finish_reason,+ )+ if cancelled:+ return+ if result.text:+ yield result.text+ if spoke:+ yield {+ "speech": base64.b64encode(wav).decode("ascii"),+ "sample_rate": result.sample_rate,+ }+def generate_with_adapter_control(self,use_adapter = None,diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 414d4e71a98..65e602f5f44 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -403,6 +403,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:"audio_type": model_info.get("audio_type"),"has_audio_input": model_info.get("has_audio_input", False),"has_video_input": model_info.get("has_video_input", False),+ "speaks": model_info.get("speaks", False),"context_length": model_info.get("context_length"),"native_context_length": model_info.get("native_context_length"),"max_context_length": model_info.get("max_context_length"),@@ -1557,6 +1558,9 @@ class InferenceOrchestrator:yield int(resp.get("row", 0)), resp.get("text", "")else:yield resp.get("text", "")+ elif rtype == "speech":+ if rows is None and not stop_sent:+ yield {"speech": resp.get("speech"), "sample_rate": resp.get("sample_rate")}elif rtype == "diffusion_frame":if frames and rows is None and not stop_sent:yield {@@ -4141,8 +4145,11 @@ class InferenceOrchestrator:stats_holder: Optional[dict] = None,stop = None,extra_audio_arrays: Optional[list] = None,+ speak: bool = False,+ tools: Optional[list] = None,) -> Generator[str, None, None]:- """Audio input generation (e.g. Gemma 3n): streams text tokens."""+ """Audio input generation (e.g. Gemma 3n): streams text tokens; with ``speak``, then one+ ``{"speech", "sample_rate"}`` item carrying the spoken reply."""yield from self._generate_audio_input_inner(audio_array = audio_array,audio_type = None, # worker uses generate_audio_input_response@@ -4159,6 +4166,8 @@ class InferenceOrchestrator:stats_holder = stats_holder,stop = stop,extra_audio_arrays = extra_audio_arrays,+ speak = speak,+ tools = tools,)def _generate_audio_input_inner(@@ -4178,6 +4187,8 @@ class InferenceOrchestrator:stats_holder: Optional[dict] = None,stop = None,extra_audio_arrays: Optional[list] = None,+ speak: bool = False,+ tools: Optional[list] = None,) -> Generator[str, None, None]:"""Shared inner logic for audio input generation (Whisper + ASR). ``stats_holder``: as ingenerate_chat_response, caller-owned and filled on gen_done with the worker's usage /@@ -4223,6 +4234,10 @@ class InferenceOrchestrator:cmd["use_adapter"] = use_adapterif stop:cmd["stop"] = stop+ if speak:+ cmd["speak"] = True+ if tools is not None:+ cmd["tools"] = toolsread_one, drain, release_mailbox = self._direct_reader(request_id, cancel_event)try:diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 8e9d5f91c2b..ea05f4fd72c 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -843,6 +843,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:"audio_type","has_audio_input","has_video_input",+ "speaks","audio_family","audio_options","gguf_variant",@@ -865,9 +866,9 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:if k in _entry})- # MLX serves a vision config mlx-lm built as text; the parent must stop accepting images.- if model_info["is_mlx"] and _entry.get("is_vision") is False:- model_info["is_vision"] = False+ # MLX decides at load: an mlx-lm text build reads no images, an mlx-vlm-only tower does.+ if model_info["is_mlx"] and isinstance(_entry.get("is_vision"), bool):+ model_info["is_vision"] = _entry["is_vision"]# Resolved MLX runtime knobs; only the backend knows what it honored.model_info.update({@@ -1945,6 +1946,10 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_evebackend, "stop", "generate_audio_input_response"):audio_kwargs["stop"] = cmd["stop"]+ if cmd.get("speak"):+ audio_kwargs["speak"] = True+ if cmd.get("tools") is not None:+ audio_kwargs["tools"] = cmd["tools"]generator = backend.generate_audio_input_response(**audio_kwargs, **extra_audio_kwargs)logger.info("Starting audio input generation for request_id=%s", request_id)@@ -1954,6 +1959,11 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_evelogger.info("Audio input generation cancelled for request %s", request_id)break+ if isinstance(text_chunk, dict):+ _send_response(+ resp_queue, {"type": "speech", "request_id": request_id, **text_chunk}+ )+ continue_send_response(resp_queue,{diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex e7059caed06..741d5d978fd 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -1579,6 +1579,9 @@ class _InferenceRuntimeFields(BaseModel):description = "Audio codec or native generation architecture.",)has_audio_input: bool = Field(False, description = "Whether model accepts audio input (ASR)")+ speaks: bool = Field(+ False, description = "Whether model can answer a spoken question in its own voice"+ )audio_family: Optional[str] = Field(None, description = "Loader family of an audio GGUF model (e.g. kokoro_tts, minimax_music3).")@@ -2880,6 +2883,17 @@ class ChatCompletionRequest(BaseModel):None,description = "[x-unsloth] Base64-encoded audio (wav/mp3/ogg/flac/m4a) for audio-input models",)+ modalities: Optional[List[Literal["text", "audio"]]] = Field(+ None,+ description = (+ 'Output modalities, as OpenAI\'s. ["text", "audio"] asks a model that speaks (status '+ "speaks) to answer the spoken question in audio_base64 in its own voice as well: the "+ "reply text, then message.audio / delta.audio {data: base64 wav, format: wav, "+ "sample_rate}. The model samples the reply its own way, so sampling settings do not "+ "apply and stop sequences are refused; max_tokens and usage count speech frames as "+ "well as text tokens; a streamed reply arrives whole, as one text delta and one audio delta."+ ),+ )extra_audio_base64: Optional[List[str]] = Field(None,description = (@@ -3668,6 +3682,7 @@ class ChoiceDelta(BaseModel):content: Optional[str] = Nonereasoning_content: Optional[str] = Nonetool_calls: Optional[list[dict]] = None+ audio: Optional[dict] = NoneOpenAIFinishReason = Literal["stop", "length", "tool_calls", "content_filter", "function_call"]@@ -3704,6 +3719,7 @@ class CompletionMessage(BaseModel):refusal: Optional[str] = Nonereasoning_content: Optional[str] = Nonetool_calls: Optional[list[dict]] = None+ audio: Optional[dict] = Noneclass CompletionChoice(BaseModel):diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 8815771a5cf..3371d64b73a 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1450,6 +1450,11 @@ def _chat_content_chunk(completion_id, created, model_name, text) -> str:)+def _speech_reply_audio(item: dict) -> dict:+ """A backend speech item as the message / delta ``audio`` part (OpenAI's field, carrying the WAV inline)."""+ return {"data": item.get("speech"), "format": "wav", "sample_rate": item.get("sample_rate")}++def _chat_reasoning_chunk(completion_id, created, model_name, text) -> str:"""Reasoning delta with empty content alongside, so strict OpenAI adapters do not drop it."""return _chat_chunk_sse(@@ -8072,6 +8077,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:# Not MLX, so the MLX runtime fields report as absent.is_mlx = False,is_npu = False,+ speaks = False,mlx_kv_quant = None,mlx_kv_quant_requested = None,mlx_kv_bits = None,@@ -18697,6 +18703,7 @@ async def _load_model_impl(audio_type = _model_info.get("audio_type"),has_audio_input = _model_info.get("has_audio_input", False),has_video_input = _model_info.get("has_video_input", False),+ speaks = bool(_model_info.get("speaks", False)),is_mlx = bool(_model_info.get("is_mlx", False)),mlx_kv_quant = _model_info.get("mlx_kv_quant"),mlx_kv_quant_requested = _model_info.get("mlx_kv_quant_requested"),@@ -19539,7 +19546,7 @@ async def _load_model_impl(status = "loaded",model = model_log_label if native_grant_backed else config.identifier,display_name = model_log_label if native_grant_backed else config.display_name,- is_vision = config.is_vision,+ is_vision = _model_info.get("is_vision", config.is_vision),is_lora = config.is_lora,is_gguf = False,is_local_model = config.is_local,@@ -19548,6 +19555,7 @@ async def _load_model_impl(audio_type = _model_info.get("audio_type", config.audio_type),has_audio_input = _model_info.get("has_audio_input", config.has_audio_input),has_video_input = _model_info.get("has_video_input", False),+ speaks = bool(_model_info.get("speaks", False)),audio_family = _model_info.get("audio_family"),audio_options = _model_info.get("audio_options"),audio_workflows = _model_info.get("audio_workflows"),@@ -22075,6 +22083,7 @@ async def _slot_status(current_subject: str):audio_type = audio_type,has_audio_input = has_audio_input,has_video_input = has_video_input,+ speaks = bool(model_info.get("speaks", False)),audio_family = model_info.get("audio_family"),audio_options = model_info.get("audio_options"),audio_workflows = model_info.get("audio_workflows"),@@ -30586,6 +30595,8 @@ async def produce_openai_chat_completions(normalized_stop = _normalize_stop_sequences(payload.stop)if using_gguf:+ if "audio" in (payload.modalities or ()):+ raise HTTPException(status_code = 400, detail = "The loaded model cannot answer in speech.")# Advertised repo id after an auto-switch load, else a clean public id,# never the absolute .gguf path.model_name = _llama_public_model_id(llama_backend, payload.model)@@ -30735,9 +30746,83 @@ async def produce_openai_chat_completions(api_monitor.fail(monitor_id, _MLX_MULTI_AUDIO_DETAIL)raise HTTPException(status_code = 400, detail = _MLX_MULTI_AUDIO_DETAIL)+ _speak = "audio" in (payload.modalities or ())+ _speak_detail = None+ if _speak and not model_info.get("speaks"):+ _speak_detail = "The loaded model cannot answer in speech."+ elif _speak and not payload.audio_base64:+ _speak_detail = "A spoken reply answers a spoken question: send it as audio_base64."+ elif _speak and _normalize_stop_sequences(payload.stop):+ # The speech cannot be cut at a text marker, so the stop would not hold.+ _speak_detail = "Stop sequences do not apply to a spoken reply."+ if _speak_detail is not None:+ api_monitor.fail(monitor_id, _speak_detail)+ raise HTTPException(status_code = 400, detail = _speak_detail)+# ── Audio INPUT path: decode WAV and route to audio input generation ──if payload.audio_base64 and model_info.get("has_audio_input"):_refuse_unused_mcp_image(_mcp_image)+ # Client tools as the text path takes them: none under tool_choice none, the forced one alone.+ _audio_tools = None+ if payload.tools and payload.tool_choice != "none":+ from core.inference.chat_template_helpers import forced_tool_name++ if model_info.get("audio_type") == "whisper" or _speak:+ _detail = "Tools do not apply to a transcription or a spoken reply."+ api_monitor.fail(monitor_id, _detail)+ _raise_unsupported_openai_parameter("tools", _detail)+ _forced = forced_tool_name(payload.tool_choice)+ _audio_tools = [+ tool+ for tool in payload.tools+ if not _forced+ or ((tool.get("function") or {}) if isinstance(tool, dict) else {}).get("name")+ == _forced+ ]+ # Heal and render the same catalogue: a tool dropped for markup was never shown (#7066).+ from core.inference.chat_template_helpers import (+ chat_render_target,+ renderable_tool_catalog_for_targets,+ )++ _audio_target = chat_render_target(+ model_info.get("processor"), model_info.get("tokenizer")+ )+ _audio_tools = await asyncio.to_thread(+ renderable_tool_catalog_for_targets,+ _audio_tools,+ (_audio_target, getattr(_audio_target, "tokenizer", _audio_target)),+ model_info,+ active_model_name = backend.active_model_name,+ )+ _tool_refusal = None+ if _forced and not _audio_tools:+ _tool_refusal = (+ "tool_choice",+ f"Invalid 'tool_choice': function '{_forced}' is not enabled.",+ )+ elif not _sf_rendered_features(backend, model_info, payload.tools)[0].get(+ "supports_tools"+ ):+ _tool_refusal = (+ "tools",+ "Client-supplied tools require a chat template with tool-call "+ "support; the current model/template does not advertise tools.",+ )+ if _tool_refusal is not None:+ api_monitor.fail(monitor_id, _tool_refusal[1])+ _raise_unsupported_openai_parameter(*_tool_refusal)+ # Tool calls and results reach the template as sent, which alternating turns would drop.+ _audio_tool_kwargs = (+ {"tools": _audio_tools or []}+ if _audio_tools or _has_openai_tool_history(payload.messages)+ else {}+ )+ _audio_heal = (+ heal_gate(payload.auto_heal_tool_calls, _audio_tools, payload.tool_choice)+ if _audio_tools+ else None+ )# No boundary to resume from: this route re-listens and answers afresh.if _continue_final_message(payload):raise HTTPException(@@ -30765,7 +30850,12 @@ async def produce_openai_chat_completions(system_prompt, chat_messages, _ = await _extract_content_parts_async(payload.messages)- system_prompt = _audio_input_system_prompt(system_prompt, request)+ # A spoken reply answers the question, so the transcription default does not apply.+ system_prompt = (+ _apply_current_date_prompt(system_prompt, request)+ if _speak+ else _audio_input_system_prompt(system_prompt, request)+ )except _DecodedAudioTooLongError as e:# A limit the caller can act on, not a server fault.api_monitor.fail(monitor_id, str(e))@@ -30813,6 +30903,8 @@ async def produce_openai_chat_completions(stats_holder = _audio_stats_holder,stop = normalized_stop,**extra_audio_kwargs,+ **({"speak": True} if _speak else {}),+ **_audio_tool_kwargs,)if payload.stream:@@ -30828,6 +30920,10 @@ async def produce_openai_chat_completions(from core.inference.llama_keepwarm import mark_current_response_failedyield _chat_role_chunk(completion_id, created, model_name)+ healer = (+ StreamToolCallHealer(_audio_heal, _audio_tools) if _audio_heal else None+ )+ heal_state = {"idx": 0}gen = audio_input_generate()gen_step = ThreadStepper(gen).next@@ -30857,14 +30953,52 @@ async def produce_openai_chat_completions({"error": {"message": _msg, "type": "server_error"}})return- if chunk_text:+ if isinstance(chunk_text, dict):+ yield _chat_chunk_sse(+ completion_id,+ created,+ model_name,+ delta = ChoiceDelta(audio = _speech_reply_audio(chunk_text)),+ finish_reason = None,+ )+ continue+ if chunk_text and healer is not None:+ for line in _sf_heal_events_to_sse(+ healer.feed(chunk_text),+ completion_id,+ created,+ model_name,+ heal_state,+ payload.parallel_tool_calls,+ monitor_id,+ ):+ yield line+ elif chunk_text:api_monitor.append_reply(monitor_id, chunk_text)yield _chat_content_chunk(completion_id, created, model_name, chunk_text)+ # A cancelled stream must not heal incomplete markup into a call.+ if healer is not None and not cancelled:+ for line in _sf_heal_events_to_sse(+ healer.finalize(),+ completion_id,+ created,+ model_name,+ heal_state,+ payload.parallel_tool_calls,+ monitor_id,+ ):+ yield line_audio_stats = _audio_stats_holder.get("stats")- _audio_finish = "stop" if cancelled else _stats_finish_reason(_audio_stats)+ _audio_finish = (+ "stop"+ if cancelled+ else "tool_calls"+ if healer is not None and healer.healed+ else _stats_finish_reason(_audio_stats)+ )# The worker fills these stats for both backends (MLX adds timings), so the row has counts and speed.if isinstance(_audio_stats, dict):_monitor_usage(@@ -30908,12 +31042,16 @@ async def produce_openai_chat_completions(_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)_tracker.__enter__()try:+ _speech_reply = {}def _drain_audio_input():text_parts = []for chunk_text in audio_input_generate():if isinstance(chunk_text, GenStreamError):return chunk_text+ if isinstance(chunk_text, dict):+ _speech_reply.update(_speech_reply_audio(chunk_text))+ continuetext_parts.append(chunk_text)return "".join(text_parts)@@ -30941,11 +31079,28 @@ async def produce_openai_chat_completions(finally:# Nested under the except arms too: api_monitor.fail() can throw, and a leaked entry 409s swaps._tracker.__exit__(None, None, None)- api_monitor.set_reply(monitor_id, full_text)_audio_json_stats = _audio_stats_holder.get("stats")_audio_json_finish = ("stop" if cancel_event.is_set() else _stats_finish_reason(_audio_json_stats))+ _audio_msg = {"role": "assistant", "content": full_text}+ if (+ _audio_heal+ and not cancel_event.is_set()+ and heal_openai_message(_audio_msg, _audio_heal, _audio_tools)+ ):+ _audio_json_finish = "tool_calls"+ if payload.parallel_tool_calls is False:+ _audio_msg["tool_calls"] = _audio_msg["tool_calls"][:1]+ _audio_calls = "; ".join(+ f"{call['function']['name']}({call['function']['arguments']})"+ for call in _audio_msg.get("tool_calls") or []+ )+ api_monitor.set_reply(+ monitor_id,+ (_audio_msg["content"] or "")+ + (f"[tool_calls] {_audio_calls}" if _audio_calls else ""),+ )_mark_cancelled_json_response_failed(request, cancel_event)if isinstance(_audio_json_stats, dict):_monitor_usage(@@ -30963,7 +31118,11 @@ async def produce_openai_chat_completions(model = model_name,choices = [CompletionChoice(- message = CompletionMessage(content = full_text),+ message = CompletionMessage(+ content = _audio_msg["content"],+ audio = _speech_reply or None,+ tool_calls = _audio_msg.get("tool_calls"),+ ),finish_reason = _audio_json_finish,)],diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.pyindex 3470e1985bd..7cd6a15160d 100644--- a/studio/backend/tests/test_chat_input_audio_part.py+++ b/studio/backend/tests/test_chat_input_audio_part.py@@ -763,3 +763,89 @@ def test_extra_clips_join_the_latest_user_turn_in_order():single = messages_with_attached_image(history, structured_content = True, image = 0, audio = "a")assert single[-1]["content"][0] == {"type": "audio", "audio": "a"}assert len(single[-1]["content"]) == 2++[email protected](("stream", "stop"), [(True, None), (False, None), (True, ["HALT"])])+def test_a_spoken_question_gets_a_spoken_reply(monkeypatch, stream, stop):+ import json+ from types import SimpleNamespace++ seen = {}++ def _generate(**kwargs):+ seen.update(kwargs)+ yield from ("Paris.", {"speech": "UklGRg==", "sample_rate": 24000})++ backend = SimpleNamespace(active_model_name = "m", generate_audio_input_response = _generate)+ backend.models = {"m": {"is_mlx": True, "has_audio_input": True, "speaks": True}}+ idle = SimpleNamespace(is_loaded = False)+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: idle)+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)+ monkeypatch.setattr(inference_route, "_decode_audio_clips", lambda clips: [[0.0]])+ body = {"model": "m", "messages": [{"role": "user", "content": ""}], "audio_base64": AUDIO_B64}+ body.update(modalities = ["text", "audio"], stream = stream, **({"stop": stop} if stop else {}))+ with _route_client() as client:+ response = client.post("/chat/completions", json = body)++ if stop:+ assert response.status_code == 400 and not seen+ assert "Stop sequences do not apply" in response.text+ return+ assert "transcribes" not in (seen["system_prompt"] or "")+ audio = {"data": "UklGRg==", "format": "wav", "sample_rate": 24000}+ if not stream:+ message = response.json()["choices"][0]["message"]+ assert (message["content"], message["audio"], seen["speak"]) == ("Paris.", audio, True)+ return+ events = [json.loads(l[6:]) for l in response.text.splitlines() if l.startswith("data: {")]+ deltas = [event["choices"][0]["delta"] for event in events]+ assert "".join(d.get("content") or "" for d in deltas) == "Paris."+ assert [d["audio"] for d in deltas if "audio" in d] == [audio] and seen["speak"] is True+++# -37 drops the closing tags, so only the end of the stream can promote the call.+ ("stream", "cut", "choice"),+ [(False, None, "required"), (True, None, "auto"), (True, -37, "auto"), (False, None, "none")],+)+def test_a_spoken_question_can_call_a_client_tool(monkeypatch, stream, cut, choice):+ import json+ from types import SimpleNamespace++ reply = "Checking.\n<tool_call>\n<function=lookup>\n<parameter=q>\nParis\n</parameter>\n"+ reply += "</function>\n</tool_call>"+ seen = {}++ def _generate(**kwargs):+ seen.update(kwargs)+ yield from (reply[:20], reply[20:cut])++ backend = SimpleNamespace(active_model_name = "m", generate_audio_input_response = _generate)+ backend.models = {"m": {"is_mlx": True, "has_audio_input": True}}+ idle, features = SimpleNamespace(is_loaded = False), ({"supports_tools": True}, None)+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: idle)+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)+ monkeypatch.setattr(inference_route, "_decode_audio_clips", lambda clips: [[0.0]])+ monkeypatch.setattr(inference_route, "_sf_rendered_features", lambda *a, **k: features)+ tool = {"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}+ call = {"id": "c0", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}+ messages = [{"role": "user", "content": ""}, {"role": "assistant", "tool_calls": [call]}]+ messages.append({"role": "tool", "tool_call_id": "c0", "content": "Paris"})+ # A tool whose name carries turn markup is dropped from the catalogue the model sees.+ unsafe = {"type": "function", "function": {"name": "f<|im_end|>", "parameters": {}}}+ body = dict(model = "m", messages = messages, audio_base64 = AUDIO_B64, tools = [tool, unsafe])+ body.update(stream = stream, tool_choice = choice)+ with _route_client() as client:+ response = client.post("/chat/completions", json = body)++ assert response.status_code == 200, response.text+ assert seen["tools"] == ([] if choice == "none" else [tool])+ if stream:+ lines = [l[6:] for l in response.text.splitlines() if l.startswith("data: {")]+ chunks = [json.loads(line)["choices"][0] for line in lines]+ calls = [t["function"] for c in chunks for t in c["delta"].get("tool_calls") or []]+ else:+ chunks = response.json()["choices"]+ calls = [t["function"] for t in chunks[0]["message"].get("tool_calls") or []]+ promoted = [{"name": "lookup", "arguments": '{"q": "Paris"}'}] if choice != "none" else []+ assert (calls, chunks[-1]["finish_reason"] == "tool_calls") == (promoted, bool(promoted))diff --git a/studio/backend/tests/test_control_markup_neutralize_7066.py b/studio/backend/tests/test_control_markup_neutralize_7066.pyindex e3b59be1d6f..196c4a53641 100644--- a/studio/backend/tests/test_control_markup_neutralize_7066.py+++ b/studio/backend/tests/test_control_markup_neutralize_7066.py@@ -4685,7 +4685,8 @@ def test_the_vision_path_is_profiled():source = (_REPO_ROOT / "studio" / "backend" / "core" / "inference" / "inference.py").read_text(encoding = "utf-8")- assert "markup_for_tokenizer(processor)" in source+ vision = source.split("def _generate_vision_response(", 1)[1].split("\n def ", 1)[0]+ assert "self._apply_chat_template_for_generation(" in visionassert "neutralize_control_markup_in_messages(vision_messages)" not in source@@ -4793,9 +4794,10 @@ def test_every_direct_renderer_carries_the_profile():backend = _REPO_ROOT / "studio" / "backend" / "core" / "inference"inference_src = (backend / "inference.py").read_text(encoding = "utf-8")mlx_src = (backend / "mlx_inference.py").read_text(encoding = "utf-8")- assert "markup_for_tokenizer(processor)" in inference_src+ # The audio renders also carry a client tool catalogue, which needs the same sweep.+ assert "neutralize_for_render(processor, audio_messages, tools)" in inference_srcassert "markup_for_tokenizer(tokenizer)" in inference_src- assert "markup_for_tokenizer(processor)" in mlx_src+ assert "neutralize_for_render(processor, messages, tools)" in mlx_srcfor source in (inference_src, mlx_src):assert "neutralize_control_markup_in_messages(messages)" not in sourceassert "neutralize_control_markup_in_messages(audio_messages)" not in sourcediff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.pyindex b3e576121ff..a13dd3baa0f 100644--- a/studio/backend/tests/test_gguf_load_cache_reuse.py+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py@@ -1187,6 +1187,7 @@ class TestLoadHubDownloadExclusion:"parallel_slots","is_mlx","is_npu",+ "speaks","mlx_kv_quant","mlx_kv_quant_requested","mlx_kv_bits",diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 8edf0d75e88..d5c1c23e371 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -4587,6 +4587,103 @@ def test_mlx_generate_audio_input_deltas_and_reject(monkeypatch):next(backend.generate_audio_input_response(**args))+ ("samples", "cancel_at", "reply"),+ [+ (4, None, ["Paris.", {"speech": "UklGRndhdg==", "sample_rate": 24000}]),+ (0, None, ["Paris."]),+ (4, 2, []),+ ],+)+def test_mlx_spoken_reply(monkeypatch, samples, cancel_at, reply):+ from core.inference import mlx_inference+ from core.inference.mlx_inference import MLXInferenceBackend++ cancel, sampled, seen = threading.Event(), [], {}++ class _SpeakingModel:+ def stream_slots(self):+ for slot in range(4):+ if slot == cancel_at:+ cancel.set()+ yield slot++ def _generate_audio(model, processor, prompt, **kwargs):+ seen.update(kwargs, prompt = prompt)+ sampled.extend(model.stream_slots())++ class _Result:+ text, sample_rate, finish_reason = "Paris.", 24000, "stop"+ audio = SimpleNamespace(size = samples)+ prompt_tokens = prompt_tps = generation_tokens = generation_tps = 1++ def to_wav_bytes(self):+ return b"RIFFwav" if samples else pytest.fail("empty WAV")++ return _Result()++ render = lambda processor, model, messages, **kwargs: messages[0]["content"]+ monkeypatch.setattr(mlx_inference, "_render_registered_vlm_prompt", render)+ generate = SimpleNamespace(generate_audio = _generate_audio)+ monkeypatch.setitem(sys.modules, "mlx_vlm.generate", generate)+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)+ backend._generation_lock = threading.Lock()+ backend._model, backend._processor = _SpeakingModel(), _audio_processor()+ backend.active_model_name, backend.last_generation_stats = "m", None+ backend.models = {"m": {"audio_type": "audio_vlm", "speaks": True}}+ args = dict(messages = [{"role": "user", "content": ""}], system_prompt = "Be brief.")+ args.update(audio_array = [0.1], cancel_event = cancel, speak = True)++ assert list(backend.generate_audio_input_response(**args)) == reply+ assert sampled == [0, 1, 2, 3][: cancel_at or 4]+ assert "stream_slots" not in vars(backend._model)+ assert (seen["generation_mode"], seen["audio"]) == ("interleaved", [[0.1]])+ assert seen["prompt"] == "Respond with interleaved text and audio.\n\nBe brief."+++ ("config", "served", "zoo", "expected"),+ [+ ({"model_type": "lfm3_audio"}, False, True, True),+ ({"model_type": "oxr"}, False, False, False),+ # mlx-lm serves it, so it stays on the text path whatever mlx-vlm declares.+ (+ {"model_type": "gemma3", "architectures": ["Gemma3ForConditionalGeneration"]},+ True,+ True,+ False,+ ),+ (None, False, True, False),+ ({}, False, True, False),+ ],+)+def test_zoo_multimodal_detection_only_for_mlx_vlm_only_families(+ monkeypatch, config, served, zoo, expected+):+ from core.inference import mlx_inference+ from utils.models import model_config++ asked = []+ monkeypatch.setattr(model_config, "_read_raw_config_json", lambda name, token: config)+ monkeypatch.setattr(mlx_inference, "_mlx_lm_serves", lambda model_type: served)+ loader = SimpleNamespace(_is_vlm = lambda cfg: asked.append(cfg) or zoo)+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)++ assert mlx_inference._zoo_declares_multimodal("repo") is expected+ assert asked == ([config] if config and not served else [])+++def test_zoo_multimodal_detection_fails_open_to_text(monkeypatch):+ from core.inference import mlx_inference+ from utils.models import model_config++ def unreadable(name, token):+ raise OSError("offline")++ monkeypatch.setattr(model_config, "_read_raw_config_json", unreadable)+ assert mlx_inference._zoo_declares_multimodal("repo") is False++def test_mlx_audio_input_normalizes_split_native_reasoning_channels(monkeypatch):from core.inference import mlx_inferencefrom core.inference.mlx_inference import MLXInferenceBackenddiff --git a/studio/backend/tests/test_pr7699_worker_audio_mirror.py b/studio/backend/tests/test_pr7699_worker_audio_mirror.pyindex bc2121a4f27..9ba7c97a523 100644--- a/studio/backend/tests/test_pr7699_worker_audio_mirror.py+++ b/studio/backend/tests/test_pr7699_worker_audio_mirror.py@@ -153,17 +153,24 @@ def test_a_classified_load_failure_crosses_the_worker_and_the_parent(monkeypatch@pytest.mark.parametrize(- "device, served_as_vision, published",- [("mlx", False, False), ("mlx", True, True), ("cuda", False, True)],+ "device, config_vision, served_as_vision, published",+ [+ ("mlx", True, False, False),+ ("mlx", True, True, True),+ ("cuda", True, False, True),+ ("mlx", False, True, True),+ ("cuda", False, True, False),+ ],)def test_an_mlx_vision_config_served_as_text_stops_accepting_images(- monkeypatch, device, served_as_vision, published+ monkeypatch, device, config_vision, served_as_vision, published):- """MLX serves a vision config mlx-lm built (Muse Spark) as text: the parent must not offer images."""+ """MLX serves a vision config mlx-lm built (Muse Spark) as text: the parent must not offer images.+ And a tower only mlx-vlm knows (OXR's vit_config) reads them although the config detector said no."""from core.inference.orchestrator import _mirrored_model_entrymc = _mc(None, is_audio = False, has_audio_input = False)- mc.is_vision = True+ mc.is_vision = config_visionentry = {"is_audio": False, "audio_type": None, "has_audio_input": False}entry["is_vision"] = served_as_visioninfo = _drive_handle_load(monkeypatch, mc, entry, device = device)diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 849d4dc9e7c..86e13866e00 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -2819,6 +2819,7 @@ function queuedResolvedModelFromStore(audioType: activeModel.audioType,hasAudioInput: activeModel.hasAudioInput,hasVideoInput: activeModel.hasVideoInput,+ speaks: activeModel.speaks,}: null,};@@ -4350,6 +4351,7 @@ async function resolveQueuedEmptyLocalModel(abortSignal: AbortSignal): Promise<{audioType: status.audio_type ?? null,hasAudioInput: status.has_audio_input ?? false,hasVideoInput: status.has_video_input ?? false,+ speaks: status.speaks ?? false,},},};@@ -6115,6 +6117,8 @@ export function createOpenAIStreamAdapter(...part,afterToolCalls: 0,})) ?? []);+ // A spoken reply's audio, kept apart from the text so the player renders on its own.+ let spokenReplyPart: { type: "text"; text: string } | null = null;const buildAssistantContent = (rawText: string) => {const positionedTools = toolCallParts.map((part, index) => {@@ -6156,6 +6160,7 @@ export function createOpenAIStreamAdapter(}}assembled.push(...runs[boundaries.length]);+ if (spokenReplyPart) assembled.push(spokenReplyPart);pinGeminiThoughtReplayParts(assembled, geminiThoughtParts);pinGeminiAnswerReplayParts(assembled, geminiAnswerParts);@@ -6815,6 +6820,10 @@ export function createOpenAIStreamAdapter(};}+ const turnAudioBase64 = findLatestUserAudioBase64(+ currentTurnMessages,+ !queuedRunSettings && !continuation,+ );return {model: params.checkpoint,messages: outboundMessages,@@ -6846,10 +6855,10 @@ export function createOpenAIStreamAdapter(: { seed: params.seed }),// Turn-scoped: a stale screenshot here made the backend refuse every later text turn's durable run.image_base64: findLatestUserImageBase64(currentTurnMessages),- audio_base64: findLatestUserAudioBase64(- currentTurnMessages,- !queuedRunSettings && !continuation,- ),+ audio_base64: turnAudioBase64,+ ...(activeModel?.speaks && turnAudioBase64+ ? { modalities: ["text", "audio"] }+ : {}),extra_audio_base64:findLatestUserExtraAudioBase64(currentTurnMessages),video_base64: findLatestUserVideoBase64(currentTurnMessages),@@ -7835,6 +7844,18 @@ export function createOpenAIStreamAdapter()?.extra_content;// State reaches the message only via a yield, so pace previews but never state changes.let replayStateChanged = false;+ const spokenAudio = (+ chunk.choices?.[0]?.delta as+ | { audio?: { data?: unknown } }+ | undefined+ )?.audio?.data;+ if (typeof spokenAudio === "string" && spokenAudio) {+ spokenReplyPart = {+ type: "text",+ text: `<audio-player src="data:audio/wav;base64,${spokenAudio}" />`,+ };+ replayStateChanged = true;+ }let geminiReplayStateChanged = false;let geminiThoughtSignature: string | undefined;let capturedGeminiThoughtPart = false;diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex d7e44bbc1fd..39674ea07b0 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -501,6 +501,7 @@ export function syncModelCapabilities(audio_type?: string | null;has_audio_input?: boolean;has_video_input?: boolean;+ speaks?: boolean;},): void {const store = useChatRuntimeStore.getState();@@ -514,6 +515,8 @@ export function syncModelCapabilities(hasAudioInput: Boolean(resp.has_audio_input),// /api/models/list omits this for the active GGUF row.hasVideoInput: Boolean(resp.has_video_input),++ speaks: Boolean(resp.speaks),};const idx = models.findIndex((m) => m.id === modelId);if (idx === -1) {diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.tsindex 4216abdbc82..5bb1e192c3f 100644--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts@@ -108,6 +108,7 @@ function ensureActiveModelInStoreList(audioType: status.audio_type ?? null,hasAudioInput: status.has_audio_input ?? false,hasVideoInput: status.has_video_input ?? false,+ speaks: status.speaks ?? false,};const existing = store.models.find((model) => model.id === checkpointId);if (existing) {diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex 3163053876d..0fc949de1b8 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -2084,6 +2084,7 @@ export function SharedComposer({audioType: resp.audio_type ?? null,hasAudioInput: Boolean(resp.has_audio_input),hasVideoInput: Boolean(resp.has_video_input),+ speaks: Boolean(resp.speaks),};if (idx === -1) {store.setModels([diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 90c17efc0a1..178e05ee94b 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -233,6 +233,7 @@ export interface LoadModelResponse {audio_device_fallback?: "cpu" | null;has_audio_input?: boolean;has_video_input?: boolean;+ speaks?: boolean;inference?: {temperature?: number;top_p?: number;@@ -359,6 +360,8 @@ export interface InferenceStatusResponse {audio_music?: unknown;has_audio_input?: boolean;has_video_input?: boolean;+ /** Answers a spoken question in its own voice when asked for modalities ["text", "audio"]. */+ speaks?: boolean;loading: string[];loaded: string[];serving?: string[];diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.tsindex 590f97d2e80..d96c38510cc 100644--- a/studio/frontend/src/features/chat/types/runtime.ts+++ b/studio/frontend/src/features/chat/types/runtime.ts@@ -103,6 +103,7 @@ export interface ChatModelSummary {audioType?: string | null;hasAudioInput?: boolean;hasVideoInput?: boolean;+ speaks?: boolean;}export interface ChatLoraSummary {diff --git a/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts b/studio/frontend/src/features/chat/utils/queued-model-capabilities.tsindex 8f0bd9a95af..8b68b2153a7 100644--- a/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts+++ b/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts@@ -9,6 +9,7 @@ export type QueuedModelCapabilities = Pick<| "audioType"| "hasAudioInput"| "hasVideoInput"+ | "speaks">;export function mergeQueuedModelCapabilities(