Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio: LFM3-Audio / LFM3-Omni on MLX, with spoken replies (#15889)

Long Yixingd7aa02beccdd
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 5dc4b7e8793..3d5c7d5aad0 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -2384,8 +2384,11 @@ class InferenceBackend:
use_adapter: Optional[Union[bool, str]] = None,
cancel_event = None,
extra_audio_arrays: Optional[list] = None,
+ tools: Optional[list] = None,
) -> Generator[str, None, None]:
- """Audio goes into the chat messages via processor.apply_chat_template (Gemma 3n pattern)."""
+ """Audio goes into the chat messages via processor.apply_chat_template (Gemma 3n pattern).
+ ``tools`` (a list, empty for tool history alone) renders the catalogue and keeps
+ ``messages`` as sent."""
import threading
import numpy as np
@@ -2402,7 +2405,7 @@ class InferenceBackend:
# Gemma 3n format: audio goes INTO apply_chat_template, one item per clip in order.
audio_messages = messages_with_attached_image(
- alternating_turns(messages),
+ alternating_turns(messages) if tools is None else messages,
system_prompt = system_prompt,
fallback_user_text = "Please transcribe this audio.",
structured_content = True,
@@ -2412,11 +2415,9 @@ class InferenceBackend:
)
# Direct processor render, so neutralize here with this processor's own profile (#7066).
- from core.inference.chat_template_helpers import markup_for_tokenizer
+ from core.inference.chat_template_helpers import neutralize_for_render
- audio_messages = neutralize_control_markup_in_messages(
- audio_messages, None, markup_for_tokenizer(processor)
- )
+ audio_messages, tools, _ = neutralize_for_render(processor, audio_messages, tools)
inputs = processor.apply_chat_template(
audio_messages,
@@ -2425,16 +2426,18 @@ class InferenceBackend:
return_dict = True,
return_tensors = "pt",
truncation = False,
+ **({"tools": tools} if tools else {}),
).to(model.device)
try:
from transformers import TextIteratorStreamer
from queue import Empty
+ # As the text path: a rendered catalogue keeps native tool-call tokens for the healer.
streamer = TextIteratorStreamer(
- raw_tokenizer,
+ NativeToolTokenDecoder(raw_tokenizer) if tools else raw_tokenizer,
skip_prompt = True,
- skip_special_tokens = True,
+ skip_special_tokens = not tools,
timeout = 0.2,
)
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 498af28c413..3124bd9ff05 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -38,10 +38,9 @@ from core.inference.chat_template_helpers import (
count_structured_videos as _count_vlm_videos,
detect_reasoning_channel_markers,
make_reasoning_normalizer,
- markup_for_tokenizer,
messages_have_tool_history as _vlm_messages_have_tool_history,
messages_with_attached_image,
- neutralize_control_markup_in_messages,
+ neutralize_for_render,
normalize_reasoning_snapshots,
prompt_opens_reasoning_channel,
strip_open_reasoning_prefill,
@@ -1490,6 +1489,7 @@ def _render_registered_vlm_prompt(
num_images,
num_audios = 0,
continue_final_message = False,
+ tools = None,
):
"""Resumes a partial from the swept messages, since a raw partial could close the turn."""
from mlx_vlm import prompt_utils
@@ -1522,7 +1522,7 @@ def _render_registered_vlm_prompt(
if _resumes_thought(messages, continue_final_message):
return None
# Recovery path: sweeps the caller's original list rather than reusing a copy (#7066).
- swept = neutralize_control_markup_in_messages(messages, None, markup_for_tokenizer(processor))
+ swept, tools, _ = neutralize_for_render(processor, messages, tools)
partial = trailing_assistant_text(swept) if continue_final_message else None
rendered = prompt_utils.apply_chat_template(
processor,
@@ -1531,6 +1531,7 @@ def _render_registered_vlm_prompt(
add_generation_prompt = True,
num_images = num_images,
num_audios = num_audios,
+ **({"tools": tools} if tools else {}),
)
if isinstance(rendered, str) and rendered.strip():
# A prefilled open "<think>" would resume the answer inside the reasoning block.
@@ -1734,6 +1735,35 @@ def _discard_video_clip(path: str) -> None:
pass
+_SPEECH_REPLY_SYSTEM_PROMPT = "Respond with interleaved text and audio."
+
+
+def _mlx_lm_serves(model_type) -> bool:
+ from importlib.util import find_spec
+
+ from mlx_lm.utils import MODEL_REMAPPING
+ return find_spec(f"mlx_lm.models.{MODEL_REMAPPING.get(model_type, model_type)}") is not None
+
+
+def _zoo_declares_multimodal(model_name, hf_token = None) -> bool:
+ """unsloth_zoo's verdict from mlx-vlm's own configs, for mlx-vlm-only families the transformers
+ registry does not know (an audio-only speech model, a tower under another name)."""
+ try:
+ from utils.models.model_config import _read_raw_config_json
+
+ config = _read_raw_config_json(model_name, hf_token)
+ model_type = (config or {}).get("model_type")
+ # A type mlx-lm serves keeps its text path, and skips zoo's scan of every mlx-vlm family.
+ if not isinstance(model_type, str) or not model_type or _mlx_lm_serves(model_type):
+ return False
+ from unsloth_zoo.mlx.loader import _is_vlm
+
+ return _is_vlm(config)
+ except Exception as exc: # noqa: BLE001 -- detection must never fail a load
+ logger.debug("unsloth_zoo multimodal detection skipped for %s (%s)", model_name, exc)
+ return False
+
+
def _classify_mlx_audio_type(
model,
processor,
@@ -3572,6 +3602,27 @@ def _stream_until_cancelled(stream_generate, cancel_event, *args, **kwargs):
return
+@contextmanager
+def _slots_until_cancelled(model, cancel_event):
+ """mlx-vlm's ``generate_audio`` takes no stop hook; LFM3 samples through ``stream_slots``, so end that loop."""
+ slots = getattr(model, "stream_slots", None)
+ if cancel_event is None or slots is None:
+ yield
+ return
+
+ def until_cancelled(*args, **kwargs):
+ for slot in slots(*args, **kwargs):
+ if cancel_event.is_set():
+ return
+ yield slot
+
+ model.stream_slots = until_cancelled
+ try:
+ yield
+ finally:
+ del model.stream_slots
+
+
class LogprobsUnsupported(ValueError):
"""``n_probs`` asked of a model that does not sample one token per step."""
@@ -5260,6 +5311,10 @@ class MLXInferenceBackend:
self._resume_load_args = (bool(load_in_4bit), repr(dtype), repr(tuple(offload or ())))
model_name = config.identifier if hasattr(config, "identifier") else str(config)
is_vision = getattr(config, "is_vision", False)
+ is_lora = getattr(config, "is_lora", False)
+ zoo_multimodal = not is_vision and _zoo_declares_multimodal(
+ getattr(config, "base_model", None) if is_lora else model_name, hf_token
+ )
kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)
self._turboquant = turboquant
if self._turboquant:
@@ -5271,7 +5326,6 @@ class MLXInferenceBackend:
) from exc
distributed_rank, distributed_size = _mlx_distributed_rank_size(distributed_group)
is_distributed = distributed_group is not None and distributed_size > 1
- is_lora = getattr(config, "is_lora", False)
self._turboquant_refusal = (
_turboquant_refusal(is_vision = is_vision, is_distributed = is_distributed, is_lora = is_lora)
if self._turboquant
@@ -5303,6 +5357,7 @@ class MLXInferenceBackend:
# Speculation runs on mlx-vlm models and caches.
use_vlm = (
is_vision
+ or zoo_multimodal
or speculates
or (self._turboquant and not self._turboquant_refusal)
or batches_quantized_text
@@ -5476,6 +5531,9 @@ class MLXInferenceBackend:
FastMLXModel.from_pretrained, load_source, load_kwargs, offload
)
+ zoo_loaded_multimodal = zoo_multimodal and not served_by_mlx_lm
+ if zoo_loaded_multimodal:
+ is_vision = getattr(model, "_unsloth_supports_images", None) is True
self._reads_vision = bool(is_vision)
if use_vlm:
processor = tokenizer_or_processor
@@ -5501,7 +5559,7 @@ class MLXInferenceBackend:
_audio_type = _classify_mlx_audio_type(
model,
self._processor,
- is_vision,
+ is_vision or zoo_loaded_multimodal,
config_audio_type = _config_audio_type,
)
_served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(
@@ -5675,6 +5733,8 @@ class MLXInferenceBackend:
"is_audio": _audio_type is not None and _audio_type != "audio_vlm",
"audio_type": _audio_type,
"has_audio_input": is_audio_input_type(_audio_type),
+ "speaks": _audio_type == "audio_vlm"
+ and getattr(self._model, "joint_audio_text_generation", False) is True,
"has_video_input": is_vision and _mlx_reads_video(self._processor),
"context_length": _served_ctx,
# Matches llama.cpp requested_n_ctx; served window alone cannot tell if a size was asked for.
@@ -7703,11 +7763,15 @@ class MLXInferenceBackend:
cancel_event = None,
stop = None,
extra_audio_arrays = None,
+ speak = False,
+ tools = None,
**_sampler,
):
"""Audio-input chat (omni models): waveform in, incremental text deltas out (the audio route
forwards deltas, unlike the snapshot-diffing text/vision paths). Greedy, so the worker's
- sampler kwargs go unused."""
+ sampler kwargs go unused. ``speak`` answers in the model's own voice instead: the reply
+ text, then one ``{"speech": <base64 wav>, "sample_rate": ...}`` item. ``tools`` (a list,
+ empty for tool history alone) renders the catalogue and keeps ``messages`` as sent."""
entry = self.models.get(self.active_model_name) or {}
if entry.get("audio_type") != "audio_vlm":
raise RuntimeError(
@@ -7719,13 +7783,19 @@ class MLXInferenceBackend:
if extra_audio_arrays:
raise RuntimeError("This MLX model takes one audio file per message.")
+ if speak:
+ yield from self._speak_audio_reply(
+ messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event
+ )
+ return
+
from mlx_vlm import stream_generate as vlm_stream
if not system_prompt:
system_prompt = "You are an assistant that transcribes speech accurately."
audio_messages = messages_with_attached_image(
- alternating_turns(messages),
+ alternating_turns(messages) if tools is None else messages,
system_prompt = system_prompt,
fallback_user_text = "Please transcribe this audio.",
image = 0,
@@ -7737,6 +7807,7 @@ class MLXInferenceBackend:
audio_messages,
num_images = 0,
num_audios = 1,
+ **({"tools": tools} if tools else {}),
)
if prompt is None:
raise RuntimeError(
@@ -7836,6 +7907,76 @@ class MLXInferenceBackend:
if stopped:
self._mark_stopped()
+ def _speak_audio_reply(
+ self,
+ messages,
+ system_prompt,
+ audio_array,
+ max_new_tokens,
+ use_adapter,
+ cancel_event = None,
+ ):
+ """One spoken question answered in interleaved text and speech, not streamed."""
+ import base64
+
+ from mlx_vlm.generate import generate_audio
+
+ if not (self.models.get(self.active_model_name) or {}).get("speaks"):
+ raise RuntimeError("The loaded model cannot answer in speech.")
+ # The model card's prompt for speech chat; a user prompt still applies after it.
+ system = "\n\n".join(p for p in (_SPEECH_REPLY_SYSTEM_PROMPT, system_prompt) if p)
+ prompt = _render_registered_vlm_prompt(
+ self._processor,
+ self._model,
+ messages_with_attached_image(
+ alternating_turns(messages),
+ system_prompt = system,
+ image = 0,
+ audio = audio_array,
+ ),
+ num_images = 0,
+ num_audios = 1,
+ )
+ max_new_tokens = self._generation_limit(
+ prompt, max_new_tokens, audio = [audio_array], cap = UNSET_GENERATION_BUDGET
+ )
+ with (
+ self._generation_lock,
+ _temporary_mlx_adapter_state(self._model, use_adapter),
+ ExitStack() as generation_scope,
+ ):
+ self._release_vlm_snapshots()
+ generation_scope.enter_context(self._generation_scopes("vision", "audio"))
+ generation_scope.enter_context(_slots_until_cancelled(self._model, cancel_event))
+ result = generate_audio(
+ self._model,
+ self._processor,
+ prompt,
+ audio = [audio_array],
+ generation_mode = "interleaved",
+ max_tokens = max_new_tokens,
+ )
+ cancelled = cancel_event is not None and cancel_event.is_set()
+ # Nothing to encode once stopped, nor for a reply that ended before its first speech frame.
+ spoke = not cancelled and result.audio is not None and result.audio.size > 0
+ wav = result.to_wav_bytes() if spoke else None
+ self.last_generation_stats = _build_generation_stats(
+ result.prompt_tokens,
+ result.prompt_tps,
+ result.generation_tokens,
+ result.generation_tps,
+ finish_reason = result.finish_reason,
+ )
+ if cancelled:
+ return
+ if result.text:
+ yield result.text
+ if spoke:
+ yield {
+ "speech": base64.b64encode(wav).decode("ascii"),
+ "sample_rate": result.sample_rate,
+ }
+
def generate_with_adapter_control(
self,
use_adapter = None,
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 414d4e71a98..65e602f5f44 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -403,6 +403,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"audio_type": model_info.get("audio_type"),
"has_audio_input": model_info.get("has_audio_input", False),
"has_video_input": model_info.get("has_video_input", False),
+ "speaks": model_info.get("speaks", False),
"context_length": model_info.get("context_length"),
"native_context_length": model_info.get("native_context_length"),
"max_context_length": model_info.get("max_context_length"),
@@ -1557,6 +1558,9 @@ class InferenceOrchestrator:
yield int(resp.get("row", 0)), resp.get("text", "")
else:
yield resp.get("text", "")
+ elif rtype == "speech":
+ if rows is None and not stop_sent:
+ yield {"speech": resp.get("speech"), "sample_rate": resp.get("sample_rate")}
elif rtype == "diffusion_frame":
if frames and rows is None and not stop_sent:
yield {
@@ -4141,8 +4145,11 @@ class InferenceOrchestrator:
stats_holder: Optional[dict] = None,
stop = None,
extra_audio_arrays: Optional[list] = None,
+ speak: bool = False,
+ tools: Optional[list] = None,
) -> Generator[str, None, None]:
- """Audio input generation (e.g. Gemma 3n): streams text tokens."""
+ """Audio input generation (e.g. Gemma 3n): streams text tokens; with ``speak``, then one
+ ``{"speech", "sample_rate"}`` item carrying the spoken reply."""
yield from self._generate_audio_input_inner(
audio_array = audio_array,
audio_type = None, # worker uses generate_audio_input_response
@@ -4159,6 +4166,8 @@ class InferenceOrchestrator:
stats_holder = stats_holder,
stop = stop,
extra_audio_arrays = extra_audio_arrays,
+ speak = speak,
+ tools = tools,
)
def _generate_audio_input_inner(
@@ -4178,6 +4187,8 @@ class InferenceOrchestrator:
stats_holder: Optional[dict] = None,
stop = None,
extra_audio_arrays: Optional[list] = None,
+ speak: bool = False,
+ tools: Optional[list] = None,
) -> Generator[str, None, None]:
"""Shared inner logic for audio input generation (Whisper + ASR). ``stats_holder``: as in
generate_chat_response, caller-owned and filled on gen_done with the worker's usage /
@@ -4223,6 +4234,10 @@ class InferenceOrchestrator:
cmd["use_adapter"] = use_adapter
if stop:
cmd["stop"] = stop
+ if speak:
+ cmd["speak"] = True
+ if tools is not None:
+ cmd["tools"] = tools
read_one, drain, release_mailbox = self._direct_reader(request_id, cancel_event)
try:
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 8e9d5f91c2b..ea05f4fd72c 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -843,6 +843,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"audio_type",
"has_audio_input",
"has_video_input",
+ "speaks",
"audio_family",
"audio_options",
"gguf_variant",
@@ -865,9 +866,9 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
if k in _entry
}
)
- # MLX serves a vision config mlx-lm built as text; the parent must stop accepting images.
- if model_info["is_mlx"] and _entry.get("is_vision") is False:
- model_info["is_vision"] = False
+ # MLX decides at load: an mlx-lm text build reads no images, an mlx-vlm-only tower does.
+ if model_info["is_mlx"] and isinstance(_entry.get("is_vision"), bool):
+ model_info["is_vision"] = _entry["is_vision"]
# Resolved MLX runtime knobs; only the backend knows what it honored.
model_info.update(
{
@@ -1945,6 +1946,10 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve
backend, "stop", "generate_audio_input_response"
):
audio_kwargs["stop"] = cmd["stop"]
+ if cmd.get("speak"):
+ audio_kwargs["speak"] = True
+ if cmd.get("tools") is not None:
+ audio_kwargs["tools"] = cmd["tools"]
generator = backend.generate_audio_input_response(**audio_kwargs, **extra_audio_kwargs)
logger.info("Starting audio input generation for request_id=%s", request_id)
@@ -1954,6 +1959,11 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve
logger.info("Audio input generation cancelled for request %s", request_id)
break
+ if isinstance(text_chunk, dict):
+ _send_response(
+ resp_queue, {"type": "speech", "request_id": request_id, **text_chunk}
+ )
+ continue
_send_response(
resp_queue,
{
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index e7059caed06..741d5d978fd 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1579,6 +1579,9 @@ class _InferenceRuntimeFields(BaseModel):
description = "Audio codec or native generation architecture.",
)
has_audio_input: bool = Field(False, description = "Whether model accepts audio input (ASR)")
+ speaks: bool = Field(
+ False, description = "Whether model can answer a spoken question in its own voice"
+ )
audio_family: Optional[str] = Field(
None, description = "Loader family of an audio GGUF model (e.g. kokoro_tts, minimax_music3)."
)
@@ -2880,6 +2883,17 @@ class ChatCompletionRequest(BaseModel):
None,
description = "[x-unsloth] Base64-encoded audio (wav/mp3/ogg/flac/m4a) for audio-input models",
)
+ modalities: Optional[List[Literal["text", "audio"]]] = Field(
+ None,
+ description = (
+ 'Output modalities, as OpenAI\'s. ["text", "audio"] asks a model that speaks (status '
+ "speaks) to answer the spoken question in audio_base64 in its own voice as well: the "
+ "reply text, then message.audio / delta.audio {data: base64 wav, format: wav, "
+ "sample_rate}. The model samples the reply its own way, so sampling settings do not "
+ "apply and stop sequences are refused; max_tokens and usage count speech frames as "
+ "well as text tokens; a streamed reply arrives whole, as one text delta and one audio delta."
+ ),
+ )
extra_audio_base64: Optional[List[str]] = Field(
None,
description = (
@@ -3668,6 +3682,7 @@ class ChoiceDelta(BaseModel):
content: Optional[str] = None
reasoning_content: Optional[str] = None
tool_calls: Optional[list[dict]] = None
+ audio: Optional[dict] = None
OpenAIFinishReason = Literal["stop", "length", "tool_calls", "content_filter", "function_call"]
@@ -3704,6 +3719,7 @@ class CompletionMessage(BaseModel):
refusal: Optional[str] = None
reasoning_content: Optional[str] = None
tool_calls: Optional[list[dict]] = None
+ audio: Optional[dict] = None
class CompletionChoice(BaseModel):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 8815771a5cf..3371d64b73a 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1450,6 +1450,11 @@ def _chat_content_chunk(completion_id, created, model_name, text) -> str:
)
+def _speech_reply_audio(item: dict) -> dict:
+ """A backend speech item as the message / delta ``audio`` part (OpenAI's field, carrying the WAV inline)."""
+ return {"data": item.get("speech"), "format": "wav", "sample_rate": item.get("sample_rate")}
+
+
def _chat_reasoning_chunk(completion_id, created, model_name, text) -> str:
"""Reasoning delta with empty content alongside, so strict OpenAI adapters do not drop it."""
return _chat_chunk_sse(
@@ -8072,6 +8077,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
# Not MLX, so the MLX runtime fields report as absent.
is_mlx = False,
is_npu = False,
+ speaks = False,
mlx_kv_quant = None,
mlx_kv_quant_requested = None,
mlx_kv_bits = None,
@@ -18697,6 +18703,7 @@ async def _load_model_impl(
audio_type = _model_info.get("audio_type"),
has_audio_input = _model_info.get("has_audio_input", False),
has_video_input = _model_info.get("has_video_input", False),
+ speaks = bool(_model_info.get("speaks", False)),
is_mlx = bool(_model_info.get("is_mlx", False)),
mlx_kv_quant = _model_info.get("mlx_kv_quant"),
mlx_kv_quant_requested = _model_info.get("mlx_kv_quant_requested"),
@@ -19539,7 +19546,7 @@ async def _load_model_impl(
status = "loaded",
model = model_log_label if native_grant_backed else config.identifier,
display_name = model_log_label if native_grant_backed else config.display_name,
- is_vision = config.is_vision,
+ is_vision = _model_info.get("is_vision", config.is_vision),
is_lora = config.is_lora,
is_gguf = False,
is_local_model = config.is_local,
@@ -19548,6 +19555,7 @@ async def _load_model_impl(
audio_type = _model_info.get("audio_type", config.audio_type),
has_audio_input = _model_info.get("has_audio_input", config.has_audio_input),
has_video_input = _model_info.get("has_video_input", False),
+ speaks = bool(_model_info.get("speaks", False)),
audio_family = _model_info.get("audio_family"),
audio_options = _model_info.get("audio_options"),
audio_workflows = _model_info.get("audio_workflows"),
@@ -22075,6 +22083,7 @@ async def _slot_status(current_subject: str):
audio_type = audio_type,
has_audio_input = has_audio_input,
has_video_input = has_video_input,
+ speaks = bool(model_info.get("speaks", False)),
audio_family = model_info.get("audio_family"),
audio_options = model_info.get("audio_options"),
audio_workflows = model_info.get("audio_workflows"),
@@ -30586,6 +30595,8 @@ async def produce_openai_chat_completions(
normalized_stop = _normalize_stop_sequences(payload.stop)
if using_gguf:
+ if "audio" in (payload.modalities or ()):
+ raise HTTPException(status_code = 400, detail = "The loaded model cannot answer in speech.")
# Advertised repo id after an auto-switch load, else a clean public id,
# never the absolute .gguf path.
model_name = _llama_public_model_id(llama_backend, payload.model)
@@ -30735,9 +30746,83 @@ async def produce_openai_chat_completions(
api_monitor.fail(monitor_id, _MLX_MULTI_AUDIO_DETAIL)
raise HTTPException(status_code = 400, detail = _MLX_MULTI_AUDIO_DETAIL)
+ _speak = "audio" in (payload.modalities or ())
+ _speak_detail = None
+ if _speak and not model_info.get("speaks"):
+ _speak_detail = "The loaded model cannot answer in speech."
+ elif _speak and not payload.audio_base64:
+ _speak_detail = "A spoken reply answers a spoken question: send it as audio_base64."
+ elif _speak and _normalize_stop_sequences(payload.stop):
+ # The speech cannot be cut at a text marker, so the stop would not hold.
+ _speak_detail = "Stop sequences do not apply to a spoken reply."
+ if _speak_detail is not None:
+ api_monitor.fail(monitor_id, _speak_detail)
+ raise HTTPException(status_code = 400, detail = _speak_detail)
+
# ── Audio INPUT path: decode WAV and route to audio input generation ──
if payload.audio_base64 and model_info.get("has_audio_input"):
_refuse_unused_mcp_image(_mcp_image)
+ # Client tools as the text path takes them: none under tool_choice none, the forced one alone.
+ _audio_tools = None
+ if payload.tools and payload.tool_choice != "none":
+ from core.inference.chat_template_helpers import forced_tool_name
+
+ if model_info.get("audio_type") == "whisper" or _speak:
+ _detail = "Tools do not apply to a transcription or a spoken reply."
+ api_monitor.fail(monitor_id, _detail)
+ _raise_unsupported_openai_parameter("tools", _detail)
+ _forced = forced_tool_name(payload.tool_choice)
+ _audio_tools = [
+ tool
+ for tool in payload.tools
+ if not _forced
+ or ((tool.get("function") or {}) if isinstance(tool, dict) else {}).get("name")
+ == _forced
+ ]
+ # Heal and render the same catalogue: a tool dropped for markup was never shown (#7066).
+ from core.inference.chat_template_helpers import (
+ chat_render_target,
+ renderable_tool_catalog_for_targets,
+ )
+
+ _audio_target = chat_render_target(
+ model_info.get("processor"), model_info.get("tokenizer")
+ )
+ _audio_tools = await asyncio.to_thread(
+ renderable_tool_catalog_for_targets,
+ _audio_tools,
+ (_audio_target, getattr(_audio_target, "tokenizer", _audio_target)),
+ model_info,
+ active_model_name = backend.active_model_name,
+ )
+ _tool_refusal = None
+ if _forced and not _audio_tools:
+ _tool_refusal = (
+ "tool_choice",
+ f"Invalid 'tool_choice': function '{_forced}' is not enabled.",
+ )
+ elif not _sf_rendered_features(backend, model_info, payload.tools)[0].get(
+ "supports_tools"
+ ):
+ _tool_refusal = (
+ "tools",
+ "Client-supplied tools require a chat template with tool-call "
+ "support; the current model/template does not advertise tools.",
+ )
+ if _tool_refusal is not None:
+ api_monitor.fail(monitor_id, _tool_refusal[1])
+ _raise_unsupported_openai_parameter(*_tool_refusal)
+ # Tool calls and results reach the template as sent, which alternating turns would drop.
+ _audio_tool_kwargs = (
+ {"tools": _audio_tools or []}
+ if _audio_tools or _has_openai_tool_history(payload.messages)
+ else {}
+ )
+ _audio_heal = (
+ heal_gate(payload.auto_heal_tool_calls, _audio_tools, payload.tool_choice)
+ if _audio_tools
+ else None
+ )
# No boundary to resume from: this route re-listens and answers afresh.
if _continue_final_message(payload):
raise HTTPException(
@@ -30765,7 +30850,12 @@ async def produce_openai_chat_completions(
system_prompt, chat_messages, _ = await _extract_content_parts_async(
payload.messages
)
- system_prompt = _audio_input_system_prompt(system_prompt, request)
+ # A spoken reply answers the question, so the transcription default does not apply.
+ system_prompt = (
+ _apply_current_date_prompt(system_prompt, request)
+ if _speak
+ else _audio_input_system_prompt(system_prompt, request)
+ )
except _DecodedAudioTooLongError as e:
# A limit the caller can act on, not a server fault.
api_monitor.fail(monitor_id, str(e))
@@ -30813,6 +30903,8 @@ async def produce_openai_chat_completions(
stats_holder = _audio_stats_holder,
stop = normalized_stop,
**extra_audio_kwargs,
+ **({"speak": True} if _speak else {}),
+ **_audio_tool_kwargs,
)
if payload.stream:
@@ -30828,6 +30920,10 @@ async def produce_openai_chat_completions(
from core.inference.llama_keepwarm import mark_current_response_failed
yield _chat_role_chunk(completion_id, created, model_name)
+ healer = (
+ StreamToolCallHealer(_audio_heal, _audio_tools) if _audio_heal else None
+ )
+ heal_state = {"idx": 0}
gen = audio_input_generate()
gen_step = ThreadStepper(gen).next
@@ -30857,14 +30953,52 @@ async def produce_openai_chat_completions(
{"error": {"message": _msg, "type": "server_error"}}
)
return
- if chunk_text:
+ if isinstance(chunk_text, dict):
+ yield _chat_chunk_sse(
+ completion_id,
+ created,
+ model_name,
+ delta = ChoiceDelta(audio = _speech_reply_audio(chunk_text)),
+ finish_reason = None,
+ )
+ continue
+ if chunk_text and healer is not None:
+ for line in _sf_heal_events_to_sse(
+ healer.feed(chunk_text),
+ completion_id,
+ created,
+ model_name,
+ heal_state,
+ payload.parallel_tool_calls,
+ monitor_id,
+ ):
+ yield line
+ elif chunk_text:
api_monitor.append_reply(monitor_id, chunk_text)
yield _chat_content_chunk(
completion_id, created, model_name, chunk_text
)
+ # A cancelled stream must not heal incomplete markup into a call.
+ if healer is not None and not cancelled:
+ for line in _sf_heal_events_to_sse(
+ healer.finalize(),
+ completion_id,
+ created,
+ model_name,
+ heal_state,
+ payload.parallel_tool_calls,
+ monitor_id,
+ ):
+ yield line
_audio_stats = _audio_stats_holder.get("stats")
- _audio_finish = "stop" if cancelled else _stats_finish_reason(_audio_stats)
+ _audio_finish = (
+ "stop"
+ if cancelled
+ else "tool_calls"
+ if healer is not None and healer.healed
+ else _stats_finish_reason(_audio_stats)
+ )
# The worker fills these stats for both backends (MLX adds timings), so the row has counts and speed.
if isinstance(_audio_stats, dict):
_monitor_usage(
@@ -30908,12 +31042,16 @@ async def produce_openai_chat_completions(
_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)
_tracker.__enter__()
try:
+ _speech_reply = {}
def _drain_audio_input():
text_parts = []
for chunk_text in audio_input_generate():
if isinstance(chunk_text, GenStreamError):
return chunk_text
+ if isinstance(chunk_text, dict):
+ _speech_reply.update(_speech_reply_audio(chunk_text))
+ continue
text_parts.append(chunk_text)
return "".join(text_parts)
@@ -30941,11 +31079,28 @@ async def produce_openai_chat_completions(
finally:
# Nested under the except arms too: api_monitor.fail() can throw, and a leaked entry 409s swaps.
_tracker.__exit__(None, None, None)
- api_monitor.set_reply(monitor_id, full_text)
_audio_json_stats = _audio_stats_holder.get("stats")
_audio_json_finish = (
"stop" if cancel_event.is_set() else _stats_finish_reason(_audio_json_stats)
)
+ _audio_msg = {"role": "assistant", "content": full_text}
+ if (
+ _audio_heal
+ and not cancel_event.is_set()
+ and heal_openai_message(_audio_msg, _audio_heal, _audio_tools)
+ ):
+ _audio_json_finish = "tool_calls"
+ if payload.parallel_tool_calls is False:
+ _audio_msg["tool_calls"] = _audio_msg["tool_calls"][:1]
+ _audio_calls = "; ".join(
+ f"{call['function']['name']}({call['function']['arguments']})"
+ for call in _audio_msg.get("tool_calls") or []
+ )
+ api_monitor.set_reply(
+ monitor_id,
+ (_audio_msg["content"] or "")
+ + (f"[tool_calls] {_audio_calls}" if _audio_calls else ""),
+ )
_mark_cancelled_json_response_failed(request, cancel_event)
if isinstance(_audio_json_stats, dict):
_monitor_usage(
@@ -30963,7 +31118,11 @@ async def produce_openai_chat_completions(
model = model_name,
choices = [
CompletionChoice(
- message = CompletionMessage(content = full_text),
+ message = CompletionMessage(
+ content = _audio_msg["content"],
+ audio = _speech_reply or None,
+ tool_calls = _audio_msg.get("tool_calls"),
+ ),
finish_reason = _audio_json_finish,
)
],
diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.py
index 3470e1985bd..7cd6a15160d 100644
--- a/studio/backend/tests/test_chat_input_audio_part.py
+++ b/studio/backend/tests/test_chat_input_audio_part.py
@@ -763,3 +763,89 @@ def test_extra_clips_join_the_latest_user_turn_in_order():
single = messages_with_attached_image(history, structured_content = True, image = 0, audio = "a")
assert single[-1]["content"][0] == {"type": "audio", "audio": "a"}
assert len(single[-1]["content"]) == 2
+
+
[email protected](("stream", "stop"), [(True, None), (False, None), (True, ["HALT"])])
+def test_a_spoken_question_gets_a_spoken_reply(monkeypatch, stream, stop):
+ import json
+ from types import SimpleNamespace
+
+ seen = {}
+
+ def _generate(**kwargs):
+ seen.update(kwargs)
+ yield from ("Paris.", {"speech": "UklGRg==", "sample_rate": 24000})
+
+ backend = SimpleNamespace(active_model_name = "m", generate_audio_input_response = _generate)
+ backend.models = {"m": {"is_mlx": True, "has_audio_input": True, "speaks": True}}
+ idle = SimpleNamespace(is_loaded = False)
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: idle)
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)
+ monkeypatch.setattr(inference_route, "_decode_audio_clips", lambda clips: [[0.0]])
+ body = {"model": "m", "messages": [{"role": "user", "content": ""}], "audio_base64": AUDIO_B64}
+ body.update(modalities = ["text", "audio"], stream = stream, **({"stop": stop} if stop else {}))
+ with _route_client() as client:
+ response = client.post("/chat/completions", json = body)
+
+ if stop:
+ assert response.status_code == 400 and not seen
+ assert "Stop sequences do not apply" in response.text
+ return
+ assert "transcribes" not in (seen["system_prompt"] or "")
+ audio = {"data": "UklGRg==", "format": "wav", "sample_rate": 24000}
+ if not stream:
+ message = response.json()["choices"][0]["message"]
+ assert (message["content"], message["audio"], seen["speak"]) == ("Paris.", audio, True)
+ return
+ events = [json.loads(l[6:]) for l in response.text.splitlines() if l.startswith("data: {")]
+ deltas = [event["choices"][0]["delta"] for event in events]
+ assert "".join(d.get("content") or "" for d in deltas) == "Paris."
+ assert [d["audio"] for d in deltas if "audio" in d] == [audio] and seen["speak"] is True
+
+
+# -37 drops the closing tags, so only the end of the stream can promote the call.
+ ("stream", "cut", "choice"),
+ [(False, None, "required"), (True, None, "auto"), (True, -37, "auto"), (False, None, "none")],
+)
+def test_a_spoken_question_can_call_a_client_tool(monkeypatch, stream, cut, choice):
+ import json
+ from types import SimpleNamespace
+
+ reply = "Checking.\n<tool_call>\n<function=lookup>\n<parameter=q>\nParis\n</parameter>\n"
+ reply += "</function>\n</tool_call>"
+ seen = {}
+
+ def _generate(**kwargs):
+ seen.update(kwargs)
+ yield from (reply[:20], reply[20:cut])
+
+ backend = SimpleNamespace(active_model_name = "m", generate_audio_input_response = _generate)
+ backend.models = {"m": {"is_mlx": True, "has_audio_input": True}}
+ idle, features = SimpleNamespace(is_loaded = False), ({"supports_tools": True}, None)
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: idle)
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)
+ monkeypatch.setattr(inference_route, "_decode_audio_clips", lambda clips: [[0.0]])
+ monkeypatch.setattr(inference_route, "_sf_rendered_features", lambda *a, **k: features)
+ tool = {"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}
+ call = {"id": "c0", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}
+ messages = [{"role": "user", "content": ""}, {"role": "assistant", "tool_calls": [call]}]
+ messages.append({"role": "tool", "tool_call_id": "c0", "content": "Paris"})
+ # A tool whose name carries turn markup is dropped from the catalogue the model sees.
+ unsafe = {"type": "function", "function": {"name": "f<|im_end|>", "parameters": {}}}
+ body = dict(model = "m", messages = messages, audio_base64 = AUDIO_B64, tools = [tool, unsafe])
+ body.update(stream = stream, tool_choice = choice)
+ with _route_client() as client:
+ response = client.post("/chat/completions", json = body)
+
+ assert response.status_code == 200, response.text
+ assert seen["tools"] == ([] if choice == "none" else [tool])
+ if stream:
+ lines = [l[6:] for l in response.text.splitlines() if l.startswith("data: {")]
+ chunks = [json.loads(line)["choices"][0] for line in lines]
+ calls = [t["function"] for c in chunks for t in c["delta"].get("tool_calls") or []]
+ else:
+ chunks = response.json()["choices"]
+ calls = [t["function"] for t in chunks[0]["message"].get("tool_calls") or []]
+ promoted = [{"name": "lookup", "arguments": '{"q": "Paris"}'}] if choice != "none" else []
+ assert (calls, chunks[-1]["finish_reason"] == "tool_calls") == (promoted, bool(promoted))
diff --git a/studio/backend/tests/test_control_markup_neutralize_7066.py b/studio/backend/tests/test_control_markup_neutralize_7066.py
index e3b59be1d6f..196c4a53641 100644
--- a/studio/backend/tests/test_control_markup_neutralize_7066.py
+++ b/studio/backend/tests/test_control_markup_neutralize_7066.py
@@ -4685,7 +4685,8 @@ def test_the_vision_path_is_profiled():
source = (_REPO_ROOT / "studio" / "backend" / "core" / "inference" / "inference.py").read_text(
encoding = "utf-8"
)
- assert "markup_for_tokenizer(processor)" in source
+ vision = source.split("def _generate_vision_response(", 1)[1].split("\n def ", 1)[0]
+ assert "self._apply_chat_template_for_generation(" in vision
assert "neutralize_control_markup_in_messages(vision_messages)" not in source
@@ -4793,9 +4794,10 @@ def test_every_direct_renderer_carries_the_profile():
backend = _REPO_ROOT / "studio" / "backend" / "core" / "inference"
inference_src = (backend / "inference.py").read_text(encoding = "utf-8")
mlx_src = (backend / "mlx_inference.py").read_text(encoding = "utf-8")
- assert "markup_for_tokenizer(processor)" in inference_src
+ # The audio renders also carry a client tool catalogue, which needs the same sweep.
+ assert "neutralize_for_render(processor, audio_messages, tools)" in inference_src
assert "markup_for_tokenizer(tokenizer)" in inference_src
- assert "markup_for_tokenizer(processor)" in mlx_src
+ assert "neutralize_for_render(processor, messages, tools)" in mlx_src
for source in (inference_src, mlx_src):
assert "neutralize_control_markup_in_messages(messages)" not in source
assert "neutralize_control_markup_in_messages(audio_messages)" not in source
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index b3e576121ff..a13dd3baa0f 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1187,6 +1187,7 @@ class TestLoadHubDownloadExclusion:
"parallel_slots",
"is_mlx",
"is_npu",
+ "speaks",
"mlx_kv_quant",
"mlx_kv_quant_requested",
"mlx_kv_bits",
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 8edf0d75e88..d5c1c23e371 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -4587,6 +4587,103 @@ def test_mlx_generate_audio_input_deltas_and_reject(monkeypatch):
next(backend.generate_audio_input_response(**args))
+ ("samples", "cancel_at", "reply"),
+ [
+ (4, None, ["Paris.", {"speech": "UklGRndhdg==", "sample_rate": 24000}]),
+ (0, None, ["Paris."]),
+ (4, 2, []),
+ ],
+)
+def test_mlx_spoken_reply(monkeypatch, samples, cancel_at, reply):
+ from core.inference import mlx_inference
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ cancel, sampled, seen = threading.Event(), [], {}
+
+ class _SpeakingModel:
+ def stream_slots(self):
+ for slot in range(4):
+ if slot == cancel_at:
+ cancel.set()
+ yield slot
+
+ def _generate_audio(model, processor, prompt, **kwargs):
+ seen.update(kwargs, prompt = prompt)
+ sampled.extend(model.stream_slots())
+
+ class _Result:
+ text, sample_rate, finish_reason = "Paris.", 24000, "stop"
+ audio = SimpleNamespace(size = samples)
+ prompt_tokens = prompt_tps = generation_tokens = generation_tps = 1
+
+ def to_wav_bytes(self):
+ return b"RIFFwav" if samples else pytest.fail("empty WAV")
+
+ return _Result()
+
+ render = lambda processor, model, messages, **kwargs: messages[0]["content"]
+ monkeypatch.setattr(mlx_inference, "_render_registered_vlm_prompt", render)
+ generate = SimpleNamespace(generate_audio = _generate_audio)
+ monkeypatch.setitem(sys.modules, "mlx_vlm.generate", generate)
+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
+ backend._generation_lock = threading.Lock()
+ backend._model, backend._processor = _SpeakingModel(), _audio_processor()
+ backend.active_model_name, backend.last_generation_stats = "m", None
+ backend.models = {"m": {"audio_type": "audio_vlm", "speaks": True}}
+ args = dict(messages = [{"role": "user", "content": ""}], system_prompt = "Be brief.")
+ args.update(audio_array = [0.1], cancel_event = cancel, speak = True)
+
+ assert list(backend.generate_audio_input_response(**args)) == reply
+ assert sampled == [0, 1, 2, 3][: cancel_at or 4]
+ assert "stream_slots" not in vars(backend._model)
+ assert (seen["generation_mode"], seen["audio"]) == ("interleaved", [[0.1]])
+ assert seen["prompt"] == "Respond with interleaved text and audio.\n\nBe brief."
+
+
+ ("config", "served", "zoo", "expected"),
+ [
+ ({"model_type": "lfm3_audio"}, False, True, True),
+ ({"model_type": "oxr"}, False, False, False),
+ # mlx-lm serves it, so it stays on the text path whatever mlx-vlm declares.
+ (
+ {"model_type": "gemma3", "architectures": ["Gemma3ForConditionalGeneration"]},
+ True,
+ True,
+ False,
+ ),
+ (None, False, True, False),
+ ({}, False, True, False),
+ ],
+)
+def test_zoo_multimodal_detection_only_for_mlx_vlm_only_families(
+ monkeypatch, config, served, zoo, expected
+):
+ from core.inference import mlx_inference
+ from utils.models import model_config
+
+ asked = []
+ monkeypatch.setattr(model_config, "_read_raw_config_json", lambda name, token: config)
+ monkeypatch.setattr(mlx_inference, "_mlx_lm_serves", lambda model_type: served)
+ loader = SimpleNamespace(_is_vlm = lambda cfg: asked.append(cfg) or zoo)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)
+
+ assert mlx_inference._zoo_declares_multimodal("repo") is expected
+ assert asked == ([config] if config and not served else [])
+
+
+def test_zoo_multimodal_detection_fails_open_to_text(monkeypatch):
+ from core.inference import mlx_inference
+ from utils.models import model_config
+
+ def unreadable(name, token):
+ raise OSError("offline")
+
+ monkeypatch.setattr(model_config, "_read_raw_config_json", unreadable)
+ assert mlx_inference._zoo_declares_multimodal("repo") is False
+
+
def test_mlx_audio_input_normalizes_split_native_reasoning_channels(monkeypatch):
from core.inference import mlx_inference
from core.inference.mlx_inference import MLXInferenceBackend
diff --git a/studio/backend/tests/test_pr7699_worker_audio_mirror.py b/studio/backend/tests/test_pr7699_worker_audio_mirror.py
index bc2121a4f27..9ba7c97a523 100644
--- a/studio/backend/tests/test_pr7699_worker_audio_mirror.py
+++ b/studio/backend/tests/test_pr7699_worker_audio_mirror.py
@@ -153,17 +153,24 @@ def test_a_classified_load_failure_crosses_the_worker_and_the_parent(monkeypatch
@pytest.mark.parametrize(
- "device, served_as_vision, published",
- [("mlx", False, False), ("mlx", True, True), ("cuda", False, True)],
+ "device, config_vision, served_as_vision, published",
+ [
+ ("mlx", True, False, False),
+ ("mlx", True, True, True),
+ ("cuda", True, False, True),
+ ("mlx", False, True, True),
+ ("cuda", False, True, False),
+ ],
)
def test_an_mlx_vision_config_served_as_text_stops_accepting_images(
- monkeypatch, device, served_as_vision, published
+ monkeypatch, device, config_vision, served_as_vision, published
):
- """MLX serves a vision config mlx-lm built (Muse Spark) as text: the parent must not offer images."""
+ """MLX serves a vision config mlx-lm built (Muse Spark) as text: the parent must not offer images.
+ And a tower only mlx-vlm knows (OXR's vit_config) reads them although the config detector said no."""
from core.inference.orchestrator import _mirrored_model_entry
mc = _mc(None, is_audio = False, has_audio_input = False)
- mc.is_vision = True
+ mc.is_vision = config_vision
entry = {"is_audio": False, "audio_type": None, "has_audio_input": False}
entry["is_vision"] = served_as_vision
info = _drive_handle_load(monkeypatch, mc, entry, device = device)
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 849d4dc9e7c..86e13866e00 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -2819,6 +2819,7 @@ function queuedResolvedModelFromStore(
audioType: activeModel.audioType,
hasAudioInput: activeModel.hasAudioInput,
hasVideoInput: activeModel.hasVideoInput,
+ speaks: activeModel.speaks,
}
: null,
};
@@ -4350,6 +4351,7 @@ async function resolveQueuedEmptyLocalModel(abortSignal: AbortSignal): Promise<{
audioType: status.audio_type ?? null,
hasAudioInput: status.has_audio_input ?? false,
hasVideoInput: status.has_video_input ?? false,
+ speaks: status.speaks ?? false,
},
},
};
@@ -6115,6 +6117,8 @@ export function createOpenAIStreamAdapter(
...part,
afterToolCalls: 0,
})) ?? []);
+ // A spoken reply's audio, kept apart from the text so the player renders on its own.
+ let spokenReplyPart: { type: "text"; text: string } | null = null;
const buildAssistantContent = (rawText: string) => {
const positionedTools = toolCallParts
.map((part, index) => {
@@ -6156,6 +6160,7 @@ export function createOpenAIStreamAdapter(
}
}
assembled.push(...runs[boundaries.length]);
+ if (spokenReplyPart) assembled.push(spokenReplyPart);
pinGeminiThoughtReplayParts(assembled, geminiThoughtParts);
pinGeminiAnswerReplayParts(assembled, geminiAnswerParts);
@@ -6815,6 +6820,10 @@ export function createOpenAIStreamAdapter(
};
}
+ const turnAudioBase64 = findLatestUserAudioBase64(
+ currentTurnMessages,
+ !queuedRunSettings && !continuation,
+ );
return {
model: params.checkpoint,
messages: outboundMessages,
@@ -6846,10 +6855,10 @@ export function createOpenAIStreamAdapter(
: { seed: params.seed }),
// Turn-scoped: a stale screenshot here made the backend refuse every later text turn's durable run.
image_base64: findLatestUserImageBase64(currentTurnMessages),
- audio_base64: findLatestUserAudioBase64(
- currentTurnMessages,
- !queuedRunSettings && !continuation,
- ),
+ audio_base64: turnAudioBase64,
+ ...(activeModel?.speaks && turnAudioBase64
+ ? { modalities: ["text", "audio"] }
+ : {}),
extra_audio_base64:
findLatestUserExtraAudioBase64(currentTurnMessages),
video_base64: findLatestUserVideoBase64(currentTurnMessages),
@@ -7835,6 +7844,18 @@ export function createOpenAIStreamAdapter(
)?.extra_content;
// State reaches the message only via a yield, so pace previews but never state changes.
let replayStateChanged = false;
+ const spokenAudio = (
+ chunk.choices?.[0]?.delta as
+ | { audio?: { data?: unknown } }
+ | undefined
+ )?.audio?.data;
+ if (typeof spokenAudio === "string" && spokenAudio) {
+ spokenReplyPart = {
+ type: "text",
+ text: `<audio-player src="data:audio/wav;base64,${spokenAudio}" />`,
+ };
+ replayStateChanged = true;
+ }
let geminiReplayStateChanged = false;
let geminiThoughtSignature: string | undefined;
let capturedGeminiThoughtPart = false;
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index d7e44bbc1fd..39674ea07b0 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -501,6 +501,7 @@ export function syncModelCapabilities(
audio_type?: string | null;
has_audio_input?: boolean;
has_video_input?: boolean;
+ speaks?: boolean;
},
): void {
const store = useChatRuntimeStore.getState();
@@ -514,6 +515,8 @@ export function syncModelCapabilities(
hasAudioInput: Boolean(resp.has_audio_input),
// /api/models/list omits this for the active GGUF row.
hasVideoInput: Boolean(resp.has_video_input),
+
+ speaks: Boolean(resp.speaks),
};
const idx = models.findIndex((m) => m.id === modelId);
if (idx === -1) {
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index 4216abdbc82..5bb1e192c3f 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -108,6 +108,7 @@ function ensureActiveModelInStoreList(
audioType: status.audio_type ?? null,
hasAudioInput: status.has_audio_input ?? false,
hasVideoInput: status.has_video_input ?? false,
+ speaks: status.speaks ?? false,
};
const existing = store.models.find((model) => model.id === checkpointId);
if (existing) {
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 3163053876d..0fc949de1b8 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -2084,6 +2084,7 @@ export function SharedComposer({
audioType: resp.audio_type ?? null,
hasAudioInput: Boolean(resp.has_audio_input),
hasVideoInput: Boolean(resp.has_video_input),
+ speaks: Boolean(resp.speaks),
};
if (idx === -1) {
store.setModels([
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 90c17efc0a1..178e05ee94b 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -233,6 +233,7 @@ export interface LoadModelResponse {
audio_device_fallback?: "cpu" | null;
has_audio_input?: boolean;
has_video_input?: boolean;
+ speaks?: boolean;
inference?: {
temperature?: number;
top_p?: number;
@@ -359,6 +360,8 @@ export interface InferenceStatusResponse {
audio_music?: unknown;
has_audio_input?: boolean;
has_video_input?: boolean;
+ /** Answers a spoken question in its own voice when asked for modalities ["text", "audio"]. */
+ speaks?: boolean;
loading: string[];
loaded: string[];
serving?: string[];
diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts
index 590f97d2e80..d96c38510cc 100644
--- a/studio/frontend/src/features/chat/types/runtime.ts
+++ b/studio/frontend/src/features/chat/types/runtime.ts
@@ -103,6 +103,7 @@ export interface ChatModelSummary {
audioType?: string | null;
hasAudioInput?: boolean;
hasVideoInput?: boolean;
+ speaks?: boolean;
}
export interface ChatLoraSummary {
diff --git a/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts b/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts
index 8f0bd9a95af..8b68b2153a7 100644
--- a/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts
+++ b/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts
@@ -9,6 +9,7 @@ export type QueuedModelCapabilities = Pick<
| "audioType"
| "hasAudioInput"
| "hasVideoInput"
+ | "speaks"
>;
export function mergeQueuedModelCapabilities(