Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep earlier messages when you send audio (#12244)

Nilay90946aace0d2
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index 684d6c8e6de..a9dd6e0f7e6 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -14,7 +14,7 @@ import re
import string
import weakref
from dataclasses import dataclass
-from typing import Optional
+from typing import Any, Optional
logger = logging.getLogger(__name__)
@@ -2919,6 +2919,34 @@ def messages_have_tool_history(messages) -> bool:
)
+def alternating_turns(messages: list) -> list:
+ """User/assistant text turns, alternating and ending on the newest user turn as sent; of two
+ same-role neighbours the later one is kept."""
+ from core.inference.message_content import content_to_text, named_turn
+
+ messages = list(messages or [])
+ newest = max(
+ (i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "user"),
+ default = -1,
+ )
+ turns = []
+ for index, message in enumerate(messages[: newest + 1]):
+ turn = message
+ if index != newest:
+ if not isinstance(message, dict) or message.get("role") not in ("user", "assistant"):
+ continue
+ text = content_to_text(message.get("content")).strip()
+ # Kept even empty: an earlier recording or picture replays as a user turn with no text.
+ if not text and message["role"] == "assistant":
+ continue
+ turn = named_turn({"role": message["role"], "content": text}, message)
+ if turns and turns[-1]["role"] == turn["role"]:
+ turns[-1] = turn
+ elif turns or turn["role"] == "user":
+ turns.append(turn)
+ return turns
+
+
def messages_with_attached_image(
messages: list,
system_prompt: str = "",
@@ -2926,11 +2954,13 @@ def messages_with_attached_image(
structured_content: bool = False,
image: int = 1,
video: bool = False,
+ audio: Any = None,
) -> list:
"""The conversation to render for a turn that carries attached media.
Prepends *system_prompt* as a leading system turn, then injects *image* ``{"type": "image"}``
- parts, or a ``{"type": "video"}`` part, into the LAST user turn and leaves every other turn --
+ parts, or a ``{"type": "video"}`` part, plus any *audio* waveform as an ``{"type": "audio"}``
+ part, into the LAST user turn and leaves every other turn --
assistant ``tool_calls`` and ``role="tool"`` results included -- exactly as the caller sent it.
Rebuilding from the newest user TEXT instead dropped the folded system instruction and the
tool history an OpenAI tool loop replays (#10092). Nothing the caller owns is mutated: callers
@@ -2982,6 +3012,8 @@ def messages_with_attached_image(
)
for _ in range(int(wanted))
]
+ if audio is not None:
+ parts.append({"type": "audio", "audio": audio})
if not parts and not fallback_user_text:
return conversation
for index in range(len(conversation) - 1, -1, -1):
@@ -2990,7 +3022,8 @@ def messages_with_attached_image(
continue
content = message.get("content", "")
if isinstance(content, str):
- content = [{"type": "text", "text": content or fallback_user_text}]
+ text = content if content.strip() else fallback_user_text or content
+ content = [{"type": "text", "text": text}]
elif not isinstance(content, list):
break
elif fallback_user_text and not last_user_text([message]):
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 5487b5671b6..671974a3856 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -40,10 +40,12 @@ from core.inference.chat_eos import (
resolve_chat_turn_end_eos_ids_using,
)
from core.inference.chat_template_helpers import (
+ alternating_turns,
build_dac_tts_prompt,
make_reasoning_normalizer,
detect_reasoning_channel_markers,
detect_think_prefill,
+ messages_with_attached_image,
neutralize_control_markup_in_messages,
neutralize_tts_prompt_text,
prompt_opens_reasoning_channel,
@@ -2010,35 +2012,18 @@ class InferenceBackend:
processor = model_info.get("processor") or model_info.get("tokenizer")
raw_tokenizer = getattr(processor, "tokenizer", processor)
- user_text = "Please transcribe this audio."
- if messages:
- for msg in reversed(messages):
- if msg["role"] == "user" and msg.get("content"):
- user_text = content_to_text(msg["content"])
- break
- # Not the caption scan above: that one falls back past a media-only turn.
- last_user = next(
- (m for m in reversed(messages or []) if m.get("role") == "user"),
- None,
- )
-
if not system_prompt:
system_prompt = "You are an assistant that transcribes speech accurately."
# Gemma 3n format — audio goes INTO apply_chat_template
- audio_messages = [
- {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
- named_turn(
- {
- "role": "user",
- "content": [
- {"type": "audio", "audio": audio_array},
- {"type": "text", "text": user_text},
- ],
- },
- last_user,
- ),
- ]
+ audio_messages = messages_with_attached_image(
+ alternating_turns(messages),
+ system_prompt = system_prompt,
+ fallback_user_text = "Please transcribe this audio.",
+ structured_content = True,
+ image = 0,
+ audio = audio_array,
+ )
# Direct processor render like the vision path, so neutralize here too, with
# this processor's own profile so another family's marker stays untouched (#7066).
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index d52bcc0a81b..ea705c4de5e 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -17,7 +17,6 @@ from contextlib import ExitStack, closing, contextmanager, nullcontext
from typing import Optional, Generator
from core.inference.worker import RowRefused
from core.inference import context_refusal
-from core.inference.message_content import content_to_text
from core.inference.native_tool_tokens import (
NativeToolTokenDecoder,
closes_an_open_envelope,
@@ -33,6 +32,7 @@ from core.inference.runtime_context import (
)
from core.inference.chat_template_helpers import (
# Aliased to this module's historic names; bodies moved to the shared helper (#10092).
+ alternating_turns,
count_structured_images as _count_vlm_images,
count_structured_videos as _count_vlm_videos,
detect_reasoning_channel_markers,
@@ -6291,22 +6291,16 @@ class MLXInferenceBackend:
from mlx_vlm import stream_generate as vlm_stream
- # Only the CURRENT user turn may caption the audio; never older history.
- user_text = ""
- for msg in reversed(messages or []):
- if isinstance(msg, dict) and msg.get("role") == "user":
- user_text = content_to_text(msg.get("content") or "").strip()
- break
- if not user_text:
- user_text = "Please transcribe this audio."
if not system_prompt:
system_prompt = "You are an assistant that transcribes speech accurately."
- # No name: mlx-vlm rebuilds non-tool turns from role and text, dropping anything else.
- audio_messages = [
- {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
- {"role": "user", "content": [{"type": "audio"}, {"type": "text", "text": user_text}]},
- ]
+ audio_messages = messages_with_attached_image(
+ alternating_turns(messages),
+ system_prompt = system_prompt,
+ fallback_user_text = "Please transcribe this audio.",
+ image = 0,
+ audio = audio_array,
+ )
prompt = _render_registered_vlm_prompt(
self._processor,
self._model,
diff --git a/studio/backend/tests/test_generation_budget.py b/studio/backend/tests/test_generation_budget.py
index e60b5a1ed70..0e896cac676 100644
--- a/studio/backend/tests/test_generation_budget.py
+++ b/studio/backend/tests/test_generation_budget.py
@@ -313,7 +313,11 @@ def test_the_audio_input_path_resolves_an_unset_budget(monkeypatch):
assert model.calls[0]["max_new_tokens"] == _WINDOW - _PROMPT_LEN
-def _audio_turns(monkeypatch, messages):
+def _rendered_audio_messages(
+ monkeypatch,
+ messages,
+ audio = None,
+):
backend, _model = _audio_backend(monkeypatch)
processor = backend.models[backend.active_model_name]["processor"]
render = processor.apply_chat_template
@@ -326,10 +330,14 @@ def _audio_turns(monkeypatch, messages):
monkeypatch.setattr(processor, "apply_chat_template", _capture, raising = False)
list(
backend.generate_audio_input_response(
- messages, "be brief", object(), 0.0, 1.0, 0, 0.0, 8, 1.0
+ messages, "be brief", audio or object(), 0.0, 1.0, 0, 0.0, 8, 1.0
)
)
- return [(m["role"], m.get("name")) for m in seen["messages"]]
+ return seen["messages"]
+
+
+def _audio_turns(monkeypatch, messages):
+ return [(m["role"], m.get("name")) for m in _rendered_audio_messages(monkeypatch, messages)]
def test_the_audio_input_turns_keep_participant_names(monkeypatch):
@@ -339,6 +347,28 @@ def test_the_audio_input_turns_keep_participant_names(monkeypatch):
]
+def test_the_audio_turn_keeps_its_text_and_ends_the_prompt(monkeypatch):
+ audio = object()
+ rendered = _rendered_audio_messages(
+ monkeypatch,
+ [
+ {"role": "user", "content": "Summarize this"},
+ {"role": "assistant", "content": "partial"},
+ ],
+ audio,
+ )
+
+ assert rendered[1:] == [
+ {
+ "role": "user",
+ "content": [
+ {"type": "audio", "audio": audio},
+ {"type": "text", "text": "Summarize this"},
+ ],
+ }
+ ]
+
+
def test_the_audio_turn_is_named_for_whoever_recorded_it(monkeypatch):
assert _audio_turns(
monkeypatch,
@@ -347,7 +377,109 @@ def test_the_audio_turn_is_named_for_whoever_recorded_it(monkeypatch):
{"role": "assistant", "content": "hello"},
{"role": "user", "name": "bob", "content": ""},
],
- ) == [("system", None), ("user", "bob")]
+ ) == [("system", None), ("user", "alice"), ("assistant", None), ("user", "bob")]
+
+
+def test_an_audio_only_turn_keeps_the_conversation_before_it(monkeypatch):
+ audio = object()
+ rendered = _rendered_audio_messages(
+ monkeypatch,
+ [
+ {"role": "user", "content": "Write a haiku about the sea"},
+ {"role": "assistant", "content": "Waves fold into foam"},
+ {"role": "user", "content": ""},
+ ],
+ audio,
+ )
+
+ assert rendered[1:] == [
+ {"role": "user", "content": [{"type": "text", "text": "Write a haiku about the sea"}]},
+ {"role": "assistant", "content": [{"type": "text", "text": "Waves fold into foam"}]},
+ {
+ "role": "user",
+ "content": [
+ {"type": "audio", "audio": audio},
+ {"type": "text", "text": "Please transcribe this audio."},
+ ],
+ },
+ ]
+
+
+def test_an_earlier_audio_turn_keeps_its_reply(monkeypatch):
+ audio = object()
+ rendered = _rendered_audio_messages(
+ monkeypatch,
+ [
+ {"role": "user", "content": "Write a haiku about the sea"},
+ {"role": "assistant", "content": "Waves fold into foam"},
+ {"role": "user", "content": ""},
+ {"role": "assistant", "content": "Please count from one to five."},
+ {"role": "user", "content": ""},
+ ],
+ audio,
+ )
+
+ assert rendered[1:] == [
+ {"role": "user", "content": [{"type": "text", "text": "Write a haiku about the sea"}]},
+ {"role": "assistant", "content": [{"type": "text", "text": "Waves fold into foam"}]},
+ {"role": "user", "content": []},
+ {
+ "role": "assistant",
+ "content": [{"type": "text", "text": "Please count from one to five."}],
+ },
+ {
+ "role": "user",
+ "content": [
+ {"type": "audio", "audio": audio},
+ {"type": "text", "text": "Please transcribe this audio."},
+ ],
+ },
+ ]
+
+
+ "history, roles",
+ [
+ ([{"role": "user", "content": "My name is Nilay"}], ["system", "user"]),
+ ([{"role": "assistant", "content": "Hi, how can I help?"}], ["system", "user"]),
+ (
+ [{"role": "user", "content": "My name is Nilay"}, {"role": "assistant", "content": ""}],
+ ["system", "user"],
+ ),
+ (
+ [
+ {"role": "user", "content": "Weather in Paris?"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {"name": "weather", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "c1", "content": "sunny"},
+ {"role": "assistant", "content": "It is sunny."},
+ ],
+ ["system", "user", "assistant", "user"],
+ ),
+ ],
+)
+def test_an_audio_turn_after_an_unanswered_or_tool_turn_still_alternates(
+ monkeypatch, history, roles
+):
+ audio = object()
+ rendered = _rendered_audio_messages(
+ monkeypatch, [*history, {"role": "user", "content": ""}], audio
+ )
+
+ assert [m["role"] for m in rendered] == roles
+ assert rendered[-1]["content"] == [
+ {"type": "audio", "audio": audio},
+ {"type": "text", "text": "Please transcribe this audio."},
+ ]
def test_an_explicit_audio_budget_is_untouched(monkeypatch):
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index f543b369e19..1bb682c95c6 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -2753,15 +2753,78 @@ def test_mlx_generate_audio_input_deltas_and_reject(monkeypatch):
assert list(backend.generate_audio_input_response(**args)) == ["H", "e", "l"]
assert calls["kwargs"]["audio"] == [audio] and calls["kwargs"]["temperature"] == 0.0
assert calls["audios"] == 1 and backend.last_generation_stats is not None
+ assert calls["messages"][-1]["content"][1:] == [{"type": "text", "text": "what is said?"}]
- # Audio-only current turn → transcribe default, never older-turn text.
+ args["messages"] = [{"role": "user", "content": " "}]
+ list(backend.generate_audio_input_response(**args))
+ assert calls["messages"][-1]["content"][1:] == [
+ {"type": "text", "text": "Please transcribe this audio."}
+ ]
+
+ # A recording without text is captioned by the transcribe default, never an older question.
args["messages"] = [
{"role": "user", "content": "old unrelated question"},
- {"role": "user", "content": [{"type": "audio"}]},
+ {"role": "assistant", "content": "old answer"},
+ {"role": "user", "content": ""},
]
list(backend.generate_audio_input_response(**args))
- assert "Please transcribe this audio." in str(calls["messages"])
- assert "old unrelated question" not in str(calls["messages"])
+ assert calls["messages"][1:] == [
+ {"role": "user", "content": "old unrelated question"},
+ {"role": "assistant", "content": "old answer"},
+ {
+ "role": "user",
+ "content": [
+ {"type": "audio", "audio": audio},
+ {"type": "text", "text": "Please transcribe this audio."},
+ ],
+ },
+ ]
+
+ audio_turn = {
+ "role": "user",
+ "content": [
+ {"type": "audio", "audio": audio},
+ {"type": "text", "text": "Please transcribe this audio."},
+ ],
+ }
+ for history, expected in (
+ ([{"role": "user", "content": "My name is Nilay"}], [audio_turn]),
+ (
+ [{"role": "user", "content": "My name is Nilay"}, {"role": "assistant", "content": ""}],
+ [audio_turn],
+ ),
+ (
+ [
+ {"role": "user", "content": "Weather in Paris?"},
+ {"role": "assistant", "content": None, "tool_calls": [{"id": "c1"}]},
+ {"role": "tool", "tool_call_id": "c1", "content": "sunny"},
+ {"role": "assistant", "content": "It is sunny."},
+ ],
+ [
+ {"role": "user", "content": "Weather in Paris?"},
+ {"role": "assistant", "content": "It is sunny."},
+ audio_turn,
+ ],
+ ),
+ (
+ [
+ {"role": "user", "content": "Write a haiku"},
+ {"role": "assistant", "content": "Waves fold into foam"},
+ {"role": "user", "content": ""},
+ {"role": "assistant", "content": "Please count from one to five."},
+ ],
+ [
+ {"role": "user", "content": "Write a haiku"},
+ {"role": "assistant", "content": "Waves fold into foam"},
+ {"role": "user", "content": ""},
+ {"role": "assistant", "content": "Please count from one to five."},
+ audio_turn,
+ ],
+ ),
+ ):
+ args["messages"] = [*history, {"role": "user", "content": ""}]
+ list(backend.generate_audio_input_response(**args))
+ assert calls["messages"][1:] == expected
backend.models["m"]["audio_type"] = None
with pytest.raises(RuntimeError, match = "not supported .* MLX"):