Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep earlier messages when you send audio (#12244)
Nilay90946aace0d2
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.pyindex 684d6c8e6de..a9dd6e0f7e6 100644--- a/studio/backend/core/inference/chat_template_helpers.py+++ b/studio/backend/core/inference/chat_template_helpers.py@@ -14,7 +14,7 @@ import reimport stringimport weakreffrom dataclasses import dataclass-from typing import Optional+from typing import Any, Optionallogger = logging.getLogger(__name__)@@ -2919,6 +2919,34 @@ def messages_have_tool_history(messages) -> bool:)+def alternating_turns(messages: list) -> list:+ """User/assistant text turns, alternating and ending on the newest user turn as sent; of two+ same-role neighbours the later one is kept."""+ from core.inference.message_content import content_to_text, named_turn++ messages = list(messages or [])+ newest = max(+ (i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "user"),+ default = -1,+ )+ turns = []+ for index, message in enumerate(messages[: newest + 1]):+ turn = message+ if index != newest:+ if not isinstance(message, dict) or message.get("role") not in ("user", "assistant"):+ continue+ text = content_to_text(message.get("content")).strip()+ # Kept even empty: an earlier recording or picture replays as a user turn with no text.+ if not text and message["role"] == "assistant":+ continue+ turn = named_turn({"role": message["role"], "content": text}, message)+ if turns and turns[-1]["role"] == turn["role"]:+ turns[-1] = turn+ elif turns or turn["role"] == "user":+ turns.append(turn)+ return turns++def messages_with_attached_image(messages: list,system_prompt: str = "",@@ -2926,11 +2954,13 @@ def messages_with_attached_image(structured_content: bool = False,image: int = 1,video: bool = False,+ audio: Any = None,) -> list:"""The conversation to render for a turn that carries attached media.Prepends *system_prompt* as a leading system turn, then injects *image* ``{"type": "image"}``- parts, or a ``{"type": "video"}`` part, into the LAST user turn and leaves every other turn --+ parts, or a ``{"type": "video"}`` part, plus any *audio* waveform as an ``{"type": "audio"}``+ part, into the LAST user turn and leaves every other turn --assistant ``tool_calls`` and ``role="tool"`` results included -- exactly as the caller sent it.Rebuilding from the newest user TEXT instead dropped the folded system instruction and thetool history an OpenAI tool loop replays (#10092). Nothing the caller owns is mutated: callers@@ -2982,6 +3012,8 @@ def messages_with_attached_image()for _ in range(int(wanted))]+ if audio is not None:+ parts.append({"type": "audio", "audio": audio})if not parts and not fallback_user_text:return conversationfor index in range(len(conversation) - 1, -1, -1):@@ -2990,7 +3022,8 @@ def messages_with_attached_image(continuecontent = message.get("content", "")if isinstance(content, str):- content = [{"type": "text", "text": content or fallback_user_text}]+ text = content if content.strip() else fallback_user_text or content+ content = [{"type": "text", "text": text}]elif not isinstance(content, list):breakelif fallback_user_text and not last_user_text([message]):diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 5487b5671b6..671974a3856 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -40,10 +40,12 @@ from core.inference.chat_eos import (resolve_chat_turn_end_eos_ids_using,)from core.inference.chat_template_helpers import (+ alternating_turns,build_dac_tts_prompt,make_reasoning_normalizer,detect_reasoning_channel_markers,detect_think_prefill,+ messages_with_attached_image,neutralize_control_markup_in_messages,neutralize_tts_prompt_text,prompt_opens_reasoning_channel,@@ -2010,35 +2012,18 @@ class InferenceBackend:processor = model_info.get("processor") or model_info.get("tokenizer")raw_tokenizer = getattr(processor, "tokenizer", processor)- user_text = "Please transcribe this audio."- if messages:- for msg in reversed(messages):- if msg["role"] == "user" and msg.get("content"):- user_text = content_to_text(msg["content"])- break- # Not the caption scan above: that one falls back past a media-only turn.- last_user = next(- (m for m in reversed(messages or []) if m.get("role") == "user"),- None,- )-if not system_prompt:system_prompt = "You are an assistant that transcribes speech accurately."# Gemma 3n format — audio goes INTO apply_chat_template- audio_messages = [- {"role": "system", "content": [{"type": "text", "text": system_prompt}]},- named_turn(- {- "role": "user",- "content": [- {"type": "audio", "audio": audio_array},- {"type": "text", "text": user_text},- ],- },- last_user,- ),- ]+ audio_messages = messages_with_attached_image(+ alternating_turns(messages),+ system_prompt = system_prompt,+ fallback_user_text = "Please transcribe this audio.",+ structured_content = True,+ image = 0,+ audio = audio_array,+ )# Direct processor render like the vision path, so neutralize here too, with# this processor's own profile so another family's marker stays untouched (#7066).diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex d52bcc0a81b..ea705c4de5e 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -17,7 +17,6 @@ from contextlib import ExitStack, closing, contextmanager, nullcontextfrom typing import Optional, Generatorfrom core.inference.worker import RowRefusedfrom core.inference import context_refusal-from core.inference.message_content import content_to_textfrom core.inference.native_tool_tokens import (NativeToolTokenDecoder,closes_an_open_envelope,@@ -33,6 +32,7 @@ from core.inference.runtime_context import ()from core.inference.chat_template_helpers import (# Aliased to this module's historic names; bodies moved to the shared helper (#10092).+ alternating_turns,count_structured_images as _count_vlm_images,count_structured_videos as _count_vlm_videos,detect_reasoning_channel_markers,@@ -6291,22 +6291,16 @@ class MLXInferenceBackend:from mlx_vlm import stream_generate as vlm_stream- # Only the CURRENT user turn may caption the audio; never older history.- user_text = ""- for msg in reversed(messages or []):- if isinstance(msg, dict) and msg.get("role") == "user":- user_text = content_to_text(msg.get("content") or "").strip()- break- if not user_text:- user_text = "Please transcribe this audio."if not system_prompt:system_prompt = "You are an assistant that transcribes speech accurately."- # No name: mlx-vlm rebuilds non-tool turns from role and text, dropping anything else.- audio_messages = [- {"role": "system", "content": [{"type": "text", "text": system_prompt}]},- {"role": "user", "content": [{"type": "audio"}, {"type": "text", "text": user_text}]},- ]+ audio_messages = messages_with_attached_image(+ alternating_turns(messages),+ system_prompt = system_prompt,+ fallback_user_text = "Please transcribe this audio.",+ image = 0,+ audio = audio_array,+ )prompt = _render_registered_vlm_prompt(self._processor,self._model,diff --git a/studio/backend/tests/test_generation_budget.py b/studio/backend/tests/test_generation_budget.pyindex e60b5a1ed70..0e896cac676 100644--- a/studio/backend/tests/test_generation_budget.py+++ b/studio/backend/tests/test_generation_budget.py@@ -313,7 +313,11 @@ def test_the_audio_input_path_resolves_an_unset_budget(monkeypatch):assert model.calls[0]["max_new_tokens"] == _WINDOW - _PROMPT_LEN-def _audio_turns(monkeypatch, messages):+def _rendered_audio_messages(+ monkeypatch,+ messages,+ audio = None,+):backend, _model = _audio_backend(monkeypatch)processor = backend.models[backend.active_model_name]["processor"]render = processor.apply_chat_template@@ -326,10 +330,14 @@ def _audio_turns(monkeypatch, messages):monkeypatch.setattr(processor, "apply_chat_template", _capture, raising = False)list(backend.generate_audio_input_response(- messages, "be brief", object(), 0.0, 1.0, 0, 0.0, 8, 1.0+ messages, "be brief", audio or object(), 0.0, 1.0, 0, 0.0, 8, 1.0))- return [(m["role"], m.get("name")) for m in seen["messages"]]+ return seen["messages"]+++def _audio_turns(monkeypatch, messages):+ return [(m["role"], m.get("name")) for m in _rendered_audio_messages(monkeypatch, messages)]def test_the_audio_input_turns_keep_participant_names(monkeypatch):@@ -339,6 +347,28 @@ def test_the_audio_input_turns_keep_participant_names(monkeypatch):]+def test_the_audio_turn_keeps_its_text_and_ends_the_prompt(monkeypatch):+ audio = object()+ rendered = _rendered_audio_messages(+ monkeypatch,+ [+ {"role": "user", "content": "Summarize this"},+ {"role": "assistant", "content": "partial"},+ ],+ audio,+ )++ assert rendered[1:] == [+ {+ "role": "user",+ "content": [+ {"type": "audio", "audio": audio},+ {"type": "text", "text": "Summarize this"},+ ],+ }+ ]++def test_the_audio_turn_is_named_for_whoever_recorded_it(monkeypatch):assert _audio_turns(monkeypatch,@@ -347,7 +377,109 @@ def test_the_audio_turn_is_named_for_whoever_recorded_it(monkeypatch):{"role": "assistant", "content": "hello"},{"role": "user", "name": "bob", "content": ""},],- ) == [("system", None), ("user", "bob")]+ ) == [("system", None), ("user", "alice"), ("assistant", None), ("user", "bob")]+++def test_an_audio_only_turn_keeps_the_conversation_before_it(monkeypatch):+ audio = object()+ rendered = _rendered_audio_messages(+ monkeypatch,+ [+ {"role": "user", "content": "Write a haiku about the sea"},+ {"role": "assistant", "content": "Waves fold into foam"},+ {"role": "user", "content": ""},+ ],+ audio,+ )++ assert rendered[1:] == [+ {"role": "user", "content": [{"type": "text", "text": "Write a haiku about the sea"}]},+ {"role": "assistant", "content": [{"type": "text", "text": "Waves fold into foam"}]},+ {+ "role": "user",+ "content": [+ {"type": "audio", "audio": audio},+ {"type": "text", "text": "Please transcribe this audio."},+ ],+ },+ ]+++def test_an_earlier_audio_turn_keeps_its_reply(monkeypatch):+ audio = object()+ rendered = _rendered_audio_messages(+ monkeypatch,+ [+ {"role": "user", "content": "Write a haiku about the sea"},+ {"role": "assistant", "content": "Waves fold into foam"},+ {"role": "user", "content": ""},+ {"role": "assistant", "content": "Please count from one to five."},+ {"role": "user", "content": ""},+ ],+ audio,+ )++ assert rendered[1:] == [+ {"role": "user", "content": [{"type": "text", "text": "Write a haiku about the sea"}]},+ {"role": "assistant", "content": [{"type": "text", "text": "Waves fold into foam"}]},+ {"role": "user", "content": []},+ {+ "role": "assistant",+ "content": [{"type": "text", "text": "Please count from one to five."}],+ },+ {+ "role": "user",+ "content": [+ {"type": "audio", "audio": audio},+ {"type": "text", "text": "Please transcribe this audio."},+ ],+ },+ ]+++ "history, roles",+ [+ ([{"role": "user", "content": "My name is Nilay"}], ["system", "user"]),+ ([{"role": "assistant", "content": "Hi, how can I help?"}], ["system", "user"]),+ (+ [{"role": "user", "content": "My name is Nilay"}, {"role": "assistant", "content": ""}],+ ["system", "user"],+ ),+ (+ [+ {"role": "user", "content": "Weather in Paris?"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "c1",+ "type": "function",+ "function": {"name": "weather", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "c1", "content": "sunny"},+ {"role": "assistant", "content": "It is sunny."},+ ],+ ["system", "user", "assistant", "user"],+ ),+ ],+)+def test_an_audio_turn_after_an_unanswered_or_tool_turn_still_alternates(+ monkeypatch, history, roles+):+ audio = object()+ rendered = _rendered_audio_messages(+ monkeypatch, [*history, {"role": "user", "content": ""}], audio+ )++ assert [m["role"] for m in rendered] == roles+ assert rendered[-1]["content"] == [+ {"type": "audio", "audio": audio},+ {"type": "text", "text": "Please transcribe this audio."},+ ]def test_an_explicit_audio_budget_is_untouched(monkeypatch):diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex f543b369e19..1bb682c95c6 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -2753,15 +2753,78 @@ def test_mlx_generate_audio_input_deltas_and_reject(monkeypatch):assert list(backend.generate_audio_input_response(**args)) == ["H", "e", "l"]assert calls["kwargs"]["audio"] == [audio] and calls["kwargs"]["temperature"] == 0.0assert calls["audios"] == 1 and backend.last_generation_stats is not None+ assert calls["messages"][-1]["content"][1:] == [{"type": "text", "text": "what is said?"}]- # Audio-only current turn → transcribe default, never older-turn text.+ args["messages"] = [{"role": "user", "content": " "}]+ list(backend.generate_audio_input_response(**args))+ assert calls["messages"][-1]["content"][1:] == [+ {"type": "text", "text": "Please transcribe this audio."}+ ]++ # A recording without text is captioned by the transcribe default, never an older question.args["messages"] = [{"role": "user", "content": "old unrelated question"},- {"role": "user", "content": [{"type": "audio"}]},+ {"role": "assistant", "content": "old answer"},+ {"role": "user", "content": ""},]list(backend.generate_audio_input_response(**args))- assert "Please transcribe this audio." in str(calls["messages"])- assert "old unrelated question" not in str(calls["messages"])+ assert calls["messages"][1:] == [+ {"role": "user", "content": "old unrelated question"},+ {"role": "assistant", "content": "old answer"},+ {+ "role": "user",+ "content": [+ {"type": "audio", "audio": audio},+ {"type": "text", "text": "Please transcribe this audio."},+ ],+ },+ ]++ audio_turn = {+ "role": "user",+ "content": [+ {"type": "audio", "audio": audio},+ {"type": "text", "text": "Please transcribe this audio."},+ ],+ }+ for history, expected in (+ ([{"role": "user", "content": "My name is Nilay"}], [audio_turn]),+ (+ [{"role": "user", "content": "My name is Nilay"}, {"role": "assistant", "content": ""}],+ [audio_turn],+ ),+ (+ [+ {"role": "user", "content": "Weather in Paris?"},+ {"role": "assistant", "content": None, "tool_calls": [{"id": "c1"}]},+ {"role": "tool", "tool_call_id": "c1", "content": "sunny"},+ {"role": "assistant", "content": "It is sunny."},+ ],+ [+ {"role": "user", "content": "Weather in Paris?"},+ {"role": "assistant", "content": "It is sunny."},+ audio_turn,+ ],+ ),+ (+ [+ {"role": "user", "content": "Write a haiku"},+ {"role": "assistant", "content": "Waves fold into foam"},+ {"role": "user", "content": ""},+ {"role": "assistant", "content": "Please count from one to five."},+ ],+ [+ {"role": "user", "content": "Write a haiku"},+ {"role": "assistant", "content": "Waves fold into foam"},+ {"role": "user", "content": ""},+ {"role": "assistant", "content": "Please count from one to five."},+ audio_turn,+ ],+ ),+ ):+ args["messages"] = [*history, {"role": "user", "content": ""}]+ list(backend.generate_audio_input_response(**args))+ assert calls["messages"][1:] == expectedbackend.models["m"]["audio_type"] = Nonewith pytest.raises(RuntimeError, match = "not supported .* MLX"):