Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: accept the standard input_audio message part (#10255)

Nilaye1f8d0cc1b2b
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index c777bfa09ef..5d7100ca98b 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1769,20 +1769,64 @@ class CompactionContentPart(BaseModel):
)
+class InputAudio(BaseModel):
+ # Non-empty: an empty payload is not lifted, so the turn would otherwise proceed as
+ # text alone and answer "transcribe this" about a recording that was never sent.
+ data: str = Field(
+ ..., min_length = 1, description = "Base64-encoded audio, without a data: prefix."
+ )
+ format: Optional[str] = Field(
+ None, description = 'Declared container, e.g. "wav"; the decoder sniffs it anyway.'
+ )
+
+
+class InputAudioContentPart(BaseModel):
+ """Audio content part in a multimodal message, in OpenAI's documented shape."""
+
+ type: Literal["input_audio"]
+ input_audio: InputAudio
+
+
+class UnknownContentPart(BaseModel):
+ """Catch-all for unmodelled part types, mirroring ``ResponsesUnknownContentPart``."""
+
+ type: str
+
+ model_config = {"extra": "allow"}
+
+
+_KNOWN_CONTENT_PART_TAGS = frozenset(
+ {
+ "text",
+ "image_url",
+ "input_audio",
+ "input_document",
+ "reasoning",
+ "image_generation_call",
+ "compaction",
+ }
+)
+
+
def _content_part_discriminator(v):
- if isinstance(v, dict):
- return v.get("type")
- return getattr(v, "type", None)
+ tag = v.get("type") if isinstance(v, dict) else getattr(v, "type", None)
+ # A list or dict tag is unhashable, so testing membership would raise TypeError out of
+ # request validation as a 500. Declining to name a member leaves pydantic to report it.
+ if not isinstance(tag, str):
+ return None
+ return tag if tag in _KNOWN_CONTENT_PART_TAGS else "unknown"
ContentPart = Annotated[
Union[
Annotated[TextContentPart, Tag("text")],
Annotated[ImageContentPart, Tag("image_url")],
+ Annotated[InputAudioContentPart, Tag("input_audio")],
Annotated[InputDocumentContentPart, Tag("input_document")],
Annotated[OpenAIReasoningContentPart, Tag("reasoning")],
Annotated[ImageGenerationCallContentPart, Tag("image_generation_call")],
Annotated[CompactionContentPart, Tag("compaction")],
+ Annotated[UnknownContentPart, Tag("unknown")],
],
Discriminator(_content_part_discriminator),
]
diff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.py
index f5846399550..56446294018 100644
--- a/studio/backend/routes/chat_generation_runs.py
+++ b/studio/backend/routes/chat_generation_runs.py
@@ -148,6 +148,12 @@ def _sanitize_request(payload: CreateChatGenerationRun) -> dict[str, Any]:
status_code = 422,
detail = safe_validation_errors(exc.errors()),
) from exc
+ # The refusal the completion route runs on entry. Without it a part no path can serve is
+ # queued at 202 and fails asynchronously in the supervisor, where the caller cannot see it.
+ from routes.inference import _messages_have_input_audio, _reject_unsupported_content_parts
+
+ _reject_unsupported_content_parts(request)
+
# Message content/reasoning are user-authored data, not routing configuration. Scan every
# other persisted field, including extra message-envelope fields, with the credential policy.
durable_config = {
@@ -172,8 +178,10 @@ def _sanitize_request(payload: CreateChatGenerationRun) -> dict[str, Any]:
detail = "Durable chat runs are available only for local inference",
)
# A media turn has no replayable transcript and its payload persists verbatim, so a base64 blob would live in
- # request_json for the life of the thread.
- if any(raw.get(field) not in (None, "") for field in _MEDIA_FIELDS):
+ # request_json for the life of the thread. A recording rides in a content part too, not just the field.
+ if any(
+ raw.get(field) not in (None, "") for field in _MEDIA_FIELDS
+ ) or _messages_have_input_audio(request.messages):
raise HTTPException(
status_code = 400,
detail = "Media chat runs use the legacy streaming path",
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index f632b86bf1c..0a2fcad9b5d 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3050,6 +3050,8 @@ from models.inference import (
InstallLatestTransformersResponse,
TextContentPart,
ImageContentPart,
+ InputAudioContentPart,
+ UnknownContentPart,
ImageUrl,
ResponsesRequest,
ResponsesInputTextPart,
@@ -17284,6 +17286,17 @@ async def generate_audio(
Works with both GGUF (llama-server) and Unsloth/transformers backends."""
import base64
+ # _extract_content_parts keeps only text, so a part this cannot voice would be spoken past
+ # in silence. Refuse it the way the completion does rather than voicing the text alone.
+ _reject_unsupported_content_parts(payload)
+ # Also the field: /chat/completions routes a TTS model here after the lift has already
+ # replaced the part with audio_base64, so a parts-only guard would not see the attachment.
+ if _messages_have_input_audio(payload.messages) or getattr(payload, "audio_base64", None):
+ _raise_unsupported_openai_parameter(
+ "messages",
+ "Audio input is not supported here; this route speaks the message text.",
+ )
+
# Extract text from the last user message
_, chat_messages, _ = _extract_content_parts(payload.messages)
if not chat_messages:
@@ -19124,6 +19137,103 @@ def _inject_video_part(messages: list[dict], video_b64: str) -> None:
return
+def _reject_unsupported_content_parts(payload) -> None:
+ """Refuse content parts no route can serve, before local or external routing.
+
+ The external proxy rebuilds messages from an allowlist, so a part left standing here is
+ dropped silently and the provider answers a prompt the caller did not send.
+ """
+ for msg in payload.messages:
+ if not isinstance(msg.content, list):
+ continue
+ for part in msg.content:
+ if isinstance(part, UnknownContentPart):
+ _raise_unsupported_openai_parameter(
+ "messages",
+ f"Message content parts of type '{part.type}' are not supported.",
+ )
+ _reject_misplaced_audio_parts(payload)
+
+
+def _reject_misplaced_audio_parts(payload) -> None:
+ """Refuse a recording the single ``audio_base64`` field cannot carry faithfully.
+
+ Placement is decided here rather than in the lift because the lift runs behind routing --
+ the preview route reaches it only after taking the preview lock and loading a checkpoint,
+ so a request that was always going to 400 would evict the resident model first.
+ """
+ last_user = None
+ for index, msg in enumerate(payload.messages):
+ if msg.role == "user":
+ last_user = index
+ # Only a user turn can carry a recording into the model, and the lift clears the part from
+ # every role. Say so rather than deleting an assistant-history clip in silence.
+ for msg in payload.messages:
+ if msg.role == "user" or not isinstance(msg.content, list):
+ continue
+ if any(isinstance(part, InputAudioContentPart) for part in msg.content):
+ _raise_unsupported_openai_parameter(
+ "messages",
+ f"Audio input is supported on a user message, not on a '{msg.role}' one.",
+ )
+ parts = [
+ (index, part)
+ for index, msg in enumerate(payload.messages)
+ if isinstance(msg.content, list) and msg.role == "user"
+ for part in msg.content
+ if isinstance(part, InputAudioContentPart)
+ ]
+ if len(parts) > 1:
+ _raise_unsupported_openai_parameter(
+ "messages",
+ "Only one audio recording per request is supported, and this one carries "
+ f"{len(parts)}.",
+ )
+ if parts and parts[0][0] != last_user:
+ _raise_unsupported_openai_parameter(
+ "messages",
+ "Audio input is supported on the latest user message, and this one carries it on an "
+ "earlier turn. Re-send the recording on the current turn.",
+ )
+
+
+def _messages_have_input_audio(messages) -> bool:
+ return any(
+ isinstance(getattr(msg, "content", None), list)
+ and any(isinstance(part, InputAudioContentPart) for part in msg.content)
+ for msg in messages
+ )
+
+
+def _normalise_chat_content_parts(payload) -> None:
+ """Lift the request's ``input_audio`` part onto ``audio_base64``, in place.
+
+ Everything that makes audio safe to serve reads that field: the capability check that keeps
+ a text-only target from being loaded for it, the size bound, the decoder, the duration limit,
+ and /chat/count_tokens' refusal. So the part is lifted rather than left standing -- a part the
+ field never sees is a recording none of those checks can act on.
+
+ ``_reject_misplaced_audio_parts`` has already refused every shape the single field cannot
+ carry faithfully, so what reaches here is one recording on the latest user turn. An explicit
+ ``audio_base64`` still wins over a part.
+ """
+ lifted = None
+ for msg in payload.messages:
+ if not isinstance(msg.content, list):
+ continue
+ kept = []
+ for part in msg.content:
+ if isinstance(part, InputAudioContentPart):
+ if msg.role == "user" and part.input_audio.data:
+ lifted = part.input_audio.data
+ continue
+ kept.append(part)
+ if len(kept) != len(msg.content):
+ msg.content = kept
+ if lifted and not getattr(payload, "audio_base64", None):
+ payload.audio_base64 = lifted
+
+
def _inject_audio_part(messages: list[dict], audio_b64: str, audio_format: str) -> None:
"""Append an input_audio part to the last user message, in place.
@@ -20932,6 +21042,11 @@ async def produce_openai_chat_completions(
"top_logprobs", "top_logprobs is not supported for chat completions."
)
+ # Before routing splits: the external proxy rebuilds messages from an allowlist and the
+ # local path flattens them, so a part neither can serve has to be refused while both are
+ # still reachable. Otherwise it is dropped in silence and the answer looks legitimate.
+ _reject_unsupported_content_parts(payload)
+
# ── External provider routing ────────────────────────────────
# encrypted_api_key is optional -- local providers (llama.cpp / vLLM / Ollama) may run without auth.
if payload.provider_id or payload.provider_type:
@@ -20958,8 +21073,19 @@ async def produce_openai_chat_completions(
status_code = 400,
detail = "Video input is only supported on a local GGUF model with video support.",
)
+ # _build_external_messages carries no input_audio case, so the recording would be
+ # stripped and the provider would answer the text alone -- a plausible reply to a
+ # question about audio nobody heard. Refuse it the way video is refused.
+ if _messages_have_input_audio(payload.messages):
+ raise HTTPException(
+ status_code = 400,
+ detail = "Audio input is only supported on a local model with audio support.",
+ )
return await _proxy_to_external_provider(payload, request, current_subject)
+ # Local path only: the lift targets audio_base64, which the proxy never reads.
+ _normalise_chat_content_parts(payload)
+
# Reject a malformed function tool here: it would otherwise reach
# llama-server and surface as an opaque 500 "Failed to parse tools".
if payload.tools:
@@ -21326,6 +21452,17 @@ async def produce_openai_chat_completions(
subject = current_subject,
)
+ # A recording the resident checkpoint cannot read. The capability check that would
+ # have caught this runs only when an automatic load could have fixed it, so with
+ # auto-switch off the branch below is simply skipped and the turn is answered from
+ # its text -- a plausible reply about audio nobody listened to.
+ if payload.audio_base64 and not model_info.get("has_audio_input"):
+ _audio_unsupported_detail = (
+ "The loaded model cannot read audio input. Load a model with audio support."
+ )
+ api_monitor.fail(monitor_id, _audio_unsupported_detail)
+ raise HTTPException(status_code = 400, detail = _audio_unsupported_detail)
+
# ── Audio INPUT path: decode WAV and route to audio input generation ──
if payload.audio_base64 and model_info.get("has_audio_input"):
# This route re-listens to the recording and answers afresh, so there is
@@ -29493,6 +29630,10 @@ async def chat_count_tokens(
for _m in payload.messages:
if _m.role == "developer":
_m.role = "system"
+ # And the same refusal and lift, so a part this cannot price is refused the way the
+ # completion refuses it, and the audio guard below sees a part the way it sees the field.
+ _reject_unsupported_content_parts(payload)
+ _normalise_chat_content_parts(payload)
# And the same default it applies to a function tool that omits the discriminator: a
# template serializing the whole entry renders it, so the count has to carry it too.
for _tool in payload.tools or []:
diff --git a/studio/backend/routes/preview.py b/studio/backend/routes/preview.py
index 34cae033aff..0faaba2c8c5 100644
--- a/studio/backend/routes/preview.py
+++ b/studio/backend/routes/preview.py
@@ -18,6 +18,7 @@ from auth.authentication import authenticated_without_credential, get_current_su
from auth.storage import DEFAULT_ADMIN_USERNAME
from models.inference import ChatCompletionRequest, LoadRequest
from routes.inference import (
+ _reject_unsupported_content_parts,
disable_openai_auto_switch_for_request,
load_model_for_preview,
openai_chat_completions,
@@ -134,6 +135,9 @@ async def _serve_chat(
run: str, checkpoint: str | None, payload: ChatCompletionRequest, request: Request
):
path = _resolve_or_4xx(run, checkpoint)
+ # Before the lock and the load: openai_chat_completions refuses these too, but only after a
+ # checkpoint load that can evict the resident model on its way to the same 400.
+ _reject_unsupported_content_parts(payload)
is_lora = (path / "adapter_config.json").exists()
payload = _sanitize_preview_payload(payload, is_lora)
scope = getattr(request, "scope", None)
diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.py
new file mode 100644
index 00000000000..2df2a937f1f
--- /dev/null
+++ b/studio/backend/tests/test_chat_input_audio_part.py
@@ -0,0 +1,511 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""OpenAI's documented `input_audio` content part must reach the audio path.
+
+`ContentPart` was a closed tagged union, so that shape 400'd with `union_tag_invalid` before any
+model ran -- though llama-server takes the part and `_inject_audio_part` builds one.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from unittest import mock
+
+import pytest
+from fastapi import HTTPException
+from pydantic import ValidationError
+
+from models.inference import ChatCompletionRequest, InputAudioContentPart, UnknownContentPart
+import routes.inference as inference_route
+from routes.inference import (
+ _normalise_chat_content_parts,
+ _reject_unsupported_content_parts,
+)
+
+
+AUDIO_B64 = "UklGRiQAAABXQVZF"
+
+
+def _request(*messages, **fields) -> ChatCompletionRequest:
+ return ChatCompletionRequest(model = "local", messages = list(messages), **fields)
+
+
+def _audio_message(
+ data = AUDIO_B64,
+ role = "user",
+ text = "what is said here?",
+):
+ return {
+ "role": role,
+ "content": [
+ {"type": "text", "text": text},
+ {"type": "input_audio", "input_audio": {"data": data, "format": "wav"}},
+ ],
+ }
+
+
+def test_input_audio_part_validates():
+ payload = _request(_audio_message())
+ assert isinstance(payload.messages[0].content[1], InputAudioContentPart)
+ assert payload.messages[0].content[1].input_audio.data == AUDIO_B64
+
+
+def test_input_audio_part_is_lifted_onto_the_audio_field():
+ payload = _request(_audio_message())
+
+ _normalise_chat_content_parts(payload)
+
+ assert payload.audio_base64 == AUDIO_B64
+ assert [p.type for p in payload.messages[0].content] == ["text"]
+
+
+def test_an_explicit_audio_base64_wins():
+ payload = _request(_audio_message(data = "b2xkZXI="), audio_base64 = AUDIO_B64)
+
+ _normalise_chat_content_parts(payload)
+
+ assert payload.audio_base64 == AUDIO_B64
+
+
+def test_two_recordings_are_refused_rather_than_reduced_to_one():
+ """``audio_base64`` holds one recording, so a second was silently discarded.
+
+ A request to compare two clips would have been answered from the last one alone.
+ """
+ payload = _request(_audio_message(data = "Zmlyc3Q="), _audio_message(data = "c2Vjb25k"))
+
+ with pytest.raises(HTTPException) as exc:
+ _reject_unsupported_content_parts(payload)
+ assert exc.value.status_code == 400
+ assert "one audio recording" in str(exc.value.detail)
+
+
+def test_an_audio_part_on_a_non_user_role_is_refused():
+ """Only a user turn carries a recording into the model, and the lift strips every role.
+
+ Dropping it in silence let a later question about an assistant-history clip be answered from
+ text alone, where this shape used to fail validation outright.
+ """
+ payload = _request(_audio_message(role = "assistant"))
+
+ with pytest.raises(HTTPException) as exc:
+ _reject_unsupported_content_parts(payload)
+ assert exc.value.status_code == 400
+ assert "'assistant'" in str(exc.value.detail)
+
+
+def test_an_unmodelled_part_type_names_itself_in_a_typed_400():
+ payload = _request(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "summarise this"},
+ {"type": "file", "file": {"file_id": "file_abc"}},
+ ],
+ }
+ )
+ assert isinstance(payload.messages[0].content[1], UnknownContentPart)
+
+ with pytest.raises(HTTPException) as exc:
+ _reject_unsupported_content_parts(payload)
+ assert exc.value.status_code == 400
+ assert "'file'" in str(exc.value.detail)
+
+
+def test_a_part_with_no_type_is_still_a_validation_error():
+ with pytest.raises(ValidationError):
+ _request({"role": "user", "content": [{"text": "hi"}]})
+
+
+def _route_client(prefix = ""):
+ """The real inference router, with only the auth dependency stubbed."""
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+ from auth.authentication import get_current_subject
+ import routes.inference as inference_route
+
+ app = FastAPI()
+ app.include_router(inference_route.router, prefix = prefix)
+ app.dependency_overrides[get_current_subject] = lambda: "test"
+ return TestClient(app, raise_server_exceptions = False)
+
+
+def _count_tokens_client():
+ return _route_client()
+
+
+def test_the_count_route_refuses_an_audio_part_the_way_it_refuses_the_field():
+ """/chat/count_tokens already refuses audio, and a part is audio.
+
+ It guards images at the part level but audio only through ``audio_base64``, which was safe
+ only while an ``input_audio`` part could not validate at all.
+ """
+ with _count_tokens_client() as client:
+ response = client.post(
+ "/chat/count_tokens",
+ json = {"model": "default", "messages": [_audio_message()]},
+ )
+
+ assert response.status_code == 503
+ assert "audio" in response.json()["detail"]
+
+
+def test_the_count_route_refuses_an_unmodelled_part_like_the_completion_does():
+ with _count_tokens_client() as client:
+ response = client.post(
+ "/chat/count_tokens",
+ json = {
+ "model": "default",
+ "messages": [
+ {"role": "user", "content": [{"type": "file", "file": {"file_id": "file_abc"}}]}
+ ],
+ },
+ )
+
+ assert response.status_code == 400
+ assert "'file'" in response.json()["detail"]["error"]["message"]
+
+
+def test_a_string_content_message_passes_through_the_lift_untouched():
+ """Only list content carries parts; a plain-string turn must not be rewritten."""
+ payload = _request({"role": "system", "content": "be terse"}, _audio_message())
+
+ _reject_unsupported_content_parts(payload)
+ _normalise_chat_content_parts(payload)
+
+ assert payload.messages[0].content == "be terse"
+ assert payload.audio_base64 == AUDIO_B64
+
+
+def test_the_completion_route_takes_the_documented_audio_part():
+ """The defect itself: the part used to be refused at body validation, before any model ran.
+
+ What happens after validation depends on which models the host has, so this pins the only
+ part that is about the union: the request is no longer rejected as an unknown tag.
+ """
+ with _route_client("/v1") as client:
+ response = client.post(
+ "/v1/chat/completions",
+ json = {"model": "local", "messages": [_audio_message()]},
+ )
+
+ assert response.status_code != 422
+ assert "union_tag_invalid" not in response.text
+
+
+def test_the_completion_route_refuses_an_unmodelled_part():
+ """Raised at the normalisation call site, so it lands before any model resolution."""
+ with _route_client("/v1") as client:
+ response = client.post(
+ "/v1/chat/completions",
+ json = {
+ "model": "local",
+ "messages": [
+ {"role": "user", "content": [{"type": "file", "file": {"file_id": "file_abc"}}]}
+ ],
+ },
+ )
+
+ assert response.status_code == 400
+ assert "'file'" in response.json()["detail"]["error"]["message"]
+
+
+# ── The four review findings on the first two commits ──────────────────────────
+
+
+def test_a_non_string_part_type_is_a_validation_error_not_a_500():
+ """A list or dict ``type`` is unhashable against the known-tag set.
+
+ Testing membership on it raised TypeError out of the discriminator, which escaped request
+ validation as a 500 where the closed union had answered 422.
+ """
+ with _route_client("/v1") as client:
+ response = client.post(
+ "/v1/chat/completions",
+ json = {
+ "model": "local",
+ "messages": [{"role": "user", "content": [{"type": [{"a": 1}], "x": 1}]}],
+ },
+ )
+
+ assert response.status_code == 422
+
+
+def test_the_external_path_refuses_audio_rather_than_dropping_it():
+ """_build_external_messages has no input_audio case, so the part would be stripped.
+
+ The provider would then answer the text alone -- a plausible reply about a recording it
+ never received. Refused the way video is refused on the same branch.
+ """
+ with _route_client("/v1") as client:
+ response = client.post(
+ "/v1/chat/completions",
+ json = {"model": "gpt-4o", "provider_type": "openai", "messages": [_audio_message()]},
+ )
+
+ assert response.status_code == 400
+ assert "Audio input is only supported" in str(response.json()["detail"])
+
+
+def test_the_external_path_refuses_an_unmodelled_part_rather_than_dropping_it():
+ with _route_client("/v1") as client:
+ response = client.post(
+ "/v1/chat/completions",
+ json = {
+ "model": "gpt-4o",
+ "provider_type": "openai",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "summarise this"},
+ {"type": "file", "file": {"file_id": "file_abc"}},
+ ],
+ }
+ ],
+ },
+ )
+
+ assert response.status_code == 400
+ assert "'file'" in response.json()["detail"]["error"]["message"]
+
+
+def test_a_recording_carried_on_an_earlier_turn_is_refused():
+ """``audio_base64`` cannot express which turn a recording came from.
+
+ _inject_audio_part appends it to the last user message, so lifting an earlier turn's audio
+ replays it against a later question -- the model is asked about something the caller did not
+ ask. Refuse instead, until the field can carry a recording with its turn.
+ """
+ payload = _request(
+ _audio_message(text = "transcribe this"),
+ {"role": "assistant", "content": "It says hello."},
+ {"role": "user", "content": [{"type": "text", "text": "who is in the background?"}]},
+ )
+
+ with pytest.raises(HTTPException) as exc:
+ _reject_unsupported_content_parts(payload)
+ assert exc.value.status_code == 400
+ assert "latest user message" in str(exc.value.detail)
+
+
+def test_a_recording_on_the_latest_user_turn_is_still_lifted():
+ """The shape the SDK documents, and the one the refusal above must not catch."""
+ payload = _request(
+ {"role": "user", "content": [{"type": "text", "text": "transcribe this"}]},
+ {"role": "assistant", "content": "Sure."},
+ _audio_message(text = "what about this one?"),
+ )
+
+ _normalise_chat_content_parts(payload)
+
+ assert payload.audio_base64 == AUDIO_B64
+ assert [p.type for p in payload.messages[2].content] == ["text"]
+
+
+def test_an_empty_audio_payload_is_refused_rather_than_dropped():
+ """``{"data": ""}`` is falsy, so it was never lifted but the part was still removed.
+
+ "transcribe this" then ran as a text-only prompt and answered about a recording nobody sent.
+ """
+ with pytest.raises(ValidationError):
+ _request(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "transcribe this"},
+ {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
+ ],
+ }
+ )
+
+
+def test_the_tts_route_refuses_an_unmodelled_part():
+ """/audio/generate shares this request model but reads only text parts.
+
+ Before the catch-all it 422'd on an unknown tag; without this it would voice the text alone.
+ """
+ with _route_client() as client:
+ response = client.post(
+ "/audio/generate",
+ json = {
+ "model": "default",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "read this out"},
+ {"type": "file", "file": {"file_id": "file_abc"}},
+ ],
+ }
+ ],
+ },
+ )
+
+ assert response.status_code == 400
+ assert "'file'" in response.json()["detail"]["error"]["message"]
+
+
+def test_the_tts_route_refuses_an_audio_part():
+ """/audio/generate voices the message text; a recording has nowhere to go there.
+
+ _extract_content_parts keeps only text, so the part was discarded and speech was returned
+ for an incomplete request that used to fail validation outright.
+ """
+ with _route_client() as client:
+ response = client.post(
+ "/audio/generate",
+ json = {"model": "default", "messages": [_audio_message(text = "read this out")]},
+ )
+
+ assert response.status_code == 400
+ assert "Audio input is not supported here" in response.json()["detail"]["error"]["message"]
+
+
+def _durable_run(content):
+ from routes.chat_generation_runs import CreateChatGenerationRun
+ return CreateChatGenerationRun(
+ runId = "run-1",
+ threadId = "thread-1",
+ userMessageId = "user-1",
+ assistantMessageId = "assistant-1",
+ requestPayload = {"model": "default", "messages": [{"role": "user", "content": content}]},
+ )
+
+
+def test_a_durable_run_refuses_a_nested_recording():
+ """The durable sanitizer refuses media because the payload persists verbatim.
+
+ It read only the top-level fields, so a recording carried in a content part would have lived
+ in ``request_json`` for the life of the thread.
+ """
+ from routes.chat_generation_runs import _sanitize_request
+
+ with pytest.raises(HTTPException) as exc:
+ _sanitize_request(
+ _durable_run(
+ [
+ {"type": "text", "text": "transcribe this"},
+ {"type": "input_audio", "input_audio": {"data": AUDIO_B64, "format": "wav"}},
+ ]
+ )
+ )
+ assert exc.value.status_code == 400
+ assert "Media chat runs" in str(exc.value.detail)
+
+
+def test_a_durable_run_refuses_an_unmodelled_part_immediately():
+ """Otherwise the create endpoint returns 202 and the supervisor fails it out of band."""
+ from routes.chat_generation_runs import _sanitize_request
+
+ with pytest.raises(HTTPException) as exc:
+ _sanitize_request(
+ _durable_run(
+ [
+ {"type": "text", "text": "summarise this"},
+ {"type": "file", "file": {"file_id": "file_abc"}},
+ ]
+ )
+ )
+ assert exc.value.status_code == 400
+ assert "'file'" in str(exc.value.detail)
+
+
+def test_a_plain_durable_run_is_still_queued():
+ from routes.chat_generation_runs import _sanitize_request
+
+ sanitized = _sanitize_request(_durable_run([{"type": "text", "text": "hello"}]))
+
+ assert sanitized["stream"] is True
+ assert sanitized["thread_id"] == "thread-1"
+
+
+def test_the_tts_route_refuses_a_recording_that_was_already_lifted():
+ """/chat/completions routes a loaded TTS model into generate_audio after normalisation.
+
+ By then the part is gone and only ``audio_base64`` is set, so a parts-only guard would let
+ the route speak the text and drop the recording.
+ """
+ payload = _request(_audio_message(text = "read this out"))
+ _normalise_chat_content_parts(payload)
+ assert payload.audio_base64 == AUDIO_B64
+
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route.generate_audio(payload, None))
+ assert exc.value.status_code == 400
+ assert "not supported here" in str(exc.value.detail)
+
+
+def test_the_preview_route_refuses_before_it_loads_a_checkpoint():
+ """_serve_chat holds the preview lock and loads the checkpoint before delegating.
+
+ The delegate refuses the part, but by then an invalid request has evicted the resident model.
+ """
+ import routes.preview as preview_route
+
+ payload = _request(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "hi"},
+ {"type": "file", "file": {"file_id": "file_abc"}},
+ ],
+ }
+ )
+ loads: list[int] = []
+ with mock.patch.object(preview_route, "_resolve_or_4xx", lambda run, cp: Path("/tmp")):
+ with mock.patch.object(
+ preview_route, "load_model_for_preview", lambda *a, **k: loads.append(1)
+ ):
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(preview_route._serve_chat("run-1", None, payload, None))
+
+ assert exc.value.status_code == 400
+ assert "'file'" in str(exc.value.detail)
+ assert loads == []
+
+
+def test_the_preview_route_refuses_misplaced_audio_before_it_loads():
+ """The placement checks used to live behind routing, so preview reached them after the load.
+
+ A request that was always going to 400 would have taken the preview lock and swapped the
+ resident checkpoint on its way there.
+ """
+ import routes.preview as preview_route
+
+ payload = _request(
+ _audio_message(text = "transcribe this"),
+ {"role": "assistant", "content": "It says hello."},
+ {"role": "user", "content": [{"type": "text", "text": "who is in the background?"}]},
+ )
+ loads: list[int] = []
+ with mock.patch.object(preview_route, "_resolve_or_4xx", lambda run, cp: Path("/tmp")):
+ with mock.patch.object(
+ preview_route, "load_model_for_preview", lambda *a, **k: loads.append(1)
+ ):
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(preview_route._serve_chat("run-1", None, payload, None))
+
+ assert exc.value.status_code == 400
+ assert "latest user message" in str(exc.value.detail)
+ assert loads == []
+
+
+def test_the_text_only_checkpoint_refusal_precedes_the_branch_that_consumes_audio():
+ """A source-order guard, not an end-to-end one: reaching that branch needs the ML stack.
+
+ The transformers path consumes audio only when the checkpoint declares audio input, and the
+ capability check that would otherwise catch a text-only one runs only when an automatic load
+ could fix it. With auto-switch off the branch is skipped and the turn is answered from its
+ text alone, so the refusal has to sit in front of it. This pins that ordering; whether the
+ refusal fires for a real checkpoint is covered by the GGUF/transformers suites, not here.
+ """
+ source = Path(inference_route.__file__).read_text(encoding = "utf-8")
+ branch = source.index('if payload.audio_base64 and not model_info.get("has_audio_input"):')
+ consume = source.index('if payload.audio_base64 and model_info.get("has_audio_input"):')
+
+ # the refusal has to precede the consuming branch, or it never runs
+ assert branch < consume
+ assert "cannot read audio input" in source[branch:consume]