Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: refuse every dropped attachment on /v1/responses, not two of them
danielhanchen701fe8b4438b
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 08ca385f439..b2f547dff88 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -27755,11 +27755,22 @@ def _responses_reasoning_output_item(return ResponsesOutputReasoning(**kwargs).model_dump()-def _reject_unsupported_responses_message_part(part) -> None:- """Refuse the two attachment shapes the local adapter cannot serve, instead of dropping them.+def _reject_unserviceable_responses_attachment(part) -> None:+ """Refuse an attachment the local adapter cannot serve, instead of dropping it.- ``_responses_tool_output_content`` already answers these exact shapes with a typed 400.- Genuinely unmodelled future part types still drop.+ ``_responses_tool_output_content`` already answers these exact shapes with a typed 400+ when they arrive as a tool result; the same four rules and the same wording apply to a+ message content part, so one server does not explain the gap in one place and hide it in+ another.++ A part reaches here only after failing its typed variant, so an ``input_image`` that got+ this far is missing its ``image_url``, or carries a ``detail`` outside the documented set,+ or both. An image carrying ``image_url`` *and* ``file_id`` validates normally and is+ served from the URL -- ``file_id`` means "instead of a URL" here, exactly as it does on+ the tool-result path.++ Called for every role, before the system/developer hoist and the assistant flatten, since+ an attachment vanishes just as quietly from those turns as it does from a user turn."""part_type = getattr(part, "type", None)if part_type == "input_file":@@ -27767,12 +27778,54 @@ def _reject_unsupported_responses_message_part(part) -> None:"input","Responses input_file message parts are not supported by the local adapter.",)- if part_type == "input_image" and getattr(part, "file_id", None):- _raise_unsupported_openai_parameter(- "input",- "Responses input_image message parts with file_id are not supported by the local "- "adapter. Use image_url instead.",- )+ if part_type == "input_image":+ image_url = getattr(part, "image_url", None)+ if not isinstance(image_url, str) or not image_url:+ if getattr(part, "file_id", None):+ _raise_unsupported_openai_parameter(+ "input",+ "Responses input_image message parts with file_id are not supported by the "+ "local adapter. Use image_url instead.",+ )+ _raise_unsupported_openai_parameter(+ "input",+ "Responses input_image message parts require an image_url string.",+ )+ detail = getattr(part, "detail", "auto")+ if detail is None:+ detail = "auto"+ if detail not in ("auto", "low", "high", "original"):+ _raise_unsupported_openai_parameter(+ "input",+ "Responses input_image message detail must be auto, low, high, or original.",+ )+++def _reject_unserviceable_responses_attachments(item) -> None:+ """Run the attachment refusal over one input message's content parts.++ Only list content carries parts; a plain string cannot hide an attachment.+ """+ if isinstance(item.content, str):+ return+ for part in item.content or []:+ _reject_unserviceable_responses_attachment(part)+++def _reject_unknown_responses_message_part(part) -> None:+ """Refuse a content part no local route can serve, naming the type.++ Mirrors ``_reject_unsupported_content_parts`` on the Chat Completions side: a part left+ standing is dropped in silence and the model answers a prompt the caller did not send.+ Applied to user turns only, so an assistant replay turn keeps the lenient text flatten+ it has always had -- clients round-trip prior assistant output verbatim and must not be+ made to fail on a part that carries no attachment.+ """+ _raise_unsupported_openai_parameter(+ "input",+ f"Responses message content parts of type '{getattr(part, 'type', None)}' "+ "are not supported.",+ )def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]:@@ -27897,7 +27950,12 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]:# don't 422.continue- # ResponsesInputMessage -- hoist system/developer to the top, merge.+ # ResponsesInputMessage. Refuse an attachment this adapter cannot serve before the+ # role branches below, each of which returns via `continue` and would otherwise carry+ # a system, developer or assistant turn's attachment away in silence.+ _reject_unserviceable_responses_attachments(item)++ # Hoist system/developer to the top, merge.if item.role in ("system", "developer"):hoisted = _responses_message_text(item.content)if hoisted:@@ -27918,8 +27976,9 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]:messages.append(ChatMessage(role = "assistant", content = text))continue- # User (and any other remaining roles) -- refuse an attachment we cannot serve,- # drop genuinely unmodelled parts silently.+ # User (and any other remaining roles). Attachments are already refused above; a part+ # still standing here is one no local route can serve, so name it rather than answer+ # a prompt the caller did not send.parts: list = []for part in item.content:if isinstance(part, (ResponsesInputTextPart, ResponsesOutputTextPart)):@@ -27932,7 +27991,7 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]:))else:- _reject_unsupported_responses_message_part(part)+ _reject_unknown_responses_message_part(part)if parts:# Collapse single-text-part content to a plain string so roles that# reject multimodal arrays (e.g. legacy templates) still accept it.diff --git a/studio/backend/tests/test_responses_message_attachments.py b/studio/backend/tests/test_responses_message_attachments.pynew file mode 100644index 00000000000..78e96df3bc4--- /dev/null+++ b/studio/backend/tests/test_responses_message_attachments.py@@ -0,0 +1,151 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""+Endpoint-level cover for the /v1/responses message-attachment refusal.++The unit tests in test_responses_tool_passthrough.py call+``_normalise_responses_input`` directly. These drive the real router instead, because+three things only the route can answer:++- the refusal has to arrive as a JSON 400, not as a 500 out of request handling;+- a streaming request has to be refused *before* the SSE stream opens, or the client+ gets a broken stream with a 200 already on the wire;+- the refusal has to land before the model switch, or an unservable request evicts the+ resident model on its way to being rejected.++No running server, no GPU, no model.+"""++import os+import sys++_backend = os.path.join(os.path.dirname(__file__), "..")+sys.path.insert(0, _backend)++import pytest+++def _route_client():+ """The real inference router, with only the auth dependency stubbed."""+ from fastapi import FastAPI+ from fastapi.testclient import TestClient+ from auth.authentication import get_current_subject+ import routes.inference as inference_route++ app = FastAPI()+ # main.py mounts this router at /v1 for the OpenAI-compatible surface.+ app.include_router(inference_route.router, prefix = "/v1")+ app.dependency_overrides[get_current_subject] = lambda: "test"+ return TestClient(app, raise_server_exceptions = False)+++def _body(part, *, role = "user", stream = False):+ message = {+ "role": role,+ "content": [{"type": "input_text", "text": "what does this say?"}, part],+ }+ body = {"model": "default", "input": [message]}+ if stream:+ body["stream"] = True+ return body+++_REFUSED_PARTS = [+ ({"type": "input_file", "file_data": "data:application/pdf;base64,AA", "filename": "r.pdf"},+ "input_file"),+ ({"type": "input_file", "file_id": "file_abc"}, "input_file"),+ ({"type": "input_file", "file_url": "https://example.com/d.pdf"}, "input_file"),+ ({"type": "input_image", "file_id": "file_abc"}, "file_id"),+ ({"type": "input_image"}, "require an image_url string"),+ ({"type": "input_image", "image_url": "https://example.com/a.png", "detail": "medium"},+ "auto, low, high, or original"),+ ({"type": "input_audio", "input_audio": {"data": "AA", "format": "wav"}}, "input_audio"),+ ({"type": "input_something_new", "value": 1}, "input_something_new"),+]+++def _part_id(case):+ part, _ = case+ return f"{part['type']}-{'+'.join(k for k in part if k != 'type') or 'bare'}"++[email protected]("case", _REFUSED_PARTS, ids = _part_id)+def test_the_route_answers_json_400_not_a_500(case):+ part, needle = case+ with _route_client() as client:+ response = client.post("/v1/responses", json = _body(part))++ assert response.status_code == 400, response.text+ error = response.json()["detail"]["error"]+ assert needle in error["message"]+ assert error["code"] == "unsupported_parameter"+ assert error["param"] == "input"++[email protected]("case", _REFUSED_PARTS, ids = _part_id)+def test_a_streaming_request_is_refused_before_the_stream_opens(case):+ """A 200 text/event-stream carrying an error frame is a broken stream, not a refusal."""+ part, needle = case+ with _route_client() as client:+ response = client.post("/v1/responses", json = _body(part, stream = True))++ assert response.status_code == 400, response.text+ assert "text/event-stream" not in response.headers.get("content-type", "")+ assert needle in response.json()["detail"]["error"]["message"]++[email protected]("role", ["system", "developer", "assistant"])+def test_the_route_refuses_an_attachment_on_a_non_user_turn(role):+ part = {"type": "input_file", "filename": "r.pdf"}+ with _route_client() as client:+ response = client.post("/v1/responses", json = _body(part, role = role))++ assert response.status_code == 400, response.text+ assert "input_file" in response.json()["detail"]["error"]["message"]++[email protected]("case", _REFUSED_PARTS, ids = _part_id)+def test_the_refusal_lands_before_the_model_switch(case, monkeypatch):+ """The route documents this invariant for its other refusals: an invalid request must+ never evict the resident model on its way to a 400."""+ import routes.inference as inference_route++ switched = []++ async def _record(*args, **kwargs):+ switched.append(args)+ raise AssertionError("model switch reached on an unservable request")++ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _record)++ part, _ = case+ with _route_client() as client:+ response = client.post("/v1/responses", json = _body(part))++ assert response.status_code == 400, response.text+ assert switched == []+++ "part",+ [+ {"type": "input_image", "image_url": "https://example.com/a.png"},+ {"type": "input_image", "image_url": "https://example.com/a.png", "detail": "original"},+ {"type": "input_image", "image_url": "https://example.com/a.png", "file_id": "file_abc"},+ {"type": "input_text", "text": "second line"},+ ],+ ids = ["url", "url+detail", "url+file_id", "text"],+)+def test_a_servable_turn_is_not_refused(part):+ """The refusal must not reach past the shapes it owns.++ There is no model here, so a servable turn stops at the load check further down the+ handler. Reaching that check is the assertion: it can only be reached by a request+ normalisation accepted.+ """+ with _route_client() as client:+ response = client.post("/v1/responses", json = _body(part))++ assert "No model loaded" in response.text, response.text+ assert "unsupported_parameter" not in response.textdiff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.pyindex 70fbfeeaaa8..809e2e54df9 100644--- a/studio/backend/tests/test_responses_tool_passthrough.py+++ b/studio/backend/tests/test_responses_tool_passthrough.py@@ -857,8 +857,10 @@ class TestNormaliseResponsesInputWithTools:assert exc.value.status_code == 400assert "file_id" in str(exc.value.detail)- def test_unmodelled_message_part_still_drops_silently(self):- # A future part type must not 400 a turn whose text is perfectly servable.+ def test_unmodelled_message_part_is_named_not_dropped(self):+ # /chat/completions refuses an unmodelled part by name in+ # _reject_unsupported_content_parts. A part this endpoint drops instead is a part+ # the caller sent and the model never saw, so the two routes answer alike.payload = ResponsesRequest(input = [{@@ -870,8 +872,265 @@ class TestNormaliseResponsesInputWithTools:}],)+ with pytest.raises(HTTPException) as exc:+ _normalise_responses_input(payload)+ assert exc.value.status_code == 400+ assert "input_something_new" in str(exc.value.detail)++ def test_image_message_part_without_any_source_rejected_clearly(self):+ # No image_url and no file_id: nothing to serve, and the tool-result path already+ # says so in these words.+ payload = ResponsesRequest(+ input = [+ {+ "role": "user",+ "content": [+ {"type": "input_text", "text": "what is in this?"},+ {"type": "input_image"},+ ],+ }+ ],+ )+ with pytest.raises(HTTPException) as exc:+ _normalise_responses_input(payload)+ assert exc.value.status_code == 400+ assert "require an image_url string" in str(exc.value.detail)++ def test_image_message_part_with_unknown_detail_rejected_clearly(self):+ # detail is a Literal, so an out-of-set value fails ResponsesInputImagePart and the+ # whole image degrades to the catch-all. Dropping it loses an image the adapter+ # could have served from the url that is sitting right there.+ payload = ResponsesRequest(+ input = [+ {+ "role": "user",+ "content": [+ {"type": "input_text", "text": "what is in this?"},+ {+ "type": "input_image",+ "image_url": "https://example.com/a.png",+ "detail": "medium",+ },+ ],+ }+ ],+ )+ with pytest.raises(HTTPException) as exc:+ _normalise_responses_input(payload)+ assert exc.value.status_code == 400+ assert "auto, low, high, or original" in str(exc.value.detail)++ def test_image_message_part_with_url_and_file_id_is_served_from_the_url(self):+ # file_id means "instead of a url" on both paths. With a url present there is+ # something to serve, so serve it rather than refuse the turn.+ payload = ResponsesRequest(+ input = [+ {+ "role": "user",+ "content": [+ {"type": "input_text", "text": "what is in this?"},+ {+ "type": "input_image",+ "image_url": "https://example.com/a.png",+ "file_id": "file_abc",+ },+ ],+ }+ ],+ )+ msgs = _normalise_responses_input(payload)+ assert len(msgs) == 1+ assert msgs[0].content[1].image_url.url == "https://example.com/a.png"++ @pytest.mark.parametrize("role", ["system", "developer", "assistant"])+ def test_attachment_refused_on_the_roles_that_exit_early(self, role):+ # system/developer hoist and assistant flatten each return via `continue`, so a+ # refusal placed in the user parts loop never sees their attachments.+ text_part = (+ {"type": "output_text", "text": "hi"}+ if role == "assistant"+ else {"type": "input_text", "text": "hi"}+ )+ payload = ResponsesRequest(+ input = [+ {+ "role": role,+ "content": [text_part, {"type": "input_file", "filename": "r.pdf"}],+ },+ {"role": "user", "content": "and now?"},+ ],+ )+ with pytest.raises(HTTPException) as exc:+ _normalise_responses_input(payload)+ assert exc.value.status_code == 400+ assert "input_file" in str(exc.value.detail)++ def test_assistant_replay_keeps_its_lenient_text_flatten(self):+ # Clients round-trip prior assistant output verbatim. A part that carries no+ # attachment must not fail a turn that used to work -- only attachments are refused+ # on a replay turn.+ payload = ResponsesRequest(+ input = [+ {+ "role": "assistant",+ "content": [+ {"type": "output_text", "text": "earlier answer"},+ {"type": "refusal", "refusal": "I cannot help with that"},+ ],+ },+ {"role": "user", "content": "why not?"},+ ],+ )+ msgs = _normalise_responses_input(payload)+ assert [(m.role, m.content) for m in msgs] == [+ ("assistant", "earlier answer"),+ ("user", "why not?"),+ ]++ def test_string_content_never_hides_an_attachment(self):+ # A plain string cannot carry a part, so the refusal must not touch it.+ payload = ResponsesRequest(+ input = [+ {"role": "system", "content": "be brief"},+ {"role": "assistant", "content": "earlier"},+ {"role": "user", "content": "hello"},+ ],+ )+ msgs = _normalise_responses_input(payload)+ assert [(m.role, m.content) for m in msgs] == [+ ("system", "be brief"),+ ("assistant", "earlier"),+ ("user", "hello"),+ ]++ def test_refusal_body_is_the_openai_unsupported_parameter_shape(self):+ # Clients branch on error.code / error.param, so the refusal has to be typed the+ # way every other refusal on this route is typed.+ payload = ResponsesRequest(+ input = [+ {"role": "user", "content": [{"type": "input_file", "filename": "r.pdf"}]}+ ],+ )+ with pytest.raises(HTTPException) as exc:+ _normalise_responses_input(payload)+ error = exc.value.detail["error"]+ assert error["code"] == "unsupported_parameter"+ assert error["param"] == "input"+ assert error["type"] == "invalid_request_error"+++# Every message-part shape this route can be handed, and the one answer it owes each of+# them. Kept as a table because the interesting failures are the combinations: a shape that+# is refused on a user turn but dropped on a system turn, or an image refused for its+# file_id when the url beside it was servable all along.+_IMG = "https://example.com/a.png"+_RESPONSES_PART_MATRIX = [+ # (part, refused, needle in the message)+ ({"type": "input_text", "text": "hi"}, False, ""),+ ({"type": "output_text", "text": "hi"}, False, ""),+ ({"type": "input_image", "image_url": _IMG}, False, ""),+ ({"type": "input_image", "image_url": _IMG, "detail": "auto"}, False, ""),+ ({"type": "input_image", "image_url": _IMG, "detail": "low"}, False, ""),+ ({"type": "input_image", "image_url": _IMG, "detail": "high"}, False, ""),+ ({"type": "input_image", "image_url": _IMG, "detail": "original"}, False, ""),+ ({"type": "input_image", "image_url": _IMG, "detail": None}, False, ""),+ ({"type": "input_image", "image_url": _IMG, "file_id": "f1"}, False, ""),+ ({"type": "input_image", "image_url": "data:image/png;base64,AAAA"}, False, ""),+ ({"type": "input_image", "file_id": "f1"}, True, "file_id"),+ ({"type": "input_image"}, True, "require an image_url string"),+ ({"type": "input_image", "image_url": None}, True, "require an image_url string"),+ ({"type": "input_image", "image_url": ""}, True, "require an image_url string"),+ ({"type": "input_image", "image_url": 17}, True, "require an image_url string"),+ ({"type": "input_image", "image_url": {"url": _IMG}}, True, "require an image_url string"),+ ({"type": "input_image", "image_url": _IMG, "detail": "medium"}, True, "auto, low, high"),+ ({"type": "input_image", "image_url": _IMG, "detail": 4}, True, "auto, low, high"),+ ({"type": "input_file", "file_id": "f1"}, True, "input_file"),+ ({"type": "input_file", "file_data": "data:application/pdf;base64,AA"}, True, "input_file"),+ ({"type": "input_file", "file_url": "https://example.com/d.pdf"}, True, "input_file"),+ ({"type": "input_file", "filename": "r.pdf"}, True, "input_file"),+ ({"type": "input_file"}, True, "input_file"),+ ({"type": "input_audio", "input_audio": {"data": "AA", "format": "wav"}}, True, "input_audio"),+ ({"type": "input_something_new", "value": 1}, True, "input_something_new"),+]+++def _matrix_id(case):+ part, refused, _ = case+ keys = "+".join(k for k in part if k != "type")+ return f"{part['type']}({keys or 'bare'})-{'400' if refused else '200'}"+++class TestResponsesMessagePartMatrix:+ """One expected answer per (part shape, role, position). No server, no GPU."""++ @pytest.mark.parametrize("case", _RESPONSES_PART_MATRIX, ids=_matrix_id)+ @pytest.mark.parametrize("alone", [True, False], ids=["alone", "with_text"])+ def test_user_turn(self, case, alone):+ part, refused, needle = case+ content = [part] if alone else [{"type": "input_text", "text": "hi"}, part]+ payload = ResponsesRequest(input = [{"role": "user", "content": content}])+ if refused:+ with pytest.raises(HTTPException) as exc:+ _normalise_responses_input(payload)+ assert exc.value.status_code == 400+ assert needle in str(exc.value.detail)+ else:+ assert _normalise_responses_input(payload)++ @pytest.mark.parametrize("case", _RESPONSES_PART_MATRIX, ids=_matrix_id)+ @pytest.mark.parametrize("role", ["system", "developer", "assistant"])+ def test_attachments_refused_on_every_role(self, case, role):+ # Only attachments are hoisted above the early exits. An unmodelled *text* part on+ # a replay turn keeps the lenient flatten, so it is expected to pass here even+ # though the same part is refused on a user turn.+ part, refused, needle = case+ attachment = part["type"] in ("input_file", "input_image")+ payload = ResponsesRequest(+ input = [+ {"role": role, "content": [{"type": "input_text", "text": "hi"}, part]},+ {"role": "user", "content": "go on"},+ ],+ )+ if refused and attachment:+ with pytest.raises(HTTPException) as exc:+ _normalise_responses_input(payload)+ assert exc.value.status_code == 400+ assert needle in str(exc.value.detail)+ else:+ assert _normalise_responses_input(payload)++ def test_instructions_still_merge_when_a_turn_is_servable(self):+ payload = ResponsesRequest(+ instructions = "be brief",+ input = [+ {"role": "developer", "content": [{"type": "input_text", "text": "and kind"}]},+ {"role": "user", "content": [{"type": "input_text", "text": "hello"}]},+ ],+ )+ msgs = _normalise_responses_input(payload)+ assert [(m.role, m.content) for m in msgs] == [+ ("system", "be brief\n\nand kind"),+ ("user", "hello"),+ ]++ def test_top_level_item_types_are_untouched_by_the_refusal(self):+ # The refusal walks message items only. Reasoning items and tool items still take+ # the paths they always took.+ payload = ResponsesRequest(+ input = [+ {"type": "reasoning", "summary": [], "id": "rs_1"},+ {+ "type": "function_call",+ "call_id": "call_1",+ "name": "get_weather",+ "arguments": "{}",+ },+ {"type": "function_call_output", "call_id": "call_1", "output": "sunny"},+ {"role": "user", "content": "thanks"},+ ],+ )msgs = _normalise_responses_input(payload)- assert [(m.role, m.content) for m in msgs] == [("user", "hello")]+ assert [m.role for m in msgs] == ["assistant", "tool", "user"]def test_empty_function_call_output_gets_no_output_sentinel(self):payload = ResponsesRequest(