Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: refuse any non-text part on a responses role that flattens
danielhanchen89f1361f490e
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 86439987568..a648976f351 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -27800,18 +27800,37 @@ def _reject_unserviceable_responses_attachment(part, *, role = "user") -> None:)+# The only part types a flatten may discard without losing something the caller sent.+# ``input_text`` / ``output_text`` survive the flatten as text; ``refusal`` and+# ``summary_text`` are the model's own output metadata, which clients round-trip and the+# prompt does not need. OpenAI's part enumeration is input_text, input_image, output_text,+# refusal, input_file, computer_screenshot, summary_text -- so an allowlist, not an+# ``input_`` prefix test, which would miss ``computer_screenshot``.+_RESPONSES_FLATTENABLE_PART_TYPES = frozenset(+ {"input_text", "output_text", "refusal", "summary_text"}+)++def _reject_unserviceable_responses_attachments(item) -> None:"""Run the attachment refusal over one input message's content parts.- Role matters for images: only a user turn keeps its parts. Every other role is flattened- by ``_responses_message_text``, which keeps text and dropped the image in silence, because- Chat Completions wants a plain string on system and assistant and strict templates reject- an array there. So an image on those roles cannot be forwarded, only refused.+ Role matters: only a user turn keeps its parts. Every other role is flattened by+ ``_responses_message_text``, which keeps text and silently dropped everything else,+ because Chat Completions wants a plain string on system and assistant and strict+ templates reject an array there. Nothing can be forwarded there, only refused."""if isinstance(item.content, str):returnfor part in item.content or []:_reject_unserviceable_responses_attachment(part, role = item.role)+ part_type = getattr(part, "type", None)+ if item.role != "user" and part_type not in _RESPONSES_FLATTENABLE_PART_TYPES:+ _raise_unsupported_openai_parameter(+ "input",+ f"Responses message content parts of type '{part_type}' are not supported on "+ f"{item.role} messages; {item.role} content is flattened to text by the local "+ "adapter.",+ )def _reject_unknown_responses_message_part(part) -> None:diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.pyindex 24084c20561..8c6da0257fc 100644--- a/studio/backend/tests/test_responses_tool_passthrough.py+++ b/studio/backend/tests/test_responses_tool_passthrough.py@@ -1071,9 +1071,8 @@ class TestResponsesMessagePartMatrix:@pytest.mark.parametrize("case", _RESPONSES_PART_MATRIX, ids=_matrix_id)@pytest.mark.parametrize("role", ["system", "developer", "assistant"])def test_attachments_refused_on_every_role(self, case, role):- # Only attachments are hoisted, so an unmodelled text part passes here while the- # same part is refused on a user turn. Every image is refused on these roles,- # servable or not: _responses_message_text keeps text and would drop it.+ # These roles flatten to text, so only a text-ish part survives; everything else,+ # servable image included, is caller content the flatten would drop.part, refused, needle = casepayload = ResponsesRequest(input = [@@ -1081,13 +1080,15 @@ class TestResponsesMessagePartMatrix:{"role": "user", "content": "go on"},],)+ if part["type"] in ("input_text", "output_text", "refusal", "summary_text"):+ assert _normalise_responses_input(payload)+ returnif part["type"] == "input_file":expected = needleelif part["type"] == "input_image":expected = needle if refused else "only supported on user messages"else:- assert _normalise_responses_input(payload)- return+ expected = part["type"]with pytest.raises(HTTPException) as exc:_normalise_responses_input(payload)assert exc.value.status_code == 400@@ -1121,6 +1122,59 @@ class TestResponsesMessagePartMatrix:assert "only supported on user messages" in str(exc.value.detail)assert role in str(exc.value.detail)+ @pytest.mark.parametrize("role", ["system", "developer", "assistant"])+ @pytest.mark.parametrize(+ "part",+ [+ {"type": "input_audio", "input_audio": {"data": "AA", "format": "wav"}},+ {"type": "computer_screenshot", "image_url": "https://example.com/a.png"},+ {"type": "input_brand_new", "value": 1},+ ],+ ids = ["input_audio", "computer_screenshot", "future"],+ )+ def test_a_non_text_part_is_refused_on_a_role_that_flattens(self, role, part):+ # The flatten keeps text and drops the rest, so anything else on these roles is+ # caller content that vanishes. A user turn already refused these by name.+ text_part = (+ {"type": "output_text", "text": "hi"}+ if role == "assistant"+ else {"type": "input_text", "text": "hi"}+ )+ payload = ResponsesRequest(+ input = [+ {"role": role, "content": [text_part, part]},+ {"role": "user", "content": "go on"},+ ],+ )+ with pytest.raises(HTTPException) as exc:+ _normalise_responses_input(payload)+ assert exc.value.status_code == 400+ assert part["type"] in str(exc.value.detail)+ assert role in str(exc.value.detail)++ @pytest.mark.parametrize("part_type", ["refusal", "summary_text"])+ def test_assistant_output_metadata_survives_the_flatten(self, part_type):+ # refusal and summary_text are the model's own output, which clients round-trip.+ # They carry nothing the prompt needs, so the flatten may drop them in silence and+ # a replay turn must not start failing.+ payload = ResponsesRequest(+ input = [+ {+ "role": "assistant",+ "content": [+ {"type": "output_text", "text": "earlier answer"},+ {"type": part_type, "text": "x", "refusal": "x"},+ ],+ },+ {"role": "user", "content": "why?"},+ ],+ )+ msgs = _normalise_responses_input(payload)+ assert [(m.role, m.content) for m in msgs] == [+ ("assistant", "earlier answer"),+ ("user", "why?"),+ ]+def test_a_servable_image_still_passes_on_a_user_turn(self):payload = ResponsesRequest(input = [