Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: refuse any non-text part on a responses role that flattens

danielhanchen89f1361f490e
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 86439987568..a648976f351 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -27800,18 +27800,37 @@ def _reject_unserviceable_responses_attachment(part, *, role = "user") -> None:
)
+# The only part types a flatten may discard without losing something the caller sent.
+# ``input_text`` / ``output_text`` survive the flatten as text; ``refusal`` and
+# ``summary_text`` are the model's own output metadata, which clients round-trip and the
+# prompt does not need. OpenAI's part enumeration is input_text, input_image, output_text,
+# refusal, input_file, computer_screenshot, summary_text -- so an allowlist, not an
+# ``input_`` prefix test, which would miss ``computer_screenshot``.
+_RESPONSES_FLATTENABLE_PART_TYPES = frozenset(
+ {"input_text", "output_text", "refusal", "summary_text"}
+)
+
+
def _reject_unserviceable_responses_attachments(item) -> None:
"""Run the attachment refusal over one input message's content parts.
- Role matters for images: only a user turn keeps its parts. Every other role is flattened
- by ``_responses_message_text``, which keeps text and dropped the image in silence, because
- Chat Completions wants a plain string on system and assistant and strict templates reject
- an array there. So an image on those roles cannot be forwarded, only refused.
+ Role matters: only a user turn keeps its parts. Every other role is flattened by
+ ``_responses_message_text``, which keeps text and silently dropped everything else,
+ because Chat Completions wants a plain string on system and assistant and strict
+ templates reject an array there. Nothing can be forwarded there, only refused.
"""
if isinstance(item.content, str):
return
for part in item.content or []:
_reject_unserviceable_responses_attachment(part, role = item.role)
+ part_type = getattr(part, "type", None)
+ if item.role != "user" and part_type not in _RESPONSES_FLATTENABLE_PART_TYPES:
+ _raise_unsupported_openai_parameter(
+ "input",
+ f"Responses message content parts of type '{part_type}' are not supported on "
+ f"{item.role} messages; {item.role} content is flattened to text by the local "
+ "adapter.",
+ )
def _reject_unknown_responses_message_part(part) -> None:
diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.py
index 24084c20561..8c6da0257fc 100644
--- a/studio/backend/tests/test_responses_tool_passthrough.py
+++ b/studio/backend/tests/test_responses_tool_passthrough.py
@@ -1071,9 +1071,8 @@ class TestResponsesMessagePartMatrix:
@pytest.mark.parametrize("case", _RESPONSES_PART_MATRIX, ids=_matrix_id)
@pytest.mark.parametrize("role", ["system", "developer", "assistant"])
def test_attachments_refused_on_every_role(self, case, role):
- # Only attachments are hoisted, so an unmodelled text part passes here while the
- # same part is refused on a user turn. Every image is refused on these roles,
- # servable or not: _responses_message_text keeps text and would drop it.
+ # These roles flatten to text, so only a text-ish part survives; everything else,
+ # servable image included, is caller content the flatten would drop.
part, refused, needle = case
payload = ResponsesRequest(
input = [
@@ -1081,13 +1080,15 @@ class TestResponsesMessagePartMatrix:
{"role": "user", "content": "go on"},
],
)
+ if part["type"] in ("input_text", "output_text", "refusal", "summary_text"):
+ assert _normalise_responses_input(payload)
+ return
if part["type"] == "input_file":
expected = needle
elif part["type"] == "input_image":
expected = needle if refused else "only supported on user messages"
else:
- assert _normalise_responses_input(payload)
- return
+ expected = part["type"]
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
@@ -1121,6 +1122,59 @@ class TestResponsesMessagePartMatrix:
assert "only supported on user messages" in str(exc.value.detail)
assert role in str(exc.value.detail)
+ @pytest.mark.parametrize("role", ["system", "developer", "assistant"])
+ @pytest.mark.parametrize(
+ "part",
+ [
+ {"type": "input_audio", "input_audio": {"data": "AA", "format": "wav"}},
+ {"type": "computer_screenshot", "image_url": "https://example.com/a.png"},
+ {"type": "input_brand_new", "value": 1},
+ ],
+ ids = ["input_audio", "computer_screenshot", "future"],
+ )
+ def test_a_non_text_part_is_refused_on_a_role_that_flattens(self, role, part):
+ # The flatten keeps text and drops the rest, so anything else on these roles is
+ # caller content that vanishes. A user turn already refused these by name.
+ text_part = (
+ {"type": "output_text", "text": "hi"}
+ if role == "assistant"
+ else {"type": "input_text", "text": "hi"}
+ )
+ payload = ResponsesRequest(
+ input = [
+ {"role": role, "content": [text_part, part]},
+ {"role": "user", "content": "go on"},
+ ],
+ )
+ with pytest.raises(HTTPException) as exc:
+ _normalise_responses_input(payload)
+ assert exc.value.status_code == 400
+ assert part["type"] in str(exc.value.detail)
+ assert role in str(exc.value.detail)
+
+ @pytest.mark.parametrize("part_type", ["refusal", "summary_text"])
+ def test_assistant_output_metadata_survives_the_flatten(self, part_type):
+ # refusal and summary_text are the model's own output, which clients round-trip.
+ # They carry nothing the prompt needs, so the flatten may drop them in silence and
+ # a replay turn must not start failing.
+ payload = ResponsesRequest(
+ input = [
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "output_text", "text": "earlier answer"},
+ {"type": part_type, "text": "x", "refusal": "x"},
+ ],
+ },
+ {"role": "user", "content": "why?"},
+ ],
+ )
+ msgs = _normalise_responses_input(payload)
+ assert [(m.role, m.content) for m in msgs] == [
+ ("assistant", "earlier answer"),
+ ("user", "why?"),
+ ]
+
def test_a_servable_image_still_passes_on_a_user_turn(self):
payload = ResponsesRequest(
input = [