Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(server): carry tool-result images on the following user turn

Xiaoze Fandb64879de40d
diff --git a/docs/cli.md b/docs/cli.md
index 2eaa9a07..bf480d42 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -103,8 +103,10 @@ See [models.md](models.md#moe-strategies) for what each strategy does.
Experimental. Needs a vision-capable checkpoint (Qwen3.6, Qwen3.8-Flash-Next, Qwen3-VL); a request carrying images is
rejected otherwise. Images are accepted on all three protocols (OpenAI `image_url`,
-Anthropic `image` blocks, Responses `input_image`) as an http(s) URL or base64. Images inside Anthropic
-`tool_result` blocks are rejected with a 400: chat templates render tool messages as plain text.
+Anthropic `image` blocks, Responses `input_image`) as an http(s) URL or base64. Images inside a tool
+result (an Anthropic `tool_result` block from Claude Code's Read, a Responses `function_call_output`
+from Codex's view_image) are moved to the user turn that follows the tool message, as vLLM does,
+because chat templates render tool messages as plain text.
`GET /v1/stats` reports what the server accepts as `model.input_modalities` (`["text"]` or `["text", "image"]`),
so a client can gate its attachment controls without reading the checkpoint config.
diff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.py
index e42e0805..0d60940c 100644
--- a/python/freetoken/server/anthropic_api.py
+++ b/python/freetoken/server/anthropic_api.py
@@ -218,21 +218,7 @@ def convert_anthropic_prompt(
# -> reasoning_content; redacted_thinking stays skipped (opaque payload).
thinking_parts.append(block.thinking)
elif block.type == "image":
- src = block.source or {}
- stype = src.get("type")
- data = src.get("data") if stype == "base64" else src.get("url")
- if not data:
- # an unsupported image source must fail the request, not degrade to a text-only answer
- raise ValueError(f"unsupported image source type: {stype!r}")
- content_parts.append(
- {
- "type": "image",
- "freetoken_ref": {
- "kind": "b64" if stype == "base64" else "url",
- "data": data,
- },
- }
- )
+ content_parts.append(_image_part(block.source))
continue
elif block.type == "tool_use":
tool_calls.append(
@@ -246,21 +232,21 @@ def convert_anthropic_prompt(
}
)
elif block.type == "tool_result":
+ text, images = _tool_result_parts(block.content)
if msg.role == "user":
other.append(
{
"role": "tool",
"tool_call_id": block.tool_use_id or block.id or "",
- "content": _tool_result_text(block.content),
+ "content": text,
}
)
+ # Chat templates render tool messages as text, so the images ride on the user turn that follows the tool messages (vLLM does the same).
+ content_parts.extend(images)
else:
- content_parts.append(
- {
- "type": "text",
- "text": f"Tool result: {_tool_result_text(block.content)}",
- }
- )
+ if images:
+ raise ValueError("images inside a tool_result are only accepted in a user message")
+ content_parts.append({"type": "text", "text": f"Tool result: {text}"})
openai_msg: dict[str, Any] = {"role": msg.role}
if thinking_parts:
@@ -353,21 +339,36 @@ def _content_text(content) -> str:
return "".join(b.text for b in content if getattr(b, "type", None) == "text" and b.text)
-def _tool_result_text(content) -> str:
+def _image_part(source: dict[str, Any] | None) -> dict[str, Any]:
+ src = source or {}
+ stype = src.get("type")
+ data = src.get("data") if stype == "base64" else src.get("url")
+ if not data:
+ # an unsupported image source must fail the request, not degrade to a text-only answer
+ raise ValueError(f"unsupported image source type: {stype!r}")
+ return {
+ "type": "image",
+ "freetoken_ref": {"kind": "b64" if stype == "base64" else "url", "data": data},
+ }
+
+
+def _tool_result_parts(content) -> tuple[str, list[dict[str, Any]]]:
+ """The text of a tool_result and its image blocks as image parts."""
if content is None:
- return ""
+ return "", []
if isinstance(content, str):
- return content
- parts: list[str] = []
+ return content, []
+ texts: list[str] = []
+ images: list[dict[str, Any]] = []
for item in content:
if isinstance(item, dict):
if item.get("type") == "image":
- # chat templates render tool messages as plain text, so an image here has nowhere to go
- raise ValueError("images inside tool results are not supported")
- parts.append(item.get("text") or "")
+ images.append(_image_part(item.get("source")))
+ else:
+ texts.append(item.get("text") or "")
else:
- parts.append(str(item))
- return "".join(parts)
+ texts.append(str(item))
+ return "".join(texts), images
# --------------------------------------------------------------------------- #
diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.py
index fc1cde9e..71d6eed8 100644
--- a/python/freetoken/server/responses_api.py
+++ b/python/freetoken/server/responses_api.py
@@ -280,13 +280,14 @@ def _convert_input_item(item: dict[str, Any]) -> list[dict[str, Any]]:
}
]
if itype == "function_call_output":
- return [
- {
- "role": "tool",
- "tool_call_id": item.get("call_id", ""),
- "content": _stringify(item.get("output")),
- }
- ]
+ output = item.get("output")
+ content = _input_content(output) if isinstance(output, list) else _stringify(output)
+ tool_msg = {"role": "tool", "tool_call_id": item.get("call_id", ""), "content": content}
+ if isinstance(content, str):
+ return [tool_msg]
+ # Chat templates render tool messages as text, so the images ride on a user turn after the tool message (the Anthropic path does the same).
+ tool_msg["content"] = "".join(p["text"] for p in content if p["type"] == "text")
+ return [tool_msg, {"role": "user", "content": [p for p in content if p["type"] == "image"]}]
if itype == "reasoning":
# Folded into its assistant turn by _merge_assistant_run; summary-only /
# encrypted items carry no recoverable text.
diff --git a/tests/server/test_anthropic_api.py b/tests/server/test_anthropic_api.py
index 59ad45da..327bfcd8 100644
--- a/tests/server/test_anthropic_api.py
+++ b/tests/server/test_anthropic_api.py
@@ -889,7 +889,8 @@ def test_count_tokens_reads_the_real_manager_return_contract():
assert r.json() == {"input_tokens": 5}
-def test_tool_result_image_is_rejected_not_dropped():
+def test_tool_result_image_moves_to_the_following_user_turn():
+ # Claude Code's Read on an image file: the tool_result carries the image, the user turn carries the reminder text.
body = {
"model": "claude-x",
"max_tokens": 16,
@@ -902,14 +903,76 @@ def test_tool_result_image_is_rejected_not_dropped():
{
"type": "tool_result",
"tool_use_id": "toolu_1",
- "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}],
- }
+ "content": [
+ {"type": "text", "text": "shot taken"},
+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}},
+ ],
+ },
+ {"type": "text", "text": "<system-reminder>look</system-reminder>"},
],
},
],
}
- with pytest.raises(ValueError, match="tool results"):
- A.convert_anthropic_to_genspec(AnthropicMessagesRequest.model_validate(body), {})
+ messages, _, _, _ = A.convert_anthropic_prompt(AnthropicMessagesRequest.model_validate(body))
+ assert messages[-2] == {"role": "tool", "tool_call_id": "toolu_1", "content": "shot taken"}
+ assert messages[-1] == {
+ "role": "user",
+ "content": [
+ {"type": "image", "freetoken_ref": {"kind": "b64", "data": "aGk="}},
+ {"type": "text", "text": "<system-reminder>look</system-reminder>"},
+ ],
+ }
+ # On a server without vision the image is refused as an image input, not dropped.
r = _client(FakeState([])).post("/v1/messages", json=body)
assert r.status_code == 400, r.text
- assert "tool results" in r.json()["error"]["message"]
+ assert "vision" in r.json()["error"]["message"]
+
+
+def test_tool_result_image_outside_a_user_message_is_rejected():
+ # Anthropic only allows tool_result in user messages; an image there has no user turn to ride on.
+ body = {
+ "model": "claude-x",
+ "max_tokens": 16,
+ "messages": [
+ {"role": "user", "content": "go"},
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_1",
+ "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}],
+ }
+ ],
+ },
+ ],
+ }
+ with pytest.raises(ValueError, match="user message"):
+ A.convert_anthropic_prompt(AnthropicMessagesRequest.model_validate(body))
+
+
+def test_image_only_tool_result_keeps_an_empty_tool_message():
+ body = {
+ "model": "claude-x",
+ "max_tokens": 16,
+ "messages": [
+ {"role": "user", "content": "take a screenshot"},
+ {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "shot", "input": {}}]},
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_1",
+ "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}],
+ }
+ ],
+ },
+ ],
+ }
+ messages, _, _, _ = A.convert_anthropic_prompt(AnthropicMessagesRequest.model_validate(body))
+ assert messages[-2] == {"role": "tool", "tool_call_id": "toolu_1", "content": ""}
+ assert messages[-1] == {
+ "role": "user",
+ "content": [{"type": "image", "freetoken_ref": {"kind": "b64", "data": "aGk="}}],
+ }
diff --git a/tests/server/test_responses_api.py b/tests/server/test_responses_api.py
index 0da62172..887e78e5 100644
--- a/tests/server/test_responses_api.py
+++ b/tests/server/test_responses_api.py
@@ -98,6 +98,33 @@ def test_convert_list_input_with_tool_roundtrip_and_tools():
assert spec.parse_tools
+def test_convert_function_call_output_image_moves_to_a_user_turn():
+ # codex's view_image returns the image as the tool output; templates render tool messages as text.
+ req = ResponsesRequest.model_validate(
+ {
+ "model": "gpt-x",
+ "input": [
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "look"}]},
+ {"type": "function_call", "call_id": "call_1", "name": "view_image", "arguments": '{"path": "a.png"}'},
+ {
+ "type": "function_call_output",
+ "call_id": "call_1",
+ "output": [
+ {"type": "input_text", "text": "viewed"},
+ {"type": "input_image", "image_url": "data:image/png;base64,aGk="},
+ ],
+ },
+ ],
+ }
+ )
+ spec = RP.convert_responses_to_genspec(req, {})
+ assert [m["role"] for m in spec.messages] == ["user", "assistant", "tool", "user"]
+ assert spec.messages[2]["tool_call_id"] == "call_1" and spec.messages[2]["content"] == "viewed"
+ assert spec.messages[3]["content"] == [
+ {"type": "image", "freetoken_ref": {"kind": "url", "data": "data:image/png;base64,aGk="}}
+ ]
+
+
def test_convert_reasoning_item_merges_into_assistant_turn():
# One turn's reasoning/message/function_call items fold into ONE assistant message.
req = ResponsesRequest.model_validate(
@@ -920,3 +947,23 @@ def test_convert_reasoning_toggle_broadcasts_every_spelling():
assert spec.chat_template_kwargs == {
"enable_thinking": False, "thinking_mode": "disabled"
}, parser
+
+
+def test_convert_function_call_output_text_list_stays_a_plain_tool_message():
+ req = ResponsesRequest.model_validate(
+ {
+ "model": "gpt-x",
+ "input": [
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "go"}]},
+ {"type": "function_call", "call_id": "call_1", "name": "f", "arguments": "{}"},
+ {
+ "type": "function_call_output",
+ "call_id": "call_1",
+ "output": [{"type": "input_text", "text": "a"}, {"type": "input_text", "text": "b"}],
+ },
+ ],
+ }
+ )
+ spec = RP.convert_responses_to_genspec(req, {})
+ assert [m["role"] for m in spec.messages] == ["user", "assistant", "tool"]
+ assert spec.messages[2]["content"] == "ab"