FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(server): carry tool-result images on the following user turn
Xiaoze Fandb64879de40d
diff --git a/docs/cli.md b/docs/cli.mdindex 2eaa9a07..bf480d42 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -103,8 +103,10 @@ See [models.md](models.md#moe-strategies) for what each strategy does.Experimental. Needs a vision-capable checkpoint (Qwen3.6, Qwen3.8-Flash-Next, Qwen3-VL); a request carrying images isrejected otherwise. Images are accepted on all three protocols (OpenAI `image_url`,-Anthropic `image` blocks, Responses `input_image`) as an http(s) URL or base64. Images inside Anthropic-`tool_result` blocks are rejected with a 400: chat templates render tool messages as plain text.+Anthropic `image` blocks, Responses `input_image`) as an http(s) URL or base64. Images inside a tool+result (an Anthropic `tool_result` block from Claude Code's Read, a Responses `function_call_output`+from Codex's view_image) are moved to the user turn that follows the tool message, as vLLM does,+because chat templates render tool messages as plain text.`GET /v1/stats` reports what the server accepts as `model.input_modalities` (`["text"]` or `["text", "image"]`),so a client can gate its attachment controls without reading the checkpoint config.diff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.pyindex e42e0805..0d60940c 100644--- a/python/freetoken/server/anthropic_api.py+++ b/python/freetoken/server/anthropic_api.py@@ -218,21 +218,7 @@ def convert_anthropic_prompt(# -> reasoning_content; redacted_thinking stays skipped (opaque payload).thinking_parts.append(block.thinking)elif block.type == "image":- src = block.source or {}- stype = src.get("type")- data = src.get("data") if stype == "base64" else src.get("url")- if not data:- # an unsupported image source must fail the request, not degrade to a text-only answer- raise ValueError(f"unsupported image source type: {stype!r}")- content_parts.append(- {- "type": "image",- "freetoken_ref": {- "kind": "b64" if stype == "base64" else "url",- "data": data,- },- }- )+ content_parts.append(_image_part(block.source))continueelif block.type == "tool_use":tool_calls.append(@@ -246,21 +232,21 @@ def convert_anthropic_prompt(})elif block.type == "tool_result":+ text, images = _tool_result_parts(block.content)if msg.role == "user":other.append({"role": "tool","tool_call_id": block.tool_use_id or block.id or "",- "content": _tool_result_text(block.content),+ "content": text,})+ # Chat templates render tool messages as text, so the images ride on the user turn that follows the tool messages (vLLM does the same).+ content_parts.extend(images)else:- content_parts.append(- {- "type": "text",- "text": f"Tool result: {_tool_result_text(block.content)}",- }- )+ if images:+ raise ValueError("images inside a tool_result are only accepted in a user message")+ content_parts.append({"type": "text", "text": f"Tool result: {text}"})openai_msg: dict[str, Any] = {"role": msg.role}if thinking_parts:@@ -353,21 +339,36 @@ def _content_text(content) -> str:return "".join(b.text for b in content if getattr(b, "type", None) == "text" and b.text)-def _tool_result_text(content) -> str:+def _image_part(source: dict[str, Any] | None) -> dict[str, Any]:+ src = source or {}+ stype = src.get("type")+ data = src.get("data") if stype == "base64" else src.get("url")+ if not data:+ # an unsupported image source must fail the request, not degrade to a text-only answer+ raise ValueError(f"unsupported image source type: {stype!r}")+ return {+ "type": "image",+ "freetoken_ref": {"kind": "b64" if stype == "base64" else "url", "data": data},+ }+++def _tool_result_parts(content) -> tuple[str, list[dict[str, Any]]]:+ """The text of a tool_result and its image blocks as image parts."""if content is None:- return ""+ return "", []if isinstance(content, str):- return content- parts: list[str] = []+ return content, []+ texts: list[str] = []+ images: list[dict[str, Any]] = []for item in content:if isinstance(item, dict):if item.get("type") == "image":- # chat templates render tool messages as plain text, so an image here has nowhere to go- raise ValueError("images inside tool results are not supported")- parts.append(item.get("text") or "")+ images.append(_image_part(item.get("source")))+ else:+ texts.append(item.get("text") or "")else:- parts.append(str(item))- return "".join(parts)+ texts.append(str(item))+ return "".join(texts), images# --------------------------------------------------------------------------- #diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.pyindex fc1cde9e..71d6eed8 100644--- a/python/freetoken/server/responses_api.py+++ b/python/freetoken/server/responses_api.py@@ -280,13 +280,14 @@ def _convert_input_item(item: dict[str, Any]) -> list[dict[str, Any]]:}]if itype == "function_call_output":- return [- {- "role": "tool",- "tool_call_id": item.get("call_id", ""),- "content": _stringify(item.get("output")),- }- ]+ output = item.get("output")+ content = _input_content(output) if isinstance(output, list) else _stringify(output)+ tool_msg = {"role": "tool", "tool_call_id": item.get("call_id", ""), "content": content}+ if isinstance(content, str):+ return [tool_msg]+ # Chat templates render tool messages as text, so the images ride on a user turn after the tool message (the Anthropic path does the same).+ tool_msg["content"] = "".join(p["text"] for p in content if p["type"] == "text")+ return [tool_msg, {"role": "user", "content": [p for p in content if p["type"] == "image"]}]if itype == "reasoning":# Folded into its assistant turn by _merge_assistant_run; summary-only /# encrypted items carry no recoverable text.diff --git a/tests/server/test_anthropic_api.py b/tests/server/test_anthropic_api.pyindex 59ad45da..327bfcd8 100644--- a/tests/server/test_anthropic_api.py+++ b/tests/server/test_anthropic_api.py@@ -889,7 +889,8 @@ def test_count_tokens_reads_the_real_manager_return_contract():assert r.json() == {"input_tokens": 5}-def test_tool_result_image_is_rejected_not_dropped():+def test_tool_result_image_moves_to_the_following_user_turn():+ # Claude Code's Read on an image file: the tool_result carries the image, the user turn carries the reminder text.body = {"model": "claude-x","max_tokens": 16,@@ -902,14 +903,76 @@ def test_tool_result_image_is_rejected_not_dropped():{"type": "tool_result","tool_use_id": "toolu_1",- "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}],- }+ "content": [+ {"type": "text", "text": "shot taken"},+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}},+ ],+ },+ {"type": "text", "text": "<system-reminder>look</system-reminder>"},],},],}- with pytest.raises(ValueError, match="tool results"):- A.convert_anthropic_to_genspec(AnthropicMessagesRequest.model_validate(body), {})+ messages, _, _, _ = A.convert_anthropic_prompt(AnthropicMessagesRequest.model_validate(body))+ assert messages[-2] == {"role": "tool", "tool_call_id": "toolu_1", "content": "shot taken"}+ assert messages[-1] == {+ "role": "user",+ "content": [+ {"type": "image", "freetoken_ref": {"kind": "b64", "data": "aGk="}},+ {"type": "text", "text": "<system-reminder>look</system-reminder>"},+ ],+ }+ # On a server without vision the image is refused as an image input, not dropped.r = _client(FakeState([])).post("/v1/messages", json=body)assert r.status_code == 400, r.text- assert "tool results" in r.json()["error"]["message"]+ assert "vision" in r.json()["error"]["message"]+++def test_tool_result_image_outside_a_user_message_is_rejected():+ # Anthropic only allows tool_result in user messages; an image there has no user turn to ride on.+ body = {+ "model": "claude-x",+ "max_tokens": 16,+ "messages": [+ {"role": "user", "content": "go"},+ {+ "role": "assistant",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "toolu_1",+ "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}],+ }+ ],+ },+ ],+ }+ with pytest.raises(ValueError, match="user message"):+ A.convert_anthropic_prompt(AnthropicMessagesRequest.model_validate(body))+++def test_image_only_tool_result_keeps_an_empty_tool_message():+ body = {+ "model": "claude-x",+ "max_tokens": 16,+ "messages": [+ {"role": "user", "content": "take a screenshot"},+ {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "shot", "input": {}}]},+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "toolu_1",+ "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}],+ }+ ],+ },+ ],+ }+ messages, _, _, _ = A.convert_anthropic_prompt(AnthropicMessagesRequest.model_validate(body))+ assert messages[-2] == {"role": "tool", "tool_call_id": "toolu_1", "content": ""}+ assert messages[-1] == {+ "role": "user",+ "content": [{"type": "image", "freetoken_ref": {"kind": "b64", "data": "aGk="}}],+ }diff --git a/tests/server/test_responses_api.py b/tests/server/test_responses_api.pyindex 0da62172..887e78e5 100644--- a/tests/server/test_responses_api.py+++ b/tests/server/test_responses_api.py@@ -98,6 +98,33 @@ def test_convert_list_input_with_tool_roundtrip_and_tools():assert spec.parse_tools+def test_convert_function_call_output_image_moves_to_a_user_turn():+ # codex's view_image returns the image as the tool output; templates render tool messages as text.+ req = ResponsesRequest.model_validate(+ {+ "model": "gpt-x",+ "input": [+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "look"}]},+ {"type": "function_call", "call_id": "call_1", "name": "view_image", "arguments": '{"path": "a.png"}'},+ {+ "type": "function_call_output",+ "call_id": "call_1",+ "output": [+ {"type": "input_text", "text": "viewed"},+ {"type": "input_image", "image_url": "data:image/png;base64,aGk="},+ ],+ },+ ],+ }+ )+ spec = RP.convert_responses_to_genspec(req, {})+ assert [m["role"] for m in spec.messages] == ["user", "assistant", "tool", "user"]+ assert spec.messages[2]["tool_call_id"] == "call_1" and spec.messages[2]["content"] == "viewed"+ assert spec.messages[3]["content"] == [+ {"type": "image", "freetoken_ref": {"kind": "url", "data": "data:image/png;base64,aGk="}}+ ]++def test_convert_reasoning_item_merges_into_assistant_turn():# One turn's reasoning/message/function_call items fold into ONE assistant message.req = ResponsesRequest.model_validate(@@ -920,3 +947,23 @@ def test_convert_reasoning_toggle_broadcasts_every_spelling():assert spec.chat_template_kwargs == {"enable_thinking": False, "thinking_mode": "disabled"}, parser+++def test_convert_function_call_output_text_list_stays_a_plain_tool_message():+ req = ResponsesRequest.model_validate(+ {+ "model": "gpt-x",+ "input": [+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "go"}]},+ {"type": "function_call", "call_id": "call_1", "name": "f", "arguments": "{}"},+ {+ "type": "function_call_output",+ "call_id": "call_1",+ "output": [{"type": "input_text", "text": "a"}, {"type": "input_text", "text": "b"}],+ },+ ],+ }+ )+ spec = RP.convert_responses_to_genspec(req, {})+ assert [m["role"] for m in spec.messages] == ["user", "assistant", "tool"]+ assert spec.messages[2]["content"] == "ab"