LLM inference in C/C++
C++ ggml
server : fix image blocks in tool_result being dropped during Anthropic OpenAI conversion (#22536)
quei6b4dc2116a92
diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cppindex 02858a2a02..31f94e0233 100644--- a/tools/server/server-chat.cpp+++ b/tools/server/server-chat.cpp@@ -431,22 +431,70 @@ json server_chat_convert_anthropic_to_oai(const json & body) {std::string tool_use_id = json_value(block, "tool_use_id", std::string());auto result_content = json_value(block, "content", json());- std::string result_text;if (result_content.is_string()) {- result_text = result_content.get<std::string>();+ tool_results.push_back({+ {"role", "tool"},+ {"tool_call_id", tool_use_id},+ {"content", result_content.get<std::string>()}+ });} else if (result_content.is_array()) {+ // Single-pass: build both text and content_parts, decide format at the end+ std::string result_text;+ json content_parts = json::array();+ bool has_images = false;+for (const auto & c : result_content) {- if (json_value(c, "type", std::string()) == "text") {- result_text += json_value(c, "text", std::string());+ std::string c_type = json_value(c, "type", std::string());+ if (c_type == "text") {+ std::string text = json_value(c, "text", std::string());+ result_text += text;+ content_parts.push_back({+ {"type", "text"},+ {"text", text}+ });+ } else if (c_type == "image") {+ has_images = true;+ json source = json_value(c, "source", json::object());+ std::string source_type = json_value(source, "type", std::string());+ if (source_type == "base64") {+ std::string media_type = json_value(source, "media_type", std::string("image/jpeg"));+ std::string data = json_value(source, "data", std::string());+ std::string url = "data:" + media_type + ";base64," + data;+ content_parts.push_back({+ {"type", "image_url"},+ {"image_url", {{"url", url}}}+ });+ } else if (source_type == "url") {+ content_parts.push_back({+ {"type", "image_url"},+ {"image_url", {{"url", json_value(source, "url", std::string())}}}+ });+ }}}- }- tool_results.push_back({- {"role", "tool"},- {"tool_call_id", tool_use_id},- {"content", result_text}- });+ if (!has_images) {+ // Text-only: collapse to a plain string for maximum compatibility+ tool_results.push_back({+ {"role", "tool"},+ {"tool_call_id", tool_use_id},+ {"content", result_text}+ });+ } else {+ // Mixed or image-only: use array content parts (OpenAI multimodal tool format)+ tool_results.push_back({+ {"role", "tool"},+ {"tool_call_id", tool_use_id},+ {"content", content_parts}+ });+ }+ } else {+ tool_results.push_back({+ {"role", "tool"},+ {"tool_call_id", tool_use_id},+ {"content", ""}+ });+ }}}diff --git a/tools/server/tests/unit/test_compat_anthropic.py b/tools/server/tests/unit/test_compat_anthropic.pyindex ef1948d4a5..e23947cdde 100644--- a/tools/server/tests/unit/test_compat_anthropic.py+++ b/tools/server/tests/unit/test_compat_anthropic.py@@ -402,6 +402,65 @@ def test_anthropic_tool_result_with_text():assert len(res.body["content"]) > 0+def test_anthropic_tool_result_with_image():+ """Test tool result containing mixed text and image blocks++ Verifies that image blocks inside Anthropic tool_result content are+ properly converted to OpenAI image_url format rather than being+ silently dropped. With a non-multimodal model, the converted image+ triggers a clear error message instead of being ignored.+ """+ server.jinja = True+ server.start()++ # Small 1x1 red PNG image in base64 (same as vision tests)+ red_pixel_png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="++ res = server.make_request("POST", "/v1/messages", data={+ "model": "test",+ "max_tokens": 100,+ "messages": [+ {"role": "user", "content": "What is in this image?"},+ {+ "role": "assistant",+ "content": [+ {+ "type": "tool_use",+ "id": "tool_1",+ "name": "read",+ "input": {"file": "test.png"}+ }+ ]+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "tool_1",+ "content": [+ {"type": "text", "text": "File: test.png"},+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": red_pixel_png+ }+ }+ ]+ }+ ]+ }+ ]+ })++ # Without the fix, image block would cause "unsupported content[].type"+ # With the fix, image is converted to image_url but tinyllama doesn't support images+ assert res.status_code == 500+ assert "image input is not supported" in res.body.get("error", {}).get("message", "").lower()++def test_anthropic_tool_result_error():"""Test tool result with error flag"""server.jinja = True