Skip to content

Your lightweight open-source personal agent.

fix(providers): preserve multimodal tool outputs

Xubin Ren07a81d70be67
diff --git a/nanobot/providers/openai_responses/converters.py b/nanobot/providers/openai_responses/converters.py
index c8b756b14..b9149d512 100644
--- a/nanobot/providers/openai_responses/converters.py
+++ b/nanobot/providers/openai_responses/converters.py
@@ -54,8 +54,8 @@ def convert_messages(messages: list[dict[str, Any]]) -> tuple[str, list[dict[str
if role == "tool":
call_id, _ = split_tool_call_id(msg.get("tool_call_id"))
- output_text = content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)
- input_items.append({"type": "function_call_output", "call_id": call_id, "output": output_text})
+ output = convert_tool_output(content)
+ input_items.append({"type": "function_call_output", "call_id": call_id, "output": output})
return system_prompt, input_items
@@ -84,6 +84,49 @@ def convert_user_message(content: Any) -> dict[str, Any]:
return {"role": "user", "content": [{"type": "input_text", "text": ""}]}
+def convert_tool_output(content: Any) -> str | list[dict[str, Any]]:
+ """Convert a tool result to Responses API function-call output content.
+
+ The Responses API accepts text, image, and file blocks as function tool
+ output. Nanobot's file tools use Chat Completions-style ``text`` and
+ ``image_url`` blocks for image reads; serializing those blocks as JSON
+ turns the image into inert text and can make the request unnecessarily
+ large. Preserve supported multimodal blocks and strip internal metadata.
+ """
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ converted: list[dict[str, Any]] = []
+ for item in content:
+ if not isinstance(item, dict):
+ continue
+ item_type = item.get("type")
+ if item_type in {"text", "input_text"}:
+ text = item.get("text")
+ if isinstance(text, str):
+ converted.append({"type": "input_text", "text": text})
+ elif item_type in {"image_url", "input_image"}:
+ image = item.get("image_url")
+ url = image.get("url") if isinstance(image, dict) else image
+ if isinstance(url, str) and url:
+ converted.append({
+ "type": "input_image",
+ "image_url": url,
+ "detail": item.get("detail", "auto"),
+ })
+ elif item_type in {"file", "input_file"}:
+ block = {"type": "input_file"}
+ for key in ("file_data", "file_id", "file_url", "filename"):
+ value = item.get(key)
+ if isinstance(value, str) and value:
+ block[key] = value
+ if len(block) > 1:
+ converted.append(block)
+ if converted:
+ return converted
+ return json.dumps(content, ensure_ascii=False)
+
+
def convert_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Convert OpenAI function-calling tool schema to Responses API flat format."""
converted: list[dict[str, Any]] = []
diff --git a/tests/providers/test_openai_responses.py b/tests/providers/test_openai_responses.py
index 4c2251c08..c36fa0981 100644
--- a/tests/providers/test_openai_responses.py
+++ b/tests/providers/test_openai_responses.py
@@ -240,6 +240,49 @@ class TestConvertMessages:
}])
assert items[0]["output"] == '{"key": "value"}'
+ def test_tool_message_preserves_image_content(self):
+ _, items = convert_messages([{
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,abc"},
+ "_meta": {"path": "/private/image.png"},
+ },
+ {"type": "text", "text": "(Image file: image.png)"},
+ ],
+ }])
+
+ assert items[0]["output"] == [
+ {
+ "type": "input_image",
+ "image_url": "data:image/png;base64,abc",
+ "detail": "auto",
+ },
+ {"type": "input_text", "text": "(Image file: image.png)"},
+ ]
+ assert "_meta" not in str(items[0])
+
+ def test_tool_message_preserves_file_content(self):
+ _, items = convert_messages([{
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": [{
+ "type": "input_file",
+ "file_id": "file_123",
+ "filename": "report.pdf",
+ "_meta": {"path": "/private/report.pdf"},
+ }],
+ }])
+
+ assert items[0]["output"] == [{
+ "type": "input_file",
+ "file_id": "file_123",
+ "filename": "report.pdf",
+ }]
+ assert "_meta" not in str(items[0])
+
def test_non_standard_keys_not_leaked(self):
"""Extra keys on messages must not appear in converted items."""
_, items = convert_messages([{