Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(formatter): keep media from tool results after all tool messages of a turn (#2613)
alanhuangyoo43deddca7afd
diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.pyindex 7040068f5..9544746ed 100644--- a/src/agentscope/formatter/_dashscope_formatter.py+++ b/src/agentscope/formatter/_dashscope_formatter.py@@ -251,7 +251,14 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):tool_calls = []thinking_parts: list[str] = []+ # Hold the promoted media until this turn's tool messages are out.+ pending_media: list[dict] = []+for block in msg.get_content_blocks():+ if pending_media and not isinstance(block, ToolResultBlock):+ formatted_msgs.extend(pending_media)+ pending_media = []+if isinstance(block, TextBlock):content_blocks.append({"type": "text", "text": block.text})@@ -369,7 +376,7 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):if fmt_item is not None:promo_content.append(fmt_item)if promo_content:- formatted_msgs.append(+ pending_media.append({"role": "user","content": promo_content,@@ -382,6 +389,8 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):type(block),)+ formatted_msgs.extend(pending_media)+msg_dashscope: dict[str, Any] = {"role": msg.role,"content": content_blocks or None,diff --git a/src/agentscope/formatter/_moonshot_formatter.py b/src/agentscope/formatter/_moonshot_formatter.pyindex 7d05b51b8..f640e81c4 100644--- a/src/agentscope/formatter/_moonshot_formatter.py+++ b/src/agentscope/formatter/_moonshot_formatter.py@@ -109,7 +109,14 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):reasoning_parts: list[str] = []tool_calls: list[dict] = []+ # Hold the promoted media until this turn's tool messages are out.+ pending_media: list[dict] = []+for block in msg.get_content_blocks():+ if pending_media and not isinstance(block, ToolResultBlock):+ messages.extend(pending_media)+ pending_media = []+if isinstance(block, ThinkingBlock):# Preserve reasoning_content for multi-turn# Preserved Thinking (kimi-k2.6 / kimi-k2-thinking)@@ -230,7 +237,7 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):if fmt_item is not None:promo_content.append(fmt_item)if promo_content:- messages.append(+ pending_media.append({"role": "user","name": "system-reminder",@@ -244,6 +251,8 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):type(block),)+ messages.extend(pending_media)+msg_moonshot = {"role": msg.role,"name": msg.name,diff --git a/src/agentscope/formatter/_ollama_formatter.py b/src/agentscope/formatter/_ollama_formatter.pyindex 5a563aa37..50b6015c8 100644--- a/src/agentscope/formatter/_ollama_formatter.py+++ b/src/agentscope/formatter/_ollama_formatter.py@@ -113,6 +113,7 @@ class OllamaChatFormatter(_OllamaFormatterBase):),)+ # pylint: disable=too-many-branchesasync def format(self,msgs: list[Msg],@@ -133,8 +134,14 @@ class OllamaChatFormatter(_OllamaFormatterBase):for msg in msgs:content_parts = []images = []+ # Hold the promoted media until this turn's tool messages are out.+ pending_media: list[dict] = []for block in msg.get_content_blocks():+ if pending_media and not isinstance(block, ToolResultBlock):+ messages.extend(pending_media)+ pending_media = []+if isinstance(block, TextBlock):content_parts.append(block.text)@@ -253,7 +260,7 @@ class OllamaChatFormatter(_OllamaFormatterBase):}if user_images:user_msg["images"] = user_images- messages.append(user_msg)+ pending_media.append(user_msg)else:logger.warning(@@ -261,6 +268,8 @@ class OllamaChatFormatter(_OllamaFormatterBase):type(block),)+ messages.extend(pending_media)+# Add the message if there's content or imagesif content_parts or images:msg_ollama: dict[str, Any] = {diff --git a/src/agentscope/formatter/_openai_formatter.py b/src/agentscope/formatter/_openai_formatter.pyindex 7d7397fde..c40311db3 100644--- a/src/agentscope/formatter/_openai_formatter.py+++ b/src/agentscope/formatter/_openai_formatter.py@@ -269,7 +269,14 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):content_blocks = []tool_calls = []+ # Hold the promoted media until this turn's tool messages are out.+ pending_media: list[dict] = []+for block in msg.get_content_blocks():+ if pending_media and not isinstance(block, ToolResultBlock):+ messages.extend(pending_media)+ pending_media = []+if isinstance(block, TextBlock):content_blocks.append({"type": "text", "text": block.text})@@ -373,7 +380,7 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):if fmt_item is not None:promo_content.append(fmt_item)if promo_content:- messages.append(+ pending_media.append({"role": "user","name": "system-reminder",@@ -392,6 +399,8 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):type(block),)+ messages.extend(pending_media)+msg_openai = {"role": msg.role,"name": msg.name,diff --git a/src/agentscope/formatter/_volcengine_formatter.py b/src/agentscope/formatter/_volcengine_formatter.pyindex c5a3f1b3b..e29d52383 100644--- a/src/agentscope/formatter/_volcengine_formatter.py+++ b/src/agentscope/formatter/_volcengine_formatter.py@@ -121,7 +121,14 @@ class VolcengineChatFormatter(_VolcengineFormatterBase):reasoning_content_blocks: list = []tool_calls = []+ # Hold the promoted media until this turn's tool messages are out.+ pending_media: list[dict] = []+for block in msg.get_content_blocks():+ if pending_media and not isinstance(block, ToolResultBlock):+ messages.extend(pending_media)+ pending_media = []+if isinstance(block, TextBlock):content_blocks.append({"type": "text", "text": block.text})@@ -247,7 +254,7 @@ class VolcengineChatFormatter(_VolcengineFormatterBase):if formatted_item is not None:promoted_content.append(formatted_item)if promoted_content:- messages.append(+ pending_media.append({"role": "user","name": "system-reminder",@@ -261,6 +268,8 @@ class VolcengineChatFormatter(_VolcengineFormatterBase):type(block),)+ messages.extend(pending_media)+msg_volcengine: dict[str, Any] = {"role": msg.role,"content": content_blocks or None,diff --git a/tests/formatter_dashscope_test.py b/tests/formatter_dashscope_test.pyindex 74584ef2b..f99563221 100644--- a/tests/formatter_dashscope_test.py+++ b/tests/formatter_dashscope_test.py@@ -1146,3 +1146,128 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):],res,)++ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(+ self,+ ) -> None:+ """Media promoted from a tool result must not split the tool+ messages that answer one assistant turn's tool calls."""+ fmt = DashScopeChatFormatter()+ msgs = [+ AssistantMsg(+ name="assistant",+ content=[+ ToolCallBlock(+ id="call_shot",+ name="screenshot",+ input="{}",+ ),+ ToolCallBlock(+ id="call_title",+ name="get_title",+ input="{}",+ ),+ ToolResultBlock(+ id="call_shot",+ name="screenshot",+ output=[+ TextBlock(text="Screenshot taken."),+ DataBlock(+ id=_FIXED_ID,+ source=URLSource(+ url="https://example.com/shot.png",+ media_type="image/png",+ ),+ ),+ ],+ state=ToolResultState.SUCCESS,+ ),+ ToolResultBlock(+ id="call_title",+ name="get_title",+ output=[TextBlock(text="Example Domain")],+ state=ToolResultState.SUCCESS,+ ),+ TextBlock(text="The page is Example Domain."),+ ],+ ),+ ]++ res = await fmt.format(msgs)++ shot_output = (+ "Screenshot taken.\n"+ "<system-reminder>A(n) image file is returned and will be "+ "presented to you with the identifier "+ f"[{_FIXED_ID}].</system-reminder>"+ )++ self.assertListEqual(+ [+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_shot",+ "type": "function",+ "function": {+ "name": "screenshot",+ "arguments": "{}",+ },+ },+ {+ "id": "call_title",+ "type": "function",+ "function": {+ "name": "get_title",+ "arguments": "{}",+ },+ },+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call_shot",+ "content": shot_output,+ "name": "screenshot",+ },+ {+ "role": "tool",+ "tool_call_id": "call_title",+ "content": "Example Domain",+ "name": "get_title",+ },+ {+ "role": "user",+ "content": [+ {+ "type": "text",+ "text": "<system-reminder>The multimodal data "+ "and their identifiers are listed as follows:",+ },+ {+ "type": "text",+ "text": f"- {_FIXED_ID} (image file): ",+ },+ {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/shot.png",+ },+ },+ {"type": "text", "text": "</system-reminder>"},+ ],+ },+ {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "The page is Example Domain.",+ },+ ],+ },+ ],+ res,+ )diff --git a/tests/formatter_moonshot_test.py b/tests/formatter_moonshot_test.pyindex de81e0ca7..825cd61a8 100644--- a/tests/formatter_moonshot_test.py+++ b/tests/formatter_moonshot_test.py@@ -820,3 +820,131 @@ class TestMoonshotFormatter(IsolatedAsyncioTestCase):],res,)++ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(+ self,+ ) -> None:+ """Media promoted from a tool result must not split the tool+ messages that answer one assistant turn's tool calls."""+ fmt = MoonshotChatFormatter()+ msgs = [+ AssistantMsg(+ name="assistant",+ content=[+ ToolCallBlock(+ id="call_shot",+ name="screenshot",+ input="{}",+ ),+ ToolCallBlock(+ id="call_title",+ name="get_title",+ input="{}",+ ),+ ToolResultBlock(+ id="call_shot",+ name="screenshot",+ output=[+ TextBlock(text="Screenshot taken."),+ DataBlock(+ id=_FIXED_ID,+ source=URLSource(+ url="https://example.com/shot.png",+ media_type="image/png",+ ),+ ),+ ],+ state=ToolResultState.SUCCESS,+ ),+ ToolResultBlock(+ id="call_title",+ name="get_title",+ output=[TextBlock(text="Example Domain")],+ state=ToolResultState.SUCCESS,+ ),+ TextBlock(text="The page is Example Domain."),+ ],+ ),+ ]++ res = await fmt.format(msgs)++ shot_output = (+ "Screenshot taken.\n"+ "<system-reminder>A(n) image file is returned and will be "+ "presented to you with the identifier "+ f"[{_FIXED_ID}].</system-reminder>"+ )++ self.assertListEqual(+ [+ {+ "role": "assistant",+ "name": "assistant",+ "reasoning_content": "",+ "content": None,+ "tool_calls": [+ {+ "id": "call_shot",+ "type": "function",+ "function": {+ "name": "screenshot",+ "arguments": "{}",+ },+ },+ {+ "id": "call_title",+ "type": "function",+ "function": {+ "name": "get_title",+ "arguments": "{}",+ },+ },+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call_shot",+ "content": shot_output,+ "name": "screenshot",+ },+ {+ "role": "tool",+ "tool_call_id": "call_title",+ "content": "Example Domain",+ "name": "get_title",+ },+ {+ "role": "user",+ "name": "system-reminder",+ "content": [+ {+ "type": "text",+ "text": "<system-reminder>The multimodal data "+ "and their identifiers are listed as follows:",+ },+ {+ "type": "text",+ "text": f"- {_FIXED_ID} (image file): ",+ },+ {+ "type": "image_url",+ "image_url": {"url": self.image_data_uri},+ },+ {"type": "text", "text": "</system-reminder>"},+ ],+ },+ {+ "role": "assistant",+ "name": "assistant",+ "reasoning_content": "",+ "content": [+ {+ "type": "text",+ "text": "The page is Example Domain.",+ },+ ],+ },+ ],+ res,+ )diff --git a/tests/formatter_ollama_test.py b/tests/formatter_ollama_test.pyindex a1b599838..f7b6a90f6 100644--- a/tests/formatter_ollama_test.py+++ b/tests/formatter_ollama_test.py@@ -375,6 +375,103 @@ class TestOllamaFormatter(IsolatedAsyncioTestCase):res,)+ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(+ self,+ ) -> None:+ """Media promoted from a tool result must not split the tool+ messages that answer one assistant turn's tool calls."""+ fmt = OllamaChatFormatter()+ msgs = [+ AssistantMsg(+ name="assistant",+ content=[+ ToolCallBlock(+ id="call_shot",+ name="screenshot",+ input="{}",+ ),+ ToolCallBlock(+ id="call_title",+ name="get_title",+ input="{}",+ ),+ ToolResultBlock(+ id="call_shot",+ name="screenshot",+ output=[+ TextBlock(text="Screenshot taken."),+ DataBlock(+ id=_FIXED_ID,+ source=Base64Source(+ data=self.image_b64,+ media_type="image/png",+ ),+ ),+ ],+ state=ToolResultState.SUCCESS,+ ),+ ToolResultBlock(+ id="call_title",+ name="get_title",+ output=[TextBlock(text="Example Domain")],+ state=ToolResultState.SUCCESS,+ ),+ TextBlock(text="The page is Example Domain."),+ ],+ ),+ ]++ res = await fmt.format(msgs)++ self.assertListEqual(+ [+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {"function": {"name": "screenshot", "arguments": {}}},+ ],+ },+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {"function": {"name": "get_title", "arguments": {}}},+ ],+ },+ {+ "role": "tool",+ "tool_name": "screenshot",+ "content": (+ "Screenshot taken.\n"+ "<system-reminder>A(n) image file is returned and "+ "will be presented to you with the identifier "+ f"[{_FIXED_ID}].</system-reminder>"+ ),+ },+ {+ "role": "tool",+ "tool_name": "get_title",+ "content": "Example Domain",+ },+ {+ "role": "user",+ "content": (+ "<system-reminder>The multimodal data "+ "and their identifiers are listed as follows:\n"+ f"- {_FIXED_ID} (image file): \n"+ "</system-reminder>"+ ),+ "images": [self.image_b64],+ },+ {+ "role": "assistant",+ "content": "The page is Example Domain.",+ },+ ],+ res,+ )+# ------------------------------------------------------------------# OllamaMultiAgentFormatter tests# ------------------------------------------------------------------diff --git a/tests/formatter_openai_chat_test.py b/tests/formatter_openai_chat_test.pyindex e1e58e7de..c3ab06596 100644--- a/tests/formatter_openai_chat_test.py+++ b/tests/formatter_openai_chat_test.py@@ -907,3 +907,131 @@ class TestOpenAIFormatter(IsolatedAsyncioTestCase):],res,)++ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(+ self,+ ) -> None:+ """Media promoted from a tool result must not split the tool+ messages that answer one assistant turn's tool calls."""+ fmt = OpenAIChatFormatter()+ msgs = [+ AssistantMsg(+ name="assistant",+ content=[+ ToolCallBlock(+ id="call_shot",+ name="screenshot",+ input="{}",+ ),+ ToolCallBlock(+ id="call_title",+ name="get_title",+ input="{}",+ ),+ ToolResultBlock(+ id="call_shot",+ name="screenshot",+ output=[+ TextBlock(text="Screenshot taken."),+ DataBlock(+ id=_FIXED_ID,+ source=URLSource(+ url="https://example.com/shot.png",+ media_type="image/png",+ ),+ ),+ ],+ state=ToolResultState.SUCCESS,+ ),+ ToolResultBlock(+ id="call_title",+ name="get_title",+ output=[TextBlock(text="Example Domain")],+ state=ToolResultState.SUCCESS,+ ),+ TextBlock(text="The page is Example Domain."),+ ],+ ),+ ]++ res = await fmt.format(msgs)++ shot_output = (+ "Screenshot taken.\n"+ "<system-reminder>A(n) image file is returned and will be "+ "presented to you with the identifier "+ f"[{_FIXED_ID}].</system-reminder>"+ )++ self.assertListEqual(+ [+ {+ "role": "assistant",+ "name": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_shot",+ "type": "function",+ "function": {+ "name": "screenshot",+ "arguments": "{}",+ },+ },+ {+ "id": "call_title",+ "type": "function",+ "function": {+ "name": "get_title",+ "arguments": "{}",+ },+ },+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call_shot",+ "content": shot_output,+ "name": "screenshot",+ },+ {+ "role": "tool",+ "tool_call_id": "call_title",+ "content": "Example Domain",+ "name": "get_title",+ },+ {+ "role": "user",+ "name": "system-reminder",+ "content": [+ {+ "type": "text",+ "text": "<system-reminder>The multimodal data "+ "and their identifiers are listed as follows:",+ },+ {+ "type": "text",+ "text": f"- {_FIXED_ID} (image file): ",+ },+ {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/shot.png",+ },+ },+ {"type": "text", "text": "</system-reminder>"},+ ],+ },+ {+ "role": "assistant",+ "name": "assistant",+ "content": [+ {+ "type": "text",+ "text": "The page is Example Domain.",+ },+ ],+ },+ ],+ res,+ )diff --git a/tests/formatter_volcengine_test.py b/tests/formatter_volcengine_test.pyindex 6b1492b34..4eb2cc4c0 100644--- a/tests/formatter_volcengine_test.py+++ b/tests/formatter_volcengine_test.py@@ -24,6 +24,8 @@ from agentscope.message import (HintBlock,)+_FIXED_ID = "TESTID1234567"+class TestVolcengineFormatter(IsolatedAsyncioTestCase):"""Comprehensive tests for Volcengine Chat and MultiAgent formatters."""@@ -694,3 +696,129 @@ class TestVolcengineFormatter(IsolatedAsyncioTestCase):],await fmt.format(msgs),)++ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(+ self,+ ) -> None:+ """Media promoted from a tool result must not split the tool+ messages that answer one assistant turn's tool calls."""+ fmt = VolcengineChatFormatter()+ msgs = [+ AssistantMsg(+ name="assistant",+ content=[+ ToolCallBlock(+ id="call_shot",+ name="screenshot",+ input="{}",+ ),+ ToolCallBlock(+ id="call_title",+ name="get_title",+ input="{}",+ ),+ ToolResultBlock(+ id="call_shot",+ name="screenshot",+ output=[+ TextBlock(text="Screenshot taken."),+ DataBlock(+ id=_FIXED_ID,+ source=URLSource(+ url="https://example.com/shot.png",+ media_type="image/png",+ ),+ ),+ ],+ state=ToolResultState.SUCCESS,+ ),+ ToolResultBlock(+ id="call_title",+ name="get_title",+ output=[TextBlock(text="Example Domain")],+ state=ToolResultState.SUCCESS,+ ),+ TextBlock(text="The page is Example Domain."),+ ],+ ),+ ]++ res = await fmt.format(msgs)++ shot_output = (+ "Screenshot taken.\n"+ "<system-reminder>A(n) image file is returned and will be "+ "presented to you with the identifier "+ f"[{_FIXED_ID}].</system-reminder>"+ )++ self.assertListEqual(+ [+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_shot",+ "type": "function",+ "function": {+ "name": "screenshot",+ "arguments": "{}",+ },+ },+ {+ "id": "call_title",+ "type": "function",+ "function": {+ "name": "get_title",+ "arguments": "{}",+ },+ },+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call_shot",+ "content": shot_output,+ "name": "screenshot",+ },+ {+ "role": "tool",+ "tool_call_id": "call_title",+ "content": "Example Domain",+ "name": "get_title",+ },+ {+ "role": "user",+ "name": "system-reminder",+ "content": [+ {+ "type": "text",+ "text": "<system-reminder>The multimodal data "+ "and their identifiers are listed as follows:",+ },+ {+ "type": "text",+ "text": f"- {_FIXED_ID} (image file): ",+ },+ {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/shot.png",+ },+ },+ {"type": "text", "text": "</system-reminder>"},+ ],+ },+ {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "The page is Example Domain.",+ },+ ],+ },+ ],+ res,+ )