Skip to content

Build and run agents you can see, understand and trust.

fix(formatter): keep media from tool results after all tool messages of a turn (#2613)

alanhuangyoo43deddca7afd
diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.py
index 7040068f5..9544746ed 100644
--- a/src/agentscope/formatter/_dashscope_formatter.py
+++ b/src/agentscope/formatter/_dashscope_formatter.py
@@ -251,7 +251,14 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):
tool_calls = []
thinking_parts: list[str] = []
+ # Hold the promoted media until this turn's tool messages are out.
+ pending_media: list[dict] = []
+
for block in msg.get_content_blocks():
+ if pending_media and not isinstance(block, ToolResultBlock):
+ formatted_msgs.extend(pending_media)
+ pending_media = []
+
if isinstance(block, TextBlock):
content_blocks.append({"type": "text", "text": block.text})
@@ -369,7 +376,7 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):
if fmt_item is not None:
promo_content.append(fmt_item)
if promo_content:
- formatted_msgs.append(
+ pending_media.append(
{
"role": "user",
"content": promo_content,
@@ -382,6 +389,8 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):
type(block),
)
+ formatted_msgs.extend(pending_media)
+
msg_dashscope: dict[str, Any] = {
"role": msg.role,
"content": content_blocks or None,
diff --git a/src/agentscope/formatter/_moonshot_formatter.py b/src/agentscope/formatter/_moonshot_formatter.py
index 7d05b51b8..f640e81c4 100644
--- a/src/agentscope/formatter/_moonshot_formatter.py
+++ b/src/agentscope/formatter/_moonshot_formatter.py
@@ -109,7 +109,14 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):
reasoning_parts: list[str] = []
tool_calls: list[dict] = []
+ # Hold the promoted media until this turn's tool messages are out.
+ pending_media: list[dict] = []
+
for block in msg.get_content_blocks():
+ if pending_media and not isinstance(block, ToolResultBlock):
+ messages.extend(pending_media)
+ pending_media = []
+
if isinstance(block, ThinkingBlock):
# Preserve reasoning_content for multi-turn
# Preserved Thinking (kimi-k2.6 / kimi-k2-thinking)
@@ -230,7 +237,7 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):
if fmt_item is not None:
promo_content.append(fmt_item)
if promo_content:
- messages.append(
+ pending_media.append(
{
"role": "user",
"name": "system-reminder",
@@ -244,6 +251,8 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):
type(block),
)
+ messages.extend(pending_media)
+
msg_moonshot = {
"role": msg.role,
"name": msg.name,
diff --git a/src/agentscope/formatter/_ollama_formatter.py b/src/agentscope/formatter/_ollama_formatter.py
index 5a563aa37..50b6015c8 100644
--- a/src/agentscope/formatter/_ollama_formatter.py
+++ b/src/agentscope/formatter/_ollama_formatter.py
@@ -113,6 +113,7 @@ class OllamaChatFormatter(_OllamaFormatterBase):
),
)
+ # pylint: disable=too-many-branches
async def format(
self,
msgs: list[Msg],
@@ -133,8 +134,14 @@ class OllamaChatFormatter(_OllamaFormatterBase):
for msg in msgs:
content_parts = []
images = []
+ # Hold the promoted media until this turn's tool messages are out.
+ pending_media: list[dict] = []
for block in msg.get_content_blocks():
+ if pending_media and not isinstance(block, ToolResultBlock):
+ messages.extend(pending_media)
+ pending_media = []
+
if isinstance(block, TextBlock):
content_parts.append(block.text)
@@ -253,7 +260,7 @@ class OllamaChatFormatter(_OllamaFormatterBase):
}
if user_images:
user_msg["images"] = user_images
- messages.append(user_msg)
+ pending_media.append(user_msg)
else:
logger.warning(
@@ -261,6 +268,8 @@ class OllamaChatFormatter(_OllamaFormatterBase):
type(block),
)
+ messages.extend(pending_media)
+
# Add the message if there's content or images
if content_parts or images:
msg_ollama: dict[str, Any] = {
diff --git a/src/agentscope/formatter/_openai_formatter.py b/src/agentscope/formatter/_openai_formatter.py
index 7d7397fde..c40311db3 100644
--- a/src/agentscope/formatter/_openai_formatter.py
+++ b/src/agentscope/formatter/_openai_formatter.py
@@ -269,7 +269,14 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):
content_blocks = []
tool_calls = []
+ # Hold the promoted media until this turn's tool messages are out.
+ pending_media: list[dict] = []
+
for block in msg.get_content_blocks():
+ if pending_media and not isinstance(block, ToolResultBlock):
+ messages.extend(pending_media)
+ pending_media = []
+
if isinstance(block, TextBlock):
content_blocks.append({"type": "text", "text": block.text})
@@ -373,7 +380,7 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):
if fmt_item is not None:
promo_content.append(fmt_item)
if promo_content:
- messages.append(
+ pending_media.append(
{
"role": "user",
"name": "system-reminder",
@@ -392,6 +399,8 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):
type(block),
)
+ messages.extend(pending_media)
+
msg_openai = {
"role": msg.role,
"name": msg.name,
diff --git a/src/agentscope/formatter/_volcengine_formatter.py b/src/agentscope/formatter/_volcengine_formatter.py
index c5a3f1b3b..e29d52383 100644
--- a/src/agentscope/formatter/_volcengine_formatter.py
+++ b/src/agentscope/formatter/_volcengine_formatter.py
@@ -121,7 +121,14 @@ class VolcengineChatFormatter(_VolcengineFormatterBase):
reasoning_content_blocks: list = []
tool_calls = []
+ # Hold the promoted media until this turn's tool messages are out.
+ pending_media: list[dict] = []
+
for block in msg.get_content_blocks():
+ if pending_media and not isinstance(block, ToolResultBlock):
+ messages.extend(pending_media)
+ pending_media = []
+
if isinstance(block, TextBlock):
content_blocks.append({"type": "text", "text": block.text})
@@ -247,7 +254,7 @@ class VolcengineChatFormatter(_VolcengineFormatterBase):
if formatted_item is not None:
promoted_content.append(formatted_item)
if promoted_content:
- messages.append(
+ pending_media.append(
{
"role": "user",
"name": "system-reminder",
@@ -261,6 +268,8 @@ class VolcengineChatFormatter(_VolcengineFormatterBase):
type(block),
)
+ messages.extend(pending_media)
+
msg_volcengine: dict[str, Any] = {
"role": msg.role,
"content": content_blocks or None,
diff --git a/tests/formatter_dashscope_test.py b/tests/formatter_dashscope_test.py
index 74584ef2b..f99563221 100644
--- a/tests/formatter_dashscope_test.py
+++ b/tests/formatter_dashscope_test.py
@@ -1146,3 +1146,128 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
],
res,
)
+
+ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(
+ self,
+ ) -> None:
+ """Media promoted from a tool result must not split the tool
+ messages that answer one assistant turn's tool calls."""
+ fmt = DashScopeChatFormatter()
+ msgs = [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(
+ id="call_shot",
+ name="screenshot",
+ input="{}",
+ ),
+ ToolCallBlock(
+ id="call_title",
+ name="get_title",
+ input="{}",
+ ),
+ ToolResultBlock(
+ id="call_shot",
+ name="screenshot",
+ output=[
+ TextBlock(text="Screenshot taken."),
+ DataBlock(
+ id=_FIXED_ID,
+ source=URLSource(
+ url="https://example.com/shot.png",
+ media_type="image/png",
+ ),
+ ),
+ ],
+ state=ToolResultState.SUCCESS,
+ ),
+ ToolResultBlock(
+ id="call_title",
+ name="get_title",
+ output=[TextBlock(text="Example Domain")],
+ state=ToolResultState.SUCCESS,
+ ),
+ TextBlock(text="The page is Example Domain."),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ shot_output = (
+ "Screenshot taken.\n"
+ "<system-reminder>A(n) image file is returned and will be "
+ "presented to you with the identifier "
+ f"[{_FIXED_ID}].</system-reminder>"
+ )
+
+ self.assertListEqual(
+ [
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_shot",
+ "type": "function",
+ "function": {
+ "name": "screenshot",
+ "arguments": "{}",
+ },
+ },
+ {
+ "id": "call_title",
+ "type": "function",
+ "function": {
+ "name": "get_title",
+ "arguments": "{}",
+ },
+ },
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_shot",
+ "content": shot_output,
+ "name": "screenshot",
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_title",
+ "content": "Example Domain",
+ "name": "get_title",
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "<system-reminder>The multimodal data "
+ "and their identifiers are listed as follows:",
+ },
+ {
+ "type": "text",
+ "text": f"- {_FIXED_ID} (image file): ",
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/shot.png",
+ },
+ },
+ {"type": "text", "text": "</system-reminder>"},
+ ],
+ },
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The page is Example Domain.",
+ },
+ ],
+ },
+ ],
+ res,
+ )
diff --git a/tests/formatter_moonshot_test.py b/tests/formatter_moonshot_test.py
index de81e0ca7..825cd61a8 100644
--- a/tests/formatter_moonshot_test.py
+++ b/tests/formatter_moonshot_test.py
@@ -820,3 +820,131 @@ class TestMoonshotFormatter(IsolatedAsyncioTestCase):
],
res,
)
+
+ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(
+ self,
+ ) -> None:
+ """Media promoted from a tool result must not split the tool
+ messages that answer one assistant turn's tool calls."""
+ fmt = MoonshotChatFormatter()
+ msgs = [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(
+ id="call_shot",
+ name="screenshot",
+ input="{}",
+ ),
+ ToolCallBlock(
+ id="call_title",
+ name="get_title",
+ input="{}",
+ ),
+ ToolResultBlock(
+ id="call_shot",
+ name="screenshot",
+ output=[
+ TextBlock(text="Screenshot taken."),
+ DataBlock(
+ id=_FIXED_ID,
+ source=URLSource(
+ url="https://example.com/shot.png",
+ media_type="image/png",
+ ),
+ ),
+ ],
+ state=ToolResultState.SUCCESS,
+ ),
+ ToolResultBlock(
+ id="call_title",
+ name="get_title",
+ output=[TextBlock(text="Example Domain")],
+ state=ToolResultState.SUCCESS,
+ ),
+ TextBlock(text="The page is Example Domain."),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ shot_output = (
+ "Screenshot taken.\n"
+ "<system-reminder>A(n) image file is returned and will be "
+ "presented to you with the identifier "
+ f"[{_FIXED_ID}].</system-reminder>"
+ )
+
+ self.assertListEqual(
+ [
+ {
+ "role": "assistant",
+ "name": "assistant",
+ "reasoning_content": "",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_shot",
+ "type": "function",
+ "function": {
+ "name": "screenshot",
+ "arguments": "{}",
+ },
+ },
+ {
+ "id": "call_title",
+ "type": "function",
+ "function": {
+ "name": "get_title",
+ "arguments": "{}",
+ },
+ },
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_shot",
+ "content": shot_output,
+ "name": "screenshot",
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_title",
+ "content": "Example Domain",
+ "name": "get_title",
+ },
+ {
+ "role": "user",
+ "name": "system-reminder",
+ "content": [
+ {
+ "type": "text",
+ "text": "<system-reminder>The multimodal data "
+ "and their identifiers are listed as follows:",
+ },
+ {
+ "type": "text",
+ "text": f"- {_FIXED_ID} (image file): ",
+ },
+ {
+ "type": "image_url",
+ "image_url": {"url": self.image_data_uri},
+ },
+ {"type": "text", "text": "</system-reminder>"},
+ ],
+ },
+ {
+ "role": "assistant",
+ "name": "assistant",
+ "reasoning_content": "",
+ "content": [
+ {
+ "type": "text",
+ "text": "The page is Example Domain.",
+ },
+ ],
+ },
+ ],
+ res,
+ )
diff --git a/tests/formatter_ollama_test.py b/tests/formatter_ollama_test.py
index a1b599838..f7b6a90f6 100644
--- a/tests/formatter_ollama_test.py
+++ b/tests/formatter_ollama_test.py
@@ -375,6 +375,103 @@ class TestOllamaFormatter(IsolatedAsyncioTestCase):
res,
)
+ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(
+ self,
+ ) -> None:
+ """Media promoted from a tool result must not split the tool
+ messages that answer one assistant turn's tool calls."""
+ fmt = OllamaChatFormatter()
+ msgs = [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(
+ id="call_shot",
+ name="screenshot",
+ input="{}",
+ ),
+ ToolCallBlock(
+ id="call_title",
+ name="get_title",
+ input="{}",
+ ),
+ ToolResultBlock(
+ id="call_shot",
+ name="screenshot",
+ output=[
+ TextBlock(text="Screenshot taken."),
+ DataBlock(
+ id=_FIXED_ID,
+ source=Base64Source(
+ data=self.image_b64,
+ media_type="image/png",
+ ),
+ ),
+ ],
+ state=ToolResultState.SUCCESS,
+ ),
+ ToolResultBlock(
+ id="call_title",
+ name="get_title",
+ output=[TextBlock(text="Example Domain")],
+ state=ToolResultState.SUCCESS,
+ ),
+ TextBlock(text="The page is Example Domain."),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ self.assertListEqual(
+ [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"function": {"name": "screenshot", "arguments": {}}},
+ ],
+ },
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"function": {"name": "get_title", "arguments": {}}},
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_name": "screenshot",
+ "content": (
+ "Screenshot taken.\n"
+ "<system-reminder>A(n) image file is returned and "
+ "will be presented to you with the identifier "
+ f"[{_FIXED_ID}].</system-reminder>"
+ ),
+ },
+ {
+ "role": "tool",
+ "tool_name": "get_title",
+ "content": "Example Domain",
+ },
+ {
+ "role": "user",
+ "content": (
+ "<system-reminder>The multimodal data "
+ "and their identifiers are listed as follows:\n"
+ f"- {_FIXED_ID} (image file): \n"
+ "</system-reminder>"
+ ),
+ "images": [self.image_b64],
+ },
+ {
+ "role": "assistant",
+ "content": "The page is Example Domain.",
+ },
+ ],
+ res,
+ )
+
# ------------------------------------------------------------------
# OllamaMultiAgentFormatter tests
# ------------------------------------------------------------------
diff --git a/tests/formatter_openai_chat_test.py b/tests/formatter_openai_chat_test.py
index e1e58e7de..c3ab06596 100644
--- a/tests/formatter_openai_chat_test.py
+++ b/tests/formatter_openai_chat_test.py
@@ -907,3 +907,131 @@ class TestOpenAIFormatter(IsolatedAsyncioTestCase):
],
res,
)
+
+ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(
+ self,
+ ) -> None:
+ """Media promoted from a tool result must not split the tool
+ messages that answer one assistant turn's tool calls."""
+ fmt = OpenAIChatFormatter()
+ msgs = [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(
+ id="call_shot",
+ name="screenshot",
+ input="{}",
+ ),
+ ToolCallBlock(
+ id="call_title",
+ name="get_title",
+ input="{}",
+ ),
+ ToolResultBlock(
+ id="call_shot",
+ name="screenshot",
+ output=[
+ TextBlock(text="Screenshot taken."),
+ DataBlock(
+ id=_FIXED_ID,
+ source=URLSource(
+ url="https://example.com/shot.png",
+ media_type="image/png",
+ ),
+ ),
+ ],
+ state=ToolResultState.SUCCESS,
+ ),
+ ToolResultBlock(
+ id="call_title",
+ name="get_title",
+ output=[TextBlock(text="Example Domain")],
+ state=ToolResultState.SUCCESS,
+ ),
+ TextBlock(text="The page is Example Domain."),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ shot_output = (
+ "Screenshot taken.\n"
+ "<system-reminder>A(n) image file is returned and will be "
+ "presented to you with the identifier "
+ f"[{_FIXED_ID}].</system-reminder>"
+ )
+
+ self.assertListEqual(
+ [
+ {
+ "role": "assistant",
+ "name": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_shot",
+ "type": "function",
+ "function": {
+ "name": "screenshot",
+ "arguments": "{}",
+ },
+ },
+ {
+ "id": "call_title",
+ "type": "function",
+ "function": {
+ "name": "get_title",
+ "arguments": "{}",
+ },
+ },
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_shot",
+ "content": shot_output,
+ "name": "screenshot",
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_title",
+ "content": "Example Domain",
+ "name": "get_title",
+ },
+ {
+ "role": "user",
+ "name": "system-reminder",
+ "content": [
+ {
+ "type": "text",
+ "text": "<system-reminder>The multimodal data "
+ "and their identifiers are listed as follows:",
+ },
+ {
+ "type": "text",
+ "text": f"- {_FIXED_ID} (image file): ",
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/shot.png",
+ },
+ },
+ {"type": "text", "text": "</system-reminder>"},
+ ],
+ },
+ {
+ "role": "assistant",
+ "name": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The page is Example Domain.",
+ },
+ ],
+ },
+ ],
+ res,
+ )
diff --git a/tests/formatter_volcengine_test.py b/tests/formatter_volcengine_test.py
index 6b1492b34..4eb2cc4c0 100644
--- a/tests/formatter_volcengine_test.py
+++ b/tests/formatter_volcengine_test.py
@@ -24,6 +24,8 @@ from agentscope.message import (
HintBlock,
)
+_FIXED_ID = "TESTID1234567"
+
class TestVolcengineFormatter(IsolatedAsyncioTestCase):
"""Comprehensive tests for Volcengine Chat and MultiAgent formatters."""
@@ -694,3 +696,129 @@ class TestVolcengineFormatter(IsolatedAsyncioTestCase):
],
await fmt.format(msgs),
)
+
+ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(
+ self,
+ ) -> None:
+ """Media promoted from a tool result must not split the tool
+ messages that answer one assistant turn's tool calls."""
+ fmt = VolcengineChatFormatter()
+ msgs = [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(
+ id="call_shot",
+ name="screenshot",
+ input="{}",
+ ),
+ ToolCallBlock(
+ id="call_title",
+ name="get_title",
+ input="{}",
+ ),
+ ToolResultBlock(
+ id="call_shot",
+ name="screenshot",
+ output=[
+ TextBlock(text="Screenshot taken."),
+ DataBlock(
+ id=_FIXED_ID,
+ source=URLSource(
+ url="https://example.com/shot.png",
+ media_type="image/png",
+ ),
+ ),
+ ],
+ state=ToolResultState.SUCCESS,
+ ),
+ ToolResultBlock(
+ id="call_title",
+ name="get_title",
+ output=[TextBlock(text="Example Domain")],
+ state=ToolResultState.SUCCESS,
+ ),
+ TextBlock(text="The page is Example Domain."),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ shot_output = (
+ "Screenshot taken.\n"
+ "<system-reminder>A(n) image file is returned and will be "
+ "presented to you with the identifier "
+ f"[{_FIXED_ID}].</system-reminder>"
+ )
+
+ self.assertListEqual(
+ [
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_shot",
+ "type": "function",
+ "function": {
+ "name": "screenshot",
+ "arguments": "{}",
+ },
+ },
+ {
+ "id": "call_title",
+ "type": "function",
+ "function": {
+ "name": "get_title",
+ "arguments": "{}",
+ },
+ },
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_shot",
+ "content": shot_output,
+ "name": "screenshot",
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_title",
+ "content": "Example Domain",
+ "name": "get_title",
+ },
+ {
+ "role": "user",
+ "name": "system-reminder",
+ "content": [
+ {
+ "type": "text",
+ "text": "<system-reminder>The multimodal data "
+ "and their identifiers are listed as follows:",
+ },
+ {
+ "type": "text",
+ "text": f"- {_FIXED_ID} (image file): ",
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/shot.png",
+ },
+ },
+ {"type": "text", "text": "</system-reminder>"},
+ ],
+ },
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The page is Example Domain.",
+ },
+ ],
+ },
+ ],
+ res,
+ )