Skip to content

Build and run agents you can see, understand and trust.

fix(formatter): keep media carried by assistant messages (#3892)

lianbied225b5775832
diff --git a/src/agentscope/formatter/_xai_formatter.py b/src/agentscope/formatter/_xai_formatter.py
index 9e158cec5..20721743b 100644
--- a/src/agentscope/formatter/_xai_formatter.py
+++ b/src/agentscope/formatter/_xai_formatter.py
@@ -7,6 +7,7 @@ other formatter, the ``format()`` method returns a list of
``chat_pb2.Message`` proto objects rather than plain dicts, because the
``xai_sdk`` chat API accepts proto messages directly.
"""
+
import base64
from fnmatch import fnmatch
from typing import Any, List
@@ -158,6 +159,46 @@ class XAIChatFormatter(FormatterBase):
xai_messages: List[Any] = []
+ def flush_assistant(
+ text_blocks: list[TextBlock],
+ tool_calls: list[ToolCallBlock],
+ media: list[DataBlock],
+ ) -> None:
+ """Emit one assistant segment before its tool results or hint."""
+ text = "\n".join(block.text for block in text_blocks)
+ media_args = _xai_user_args_from_blocks(
+ media,
+ image,
+ self.supported_input_media_types,
+ )
+ if media_args:
+ if text:
+ xai_messages.append(assistant(text))
+ text = ""
+ # Keep promoted media ahead of the tool call/result sequence.
+ xai_messages.append(user(*media_args))
+
+ if tool_calls:
+ msg_proto = chat_pb2.Message()
+ msg_proto.role = chat_pb2.MessageRole.Value("ROLE_ASSISTANT")
+ if text:
+ msg_proto.content.add().text = text
+ for call in tool_calls:
+ proto_call = msg_proto.tool_calls.add()
+ proto_call.id = call.id
+ proto_call.type = chat_pb2.ToolCallType.Value(
+ "TOOL_CALL_TYPE_CLIENT_SIDE_TOOL",
+ )
+ proto_call.function.name = call.name
+ proto_call.function.arguments = call.input
+ xai_messages.append(msg_proto)
+ elif text:
+ xai_messages.append(assistant(text))
+
+ text_blocks.clear()
+ tool_calls.clear()
+ media.clear()
+
for msg in msgs:
blocks = msg.get_content_blocks()
@@ -210,6 +251,7 @@ class XAIChatFormatter(FormatterBase):
pending_text: list[TextBlock] = []
pending_tool_calls: list[ToolCallBlock] = []
pending_media: list[Any] = []
+ assistant_media: list[DataBlock] = []
for block in blocks:
if pending_media and not isinstance(
@@ -220,34 +262,11 @@ class XAIChatFormatter(FormatterBase):
pending_media = []
if isinstance(block, ToolResultBlock):
- # Convert each ToolResultBlock to a tool_result
- # message.
- if pending_tool_calls:
- msg_proto = chat_pb2.Message()
- msg_proto.role = chat_pb2.MessageRole.Value(
- "ROLE_ASSISTANT",
- )
- if pending_text:
- c = msg_proto.content.add()
- c.text = "\n".join(
- b.text for b in pending_text
- )
- pending_text = []
- for tc in pending_tool_calls:
- proto_tc = msg_proto.tool_calls.add()
- proto_tc.id = tc.id
- proto_tc.type = chat_pb2.ToolCallType.Value(
- "TOOL_CALL_TYPE_CLIENT_SIDE_TOOL",
- )
- proto_tc.function.name = tc.name
- proto_tc.function.arguments = tc.input
- xai_messages.append(msg_proto)
- pending_tool_calls = []
- elif pending_text:
- text = "\n".join(b.text for b in pending_text)
- if text:
- xai_messages.append(assistant(text))
- pending_text = []
+ flush_assistant(
+ pending_text,
+ pending_tool_calls,
+ assistant_media,
+ )
(
textual_output,
@@ -262,12 +281,7 @@ class XAIChatFormatter(FormatterBase):
),
)
- # Promote media that came back inside the tool
- # result into a following user message, which is what
- # the other formatters do via the same shared helper.
- # ``xai_sdk`` re-hydrates binary tool results and
- # ``input_types`` already advertises images, so the
- # textual hint alone would lose the picture.
+ # Keep tool results together before emitting media.
if multimodal_data:
promo_args = _xai_user_args_from_blocks(
multimodal_data,
@@ -286,36 +300,16 @@ class XAIChatFormatter(FormatterBase):
elif isinstance(block, ThinkingBlock):
pass
+ elif isinstance(block, DataBlock):
+ # xAI requires media in user messages.
+ assistant_media.append(block)
+
elif isinstance(block, HintBlock):
- if pending_tool_calls or pending_text:
- if pending_tool_calls:
- msg_proto = chat_pb2.Message()
- msg_proto.role = chat_pb2.MessageRole.Value(
- "ROLE_ASSISTANT",
- )
- if pending_text:
- c = msg_proto.content.add()
- c.text = "\n".join(
- b.text for b in pending_text
- )
- pending_text = []
- for tc in pending_tool_calls:
- proto_tc = msg_proto.tool_calls.add()
- proto_tc.id = tc.id
- proto_tc.type = (
- chat_pb2.ToolCallType.Value(
- "TOOL_CALL_TYPE_CLIENT_SIDE_TOOL",
- )
- )
- proto_tc.function.name = tc.name
- proto_tc.function.arguments = tc.input
- xai_messages.append(msg_proto)
- pending_tool_calls = []
- elif pending_text:
- text = "\n".join(b.text for b in pending_text)
- if text:
- xai_messages.append(assistant(text))
- pending_text = []
+ flush_assistant(
+ pending_text,
+ pending_tool_calls,
+ assistant_media,
+ )
if isinstance(block.hint, str):
xai_messages.append(user(block.hint))
else:
@@ -328,31 +322,11 @@ class XAIChatFormatter(FormatterBase):
xai_messages.append(user(*hint_args))
xai_messages.extend(pending_media)
-
- if pending_tool_calls:
- # Assistant turn that triggered tool calls (history).
- msg_proto = chat_pb2.Message()
- msg_proto.role = chat_pb2.MessageRole.Value(
- "ROLE_ASSISTANT",
- )
- if pending_text:
- c = msg_proto.content.add()
- c.text = "\n".join(b.text for b in pending_text)
- pending_text = []
- for tc in pending_tool_calls:
- proto_tc = msg_proto.tool_calls.add()
- proto_tc.id = tc.id
- proto_tc.type = chat_pb2.ToolCallType.Value(
- "TOOL_CALL_TYPE_CLIENT_SIDE_TOOL",
- )
- proto_tc.function.name = tc.name
- proto_tc.function.arguments = tc.input
- xai_messages.append(msg_proto)
- elif pending_text:
- # Regular assistant text message.
- text = "\n".join(b.text for b in pending_text)
- if text:
- xai_messages.append(assistant(text))
+ flush_assistant(
+ pending_text,
+ pending_tool_calls,
+ assistant_media,
+ )
else:
logger.warning(
diff --git a/tests/formatter_xai_test.py b/tests/formatter_xai_test.py
index 027c81ff5..d45dd31cc 100644
--- a/tests/formatter_xai_test.py
+++ b/tests/formatter_xai_test.py
@@ -290,6 +290,43 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
res,
)
+ async def test_chat_formatter_assistant_media(self) -> None:
+ """Assistant images follow their text and precede the next user."""
+ picture = DataBlock(
+ source=Base64Source(
+ data="SU1HX0RBVEE=",
+ media_type="image/png",
+ ),
+ detail="high",
+ )
+ for text in ("Here is the chart.", ""):
+ with self.subTest(text=text):
+ content = (
+ [TextBlock(text=text), picture] if text else [picture]
+ )
+ res = await XAIChatFormatter().format(
+ [
+ UserMsg(name="user", content="Draw a chart."),
+ AssistantMsg(name="assistant", content=content),
+ UserMsg(name="user", content="Explain the chart."),
+ ],
+ )
+ expected = [user("Draw a chart.")]
+ if text:
+ expected.append(assistant(text))
+ expected.extend(
+ [
+ user(
+ image(
+ "data:image/png;base64,SU1HX0RBVEE=",
+ detail="high",
+ ),
+ ),
+ user("Explain the chart."),
+ ],
+ )
+ self.assertListEqual(expected, res)
+
async def test_chat_formatter_system_message(self) -> None:
"""System message becomes a system() stub."""
fmt = XAIChatFormatter()
@@ -519,7 +556,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
async def test_chat_formatter_parallel_tool_media_after_tool_msgs(
self,
) -> None:
- """Promoted media does not split parallel tool-result messages."""
+ """Assistant media precedes tools; result media follows all results."""
fmt = XAIChatFormatter()
picture = DataBlock(
id="image_1",
@@ -533,6 +570,13 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
AssistantMsg(
name="assistant",
content=[
+ TextBlock(text="Original chart."),
+ DataBlock(
+ source=URLSource(
+ url="https://example.com/original.png",
+ media_type="image/png",
+ ),
+ ),
ToolCallBlock(
id="call_shot",
name="screenshot",
@@ -558,6 +602,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
output=[TextBlock(text="Example Domain")],
state=ToolResultState.SUCCESS,
),
+ TextBlock(text="New chart analysis."),
],
),
],
@@ -565,8 +610,8 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
normalized_res = [
(
- res[0].role,
- [part.text for part in res[0].content],
+ msg.role,
+ [part.text for part in msg.content],
[
(
tool_call.id,
@@ -574,14 +619,18 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
tool_call.function.name,
tool_call.function.arguments,
)
- for tool_call in res[0].tool_calls
+ for tool_call in msg.tool_calls
],
- ),
- *res[1:],
+ )
+ if hasattr(msg, "tool_calls")
+ else msg
+ for msg in res
]
self.assertListEqual(
normalized_res,
[
+ assistant("Original chart."),
+ user(image("https://example.com/original.png")),
(
2,
[],
@@ -618,6 +667,65 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
image("https://example.com/shot.png"),
"</system-reminder>",
),
+ assistant("New chart analysis."),
+ ],
+ )
+
+ async def test_chat_formatter_media_before_separate_tool_result(
+ self,
+ ) -> None:
+ """Media must not separate a tool call from a later result message."""
+ res = await XAIChatFormatter().format(
+ [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ DataBlock(
+ source=URLSource(
+ url="https://example.com/original.png",
+ media_type="image/png",
+ ),
+ ),
+ ToolCallBlock(id="call_1", name="edit", input="{}"),
+ ],
+ ),
+ AssistantMsg(
+ name="assistant",
+ content=[
+ ToolResultBlock(
+ id="call_1",
+ name="edit",
+ output="Done.",
+ state=ToolResultState.SUCCESS,
+ ),
+ ],
+ ),
+ ],
+ )
+ normalized_res = [
+ (
+ msg.role,
+ [part.text for part in msg.content],
+ [
+ (
+ call.id,
+ call.type,
+ call.function.name,
+ call.function.arguments,
+ )
+ for call in msg.tool_calls
+ ],
+ )
+ if hasattr(msg, "tool_calls")
+ else msg
+ for msg in res
+ ]
+ self.assertListEqual(
+ normalized_res,
+ [
+ user(image("https://example.com/original.png")),
+ (2, [], [("call_1", 1, "edit", "{}")]),
+ tool_result("Done.", tool_call_id="call_1"),
],
)
@@ -1030,6 +1138,52 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
res,
)
+ async def test_chat_formatter_assistant_media_around_hint(self) -> None:
+ """Assistant images stay with their segment on either side of hints."""
+ before = DataBlock(
+ source=URLSource(
+ url="https://example.com/before.png",
+ media_type="image/png",
+ ),
+ )
+ after = DataBlock(
+ source=URLSource(
+ url="https://example.com/after.png",
+ media_type="image/png",
+ ),
+ )
+ for hint in (
+ "Revise the chart.",
+ [TextBlock(text="Revise the chart.")],
+ ):
+ for text in ("Original chart.", ""):
+ with self.subTest(hint=hint, text=text):
+ content = [TextBlock(text=text)] if text else []
+ res = await XAIChatFormatter().format(
+ [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ *content,
+ before,
+ HintBlock(hint=hint),
+ TextBlock(text="Revised chart."),
+ after,
+ ],
+ ),
+ ],
+ )
+ expected = [assistant(text)] if text else []
+ expected.extend(
+ [
+ user(image("https://example.com/before.png")),
+ user("Revise the chart."),
+ assistant("Revised chart."),
+ user(image("https://example.com/after.png")),
+ ],
+ )
+ self.assertListEqual(expected, res)
+
async def test_chat_formatter_hint_block_multimodal(self) -> None:
"""Multimodal HintBlock becomes one user() stub carrying text +
image."""