Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(formatter): keep media carried by assistant messages (#3892)
lianbied225b5775832
diff --git a/src/agentscope/formatter/_xai_formatter.py b/src/agentscope/formatter/_xai_formatter.pyindex 9e158cec5..20721743b 100644--- a/src/agentscope/formatter/_xai_formatter.py+++ b/src/agentscope/formatter/_xai_formatter.py@@ -7,6 +7,7 @@ other formatter, the ``format()`` method returns a list of``chat_pb2.Message`` proto objects rather than plain dicts, because the``xai_sdk`` chat API accepts proto messages directly."""+import base64from fnmatch import fnmatchfrom typing import Any, List@@ -158,6 +159,46 @@ class XAIChatFormatter(FormatterBase):xai_messages: List[Any] = []+ def flush_assistant(+ text_blocks: list[TextBlock],+ tool_calls: list[ToolCallBlock],+ media: list[DataBlock],+ ) -> None:+ """Emit one assistant segment before its tool results or hint."""+ text = "\n".join(block.text for block in text_blocks)+ media_args = _xai_user_args_from_blocks(+ media,+ image,+ self.supported_input_media_types,+ )+ if media_args:+ if text:+ xai_messages.append(assistant(text))+ text = ""+ # Keep promoted media ahead of the tool call/result sequence.+ xai_messages.append(user(*media_args))++ if tool_calls:+ msg_proto = chat_pb2.Message()+ msg_proto.role = chat_pb2.MessageRole.Value("ROLE_ASSISTANT")+ if text:+ msg_proto.content.add().text = text+ for call in tool_calls:+ proto_call = msg_proto.tool_calls.add()+ proto_call.id = call.id+ proto_call.type = chat_pb2.ToolCallType.Value(+ "TOOL_CALL_TYPE_CLIENT_SIDE_TOOL",+ )+ proto_call.function.name = call.name+ proto_call.function.arguments = call.input+ xai_messages.append(msg_proto)+ elif text:+ xai_messages.append(assistant(text))++ text_blocks.clear()+ tool_calls.clear()+ media.clear()+for msg in msgs:blocks = msg.get_content_blocks()@@ -210,6 +251,7 @@ class XAIChatFormatter(FormatterBase):pending_text: list[TextBlock] = []pending_tool_calls: list[ToolCallBlock] = []pending_media: list[Any] = []+ assistant_media: list[DataBlock] = []for block in blocks:if pending_media and not isinstance(@@ -220,34 +262,11 @@ class XAIChatFormatter(FormatterBase):pending_media = []if isinstance(block, ToolResultBlock):- # Convert each ToolResultBlock to a tool_result- # message.- if pending_tool_calls:- msg_proto = chat_pb2.Message()- msg_proto.role = chat_pb2.MessageRole.Value(- "ROLE_ASSISTANT",- )- if pending_text:- c = msg_proto.content.add()- c.text = "\n".join(- b.text for b in pending_text- )- pending_text = []- for tc in pending_tool_calls:- proto_tc = msg_proto.tool_calls.add()- proto_tc.id = tc.id- proto_tc.type = chat_pb2.ToolCallType.Value(- "TOOL_CALL_TYPE_CLIENT_SIDE_TOOL",- )- proto_tc.function.name = tc.name- proto_tc.function.arguments = tc.input- xai_messages.append(msg_proto)- pending_tool_calls = []- elif pending_text:- text = "\n".join(b.text for b in pending_text)- if text:- xai_messages.append(assistant(text))- pending_text = []+ flush_assistant(+ pending_text,+ pending_tool_calls,+ assistant_media,+ )(textual_output,@@ -262,12 +281,7 @@ class XAIChatFormatter(FormatterBase):),)- # Promote media that came back inside the tool- # result into a following user message, which is what- # the other formatters do via the same shared helper.- # ``xai_sdk`` re-hydrates binary tool results and- # ``input_types`` already advertises images, so the- # textual hint alone would lose the picture.+ # Keep tool results together before emitting media.if multimodal_data:promo_args = _xai_user_args_from_blocks(multimodal_data,@@ -286,36 +300,16 @@ class XAIChatFormatter(FormatterBase):elif isinstance(block, ThinkingBlock):pass+ elif isinstance(block, DataBlock):+ # xAI requires media in user messages.+ assistant_media.append(block)+elif isinstance(block, HintBlock):- if pending_tool_calls or pending_text:- if pending_tool_calls:- msg_proto = chat_pb2.Message()- msg_proto.role = chat_pb2.MessageRole.Value(- "ROLE_ASSISTANT",- )- if pending_text:- c = msg_proto.content.add()- c.text = "\n".join(- b.text for b in pending_text- )- pending_text = []- for tc in pending_tool_calls:- proto_tc = msg_proto.tool_calls.add()- proto_tc.id = tc.id- proto_tc.type = (- chat_pb2.ToolCallType.Value(- "TOOL_CALL_TYPE_CLIENT_SIDE_TOOL",- )- )- proto_tc.function.name = tc.name- proto_tc.function.arguments = tc.input- xai_messages.append(msg_proto)- pending_tool_calls = []- elif pending_text:- text = "\n".join(b.text for b in pending_text)- if text:- xai_messages.append(assistant(text))- pending_text = []+ flush_assistant(+ pending_text,+ pending_tool_calls,+ assistant_media,+ )if isinstance(block.hint, str):xai_messages.append(user(block.hint))else:@@ -328,31 +322,11 @@ class XAIChatFormatter(FormatterBase):xai_messages.append(user(*hint_args))xai_messages.extend(pending_media)-- if pending_tool_calls:- # Assistant turn that triggered tool calls (history).- msg_proto = chat_pb2.Message()- msg_proto.role = chat_pb2.MessageRole.Value(- "ROLE_ASSISTANT",- )- if pending_text:- c = msg_proto.content.add()- c.text = "\n".join(b.text for b in pending_text)- pending_text = []- for tc in pending_tool_calls:- proto_tc = msg_proto.tool_calls.add()- proto_tc.id = tc.id- proto_tc.type = chat_pb2.ToolCallType.Value(- "TOOL_CALL_TYPE_CLIENT_SIDE_TOOL",- )- proto_tc.function.name = tc.name- proto_tc.function.arguments = tc.input- xai_messages.append(msg_proto)- elif pending_text:- # Regular assistant text message.- text = "\n".join(b.text for b in pending_text)- if text:- xai_messages.append(assistant(text))+ flush_assistant(+ pending_text,+ pending_tool_calls,+ assistant_media,+ )else:logger.warning(diff --git a/tests/formatter_xai_test.py b/tests/formatter_xai_test.pyindex 027c81ff5..d45dd31cc 100644--- a/tests/formatter_xai_test.py+++ b/tests/formatter_xai_test.py@@ -290,6 +290,43 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsres,)+ async def test_chat_formatter_assistant_media(self) -> None:+ """Assistant images follow their text and precede the next user."""+ picture = DataBlock(+ source=Base64Source(+ data="SU1HX0RBVEE=",+ media_type="image/png",+ ),+ detail="high",+ )+ for text in ("Here is the chart.", ""):+ with self.subTest(text=text):+ content = (+ [TextBlock(text=text), picture] if text else [picture]+ )+ res = await XAIChatFormatter().format(+ [+ UserMsg(name="user", content="Draw a chart."),+ AssistantMsg(name="assistant", content=content),+ UserMsg(name="user", content="Explain the chart."),+ ],+ )+ expected = [user("Draw a chart.")]+ if text:+ expected.append(assistant(text))+ expected.extend(+ [+ user(+ image(+ "data:image/png;base64,SU1HX0RBVEE=",+ detail="high",+ ),+ ),+ user("Explain the chart."),+ ],+ )+ self.assertListEqual(expected, res)+async def test_chat_formatter_system_message(self) -> None:"""System message becomes a system() stub."""fmt = XAIChatFormatter()@@ -519,7 +556,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsasync def test_chat_formatter_parallel_tool_media_after_tool_msgs(self,) -> None:- """Promoted media does not split parallel tool-result messages."""+ """Assistant media precedes tools; result media follows all results."""fmt = XAIChatFormatter()picture = DataBlock(id="image_1",@@ -533,6 +570,13 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsAssistantMsg(name="assistant",content=[+ TextBlock(text="Original chart."),+ DataBlock(+ source=URLSource(+ url="https://example.com/original.png",+ media_type="image/png",+ ),+ ),ToolCallBlock(id="call_shot",name="screenshot",@@ -558,6 +602,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsoutput=[TextBlock(text="Example Domain")],state=ToolResultState.SUCCESS,),+ TextBlock(text="New chart analysis."),],),],@@ -565,8 +610,8 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsnormalized_res = [(- res[0].role,- [part.text for part in res[0].content],+ msg.role,+ [part.text for part in msg.content],[(tool_call.id,@@ -574,14 +619,18 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodstool_call.function.name,tool_call.function.arguments,)- for tool_call in res[0].tool_calls+ for tool_call in msg.tool_calls],- ),- *res[1:],+ )+ if hasattr(msg, "tool_calls")+ else msg+ for msg in res]self.assertListEqual(normalized_res,[+ assistant("Original chart."),+ user(image("https://example.com/original.png")),(2,[],@@ -618,6 +667,65 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsimage("https://example.com/shot.png"),"</system-reminder>",),+ assistant("New chart analysis."),+ ],+ )++ async def test_chat_formatter_media_before_separate_tool_result(+ self,+ ) -> None:+ """Media must not separate a tool call from a later result message."""+ res = await XAIChatFormatter().format(+ [+ AssistantMsg(+ name="assistant",+ content=[+ DataBlock(+ source=URLSource(+ url="https://example.com/original.png",+ media_type="image/png",+ ),+ ),+ ToolCallBlock(id="call_1", name="edit", input="{}"),+ ],+ ),+ AssistantMsg(+ name="assistant",+ content=[+ ToolResultBlock(+ id="call_1",+ name="edit",+ output="Done.",+ state=ToolResultState.SUCCESS,+ ),+ ],+ ),+ ],+ )+ normalized_res = [+ (+ msg.role,+ [part.text for part in msg.content],+ [+ (+ call.id,+ call.type,+ call.function.name,+ call.function.arguments,+ )+ for call in msg.tool_calls+ ],+ )+ if hasattr(msg, "tool_calls")+ else msg+ for msg in res+ ]+ self.assertListEqual(+ normalized_res,+ [+ user(image("https://example.com/original.png")),+ (2, [], [("call_1", 1, "edit", "{}")]),+ tool_result("Done.", tool_call_id="call_1"),],)@@ -1030,6 +1138,52 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsres,)+ async def test_chat_formatter_assistant_media_around_hint(self) -> None:+ """Assistant images stay with their segment on either side of hints."""+ before = DataBlock(+ source=URLSource(+ url="https://example.com/before.png",+ media_type="image/png",+ ),+ )+ after = DataBlock(+ source=URLSource(+ url="https://example.com/after.png",+ media_type="image/png",+ ),+ )+ for hint in (+ "Revise the chart.",+ [TextBlock(text="Revise the chart.")],+ ):+ for text in ("Original chart.", ""):+ with self.subTest(hint=hint, text=text):+ content = [TextBlock(text=text)] if text else []+ res = await XAIChatFormatter().format(+ [+ AssistantMsg(+ name="assistant",+ content=[+ *content,+ before,+ HintBlock(hint=hint),+ TextBlock(text="Revised chart."),+ after,+ ],+ ),+ ],+ )+ expected = [assistant(text)] if text else []+ expected.extend(+ [+ user(image("https://example.com/before.png")),+ user("Revise the chart."),+ assistant("Revised chart."),+ user(image("https://example.com/after.png")),+ ],+ )+ self.assertListEqual(expected, res)+async def test_chat_formatter_hint_block_multimodal(self) -> None:"""Multimodal HintBlock becomes one user() stub carrying text +image."""