Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(formatter): forward the images of collapsed messages in xAI multi-agent history (#2808)
Grapette.L2c885b58aa79
diff --git a/src/agentscope/formatter/_xai_formatter.py b/src/agentscope/formatter/_xai_formatter.pyindex ff802da7a..8df200e89 100644--- a/src/agentscope/formatter/_xai_formatter.py+++ b/src/agentscope/formatter/_xai_formatter.py@@ -27,6 +27,68 @@ from ..message import ()+def _xai_user_args_from_blocks(+ blocks: list,+ image: Any,+) -> list:+ """Convert a list of ``TextBlock | DataBlock`` into the positional+ args expected by ``xai_sdk.chat.user(*args)``.++ DataBlocks that are not images (or use an unsupported media type) are+ dropped with a warning, matching the user-role DataBlock handling in+ :meth:`XAIChatFormatter.format`.++ Args:+ blocks (`list`):+ The ``TextBlock`` / ``DataBlock`` list to convert.+ image (`Any`):+ The ``xai_sdk.chat.image`` constructor, passed in to keep+ the local import contract identical to ``format``.++ Returns:+ `list`:+ Positional args for ``user(*args)``; empty if nothing+ survived.+ """+ args: list = []+ for sub in blocks:+ if isinstance(sub, TextBlock):+ args.append(sub.text)+ elif isinstance(sub, DataBlock):+ if not sub.source.media_type.startswith("image/"):+ logger.warning(+ "Unsupported media type %s for xAI API. "+ "Only image/jpeg and image/png are supported. "+ "This block will be skipped.",+ sub.source.media_type,+ )+ continue+ if isinstance(sub.source, URLSource):+ url_str = str(sub.source.url)+ if url_str.startswith("file://"):+ local_path = url_str.removeprefix("file://")+ with open(local_path, "rb") as f:+ encoded = base64.b64encode(f.read()).decode(+ "utf-8",+ )+ args.append(+ image(+ f"data:{sub.source.media_type};"+ f"base64,{encoded}",+ ),+ )+ else:+ args.append(image(url_str))+ elif isinstance(sub.source, Base64Source):+ args.append(+ image(+ f"data:{sub.source.media_type};"+ f"base64,{sub.source.data}",+ ),+ )+ return args++class XAIChatFormatter(FormatterBase):"""Formatter for the xAI chat model.@@ -100,7 +162,7 @@ class XAIChatFormatter(FormatterBase):if isinstance(block.hint, str):xai_messages.append(user(block.hint))else:- hint_args = self._xai_user_args_from_blocks(+ hint_args = _xai_user_args_from_blocks(block.hint,image,)@@ -240,7 +302,7 @@ class XAIChatFormatter(FormatterBase):if isinstance(block.hint, str):xai_messages.append(user(block.hint))else:- hint_args = self._xai_user_args_from_blocks(+ hint_args = _xai_user_args_from_blocks(block.hint,image,)@@ -280,68 +342,6 @@ class XAIChatFormatter(FormatterBase):return xai_messages- def _xai_user_args_from_blocks(- self,- blocks: list,- image: Any,- ) -> list:- """Convert a list of ``TextBlock | DataBlock`` into the positional- args expected by ``xai_sdk.chat.user(*args)``.-- DataBlocks that are not images (or use an unsupported media type)- are dropped with a warning, matching the existing user-role- DataBlock handling above.-- Args:- blocks (`list`):- The ``hint`` list from a :class:`HintBlock`.- image (`Any`):- The ``xai_sdk.chat.image`` constructor, passed in to keep- the local import contract identical to ``format``.-- Returns:- `list`:- Positional args for ``user(*args)``; empty if nothing- survived.- """- args: list = []- for sub in blocks:- if isinstance(sub, TextBlock):- args.append(sub.text)- elif isinstance(sub, DataBlock):- if not sub.source.media_type.startswith("image/"):- logger.warning(- "Unsupported media type %s for xAI API. "- "Only image/jpeg and image/png are supported. "- "This hint sub-block will be skipped.",- sub.source.media_type,- )- continue- if isinstance(sub.source, URLSource):- url_str = str(sub.source.url)- if url_str.startswith("file://"):- local_path = url_str.removeprefix("file://")- with open(local_path, "rb") as f:- encoded = base64.b64encode(f.read()).decode(- "utf-8",- )- args.append(- image(- f"data:{sub.source.media_type};"- f"base64,{encoded}",- ),- )- else:- args.append(image(url_str))- elif isinstance(sub.source, Base64Source):- args.append(- image(- f"data:{sub.source.media_type};"- f"base64,{sub.source.data}",- ),- )- return args-def _extract_result_text(self, output: Any) -> str:"""Extract a plain-text string from a ``ToolResultBlock`` output.@@ -381,8 +381,9 @@ class XAIMultiAgentFormatter(FormatterBase):Produces ``xai_sdk`` protobuf ``Message`` objects (same as:class:`XAIChatFormatter`). Prior agent-to-agent messages are collapsed- into a single ``user`` message with ``<history></history>`` tags; tool- call / result sequences are delegated to :class:`XAIChatFormatter`.+ into a single ``user`` message with ``<history></history>`` tags; the+ images of those messages are appended to the same ``user`` message, and+ tool call / result sequences are delegated to :class:`XAIChatFormatter`... note:: ``format()`` returns ``list[Any]`` (protobuf messages), not``list[dict]``.@@ -413,8 +414,9 @@ class XAIMultiAgentFormatter(FormatterBase):"""Convert a list of ``Msg`` objects to ``xai_sdk`` proto messages.Conversation history (non-tool messages) is collapsed into a single- ``user`` protobuf message containing ``<history></history>`` tags.- Tool sequences are formatted by :class:`XAIChatFormatter`.+ ``user`` protobuf message containing ``<history></history>`` tags,+ followed by the images of those messages. Tool sequences are+ formatted by :class:`XAIChatFormatter`.Args:msgs (`list[Msg]`):@@ -426,7 +428,7 @@ class XAIMultiAgentFormatter(FormatterBase):`list[Any]`:A list of ``chat_pb2.Message`` proto objects."""- from xai_sdk.chat import system, user+ from xai_sdk.chat import image, system, userself.assert_list_of_msgs(msgs)@@ -451,9 +453,21 @@ class XAIMultiAgentFormatter(FormatterBase):group,is_first=is_first_agent_message,)+ user_args = _xai_user_args_from_blocks(+ [+ block+ for msg in group+ for block in msg.get_content_blocks()+ if isinstance(block, DataBlock)+ ],+ image,+ )if history_text:- xai_messages.append(user(history_text))- is_first_agent_message = False+ user_args.insert(0, history_text)+ if user_args:+ xai_messages.append(user(*user_args))+ if history_text:+ is_first_agent_message = Falsereturn xai_messagesdiff --git a/tests/formatter_xai_test.py b/tests/formatter_xai_test.pyindex 5683bfa9c..508bfaed0 100644--- a/tests/formatter_xai_test.py+++ b/tests/formatter_xai_test.py@@ -500,6 +500,51 @@ class TestXAIFormatter(IsolatedAsyncioTestCase):res = await fmt.format([])self.assertListEqual([], res)+ async def test_multiagent_formatter_history_keeps_images(self) -> None:+ """Images of the collapsed messages ride along with the history+ text, in message order."""+ fmt = XAIMultiAgentFormatter()+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Compare these two charts."),+ DataBlock(+ source=Base64Source(+ data="Zmlyc3Q=",+ media_type="image/jpeg",+ ),+ ),+ ],+ ),+ AssistantMsg(+ name="agent",+ content=[+ TextBlock(text="The second one differs."),+ DataBlock(+ source=URLSource(+ url="https://example.com/second.png",+ media_type="image/png",+ ),+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ user(+ self._hist_prompt + "<history>\n"+ "user: Compare these two charts.\n"+ "agent: The second one differs.\n"+ "</history>",+ image("data:image/jpeg;base64,Zmlyc3Q="),+ image("https://example.com/second.png"),+ ),+ ],+ res,+ )+async def test_chat_formatter_complex_multi_step(self) -> None:"""Complex multi-step sequence with interleaved thinking, text,tool calls, and tool results."""