Skip to content

Build and run agents you can see, understand and trust.

fix(formatter): forward the images of collapsed messages in xAI multi-agent history (#2808)

Grapette.L2c885b58aa79
diff --git a/src/agentscope/formatter/_xai_formatter.py b/src/agentscope/formatter/_xai_formatter.py
index ff802da7a..8df200e89 100644
--- a/src/agentscope/formatter/_xai_formatter.py
+++ b/src/agentscope/formatter/_xai_formatter.py
@@ -27,6 +27,68 @@ from ..message import (
)
+def _xai_user_args_from_blocks(
+ blocks: list,
+ image: Any,
+) -> list:
+ """Convert a list of ``TextBlock | DataBlock`` into the positional
+ args expected by ``xai_sdk.chat.user(*args)``.
+
+ DataBlocks that are not images (or use an unsupported media type) are
+ dropped with a warning, matching the user-role DataBlock handling in
+ :meth:`XAIChatFormatter.format`.
+
+ Args:
+ blocks (`list`):
+ The ``TextBlock`` / ``DataBlock`` list to convert.
+ image (`Any`):
+ The ``xai_sdk.chat.image`` constructor, passed in to keep
+ the local import contract identical to ``format``.
+
+ Returns:
+ `list`:
+ Positional args for ``user(*args)``; empty if nothing
+ survived.
+ """
+ args: list = []
+ for sub in blocks:
+ if isinstance(sub, TextBlock):
+ args.append(sub.text)
+ elif isinstance(sub, DataBlock):
+ if not sub.source.media_type.startswith("image/"):
+ logger.warning(
+ "Unsupported media type %s for xAI API. "
+ "Only image/jpeg and image/png are supported. "
+ "This block will be skipped.",
+ sub.source.media_type,
+ )
+ continue
+ if isinstance(sub.source, URLSource):
+ url_str = str(sub.source.url)
+ if url_str.startswith("file://"):
+ local_path = url_str.removeprefix("file://")
+ with open(local_path, "rb") as f:
+ encoded = base64.b64encode(f.read()).decode(
+ "utf-8",
+ )
+ args.append(
+ image(
+ f"data:{sub.source.media_type};"
+ f"base64,{encoded}",
+ ),
+ )
+ else:
+ args.append(image(url_str))
+ elif isinstance(sub.source, Base64Source):
+ args.append(
+ image(
+ f"data:{sub.source.media_type};"
+ f"base64,{sub.source.data}",
+ ),
+ )
+ return args
+
+
class XAIChatFormatter(FormatterBase):
"""Formatter for the xAI chat model.
@@ -100,7 +162,7 @@ class XAIChatFormatter(FormatterBase):
if isinstance(block.hint, str):
xai_messages.append(user(block.hint))
else:
- hint_args = self._xai_user_args_from_blocks(
+ hint_args = _xai_user_args_from_blocks(
block.hint,
image,
)
@@ -240,7 +302,7 @@ class XAIChatFormatter(FormatterBase):
if isinstance(block.hint, str):
xai_messages.append(user(block.hint))
else:
- hint_args = self._xai_user_args_from_blocks(
+ hint_args = _xai_user_args_from_blocks(
block.hint,
image,
)
@@ -280,68 +342,6 @@ class XAIChatFormatter(FormatterBase):
return xai_messages
- def _xai_user_args_from_blocks(
- self,
- blocks: list,
- image: Any,
- ) -> list:
- """Convert a list of ``TextBlock | DataBlock`` into the positional
- args expected by ``xai_sdk.chat.user(*args)``.
-
- DataBlocks that are not images (or use an unsupported media type)
- are dropped with a warning, matching the existing user-role
- DataBlock handling above.
-
- Args:
- blocks (`list`):
- The ``hint`` list from a :class:`HintBlock`.
- image (`Any`):
- The ``xai_sdk.chat.image`` constructor, passed in to keep
- the local import contract identical to ``format``.
-
- Returns:
- `list`:
- Positional args for ``user(*args)``; empty if nothing
- survived.
- """
- args: list = []
- for sub in blocks:
- if isinstance(sub, TextBlock):
- args.append(sub.text)
- elif isinstance(sub, DataBlock):
- if not sub.source.media_type.startswith("image/"):
- logger.warning(
- "Unsupported media type %s for xAI API. "
- "Only image/jpeg and image/png are supported. "
- "This hint sub-block will be skipped.",
- sub.source.media_type,
- )
- continue
- if isinstance(sub.source, URLSource):
- url_str = str(sub.source.url)
- if url_str.startswith("file://"):
- local_path = url_str.removeprefix("file://")
- with open(local_path, "rb") as f:
- encoded = base64.b64encode(f.read()).decode(
- "utf-8",
- )
- args.append(
- image(
- f"data:{sub.source.media_type};"
- f"base64,{encoded}",
- ),
- )
- else:
- args.append(image(url_str))
- elif isinstance(sub.source, Base64Source):
- args.append(
- image(
- f"data:{sub.source.media_type};"
- f"base64,{sub.source.data}",
- ),
- )
- return args
-
def _extract_result_text(self, output: Any) -> str:
"""Extract a plain-text string from a ``ToolResultBlock`` output.
@@ -381,8 +381,9 @@ class XAIMultiAgentFormatter(FormatterBase):
Produces ``xai_sdk`` protobuf ``Message`` objects (same as
:class:`XAIChatFormatter`). Prior agent-to-agent messages are collapsed
- into a single ``user`` message with ``<history></history>`` tags; tool
- call / result sequences are delegated to :class:`XAIChatFormatter`.
+ into a single ``user`` message with ``<history></history>`` tags; the
+ images of those messages are appended to the same ``user`` message, and
+ tool call / result sequences are delegated to :class:`XAIChatFormatter`.
.. note:: ``format()`` returns ``list[Any]`` (protobuf messages), not
``list[dict]``.
@@ -413,8 +414,9 @@ class XAIMultiAgentFormatter(FormatterBase):
"""Convert a list of ``Msg`` objects to ``xai_sdk`` proto messages.
Conversation history (non-tool messages) is collapsed into a single
- ``user`` protobuf message containing ``<history></history>`` tags.
- Tool sequences are formatted by :class:`XAIChatFormatter`.
+ ``user`` protobuf message containing ``<history></history>`` tags,
+ followed by the images of those messages. Tool sequences are
+ formatted by :class:`XAIChatFormatter`.
Args:
msgs (`list[Msg]`):
@@ -426,7 +428,7 @@ class XAIMultiAgentFormatter(FormatterBase):
`list[Any]`:
A list of ``chat_pb2.Message`` proto objects.
"""
- from xai_sdk.chat import system, user
+ from xai_sdk.chat import image, system, user
self.assert_list_of_msgs(msgs)
@@ -451,9 +453,21 @@ class XAIMultiAgentFormatter(FormatterBase):
group,
is_first=is_first_agent_message,
)
+ user_args = _xai_user_args_from_blocks(
+ [
+ block
+ for msg in group
+ for block in msg.get_content_blocks()
+ if isinstance(block, DataBlock)
+ ],
+ image,
+ )
if history_text:
- xai_messages.append(user(history_text))
- is_first_agent_message = False
+ user_args.insert(0, history_text)
+ if user_args:
+ xai_messages.append(user(*user_args))
+ if history_text:
+ is_first_agent_message = False
return xai_messages
diff --git a/tests/formatter_xai_test.py b/tests/formatter_xai_test.py
index 5683bfa9c..508bfaed0 100644
--- a/tests/formatter_xai_test.py
+++ b/tests/formatter_xai_test.py
@@ -500,6 +500,51 @@ class TestXAIFormatter(IsolatedAsyncioTestCase):
res = await fmt.format([])
self.assertListEqual([], res)
+ async def test_multiagent_formatter_history_keeps_images(self) -> None:
+ """Images of the collapsed messages ride along with the history
+ text, in message order."""
+ fmt = XAIMultiAgentFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Compare these two charts."),
+ DataBlock(
+ source=Base64Source(
+ data="Zmlyc3Q=",
+ media_type="image/jpeg",
+ ),
+ ),
+ ],
+ ),
+ AssistantMsg(
+ name="agent",
+ content=[
+ TextBlock(text="The second one differs."),
+ DataBlock(
+ source=URLSource(
+ url="https://example.com/second.png",
+ media_type="image/png",
+ ),
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ user(
+ self._hist_prompt + "<history>\n"
+ "user: Compare these two charts.\n"
+ "agent: The second one differs.\n"
+ "</history>",
+ image("data:image/jpeg;base64,Zmlyc3Q="),
+ image("https://example.com/second.png"),
+ ),
+ ],
+ res,
+ )
+
async def test_chat_formatter_complex_multi_step(self) -> None:
"""Complex multi-step sequence with interleaved thinking, text,
tool calls, and tool results."""