Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(formatter): respect xAI image input types (#2823)
wangtaoab42529dbb0b
diff --git a/src/agentscope/formatter/_xai_formatter.py b/src/agentscope/formatter/_xai_formatter.pyindex 8df200e89..1f2bb8821 100644--- a/src/agentscope/formatter/_xai_formatter.py+++ b/src/agentscope/formatter/_xai_formatter.py@@ -8,6 +8,7 @@ other formatter, the ``format()`` method returns a list of``xai_sdk`` chat API accepts proto messages directly."""import base64+from fnmatch import fnmatchfrom typing import Any, Listfrom pydantic import Field@@ -30,6 +31,7 @@ from ..message import (def _xai_user_args_from_blocks(blocks: list,image: Any,+ supported_media_types: list[str],) -> list:"""Convert a list of ``TextBlock | DataBlock`` into the positionalargs expected by ``xai_sdk.chat.user(*args)``.@@ -44,6 +46,8 @@ def _xai_user_args_from_blocks(image (`Any`):The ``xai_sdk.chat.image`` constructor, passed in to keepthe local import contract identical to ``format``.+ supported_media_types (`list[str]`):+ Glob-style media-type patterns accepted by the formatter.Returns:`list`:@@ -55,12 +59,16 @@ def _xai_user_args_from_blocks(if isinstance(sub, TextBlock):args.append(sub.text)elif isinstance(sub, DataBlock):- if not sub.source.media_type.startswith("image/"):+ if not sub.source.media_type.startswith("image/") or not any(+ fnmatch(sub.source.media_type, pattern)+ for pattern in supported_media_types+ ):logger.warning("Unsupported media type %s for xAI API. "- "Only image/jpeg and image/png are supported. "+ "Supported media types are: %s. ""This block will be skipped.",sub.source.media_type,+ ", ".join(supported_media_types),)continueif isinstance(sub.source, URLSource):@@ -165,46 +173,20 @@ class XAIChatFormatter(FormatterBase):hint_args = _xai_user_args_from_blocks(block.hint,image,+ self.supported_input_media_types,)if hint_args:xai_messages.append(user(*hint_args))elif isinstance(block, TextBlock):content_args.append(block.text)elif isinstance(block, DataBlock):- if block.source.media_type.startswith("image/"):- if isinstance(block.source, URLSource):- url_str = str(block.source.url)- if url_str.startswith("file://"):- # Local file — read and encode as data URI- local_path = url_str.removeprefix(- "file://",- )- with open(local_path, "rb") as f:- encoded = base64.b64encode(- f.read(),- ).decode("utf-8")- content_args.append(- image(- f"data:{block.source.media_type};"- f"base64,{encoded}",- ),- )- else:- content_args.append(image(url_str))- elif isinstance(block.source, Base64Source):- content_args.append(- image(- f"data:{block.source.media_type};"- f"base64,{block.source.data}",- ),- )- else:- logger.warning(- "Unsupported media type %s for xAI API. "- "Only image/jpeg and image/png are supported. "- "This block will be skipped.",- block.source.media_type,- )+ content_args.extend(+ _xai_user_args_from_blocks(+ [block],+ image,+ self.supported_input_media_types,+ ),+ )else:logger.warning("Unsupported block type %s in user message, "@@ -305,6 +287,7 @@ class XAIChatFormatter(FormatterBase):hint_args = _xai_user_args_from_blocks(block.hint,image,+ self.supported_input_media_types,)if hint_args:xai_messages.append(user(*hint_args))@@ -461,6 +444,7 @@ class XAIMultiAgentFormatter(FormatterBase):if isinstance(block, DataBlock)],image,+ self.supported_input_media_types,)if history_text:user_args.insert(0, history_text)diff --git a/tests/formatter_xai_test.py b/tests/formatter_xai_test.pyindex 508bfaed0..30c60fbc1 100644--- a/tests/formatter_xai_test.py+++ b/tests/formatter_xai_test.py@@ -184,7 +184,9 @@ def _build_xai_sdk_stub() -> None:_build_xai_sdk_stub()-class TestXAIFormatter(IsolatedAsyncioTestCase):+class TestXAIFormatter( # pylint: disable=too-many-public-methods+ IsolatedAsyncioTestCase,+):"""Comprehensive tests for XAI Chat and MultiAgent formatters.The stub objects support __eq__, so full assertListEqual works for@@ -417,6 +419,28 @@ class TestXAIFormatter(IsolatedAsyncioTestCase):res = await fmt.format([])self.assertListEqual([], res)+ async def test_chat_formatter_respects_image_input_types(self) -> None:+ """Only image media types declared in input_types are forwarded."""+ msg = UserMsg(+ name="user",+ content=[+ DataBlock(+ source=Base64Source(+ data="R0lGODlh",+ media_type="image/gif",+ ),+ ),+ ],+ )++ self.assertListEqual([], await XAIChatFormatter().format([msg]))+ self.assertListEqual(+ [user(image("data:image/gif;base64,R0lGODlh"))],+ await XAIChatFormatter(+ input_types=["text/plain", "image/*"],+ ).format([msg]),+ )+# -------------------------------------------------------------------# XAIMultiAgentFormatter tests# -------------------------------------------------------------------@@ -545,6 +569,36 @@ class TestXAIFormatter(IsolatedAsyncioTestCase):res,)+ async def test_multiagent_history_respects_image_input_types(self) -> None:+ """Collapsed history does not forward undeclared image types."""+ fmt = XAIMultiAgentFormatter()+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Inspect this animation."),+ DataBlock(+ source=Base64Source(+ data="R0lGODlh",+ media_type="image/gif",+ ),+ ),+ ],+ ),+ ]++ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ user(+ self._hist_prompt + "<history>\n"+ "user: Inspect this animation.\n"+ "</history>",+ ),+ ],+ res,+ )+async def test_chat_formatter_complex_multi_step(self) -> None:"""Complex multi-step sequence with interleaved thinking, text,tool calls, and tool results."""