Skip to content

Build and run agents you can see, understand and trust.

fix(formatter): respect xAI image input types (#2823)

wangtaoab42529dbb0b
diff --git a/src/agentscope/formatter/_xai_formatter.py b/src/agentscope/formatter/_xai_formatter.py
index 8df200e89..1f2bb8821 100644
--- a/src/agentscope/formatter/_xai_formatter.py
+++ b/src/agentscope/formatter/_xai_formatter.py
@@ -8,6 +8,7 @@ other formatter, the ``format()`` method returns a list of
``xai_sdk`` chat API accepts proto messages directly.
"""
import base64
+from fnmatch import fnmatch
from typing import Any, List
from pydantic import Field
@@ -30,6 +31,7 @@ from ..message import (
def _xai_user_args_from_blocks(
blocks: list,
image: Any,
+ supported_media_types: list[str],
) -> list:
"""Convert a list of ``TextBlock | DataBlock`` into the positional
args expected by ``xai_sdk.chat.user(*args)``.
@@ -44,6 +46,8 @@ def _xai_user_args_from_blocks(
image (`Any`):
The ``xai_sdk.chat.image`` constructor, passed in to keep
the local import contract identical to ``format``.
+ supported_media_types (`list[str]`):
+ Glob-style media-type patterns accepted by the formatter.
Returns:
`list`:
@@ -55,12 +59,16 @@ def _xai_user_args_from_blocks(
if isinstance(sub, TextBlock):
args.append(sub.text)
elif isinstance(sub, DataBlock):
- if not sub.source.media_type.startswith("image/"):
+ if not sub.source.media_type.startswith("image/") or not any(
+ fnmatch(sub.source.media_type, pattern)
+ for pattern in supported_media_types
+ ):
logger.warning(
"Unsupported media type %s for xAI API. "
- "Only image/jpeg and image/png are supported. "
+ "Supported media types are: %s. "
"This block will be skipped.",
sub.source.media_type,
+ ", ".join(supported_media_types),
)
continue
if isinstance(sub.source, URLSource):
@@ -165,46 +173,20 @@ class XAIChatFormatter(FormatterBase):
hint_args = _xai_user_args_from_blocks(
block.hint,
image,
+ self.supported_input_media_types,
)
if hint_args:
xai_messages.append(user(*hint_args))
elif isinstance(block, TextBlock):
content_args.append(block.text)
elif isinstance(block, DataBlock):
- if block.source.media_type.startswith("image/"):
- if isinstance(block.source, URLSource):
- url_str = str(block.source.url)
- if url_str.startswith("file://"):
- # Local file — read and encode as data URI
- local_path = url_str.removeprefix(
- "file://",
- )
- with open(local_path, "rb") as f:
- encoded = base64.b64encode(
- f.read(),
- ).decode("utf-8")
- content_args.append(
- image(
- f"data:{block.source.media_type};"
- f"base64,{encoded}",
- ),
- )
- else:
- content_args.append(image(url_str))
- elif isinstance(block.source, Base64Source):
- content_args.append(
- image(
- f"data:{block.source.media_type};"
- f"base64,{block.source.data}",
- ),
- )
- else:
- logger.warning(
- "Unsupported media type %s for xAI API. "
- "Only image/jpeg and image/png are supported. "
- "This block will be skipped.",
- block.source.media_type,
- )
+ content_args.extend(
+ _xai_user_args_from_blocks(
+ [block],
+ image,
+ self.supported_input_media_types,
+ ),
+ )
else:
logger.warning(
"Unsupported block type %s in user message, "
@@ -305,6 +287,7 @@ class XAIChatFormatter(FormatterBase):
hint_args = _xai_user_args_from_blocks(
block.hint,
image,
+ self.supported_input_media_types,
)
if hint_args:
xai_messages.append(user(*hint_args))
@@ -461,6 +444,7 @@ class XAIMultiAgentFormatter(FormatterBase):
if isinstance(block, DataBlock)
],
image,
+ self.supported_input_media_types,
)
if history_text:
user_args.insert(0, history_text)
diff --git a/tests/formatter_xai_test.py b/tests/formatter_xai_test.py
index 508bfaed0..30c60fbc1 100644
--- a/tests/formatter_xai_test.py
+++ b/tests/formatter_xai_test.py
@@ -184,7 +184,9 @@ def _build_xai_sdk_stub() -> None:
_build_xai_sdk_stub()
-class TestXAIFormatter(IsolatedAsyncioTestCase):
+class TestXAIFormatter( # pylint: disable=too-many-public-methods
+ IsolatedAsyncioTestCase,
+):
"""Comprehensive tests for XAI Chat and MultiAgent formatters.
The stub objects support __eq__, so full assertListEqual works for
@@ -417,6 +419,28 @@ class TestXAIFormatter(IsolatedAsyncioTestCase):
res = await fmt.format([])
self.assertListEqual([], res)
+ async def test_chat_formatter_respects_image_input_types(self) -> None:
+ """Only image media types declared in input_types are forwarded."""
+ msg = UserMsg(
+ name="user",
+ content=[
+ DataBlock(
+ source=Base64Source(
+ data="R0lGODlh",
+ media_type="image/gif",
+ ),
+ ),
+ ],
+ )
+
+ self.assertListEqual([], await XAIChatFormatter().format([msg]))
+ self.assertListEqual(
+ [user(image("data:image/gif;base64,R0lGODlh"))],
+ await XAIChatFormatter(
+ input_types=["text/plain", "image/*"],
+ ).format([msg]),
+ )
+
# -------------------------------------------------------------------
# XAIMultiAgentFormatter tests
# -------------------------------------------------------------------
@@ -545,6 +569,36 @@ class TestXAIFormatter(IsolatedAsyncioTestCase):
res,
)
+ async def test_multiagent_history_respects_image_input_types(self) -> None:
+ """Collapsed history does not forward undeclared image types."""
+ fmt = XAIMultiAgentFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Inspect this animation."),
+ DataBlock(
+ source=Base64Source(
+ data="R0lGODlh",
+ media_type="image/gif",
+ ),
+ ),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ user(
+ self._hist_prompt + "<history>\n"
+ "user: Inspect this animation.\n"
+ "</history>",
+ ),
+ ],
+ res,
+ )
+
async def test_chat_formatter_complex_multi_step(self) -> None:
"""Complex multi-step sequence with interleaved thinking, text,
tool calls, and tool results."""