Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(formatter): support provider-specific media parameters in DataBlock (#2103)
qbcd4d01e8e5658
diff --git a/src/agentscope/formatter/_anthropic_formatter.py b/src/agentscope/formatter/_anthropic_formatter.pyindex f6995666c..3bb09d69c 100644--- a/src/agentscope/formatter/_anthropic_formatter.py+++ b/src/agentscope/formatter/_anthropic_formatter.py@@ -285,7 +285,8 @@ class _AnthropicFormatterBase(FormatterBase, ABC):Returns:`dict[str, Any] | None`:- The formatted data block, or None if the media type is not+ The formatted data block with extra fields placed beside+ ``type`` and ``source``, or None if the media type is notsupported."""source = block.source@@ -304,9 +305,17 @@ class _AnthropicFormatterBase(FormatterBase, ABC):# Anthropic supports images and PDF documentsif media_type.startswith("image/"):- return self._format_source(source, "image")+ return self._format_source(+ source,+ "image",+ block.model_extra,+ )if media_type == "application/pdf":- return self._format_source(source, "document")+ return self._format_source(+ source,+ "document",+ block.model_extra,+ )logger.warning("Anthropic only supports image and PDF data, got %s, skipped.",@@ -318,16 +327,24 @@ class _AnthropicFormatterBase(FormatterBase, ABC):def _format_source(source: URLSource | Base64Source,block_type: str,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:"""Format an image or document source into an Anthropic base64content block.+ Extra fields are merged into the content block itself, at the same+ level as ``type`` and ``source`` (for example provider-specific cache+ controls).+Args:source (`URLSource | Base64Source`):The source to format. Local ``file://`` URLs are read fromdisk and remote URLs are downloaded.block_type (`str`):The Anthropic block type, ``"image"`` or ``"document"``.+ extra (`dict[str, Any] | None`, optional):+ Provider-specific parameters to merge into the content block,+ at the same level as ``type`` and ``source``.Returns:`dict[str, Any]`:@@ -350,7 +367,7 @@ class _AnthropicFormatterBase(FormatterBase, ABC):else:raise ValueError(f"Unsupported source type: {type(source)}")- return {+ result = {"type": block_type,"source": {"type": "base64",@@ -358,6 +375,9 @@ class _AnthropicFormatterBase(FormatterBase, ABC):"data": data,},}+ if extra:+ result.update(extra)+ return resultclass AnthropicChatFormatter(_AnthropicFormatterBase):diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.pyindex 9ee200f78..d7c866f9f 100644--- a/src/agentscope/formatter/_dashscope_formatter.py+++ b/src/agentscope/formatter/_dashscope_formatter.py@@ -73,6 +73,11 @@ class _DashScopeFormatterBase(FormatterBase, ABC):- Videos: ``{"type": "video_url", "video_url": {"url": ...}}``- Audio: ``{"type": "input_audio", "input_audio": {...}}``+ ``DataBlock`` extra fields are merged into the content item itself,+ at the same level as ``image_url``, ``video_url`` or ``input_audio``.+ This matches DashScope's OpenAI-compatible multimodal extensions, for+ example video ``fps`` and ``snippet``.+Args:block (`DataBlock`):The DataBlock to format.@@ -97,13 +102,13 @@ class _DashScopeFormatterBase(FormatterBase, ABC):main_type = block.source.media_type.split("/")[0]if main_type == "image":- return self._format_image_source(block.source)+ return self._format_image_source(block.source, block.model_extra)if main_type == "video":- return self._format_video_source(block.source)+ return self._format_video_source(block.source, block.model_extra)if main_type == "audio":- return self._format_audio_source(block.source)+ return self._format_audio_source(block.source, block.model_extra)logger.warning("Unsupported main media type %s for DashScope API. "@@ -115,11 +120,24 @@ class _DashScopeFormatterBase(FormatterBase, ABC):@staticmethoddef _format_image_source(source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:"""Convert an image source to OpenAI-compatible ``image_url`` format.Local ``file://`` URLs are read from disk and converted to base64- data URIs. Remote URLs are passed through unchanged.+ data URIs. Remote URLs are passed through unchanged. Extra fields are+ merged at the content-item level, not inside ``image_url``.++ Args:+ source (`URLSource | Base64Source`):+ The image source to convert.+ extra (`dict[str, Any] | None`, optional):+ Provider-specific image parameters to merge into the content+ item, at the same level as ``type`` and ``image_url``.++ Returns:+ `dict[str, Any]`:+ A DashScope image content item."""if isinstance(source, Base64Source):url = f"data:{source.media_type};base64,{source.data}"@@ -135,20 +153,37 @@ class _DashScopeFormatterBase(FormatterBase, ABC):else:raise ValueError(f"Unsupported image source type: {type(source)}")- return {+ result = {"type": "image_url","image_url": {"url": url},}+ if extra:+ result.update(extra)+ return result@staticmethoddef _format_video_source(source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:"""Convert a video source to DashScope's ``video_url`` format(OpenAI-compatible extension).Local ``file://`` URLs are read from disk and converted to base64- data URIs. Remote URLs are passed through unchanged.+ data URIs. Remote URLs are passed through unchanged. Extra fields such+ as ``fps`` and ``snippet`` are merged at the content-item level, not+ inside ``video_url``.++ Args:+ source (`URLSource | Base64Source`):+ The video source to convert.+ extra (`dict[str, Any] | None`, optional):+ Provider-specific video parameters to merge into the content+ item, such as DashScope's ``fps`` and ``snippet``.++ Returns:+ `dict[str, Any]`:+ A DashScope video content item."""if isinstance(source, Base64Source):url = f"data:{source.media_type};base64,{source.data}"@@ -164,57 +199,67 @@ class _DashScopeFormatterBase(FormatterBase, ABC):else:raise ValueError(f"Unsupported video source type: {type(source)}")- return {+ result = {"type": "video_url","video_url": {"url": url},}+ if extra:+ result.update(extra)+ return result@staticmethoddef _format_audio_source(source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:"""Convert an audio source to DashScope ``input_audio`` format.DashScope's compatible API accepts URLs directly in the ``data``- field. Base64-encoded audio must be wrapped in a data URL. Local- files are read, base64-encoded, and wrapped in the same form.+ field. Base64-encoded audio and local files are encoded as data URLs.+ Extra fields are merged at the content-item level, not inside+ ``input_audio``.++ Args:+ source (`URLSource | Base64Source`):+ The audio source to convert.+ extra (`dict[str, Any] | None`, optional):+ Provider-specific audio parameters to merge into the content+ item, at the same level as ``type`` and ``input_audio``.++ Returns:+ `dict[str, Any]`:+ A DashScope audio content item."""fmt = source.media_type.split("/")[-1]if fmt == "mpeg":fmt = "mp3"if isinstance(source, Base64Source):- return {- "type": "input_audio",- "input_audio": {- "data": f"data:;base64,{source.data}",- "format": fmt,- },- }+ data = f"data:;base64,{source.data}"- if isinstance(source, URLSource):+ elif isinstance(source, URLSource):url_str = str(source.url)if url_str.startswith("file://"):local_path = url2pathname(url_str.removeprefix("file://"))with open(local_path, "rb") as f:encoded = base64.b64encode(f.read()).decode("utf-8")- return {- "type": "input_audio",- "input_audio": {- "data": f"data:;base64,{encoded}",- "format": fmt,- },- }+ data = f"data:;base64,{encoded}"else:- return {- "type": "input_audio",- "input_audio": {- "data": url_str,- "format": fmt,- },- }-- raise ValueError(f"Unsupported audio source type: {type(source)}")+ data = url_str++ else:+ raise ValueError(f"Unsupported audio source type: {type(source)}")++ result = {+ "type": "input_audio",+ "input_audio": {+ "data": data,+ "format": fmt,+ },+ }+ if extra:+ result.update(extra)+ return resultclass DashScopeChatFormatter(_DashScopeFormatterBase):diff --git a/src/agentscope/formatter/_gemini_formatter.py b/src/agentscope/formatter/_gemini_formatter.pyindex de789d391..5529468c3 100644--- a/src/agentscope/formatter/_gemini_formatter.py+++ b/src/agentscope/formatter/_gemini_formatter.py@@ -42,7 +42,8 @@ class _GeminiFormatterBase(FormatterBase, ABC):Returns:`dict[str, Any] | None`:The formatted data block in Gemini ``inline_data`` format,- or None if the media type is not supported.+ with extra fields placed beside ``inline_data``, or None if+ the media type is not supported."""source = block.sourcemedia_type = source.media_type@@ -58,29 +59,31 @@ class _GeminiFormatterBase(FormatterBase, ABC):)return None- return self._format_media_source(source)+ return self._format_media_source(source, block.model_extra)@staticmethoddef _format_media_source(source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:"""Format a media source into Gemini API ``inline_data`` format.+ Extra fields are merged into the outer part object, at the same level+ as ``inline_data`` (for example Gemini video metadata).+Args:source (`URLSource | Base64Source`):The media source to format.+ extra (`dict[str, Any] | None`, optional):+ Provider-specific media parameters to merge into the outer+ part object, at the same level as ``inline_data``.Returns:`dict[str, Any]`:The formatted media source."""if isinstance(source, Base64Source):- return {- "inline_data": {- "data": source.data,- "mime_type": source.media_type,- },- }+ data = source.dataelif isinstance(source, URLSource):url = str(source.url)if url.startswith("file://"):@@ -88,26 +91,24 @@ class _GeminiFormatterBase(FormatterBase, ABC):file_path = url2pathname(url.removeprefix("file://"))with open(file_path, "rb") as f:data = base64.b64encode(f.read()).decode("utf-8")- return {- "inline_data": {- "data": data,- "mime_type": source.media_type,- },- }else:# Remote URL - download and convert to base64response = requests.get(url, timeout=30)response.raise_for_status()data = base64.b64encode(response.content).decode("utf-8")- return {- "inline_data": {- "data": data,- "mime_type": source.media_type,- },- }else:raise ValueError(f"Unsupported source type: {type(source)}")+ result = {+ "inline_data": {+ "data": data,+ "mime_type": source.media_type,+ },+ }+ if extra:+ result.update(extra)+ return result+class GeminiChatFormatter(_GeminiFormatterBase):"""The Gemini formatter class for chatbot scenario, where only a userdiff --git a/src/agentscope/formatter/_moonshot_formatter.py b/src/agentscope/formatter/_moonshot_formatter.pyindex af64fb59f..4d2098dba 100644--- a/src/agentscope/formatter/_moonshot_formatter.py+++ b/src/agentscope/formatter/_moonshot_formatter.py@@ -26,6 +26,7 @@ from ..message import (def _moonshot_format_media_source(source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:"""Convert an image or video source to Moonshot ``image_url`` or``video_url`` format.@@ -33,7 +34,20 @@ def _moonshot_format_media_source(Moonshot's vision API only accepts base64 data URIs or file IDs — rawremote URLs are rejected. This helper downloads remote ``http(s)://``URLs and converts them to base64 data URIs, while ``file://`` URLs and- ``Base64Source`` go through the same conversion as the OpenAI base.+ ``Base64Source`` go through the same conversion as the OpenAI base. Extra+ fields are merged into the nested ``image_url`` or ``video_url``+ object.++ Args:+ source (`URLSource | Base64Source`):+ The image or video source to convert.+ extra (`dict[str, Any] | None`, optional):+ Provider-specific media parameters to merge into the nested+ ``image_url`` or ``video_url`` object.++ Returns:+ `dict[str, Any]`:+ A Moonshot image or video content item."""if isinstance(source, Base64Source):url = f"data:{source.media_type};base64,{source.data}"@@ -57,7 +71,10 @@ def _moonshot_format_media_source(key = ("video_url" if source.media_type.startswith("video/") else "image_url")- return {"type": key, key: {"url": url}}+ payload: dict[str, Any] = {"url": url}+ if extra:+ payload.update(extra)+ return {"type": key, key: payload}class MoonshotChatFormatter(_OpenAIFormatterBase):@@ -86,8 +103,9 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):def _format_image_source(self,source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:- return _moonshot_format_media_source(source)+ return _moonshot_format_media_source(source, extra)def _format_openai_data_block(self,@@ -98,7 +116,10 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):fnmatch(media_type, pattern)for pattern in self.supported_input_media_types):- return _moonshot_format_media_source(block.source)+ return _moonshot_format_media_source(+ block.source,+ block.model_extra,+ )return super()._format_openai_data_block(block)# pylint: disable=too-many-branches@@ -348,8 +369,9 @@ class MoonshotMultiAgentFormatter(_OpenAIFormatterBase):def _format_image_source(self,source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:- return _moonshot_format_media_source(source)+ return _moonshot_format_media_source(source, extra)def _format_openai_data_block(self,@@ -360,7 +382,10 @@ class MoonshotMultiAgentFormatter(_OpenAIFormatterBase):fnmatch(media_type, pattern)for pattern in self.supported_input_media_types):- return _moonshot_format_media_source(block.source)+ return _moonshot_format_media_source(+ block.source,+ block.model_extra,+ )return super()._format_openai_data_block(block)async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:diff --git a/src/agentscope/formatter/_openai_formatter.py b/src/agentscope/formatter/_openai_formatter.pyindex cc58709e0..ea8e1fe9f 100644--- a/src/agentscope/formatter/_openai_formatter.py+++ b/src/agentscope/formatter/_openai_formatter.py@@ -63,8 +63,11 @@ class _OpenAIFormatterBase(FormatterBase, ABC):"""Format a DataBlock into the required format for OpenAI API.For image blocks, URLs are returned as-is (or converted to base64 for- local ``file://`` paths). For audio blocks, data is always converted- to base64 as required by the OpenAI input_audio format.+ local ``file://`` paths). Image extras such as ``detail`` are merged+ into the nested ``image_url`` payload as required by OpenAI Chat+ Completions. For audio blocks, data is always converted to base64 as+ required by the OpenAI ``input_audio`` format, and extras are merged+ into the nested ``input_audio`` payload.Args:block (`DataBlock`):@@ -90,10 +93,10 @@ class _OpenAIFormatterBase(FormatterBase, ABC):main_type = block.source.media_type.split("/")[0]if main_type == "image":- return self._format_image_source(block.source)+ return self._format_image_source(block.source, block.model_extra)if main_type == "audio":- return self._format_audio_source(block.source)+ return self._format_audio_source(block.source, block.model_extra)if block.source.media_type == "application/pdf":return self._format_file_source(block.source, block.name)@@ -108,22 +111,29 @@ class _OpenAIFormatterBase(FormatterBase, ABC):def _format_image_source(self,source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:"""Convert an image source to OpenAI image_url format.Local ``file://`` URLs are read from disk and converted to base64- data URIs. Remote URLs are passed through unchanged. Subclasses may- override this to apply provider-specific handling (e.g. forcing- remote URLs to be downloaded and base64-encoded for APIs that don't- accept raw HTTPS URLs).+ data URIs. Remote URLs are passed through unchanged. Extra media+ parameters (for example ``detail``) are merged into the nested+ ``image_url`` object, matching OpenAI Chat Completions' image input+ schema. Subclasses may override this to apply provider-specific+ handling (e.g. forcing remote URLs to be downloaded and base64-encoded+ for APIs that don't accept raw HTTPS URLs).Args:source (`URLSource | Base64Source`):The image source to convert.+ extra (`dict[str, Any] | None`, optional):+ Provider-specific image parameters to merge into the nested+ ``image_url`` object, such as OpenAI's ``detail``.Returns:`dict[str, Any]`:A dictionary with ``"type": "image_url"`` in OpenAI format.+ Extra fields are nested under the ``image_url`` object."""if isinstance(source, Base64Source):url = f"data:{source.media_type};base64,{source.data}"@@ -143,23 +153,31 @@ class _OpenAIFormatterBase(FormatterBase, ABC):else:raise ValueError(f"Unsupported image source type: {type(source)}")+ image_url_payload = {"url": url}+ if extra:+ image_url_payload.update(extra)return {"type": "image_url",- "image_url": {"url": url},+ "image_url": image_url_payload,}@staticmethoddef _format_audio_source(source: URLSource | Base64Source,+ extra: dict[str, Any] | None = None,) -> dict[str, Any]:"""Convert an audio source to OpenAI input_audio format.Local ``file://`` URLs are read from disk. Remote URLs are downloaded.Only ``wav`` and ``mp3`` formats are supported by the OpenAI API.+ Extra fields are merged into the nested ``input_audio`` object.Args:source (`URLSource | Base64Source`):The audio source to convert.+ extra (`dict[str, Any] | None`, optional):+ Provider-specific audio parameters to merge into the nested+ ``input_audio`` object.Returns:`dict[str, Any]`:@@ -179,12 +197,15 @@ class _OpenAIFormatterBase(FormatterBase, ABC):audio_format = media_type_to_format[media_type]if isinstance(source, Base64Source):+ input_audio_payload = {+ "data": source.data,+ "format": audio_format,+ }+ if extra:+ input_audio_payload.update(extra)return {"type": "input_audio",- "input_audio": {- "data": source.data,- "format": audio_format,- },+ "input_audio": input_audio_payload,}if isinstance(source, URLSource):@@ -200,12 +221,15 @@ class _OpenAIFormatterBase(FormatterBase, ABC):response.raise_for_status()data = base64.b64encode(response.content).decode("utf-8")+ input_audio_payload = {+ "data": data,+ "format": audio_format,+ }+ if extra:+ input_audio_payload.update(extra)return {"type": "input_audio",- "input_audio": {- "data": data,- "format": audio_format,- },+ "input_audio": input_audio_payload,}raise TypeError(f"Unsupported audio source type: {type(source)}.")diff --git a/src/agentscope/formatter/_openai_response_formatter.py b/src/agentscope/formatter/_openai_response_formatter.pyindex 5f337c629..914c01fac 100644--- a/src/agentscope/formatter/_openai_response_formatter.py+++ b/src/agentscope/formatter/_openai_response_formatter.py@@ -52,6 +52,11 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):audio input yet; use Chat Completions API instead). Seehttps://developers.openai.com/api/docs/guides/audio+ For image blocks, ``DataBlock`` extra fields are merged into the+ ``input_image`` item itself (for example ``detail``), because the+ Responses API uses a flat ``image_url`` string instead of the nested+ Chat Completions ``image_url`` object.+Args:block (`DataBlock`):The DataBlock to format.@@ -80,10 +85,14 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):return Noneif base_result.get("type") == "image_url":- return {+ result = {"type": "input_image","image_url": base_result["image_url"]["url"],}+ extra = block.model_extra or {}+ if extra:+ result.update(extra)+ return resultif base_result.get("type") == "file":return {"type": "input_file", **base_result["file"]}diff --git a/src/agentscope/formatter/_xai_formatter.py b/src/agentscope/formatter/_xai_formatter.pyindex 11f3209de..0032e43c3 100644--- a/src/agentscope/formatter/_xai_formatter.py+++ b/src/agentscope/formatter/_xai_formatter.py@@ -39,7 +39,8 @@ def _xai_user_args_from_blocks(DataBlocks that are not images (or use an unsupported media type) aredropped with a warning, matching the user-role DataBlock handling in- :meth:`XAIChatFormatter.format`.+ :meth:`XAIChatFormatter.format`. Image extras are forwarded to+ ``xai_sdk.chat.image`` as keyword arguments.Args:blocks (`list`):@@ -84,15 +85,19 @@ def _xai_user_args_from_blocks(image(f"data:{sub.source.media_type};"f"base64,{encoded}",+ **(sub.model_extra or {}),),)else:- args.append(image(url_str))+ args.append(+ image(url_str, **(sub.model_extra or {})),+ )elif isinstance(sub.source, Base64Source):args.append(image(f"data:{sub.source.media_type};"f"base64,{sub.source.data}",+ **(sub.model_extra or {}),),)return args@@ -102,7 +107,9 @@ class XAIChatFormatter(FormatterBase):"""Formatter for the xAI chat model.Converts ``Msg`` objects into ``xai_sdk`` protobuf ``Message`` objects- that can be appended directly to a ``xai_sdk`` chat session.+ that can be appended directly to a ``xai_sdk`` chat session. Image+ ``DataBlock`` extras are passed as keyword arguments to+ ``xai_sdk.chat.image`` (for example ``detail``).Unlike other formatters whose ``format()`` returns ``list[dict]``, thisformatter returns ``list[chat_pb2.Message]``. The type annotation is@@ -134,8 +141,9 @@ class XAIChatFormatter(FormatterBase):Returns:`list[Any]`:A list of ``chat_pb2.Message`` proto objects, ready to be- appended to a ``xai_sdk`` chat session via- ``chat.append()``.+ appended to a ``xai_sdk`` chat session via ``chat.append()``.+ Image ``DataBlock`` extras are forwarded to+ ``xai_sdk.chat.image`` as keyword arguments."""from xai_sdk.chat import (assistant,diff --git a/src/agentscope/message/_block.py b/src/agentscope/message/_block.pyindex dd0a7aad1..cdb167810 100644--- a/src/agentscope/message/_block.py+++ b/src/agentscope/message/_block.py@@ -81,7 +81,16 @@ class URLSource(BaseModel):class DataBlock(BaseModel):- """The data block for binary content (images, audio, video, etc.)."""+ """The data block for binary content (images, audio, video, etc.).++ Allows extra provider-specific fields (e.g. OpenAI's ``detail``,+ DashScope's ``fps``) via ``extra="allow"`` so users can attach media+ processing parameters without subclassing. Formatters decide where these+ extras belong in the target API payload because providers use different+ nesting rules.+ """++ model_config = ConfigDict(extra="allow")type: Literal["data"] = "data""""The type of the data block, which is always 'data'."""diff --git a/tests/formatter_anthropic_test.py b/tests/formatter_anthropic_test.pyindex 903b71a3a..ac2e5b7ac 100644--- a/tests/formatter_anthropic_test.py+++ b/tests/formatter_anthropic_test.py@@ -1117,6 +1117,49 @@ class TestAnthropicFormatter( # pylint: disable=too-many-public-methodsres,)+ async def test_chat_formatter_image_extra_params(self) -> None:+ """DataBlock extra params are forwarded to Anthropic image block."""+ fmt = AnthropicChatFormatter()+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Inspect this screenshot:"),+ DataBlock(+ source=Base64Source(+ data=self.image_b64,+ media_type="image/png",+ ),+ cache_control={"type": "ephemeral"},+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ {+ "role": "user",+ "content": [+ {+ "type": "text",+ "text": "Inspect this screenshot:",+ },+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": self.image_b64,+ },+ "cache_control": {"type": "ephemeral"},+ },+ ],+ },+ ],+ res,+ )+async def test_chat_formatter_parallel_tool_results_merged(self,) -> None:diff --git a/tests/formatter_dashscope_test.py b/tests/formatter_dashscope_test.pyindex 14d4e259b..71b365efa 100644--- a/tests/formatter_dashscope_test.py+++ b/tests/formatter_dashscope_test.py@@ -313,6 +313,50 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):# DashScopeChatFormatter tests# -------------------------------------------------------------------+ async def test_chat_formatter_video_extra_params(self) -> None:+ """DataBlock extra params are forwarded at content-item level."""+ fmt = DashScopeChatFormatter()+ video_url = str(+ URLSource(+ url="https://example.com/video.mp4",+ media_type="video/mp4",+ ).url,+ )+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Summarize this video."),+ DataBlock(+ source=URLSource(+ url=video_url,+ media_type="video/mp4",+ ),+ fps=2,+ snippet=[0, 30],+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Summarize this video."},+ {+ "type": "video_url",+ "video_url": {"url": video_url},+ "fps": 2,+ "snippet": [0, 30],+ },+ ],+ },+ ],+ res,+ )+async def test_chat_formatter(self) -> None:"""Chat formatter produces exact output for various subsets."""fmt = DashScopeChatFormatter()diff --git a/tests/formatter_gemini_test.py b/tests/formatter_gemini_test.pyindex ed6bf9260..b2a9b05a6 100644--- a/tests/formatter_gemini_test.py+++ b/tests/formatter_gemini_test.py@@ -1160,6 +1160,44 @@ class TestGeminiFormatter(IsolatedAsyncioTestCase):res,)+ async def test_chat_formatter_media_extra_params(self) -> None:+ """DataBlock extra params are forwarded beside inline_data."""+ fmt = GeminiChatFormatter()+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Summarize this video."),+ DataBlock(+ source=Base64Source(+ data=self.image_b64,+ media_type="video/mp4",+ ),+ video_metadata={"fps": 2},+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ {+ "role": "user",+ "parts": [+ {"text": "Summarize this video."},+ {+ "inline_data": {+ "data": self.image_b64,+ "mime_type": "video/mp4",+ },+ "video_metadata": {"fps": 2},+ },+ ],+ },+ ],+ res,+ )+async def test_tool_call_with_incomplete_input_does_not_crash(self,) -> None:diff --git a/tests/formatter_moonshot_test.py b/tests/formatter_moonshot_test.pyindex 14bc43ac9..1457a4d90 100644--- a/tests/formatter_moonshot_test.py+++ b/tests/formatter_moonshot_test.py@@ -333,6 +333,45 @@ class TestMoonshotFormatter(IsolatedAsyncioTestCase):res = await fmt.format([])self.assertListEqual([], res)+ async def test_chat_formatter_image_extra_params(self) -> None:+ """DataBlock extra params are nested inside image_url payload."""+ fmt = MoonshotChatFormatter()+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Inspect this image."),+ DataBlock(+ source=URLSource(+ url=self.image_url,+ media_type="image/png",+ ),+ detail="high",+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ {+ "role": "user",+ "name": "user",+ "content": [+ {"type": "text", "text": "Inspect this image."},+ {+ "type": "image_url",+ "image_url": {+ "url": self.image_data_uri,+ "detail": "high",+ },+ },+ ],+ },+ ],+ res,+ )+async def test_chat_formatter_thinking_to_reasoning_content(self,) -> None:@@ -514,6 +553,34 @@ class TestMoonshotFormatter(IsolatedAsyncioTestCase):await fmt.format(msgs),)+ async def test_video_extra_params(self) -> None:+ """Both Moonshot formatters preserve video-specific parameters."""+ block = DataBlock(+ source=Base64Source(+ data="ZmFrZSB2aWRlbw==",+ media_type="video/mp4",+ ),+ fps=2,+ )+ for formatter_type in (+ MoonshotChatFormatter,+ MoonshotMultiAgentFormatter,+ ):+ with self.subTest(formatter=formatter_type.__name__):+ result = await formatter_type().format(+ [UserMsg(name="user", content=[block])],+ )+ self.assertEqual(+ result[0]["content"][-1],+ {+ "type": "video_url",+ "video_url": {+ "url": "data:video/mp4;base64,ZmFrZSB2aWRlbw==",+ "fps": 2,+ },+ },+ )+async def test_chat_formatter_url_image_in_tool_result(self,) -> None:diff --git a/tests/formatter_openai_chat_test.py b/tests/formatter_openai_chat_test.pyindex 1d1e67f0f..94c0d9c90 100644--- a/tests/formatter_openai_chat_test.py+++ b/tests/formatter_openai_chat_test.py@@ -287,6 +287,85 @@ class TestOpenAIFormatter(IsolatedAsyncioTestCase):# OpenAIChatFormatter tests# -------------------------------------------------------------------+ async def test_chat_formatter_image_extra_params(self) -> None:+ """DataBlock extra params are nested inside image_url payload."""+ fmt = OpenAIChatFormatter()+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Inspect this image."),+ DataBlock(+ source=URLSource(+ url=self.image_url,+ media_type="image/png",+ ),+ detail="high",+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ {+ "role": "user",+ "name": "user",+ "content": [+ {"type": "text", "text": "Inspect this image."},+ {+ "type": "image_url",+ "image_url": {+ "url": self.image_url,+ "detail": "high",+ },+ },+ ],+ },+ ],+ res,+ )++ async def test_chat_formatter_audio_extra_params(self) -> None:+ """DataBlock extra params are nested inside input_audio payload."""+ fmt = OpenAIChatFormatter()+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Transcribe this audio."),+ DataBlock(+ source=Base64Source(+ data="ZmFrZSBhdWRpbyBkYXRh",+ media_type="audio/wav",+ ),+ extra_audio_param="value",+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ {+ "role": "user",+ "name": "user",+ "content": [+ {"type": "text", "text": "Transcribe this audio."},+ {+ "type": "input_audio",+ "input_audio": {+ "data": "ZmFrZSBhdWRpbyBkYXRh",+ "format": "wav",+ "extra_audio_param": "value",+ },+ },+ ],+ },+ ],+ res,+ )+async def test_chat_formatter(self) -> None:"""Chat formatter produces exact output for various subsets."""fmt = OpenAIChatFormatter()diff --git a/tests/formatter_openai_response_test.py b/tests/formatter_openai_response_test.pyindex b6899e3a0..a48589844 100644--- a/tests/formatter_openai_response_test.py+++ b/tests/formatter_openai_response_test.py@@ -332,6 +332,47 @@ class TestOpenAIResponseFormatter(IsolatedAsyncioTestCase):res,)+ async def test_chat_formatter_image_detail_extra_param(self) -> None:+ """Responses API forwards image detail at input_image top level."""+ fmt = OpenAIResponseFormatter()+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Inspect this image."),+ DataBlock(+ source=URLSource(+ url=self.image_url,+ media_type="image/png",+ ),+ detail="high",+ custom_param="value",+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ {+ "role": "user",+ "content": [+ {+ "type": "input_text",+ "text": "Inspect this image.",+ },+ {+ "type": "input_image",+ "image_url": self.image_url,+ "detail": "high",+ "custom_param": "value",+ },+ ],+ },+ ],+ res,+ )+async def test_chat_formatter_base64_pdf(self) -> None:"""Base64-encoded PDF becomes an ``input_file`` item."""fmt = OpenAIResponseFormatter()diff --git a/tests/formatter_xai_test.py b/tests/formatter_xai_test.pyindex a799caf7e..d2ab7a9ba 100644--- a/tests/formatter_xai_test.py+++ b/tests/formatter_xai_test.py@@ -70,17 +70,18 @@ class _StubMessage:class _StubImage:"""Comparable stub for xai_sdk.chat.image()."""- def __init__(self, url: str) -> None:+ def __init__(self, url: str, detail: str = "auto") -> None:self.type = "image"self.url = url+ self.detail = detaildef __eq__(self, other: object) -> bool:if not isinstance(other, _StubImage):return NotImplemented- return self.url == other.url+ return self.url == other.url and self.detail == other.detaildef __repr__(self) -> str:- return f"_StubImage(url={self.url!r})"+ return f"_StubImage(url={self.url!r}, detail={self.detail!r})"def user(*args: Any) -> _StubMessage:@@ -103,9 +104,9 @@ def tool_result(*args: Any, **kwargs: Any) -> _StubMessage:return _StubMessage(role="tool", args=args, kwargs=kwargs)-def image(url: str) -> _StubImage:+def image(url: str, *, detail: str = "auto") -> _StubImage:"""Create a comparable stub image object."""- return _StubImage(url=url)+ return _StubImage(url=url, detail=detail)# ---------------------------------------------------------------------------@@ -256,6 +257,39 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods# XAIChatFormatter tests# -------------------------------------------------------------------+ async def test_chat_formatter_image_detail_extra_param(self) -> None:+ """DataBlock detail extra param is forwarded to xAI image()."""+ fmt = XAIChatFormatter()+ image_b64 = "ZmFrZSBpbWFnZSBkYXRh"+ msgs = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Inspect this image."),+ DataBlock(+ source=Base64Source(+ data=image_b64,+ media_type="image/png",+ ),+ detail="high",+ ),+ ],+ ),+ ]+ res = await fmt.format(msgs)+ self.assertListEqual(+ [+ user(+ "Inspect this image.",+ image(+ f"data:image/png;base64,{image_b64}",+ detail="high",+ ),+ ),+ ],+ res,+ )+async def test_chat_formatter_system_message(self) -> None:"""System message becomes a system() stub."""fmt = XAIChatFormatter()@@ -299,6 +333,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsurl=path.as_uri(),media_type="image/png",),+ detail="high",),],),@@ -308,7 +343,10 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods[user("What's in this image?",- image(f"data:image/png;base64,{expected_b64}"),+ image(+ f"data:image/png;base64,{expected_b64}",+ detail="high",+ ),),],res,@@ -731,6 +769,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsdata="Zmlyc3Q=",media_type="image/jpeg",),+ detail="high",),],),@@ -743,6 +782,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsurl="https://example.com/second.png",media_type="image/png",),+ detail="low",),],),@@ -755,8 +795,8 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods"user: Compare these two charts.\n""agent: The second one differs.\n""</history>",- image("data:image/jpeg;base64,Zmlyc3Q="),- image("https://example.com/second.png"),+ image("data:image/jpeg;base64,Zmlyc3Q=", detail="high"),+ image("https://example.com/second.png", detail="low"),),],res,@@ -954,6 +994,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methodsdata=image_b64,media_type="image/png",),+ detail="high",),],),@@ -965,7 +1006,10 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods[user("Inspect this screenshot:",- image(f"data:image/png;base64,{image_b64}"),+ image(+ f"data:image/png;base64,{image_b64}",+ detail="high",+ ),),],res,diff --git a/tests/message_test.py b/tests/message_test.pyindex 6285308ed..1235a04e9 100644--- a/tests/message_test.py+++ b/tests/message_test.py@@ -335,6 +335,7 @@ class MessageTest(IsolatedAsyncioTestCase):url="https://example.com/image.png",media_type="image/png",),+ detail="high",),DataBlock(source=Base64Source(@@ -368,6 +369,7 @@ class MessageTest(IsolatedAsyncioTestCase):"media_type": "image/png",},"name": None,+ "detail": "high","created_at": AnyString(),"finished_at": None,},