Skip to content

Build and run agents you can see, understand and trust.

feat(formatter): support provider-specific media parameters in DataBlock (#2103)

qbcd4d01e8e5658
diff --git a/src/agentscope/formatter/_anthropic_formatter.py b/src/agentscope/formatter/_anthropic_formatter.py
index f6995666c..3bb09d69c 100644
--- a/src/agentscope/formatter/_anthropic_formatter.py
+++ b/src/agentscope/formatter/_anthropic_formatter.py
@@ -285,7 +285,8 @@ class _AnthropicFormatterBase(FormatterBase, ABC):
Returns:
`dict[str, Any] | None`:
- The formatted data block, or None if the media type is not
+ The formatted data block with extra fields placed beside
+ ``type`` and ``source``, or None if the media type is not
supported.
"""
source = block.source
@@ -304,9 +305,17 @@ class _AnthropicFormatterBase(FormatterBase, ABC):
# Anthropic supports images and PDF documents
if media_type.startswith("image/"):
- return self._format_source(source, "image")
+ return self._format_source(
+ source,
+ "image",
+ block.model_extra,
+ )
if media_type == "application/pdf":
- return self._format_source(source, "document")
+ return self._format_source(
+ source,
+ "document",
+ block.model_extra,
+ )
logger.warning(
"Anthropic only supports image and PDF data, got %s, skipped.",
@@ -318,16 +327,24 @@ class _AnthropicFormatterBase(FormatterBase, ABC):
def _format_source(
source: URLSource | Base64Source,
block_type: str,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Format an image or document source into an Anthropic base64
content block.
+ Extra fields are merged into the content block itself, at the same
+ level as ``type`` and ``source`` (for example provider-specific cache
+ controls).
+
Args:
source (`URLSource | Base64Source`):
The source to format. Local ``file://`` URLs are read from
disk and remote URLs are downloaded.
block_type (`str`):
The Anthropic block type, ``"image"`` or ``"document"``.
+ extra (`dict[str, Any] | None`, optional):
+ Provider-specific parameters to merge into the content block,
+ at the same level as ``type`` and ``source``.
Returns:
`dict[str, Any]`:
@@ -350,7 +367,7 @@ class _AnthropicFormatterBase(FormatterBase, ABC):
else:
raise ValueError(f"Unsupported source type: {type(source)}")
- return {
+ result = {
"type": block_type,
"source": {
"type": "base64",
@@ -358,6 +375,9 @@ class _AnthropicFormatterBase(FormatterBase, ABC):
"data": data,
},
}
+ if extra:
+ result.update(extra)
+ return result
class AnthropicChatFormatter(_AnthropicFormatterBase):
diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.py
index 9ee200f78..d7c866f9f 100644
--- a/src/agentscope/formatter/_dashscope_formatter.py
+++ b/src/agentscope/formatter/_dashscope_formatter.py
@@ -73,6 +73,11 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
- Videos: ``{"type": "video_url", "video_url": {"url": ...}}``
- Audio: ``{"type": "input_audio", "input_audio": {...}}``
+ ``DataBlock`` extra fields are merged into the content item itself,
+ at the same level as ``image_url``, ``video_url`` or ``input_audio``.
+ This matches DashScope's OpenAI-compatible multimodal extensions, for
+ example video ``fps`` and ``snippet``.
+
Args:
block (`DataBlock`):
The DataBlock to format.
@@ -97,13 +102,13 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
main_type = block.source.media_type.split("/")[0]
if main_type == "image":
- return self._format_image_source(block.source)
+ return self._format_image_source(block.source, block.model_extra)
if main_type == "video":
- return self._format_video_source(block.source)
+ return self._format_video_source(block.source, block.model_extra)
if main_type == "audio":
- return self._format_audio_source(block.source)
+ return self._format_audio_source(block.source, block.model_extra)
logger.warning(
"Unsupported main media type %s for DashScope API. "
@@ -115,11 +120,24 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
@staticmethod
def _format_image_source(
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Convert an image source to OpenAI-compatible ``image_url`` format.
Local ``file://`` URLs are read from disk and converted to base64
- data URIs. Remote URLs are passed through unchanged.
+ data URIs. Remote URLs are passed through unchanged. Extra fields are
+ merged at the content-item level, not inside ``image_url``.
+
+ Args:
+ source (`URLSource | Base64Source`):
+ The image source to convert.
+ extra (`dict[str, Any] | None`, optional):
+ Provider-specific image parameters to merge into the content
+ item, at the same level as ``type`` and ``image_url``.
+
+ Returns:
+ `dict[str, Any]`:
+ A DashScope image content item.
"""
if isinstance(source, Base64Source):
url = f"data:{source.media_type};base64,{source.data}"
@@ -135,20 +153,37 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
else:
raise ValueError(f"Unsupported image source type: {type(source)}")
- return {
+ result = {
"type": "image_url",
"image_url": {"url": url},
}
+ if extra:
+ result.update(extra)
+ return result
@staticmethod
def _format_video_source(
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Convert a video source to DashScope's ``video_url`` format
(OpenAI-compatible extension).
Local ``file://`` URLs are read from disk and converted to base64
- data URIs. Remote URLs are passed through unchanged.
+ data URIs. Remote URLs are passed through unchanged. Extra fields such
+ as ``fps`` and ``snippet`` are merged at the content-item level, not
+ inside ``video_url``.
+
+ Args:
+ source (`URLSource | Base64Source`):
+ The video source to convert.
+ extra (`dict[str, Any] | None`, optional):
+ Provider-specific video parameters to merge into the content
+ item, such as DashScope's ``fps`` and ``snippet``.
+
+ Returns:
+ `dict[str, Any]`:
+ A DashScope video content item.
"""
if isinstance(source, Base64Source):
url = f"data:{source.media_type};base64,{source.data}"
@@ -164,57 +199,67 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
else:
raise ValueError(f"Unsupported video source type: {type(source)}")
- return {
+ result = {
"type": "video_url",
"video_url": {"url": url},
}
+ if extra:
+ result.update(extra)
+ return result
@staticmethod
def _format_audio_source(
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Convert an audio source to DashScope ``input_audio`` format.
DashScope's compatible API accepts URLs directly in the ``data``
- field. Base64-encoded audio must be wrapped in a data URL. Local
- files are read, base64-encoded, and wrapped in the same form.
+ field. Base64-encoded audio and local files are encoded as data URLs.
+ Extra fields are merged at the content-item level, not inside
+ ``input_audio``.
+
+ Args:
+ source (`URLSource | Base64Source`):
+ The audio source to convert.
+ extra (`dict[str, Any] | None`, optional):
+ Provider-specific audio parameters to merge into the content
+ item, at the same level as ``type`` and ``input_audio``.
+
+ Returns:
+ `dict[str, Any]`:
+ A DashScope audio content item.
"""
fmt = source.media_type.split("/")[-1]
if fmt == "mpeg":
fmt = "mp3"
if isinstance(source, Base64Source):
- return {
- "type": "input_audio",
- "input_audio": {
- "data": f"data:;base64,{source.data}",
- "format": fmt,
- },
- }
+ data = f"data:;base64,{source.data}"
- if isinstance(source, URLSource):
+ elif isinstance(source, URLSource):
url_str = str(source.url)
if url_str.startswith("file://"):
local_path = url2pathname(url_str.removeprefix("file://"))
with open(local_path, "rb") as f:
encoded = base64.b64encode(f.read()).decode("utf-8")
- return {
- "type": "input_audio",
- "input_audio": {
- "data": f"data:;base64,{encoded}",
- "format": fmt,
- },
- }
+ data = f"data:;base64,{encoded}"
else:
- return {
- "type": "input_audio",
- "input_audio": {
- "data": url_str,
- "format": fmt,
- },
- }
-
- raise ValueError(f"Unsupported audio source type: {type(source)}")
+ data = url_str
+
+ else:
+ raise ValueError(f"Unsupported audio source type: {type(source)}")
+
+ result = {
+ "type": "input_audio",
+ "input_audio": {
+ "data": data,
+ "format": fmt,
+ },
+ }
+ if extra:
+ result.update(extra)
+ return result
class DashScopeChatFormatter(_DashScopeFormatterBase):
diff --git a/src/agentscope/formatter/_gemini_formatter.py b/src/agentscope/formatter/_gemini_formatter.py
index de789d391..5529468c3 100644
--- a/src/agentscope/formatter/_gemini_formatter.py
+++ b/src/agentscope/formatter/_gemini_formatter.py
@@ -42,7 +42,8 @@ class _GeminiFormatterBase(FormatterBase, ABC):
Returns:
`dict[str, Any] | None`:
The formatted data block in Gemini ``inline_data`` format,
- or None if the media type is not supported.
+ with extra fields placed beside ``inline_data``, or None if
+ the media type is not supported.
"""
source = block.source
media_type = source.media_type
@@ -58,29 +59,31 @@ class _GeminiFormatterBase(FormatterBase, ABC):
)
return None
- return self._format_media_source(source)
+ return self._format_media_source(source, block.model_extra)
@staticmethod
def _format_media_source(
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Format a media source into Gemini API ``inline_data`` format.
+ Extra fields are merged into the outer part object, at the same level
+ as ``inline_data`` (for example Gemini video metadata).
+
Args:
source (`URLSource | Base64Source`):
The media source to format.
+ extra (`dict[str, Any] | None`, optional):
+ Provider-specific media parameters to merge into the outer
+ part object, at the same level as ``inline_data``.
Returns:
`dict[str, Any]`:
The formatted media source.
"""
if isinstance(source, Base64Source):
- return {
- "inline_data": {
- "data": source.data,
- "mime_type": source.media_type,
- },
- }
+ data = source.data
elif isinstance(source, URLSource):
url = str(source.url)
if url.startswith("file://"):
@@ -88,26 +91,24 @@ class _GeminiFormatterBase(FormatterBase, ABC):
file_path = url2pathname(url.removeprefix("file://"))
with open(file_path, "rb") as f:
data = base64.b64encode(f.read()).decode("utf-8")
- return {
- "inline_data": {
- "data": data,
- "mime_type": source.media_type,
- },
- }
else:
# Remote URL - download and convert to base64
response = requests.get(url, timeout=30)
response.raise_for_status()
data = base64.b64encode(response.content).decode("utf-8")
- return {
- "inline_data": {
- "data": data,
- "mime_type": source.media_type,
- },
- }
else:
raise ValueError(f"Unsupported source type: {type(source)}")
+ result = {
+ "inline_data": {
+ "data": data,
+ "mime_type": source.media_type,
+ },
+ }
+ if extra:
+ result.update(extra)
+ return result
+
class GeminiChatFormatter(_GeminiFormatterBase):
"""The Gemini formatter class for chatbot scenario, where only a user
diff --git a/src/agentscope/formatter/_moonshot_formatter.py b/src/agentscope/formatter/_moonshot_formatter.py
index af64fb59f..4d2098dba 100644
--- a/src/agentscope/formatter/_moonshot_formatter.py
+++ b/src/agentscope/formatter/_moonshot_formatter.py
@@ -26,6 +26,7 @@ from ..message import (
def _moonshot_format_media_source(
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Convert an image or video source to Moonshot ``image_url`` or
``video_url`` format.
@@ -33,7 +34,20 @@ def _moonshot_format_media_source(
Moonshot's vision API only accepts base64 data URIs or file IDs — raw
remote URLs are rejected. This helper downloads remote ``http(s)://``
URLs and converts them to base64 data URIs, while ``file://`` URLs and
- ``Base64Source`` go through the same conversion as the OpenAI base.
+ ``Base64Source`` go through the same conversion as the OpenAI base. Extra
+ fields are merged into the nested ``image_url`` or ``video_url``
+ object.
+
+ Args:
+ source (`URLSource | Base64Source`):
+ The image or video source to convert.
+ extra (`dict[str, Any] | None`, optional):
+ Provider-specific media parameters to merge into the nested
+ ``image_url`` or ``video_url`` object.
+
+ Returns:
+ `dict[str, Any]`:
+ A Moonshot image or video content item.
"""
if isinstance(source, Base64Source):
url = f"data:{source.media_type};base64,{source.data}"
@@ -57,7 +71,10 @@ def _moonshot_format_media_source(
key = (
"video_url" if source.media_type.startswith("video/") else "image_url"
)
- return {"type": key, key: {"url": url}}
+ payload: dict[str, Any] = {"url": url}
+ if extra:
+ payload.update(extra)
+ return {"type": key, key: payload}
class MoonshotChatFormatter(_OpenAIFormatterBase):
@@ -86,8 +103,9 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):
def _format_image_source(
self,
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
- return _moonshot_format_media_source(source)
+ return _moonshot_format_media_source(source, extra)
def _format_openai_data_block(
self,
@@ -98,7 +116,10 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):
fnmatch(media_type, pattern)
for pattern in self.supported_input_media_types
):
- return _moonshot_format_media_source(block.source)
+ return _moonshot_format_media_source(
+ block.source,
+ block.model_extra,
+ )
return super()._format_openai_data_block(block)
# pylint: disable=too-many-branches
@@ -348,8 +369,9 @@ class MoonshotMultiAgentFormatter(_OpenAIFormatterBase):
def _format_image_source(
self,
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
- return _moonshot_format_media_source(source)
+ return _moonshot_format_media_source(source, extra)
def _format_openai_data_block(
self,
@@ -360,7 +382,10 @@ class MoonshotMultiAgentFormatter(_OpenAIFormatterBase):
fnmatch(media_type, pattern)
for pattern in self.supported_input_media_types
):
- return _moonshot_format_media_source(block.source)
+ return _moonshot_format_media_source(
+ block.source,
+ block.model_extra,
+ )
return super()._format_openai_data_block(block)
async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
diff --git a/src/agentscope/formatter/_openai_formatter.py b/src/agentscope/formatter/_openai_formatter.py
index cc58709e0..ea8e1fe9f 100644
--- a/src/agentscope/formatter/_openai_formatter.py
+++ b/src/agentscope/formatter/_openai_formatter.py
@@ -63,8 +63,11 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
"""Format a DataBlock into the required format for OpenAI API.
For image blocks, URLs are returned as-is (or converted to base64 for
- local ``file://`` paths). For audio blocks, data is always converted
- to base64 as required by the OpenAI input_audio format.
+ local ``file://`` paths). Image extras such as ``detail`` are merged
+ into the nested ``image_url`` payload as required by OpenAI Chat
+ Completions. For audio blocks, data is always converted to base64 as
+ required by the OpenAI ``input_audio`` format, and extras are merged
+ into the nested ``input_audio`` payload.
Args:
block (`DataBlock`):
@@ -90,10 +93,10 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
main_type = block.source.media_type.split("/")[0]
if main_type == "image":
- return self._format_image_source(block.source)
+ return self._format_image_source(block.source, block.model_extra)
if main_type == "audio":
- return self._format_audio_source(block.source)
+ return self._format_audio_source(block.source, block.model_extra)
if block.source.media_type == "application/pdf":
return self._format_file_source(block.source, block.name)
@@ -108,22 +111,29 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
def _format_image_source(
self,
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Convert an image source to OpenAI image_url format.
Local ``file://`` URLs are read from disk and converted to base64
- data URIs. Remote URLs are passed through unchanged. Subclasses may
- override this to apply provider-specific handling (e.g. forcing
- remote URLs to be downloaded and base64-encoded for APIs that don't
- accept raw HTTPS URLs).
+ data URIs. Remote URLs are passed through unchanged. Extra media
+ parameters (for example ``detail``) are merged into the nested
+ ``image_url`` object, matching OpenAI Chat Completions' image input
+ schema. Subclasses may override this to apply provider-specific
+ handling (e.g. forcing remote URLs to be downloaded and base64-encoded
+ for APIs that don't accept raw HTTPS URLs).
Args:
source (`URLSource | Base64Source`):
The image source to convert.
+ extra (`dict[str, Any] | None`, optional):
+ Provider-specific image parameters to merge into the nested
+ ``image_url`` object, such as OpenAI's ``detail``.
Returns:
`dict[str, Any]`:
A dictionary with ``"type": "image_url"`` in OpenAI format.
+ Extra fields are nested under the ``image_url`` object.
"""
if isinstance(source, Base64Source):
url = f"data:{source.media_type};base64,{source.data}"
@@ -143,23 +153,31 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
else:
raise ValueError(f"Unsupported image source type: {type(source)}")
+ image_url_payload = {"url": url}
+ if extra:
+ image_url_payload.update(extra)
return {
"type": "image_url",
- "image_url": {"url": url},
+ "image_url": image_url_payload,
}
@staticmethod
def _format_audio_source(
source: URLSource | Base64Source,
+ extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Convert an audio source to OpenAI input_audio format.
Local ``file://`` URLs are read from disk. Remote URLs are downloaded.
Only ``wav`` and ``mp3`` formats are supported by the OpenAI API.
+ Extra fields are merged into the nested ``input_audio`` object.
Args:
source (`URLSource | Base64Source`):
The audio source to convert.
+ extra (`dict[str, Any] | None`, optional):
+ Provider-specific audio parameters to merge into the nested
+ ``input_audio`` object.
Returns:
`dict[str, Any]`:
@@ -179,12 +197,15 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
audio_format = media_type_to_format[media_type]
if isinstance(source, Base64Source):
+ input_audio_payload = {
+ "data": source.data,
+ "format": audio_format,
+ }
+ if extra:
+ input_audio_payload.update(extra)
return {
"type": "input_audio",
- "input_audio": {
- "data": source.data,
- "format": audio_format,
- },
+ "input_audio": input_audio_payload,
}
if isinstance(source, URLSource):
@@ -200,12 +221,15 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
response.raise_for_status()
data = base64.b64encode(response.content).decode("utf-8")
+ input_audio_payload = {
+ "data": data,
+ "format": audio_format,
+ }
+ if extra:
+ input_audio_payload.update(extra)
return {
"type": "input_audio",
- "input_audio": {
- "data": data,
- "format": audio_format,
- },
+ "input_audio": input_audio_payload,
}
raise TypeError(f"Unsupported audio source type: {type(source)}.")
diff --git a/src/agentscope/formatter/_openai_response_formatter.py b/src/agentscope/formatter/_openai_response_formatter.py
index 5f337c629..914c01fac 100644
--- a/src/agentscope/formatter/_openai_response_formatter.py
+++ b/src/agentscope/formatter/_openai_response_formatter.py
@@ -52,6 +52,11 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):
audio input yet; use Chat Completions API instead). See
https://developers.openai.com/api/docs/guides/audio
+ For image blocks, ``DataBlock`` extra fields are merged into the
+ ``input_image`` item itself (for example ``detail``), because the
+ Responses API uses a flat ``image_url`` string instead of the nested
+ Chat Completions ``image_url`` object.
+
Args:
block (`DataBlock`):
The DataBlock to format.
@@ -80,10 +85,14 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):
return None
if base_result.get("type") == "image_url":
- return {
+ result = {
"type": "input_image",
"image_url": base_result["image_url"]["url"],
}
+ extra = block.model_extra or {}
+ if extra:
+ result.update(extra)
+ return result
if base_result.get("type") == "file":
return {"type": "input_file", **base_result["file"]}
diff --git a/src/agentscope/formatter/_xai_formatter.py b/src/agentscope/formatter/_xai_formatter.py
index 11f3209de..0032e43c3 100644
--- a/src/agentscope/formatter/_xai_formatter.py
+++ b/src/agentscope/formatter/_xai_formatter.py
@@ -39,7 +39,8 @@ def _xai_user_args_from_blocks(
DataBlocks that are not images (or use an unsupported media type) are
dropped with a warning, matching the user-role DataBlock handling in
- :meth:`XAIChatFormatter.format`.
+ :meth:`XAIChatFormatter.format`. Image extras are forwarded to
+ ``xai_sdk.chat.image`` as keyword arguments.
Args:
blocks (`list`):
@@ -84,15 +85,19 @@ def _xai_user_args_from_blocks(
image(
f"data:{sub.source.media_type};"
f"base64,{encoded}",
+ **(sub.model_extra or {}),
),
)
else:
- args.append(image(url_str))
+ args.append(
+ image(url_str, **(sub.model_extra or {})),
+ )
elif isinstance(sub.source, Base64Source):
args.append(
image(
f"data:{sub.source.media_type};"
f"base64,{sub.source.data}",
+ **(sub.model_extra or {}),
),
)
return args
@@ -102,7 +107,9 @@ class XAIChatFormatter(FormatterBase):
"""Formatter for the xAI chat model.
Converts ``Msg`` objects into ``xai_sdk`` protobuf ``Message`` objects
- that can be appended directly to a ``xai_sdk`` chat session.
+ that can be appended directly to a ``xai_sdk`` chat session. Image
+ ``DataBlock`` extras are passed as keyword arguments to
+ ``xai_sdk.chat.image`` (for example ``detail``).
Unlike other formatters whose ``format()`` returns ``list[dict]``, this
formatter returns ``list[chat_pb2.Message]``. The type annotation is
@@ -134,8 +141,9 @@ class XAIChatFormatter(FormatterBase):
Returns:
`list[Any]`:
A list of ``chat_pb2.Message`` proto objects, ready to be
- appended to a ``xai_sdk`` chat session via
- ``chat.append()``.
+ appended to a ``xai_sdk`` chat session via ``chat.append()``.
+ Image ``DataBlock`` extras are forwarded to
+ ``xai_sdk.chat.image`` as keyword arguments.
"""
from xai_sdk.chat import (
assistant,
diff --git a/src/agentscope/message/_block.py b/src/agentscope/message/_block.py
index dd0a7aad1..cdb167810 100644
--- a/src/agentscope/message/_block.py
+++ b/src/agentscope/message/_block.py
@@ -81,7 +81,16 @@ class URLSource(BaseModel):
class DataBlock(BaseModel):
- """The data block for binary content (images, audio, video, etc.)."""
+ """The data block for binary content (images, audio, video, etc.).
+
+ Allows extra provider-specific fields (e.g. OpenAI's ``detail``,
+ DashScope's ``fps``) via ``extra="allow"`` so users can attach media
+ processing parameters without subclassing. Formatters decide where these
+ extras belong in the target API payload because providers use different
+ nesting rules.
+ """
+
+ model_config = ConfigDict(extra="allow")
type: Literal["data"] = "data"
"""The type of the data block, which is always 'data'."""
diff --git a/tests/formatter_anthropic_test.py b/tests/formatter_anthropic_test.py
index 903b71a3a..ac2e5b7ac 100644
--- a/tests/formatter_anthropic_test.py
+++ b/tests/formatter_anthropic_test.py
@@ -1117,6 +1117,49 @@ class TestAnthropicFormatter( # pylint: disable=too-many-public-methods
res,
)
+ async def test_chat_formatter_image_extra_params(self) -> None:
+ """DataBlock extra params are forwarded to Anthropic image block."""
+ fmt = AnthropicChatFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Inspect this screenshot:"),
+ DataBlock(
+ source=Base64Source(
+ data=self.image_b64,
+ media_type="image/png",
+ ),
+ cache_control={"type": "ephemeral"},
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Inspect this screenshot:",
+ },
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": self.image_b64,
+ },
+ "cache_control": {"type": "ephemeral"},
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
async def test_chat_formatter_parallel_tool_results_merged(
self,
) -> None:
diff --git a/tests/formatter_dashscope_test.py b/tests/formatter_dashscope_test.py
index 14d4e259b..71b365efa 100644
--- a/tests/formatter_dashscope_test.py
+++ b/tests/formatter_dashscope_test.py
@@ -313,6 +313,50 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
# DashScopeChatFormatter tests
# -------------------------------------------------------------------
+ async def test_chat_formatter_video_extra_params(self) -> None:
+ """DataBlock extra params are forwarded at content-item level."""
+ fmt = DashScopeChatFormatter()
+ video_url = str(
+ URLSource(
+ url="https://example.com/video.mp4",
+ media_type="video/mp4",
+ ).url,
+ )
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Summarize this video."),
+ DataBlock(
+ source=URLSource(
+ url=video_url,
+ media_type="video/mp4",
+ ),
+ fps=2,
+ snippet=[0, 30],
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Summarize this video."},
+ {
+ "type": "video_url",
+ "video_url": {"url": video_url},
+ "fps": 2,
+ "snippet": [0, 30],
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
async def test_chat_formatter(self) -> None:
"""Chat formatter produces exact output for various subsets."""
fmt = DashScopeChatFormatter()
diff --git a/tests/formatter_gemini_test.py b/tests/formatter_gemini_test.py
index ed6bf9260..b2a9b05a6 100644
--- a/tests/formatter_gemini_test.py
+++ b/tests/formatter_gemini_test.py
@@ -1160,6 +1160,44 @@ class TestGeminiFormatter(IsolatedAsyncioTestCase):
res,
)
+ async def test_chat_formatter_media_extra_params(self) -> None:
+ """DataBlock extra params are forwarded beside inline_data."""
+ fmt = GeminiChatFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Summarize this video."),
+ DataBlock(
+ source=Base64Source(
+ data=self.image_b64,
+ media_type="video/mp4",
+ ),
+ video_metadata={"fps": 2},
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "parts": [
+ {"text": "Summarize this video."},
+ {
+ "inline_data": {
+ "data": self.image_b64,
+ "mime_type": "video/mp4",
+ },
+ "video_metadata": {"fps": 2},
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
async def test_tool_call_with_incomplete_input_does_not_crash(
self,
) -> None:
diff --git a/tests/formatter_moonshot_test.py b/tests/formatter_moonshot_test.py
index 14bc43ac9..1457a4d90 100644
--- a/tests/formatter_moonshot_test.py
+++ b/tests/formatter_moonshot_test.py
@@ -333,6 +333,45 @@ class TestMoonshotFormatter(IsolatedAsyncioTestCase):
res = await fmt.format([])
self.assertListEqual([], res)
+ async def test_chat_formatter_image_extra_params(self) -> None:
+ """DataBlock extra params are nested inside image_url payload."""
+ fmt = MoonshotChatFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Inspect this image."),
+ DataBlock(
+ source=URLSource(
+ url=self.image_url,
+ media_type="image/png",
+ ),
+ detail="high",
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "name": "user",
+ "content": [
+ {"type": "text", "text": "Inspect this image."},
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": self.image_data_uri,
+ "detail": "high",
+ },
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
async def test_chat_formatter_thinking_to_reasoning_content(
self,
) -> None:
@@ -514,6 +553,34 @@ class TestMoonshotFormatter(IsolatedAsyncioTestCase):
await fmt.format(msgs),
)
+ async def test_video_extra_params(self) -> None:
+ """Both Moonshot formatters preserve video-specific parameters."""
+ block = DataBlock(
+ source=Base64Source(
+ data="ZmFrZSB2aWRlbw==",
+ media_type="video/mp4",
+ ),
+ fps=2,
+ )
+ for formatter_type in (
+ MoonshotChatFormatter,
+ MoonshotMultiAgentFormatter,
+ ):
+ with self.subTest(formatter=formatter_type.__name__):
+ result = await formatter_type().format(
+ [UserMsg(name="user", content=[block])],
+ )
+ self.assertEqual(
+ result[0]["content"][-1],
+ {
+ "type": "video_url",
+ "video_url": {
+ "url": "data:video/mp4;base64,ZmFrZSB2aWRlbw==",
+ "fps": 2,
+ },
+ },
+ )
+
async def test_chat_formatter_url_image_in_tool_result(
self,
) -> None:
diff --git a/tests/formatter_openai_chat_test.py b/tests/formatter_openai_chat_test.py
index 1d1e67f0f..94c0d9c90 100644
--- a/tests/formatter_openai_chat_test.py
+++ b/tests/formatter_openai_chat_test.py
@@ -287,6 +287,85 @@ class TestOpenAIFormatter(IsolatedAsyncioTestCase):
# OpenAIChatFormatter tests
# -------------------------------------------------------------------
+ async def test_chat_formatter_image_extra_params(self) -> None:
+ """DataBlock extra params are nested inside image_url payload."""
+ fmt = OpenAIChatFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Inspect this image."),
+ DataBlock(
+ source=URLSource(
+ url=self.image_url,
+ media_type="image/png",
+ ),
+ detail="high",
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "name": "user",
+ "content": [
+ {"type": "text", "text": "Inspect this image."},
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": self.image_url,
+ "detail": "high",
+ },
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
+ async def test_chat_formatter_audio_extra_params(self) -> None:
+ """DataBlock extra params are nested inside input_audio payload."""
+ fmt = OpenAIChatFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Transcribe this audio."),
+ DataBlock(
+ source=Base64Source(
+ data="ZmFrZSBhdWRpbyBkYXRh",
+ media_type="audio/wav",
+ ),
+ extra_audio_param="value",
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "name": "user",
+ "content": [
+ {"type": "text", "text": "Transcribe this audio."},
+ {
+ "type": "input_audio",
+ "input_audio": {
+ "data": "ZmFrZSBhdWRpbyBkYXRh",
+ "format": "wav",
+ "extra_audio_param": "value",
+ },
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
async def test_chat_formatter(self) -> None:
"""Chat formatter produces exact output for various subsets."""
fmt = OpenAIChatFormatter()
diff --git a/tests/formatter_openai_response_test.py b/tests/formatter_openai_response_test.py
index b6899e3a0..a48589844 100644
--- a/tests/formatter_openai_response_test.py
+++ b/tests/formatter_openai_response_test.py
@@ -332,6 +332,47 @@ class TestOpenAIResponseFormatter(IsolatedAsyncioTestCase):
res,
)
+ async def test_chat_formatter_image_detail_extra_param(self) -> None:
+ """Responses API forwards image detail at input_image top level."""
+ fmt = OpenAIResponseFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Inspect this image."),
+ DataBlock(
+ source=URLSource(
+ url=self.image_url,
+ media_type="image/png",
+ ),
+ detail="high",
+ custom_param="value",
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_text",
+ "text": "Inspect this image.",
+ },
+ {
+ "type": "input_image",
+ "image_url": self.image_url,
+ "detail": "high",
+ "custom_param": "value",
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
async def test_chat_formatter_base64_pdf(self) -> None:
"""Base64-encoded PDF becomes an ``input_file`` item."""
fmt = OpenAIResponseFormatter()
diff --git a/tests/formatter_xai_test.py b/tests/formatter_xai_test.py
index a799caf7e..d2ab7a9ba 100644
--- a/tests/formatter_xai_test.py
+++ b/tests/formatter_xai_test.py
@@ -70,17 +70,18 @@ class _StubMessage:
class _StubImage:
"""Comparable stub for xai_sdk.chat.image()."""
- def __init__(self, url: str) -> None:
+ def __init__(self, url: str, detail: str = "auto") -> None:
self.type = "image"
self.url = url
+ self.detail = detail
def __eq__(self, other: object) -> bool:
if not isinstance(other, _StubImage):
return NotImplemented
- return self.url == other.url
+ return self.url == other.url and self.detail == other.detail
def __repr__(self) -> str:
- return f"_StubImage(url={self.url!r})"
+ return f"_StubImage(url={self.url!r}, detail={self.detail!r})"
def user(*args: Any) -> _StubMessage:
@@ -103,9 +104,9 @@ def tool_result(*args: Any, **kwargs: Any) -> _StubMessage:
return _StubMessage(role="tool", args=args, kwargs=kwargs)
-def image(url: str) -> _StubImage:
+def image(url: str, *, detail: str = "auto") -> _StubImage:
"""Create a comparable stub image object."""
- return _StubImage(url=url)
+ return _StubImage(url=url, detail=detail)
# ---------------------------------------------------------------------------
@@ -256,6 +257,39 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
# XAIChatFormatter tests
# -------------------------------------------------------------------
+ async def test_chat_formatter_image_detail_extra_param(self) -> None:
+ """DataBlock detail extra param is forwarded to xAI image()."""
+ fmt = XAIChatFormatter()
+ image_b64 = "ZmFrZSBpbWFnZSBkYXRh"
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Inspect this image."),
+ DataBlock(
+ source=Base64Source(
+ data=image_b64,
+ media_type="image/png",
+ ),
+ detail="high",
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ user(
+ "Inspect this image.",
+ image(
+ f"data:image/png;base64,{image_b64}",
+ detail="high",
+ ),
+ ),
+ ],
+ res,
+ )
+
async def test_chat_formatter_system_message(self) -> None:
"""System message becomes a system() stub."""
fmt = XAIChatFormatter()
@@ -299,6 +333,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
url=path.as_uri(),
media_type="image/png",
),
+ detail="high",
),
],
),
@@ -308,7 +343,10 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
[
user(
"What's in this image?",
- image(f"data:image/png;base64,{expected_b64}"),
+ image(
+ f"data:image/png;base64,{expected_b64}",
+ detail="high",
+ ),
),
],
res,
@@ -731,6 +769,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
data="Zmlyc3Q=",
media_type="image/jpeg",
),
+ detail="high",
),
],
),
@@ -743,6 +782,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
url="https://example.com/second.png",
media_type="image/png",
),
+ detail="low",
),
],
),
@@ -755,8 +795,8 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
"user: Compare these two charts.\n"
"agent: The second one differs.\n"
"</history>",
- image("data:image/jpeg;base64,Zmlyc3Q="),
- image("https://example.com/second.png"),
+ image("data:image/jpeg;base64,Zmlyc3Q=", detail="high"),
+ image("https://example.com/second.png", detail="low"),
),
],
res,
@@ -954,6 +994,7 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
data=image_b64,
media_type="image/png",
),
+ detail="high",
),
],
),
@@ -965,7 +1006,10 @@ class TestXAIFormatter( # pylint: disable=too-many-public-methods
[
user(
"Inspect this screenshot:",
- image(f"data:image/png;base64,{image_b64}"),
+ image(
+ f"data:image/png;base64,{image_b64}",
+ detail="high",
+ ),
),
],
res,
diff --git a/tests/message_test.py b/tests/message_test.py
index 6285308ed..1235a04e9 100644
--- a/tests/message_test.py
+++ b/tests/message_test.py
@@ -335,6 +335,7 @@ class MessageTest(IsolatedAsyncioTestCase):
url="https://example.com/image.png",
media_type="image/png",
),
+ detail="high",
),
DataBlock(
source=Base64Source(
@@ -368,6 +369,7 @@ class MessageTest(IsolatedAsyncioTestCase):
"media_type": "image/png",
},
"name": None,
+ "detail": "high",
"created_at": AnyString(),
"finished_at": None,
},