Skip to content

Build and run agents you can see, understand and trust.

feat(formatter): support video block promoting in DashScope formatters (#964)

qbcbd5d926b678f
diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.py
index 904cb74ed..ed33a99ee 100644
--- a/src/agentscope/formatter/_dashscope_formatter.py
+++ b/src/agentscope/formatter/_dashscope_formatter.py
@@ -14,6 +14,7 @@ from ..message import (
TextBlock,
ImageBlock,
AudioBlock,
+ VideoBlock,
ToolUseBlock,
ToolResultBlock,
URLSource,
@@ -162,6 +163,7 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
TextBlock,
ImageBlock,
AudioBlock,
+ VideoBlock,
ToolUseBlock,
ToolResultBlock,
]
@@ -169,6 +171,8 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
def __init__(
self,
promote_tool_result_images: bool = False,
+ promote_tool_result_audios: bool = False,
+ promote_tool_result_videos: bool = False,
token_counter: TokenCounterBase | None = None,
max_tokens: int | None = None,
) -> None:
@@ -181,6 +185,18 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
do support them in user message blocks. When `True`, images are
extracted and appended as a separate user message with
explanatory text indicating their source.
+ promote_tool_result_audios (`bool`, defaults to `False`):
+ Whether to promote audios from tool results to user messages.
+ Most LLM APIs don't support audios in tool result blocks, but
+ do support them in user message blocks. When `True`, audios are
+ extracted and appended as a separate user message with
+ explanatory text indicating their source.
+ promote_tool_result_videos (`bool`, defaults to `False`):
+ Whether to promote videos from tool results to user messages.
+ Most LLM APIs don't support videos in tool result blocks, but
+ do support them in user message blocks. When `True`, videos are
+ extracted and appended as a separate user message with
+ explanatory text indicating their source.
token_counter (`TokenCounterBase | None`, optional):
A token counter instance used to count tokens in the messages.
If not provided, the formatter will format the messages
@@ -192,6 +208,8 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
"""
super().__init__(token_counter, max_tokens)
self.promote_tool_result_images = promote_tool_result_images
+ self.promote_tool_result_audios = promote_tool_result_audios
+ self.promote_tool_result_videos = promote_tool_result_videos
async def _format(
self,
@@ -227,7 +245,7 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
},
)
- elif typ in ["image", "audio"]:
+ elif typ in ["image", "audio", "video"]:
content_blocks.append(
_format_dashscope_media_block(
block, # type: ignore[arg-type]
@@ -287,6 +305,44 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
),
],
)
+ elif (
+ multimodal_block["type"] == "audio"
+ and self.promote_tool_result_audios
+ ):
+ promoted_blocks.extend(
+ [
+ TextBlock(
+ type="text",
+ text=f"\n- The audio from '{url}': ",
+ ),
+ AudioBlock(
+ type="audio",
+ source=URLSource(
+ type="url",
+ url=url,
+ ),
+ ),
+ ],
+ )
+ elif (
+ multimodal_block["type"] == "video"
+ and self.promote_tool_result_videos
+ ):
+ promoted_blocks.extend(
+ [
+ TextBlock(
+ type="text",
+ text=f"\n- The video from '{url}': ",
+ ),
+ VideoBlock(
+ type="video",
+ source=URLSource(
+ type="url",
+ url=url,
+ ),
+ ),
+ ],
+ )
if promoted_blocks:
# Insert promoted blocks as new user message(s)
@@ -294,7 +350,7 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
TextBlock(
type="text",
text="<system-info>The following are "
- "the image contents from the tool "
+ f"the media contents from the tool "
f"result of '{block['name']}':",
),
*promoted_blocks,
@@ -373,6 +429,7 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
# Multimodal
ImageBlock,
AudioBlock,
+ VideoBlock,
# Tool use
ToolUseBlock,
ToolResultBlock,
@@ -387,6 +444,8 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
"your conversation history\n"
),
promote_tool_result_images: bool = False,
+ promote_tool_result_audios: bool = False,
+ promote_tool_result_videos: bool = False,
token_counter: TokenCounterBase | None = None,
max_tokens: int | None = None,
) -> None:
@@ -401,6 +460,18 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
do support them in user message blocks. When `True`, images are
extracted and appended as a separate user message with
explanatory text indicating their source.
+ promote_tool_result_audios (`bool`, defaults to `False`):
+ Whether to promote audios from tool results to user messages.
+ Most LLM APIs don't support audios in tool result blocks, but
+ do support them in user message blocks. When `True`, audios are
+ extracted and appended as a separate user message with
+ explanatory text indicating their source.
+ promote_tool_result_videos (`bool`, defaults to `False`):
+ Whether to promote videos from tool results to user messages.
+ Most LLM APIs don't support videos in tool result blocks, but
+ do support them in user message blocks. When `True`, videos are
+ extracted and appended as a separate user message with
+ explanatory text indicating their source.
token_counter (`TokenCounterBase | None`, optional):
The token counter used for truncation.
max_tokens (`int | None`, optional):
@@ -410,6 +481,8 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
super().__init__(token_counter=token_counter, max_tokens=max_tokens)
self.conversation_history_prompt = conversation_history_prompt
self.promote_tool_result_images = promote_tool_result_images
+ self.promote_tool_result_audios = promote_tool_result_audios
+ self.promote_tool_result_videos = promote_tool_result_videos
async def _format_tool_sequence(
self,
@@ -428,6 +501,8 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
"""
return await DashScopeChatFormatter(
promote_tool_result_images=self.promote_tool_result_images,
+ promote_tool_result_audios=self.promote_tool_result_audios,
+ promote_tool_result_videos=self.promote_tool_result_videos,
).format(msgs)
async def _format_agent_message(
@@ -467,7 +542,7 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
if block["type"] == "text":
accumulated_text.append(f"{msg.name}: {block['text']}")
- elif block["type"] in ["image", "audio"]:
+ elif block["type"] in ["image", "audio", "video"]:
# Handle the accumulated text as a single block
if accumulated_text:
conversation_blocks.append(
diff --git a/src/agentscope/formatter/_formatter_base.py b/src/agentscope/formatter/_formatter_base.py
index ba28c3032..6abb05a99 100644
--- a/src/agentscope/formatter/_formatter_base.py
+++ b/src/agentscope/formatter/_formatter_base.py
@@ -5,7 +5,7 @@ from abc import abstractmethod
from typing import Any, List, Tuple, Sequence
from .._utils._common import _save_base64_data
-from ..message import Msg, AudioBlock, ImageBlock, TextBlock
+from ..message import Msg, AudioBlock, ImageBlock, TextBlock, VideoBlock
class FormatterBase:
@@ -35,8 +35,16 @@ class FormatterBase:
@staticmethod
def convert_tool_result_to_string(
- output: str | List[TextBlock | ImageBlock | AudioBlock],
- ) -> tuple[str, Sequence[Tuple[str, ImageBlock | AudioBlock | TextBlock]]]:
+ output: str | List[TextBlock | ImageBlock | AudioBlock | VideoBlock],
+ ) -> tuple[
+ str,
+ Sequence[
+ Tuple[
+ str,
+ ImageBlock | AudioBlock | TextBlock | VideoBlock,
+ ]
+ ],
+ ]:
"""Turn the tool result list into a textual output to be compatible
with the LLM API that doesn't support multimodal data in the tool
result.
@@ -46,12 +54,13 @@ class FormatterBase:
is included in the returned list.
Args:
- output (`str | List[TextBlock | ImageBlock | AudioBlock]`):
+ output (`str | List[TextBlock | ImageBlock | AudioBlock | \
+ VideoBlock]`):
The output of the tool response, including text and multimodal
data like images and audio.
Returns:
- `tuple[str, list[Tuple[str, ImageBlock | AudioBlock | \
+ `tuple[str, list[Tuple[str, ImageBlock | AudioBlock | VideoBlock \
TextBlock]]]`:
A tuple containing the textual representation of the tool
result and a list of tuples. The first element of each tuple
@@ -66,8 +75,8 @@ class FormatterBase:
multimodal_data = []
for block in output:
assert isinstance(block, dict) and "type" in block, (
- f"Invalid block: {block}, a TextBlock, ImageBlock, or "
- f"AudioBlock is expected."
+ f"Invalid block: {block}, a TextBlock, ImageBlock, "
+ f"AudioBlock, or VideoBlock is expected."
)
if block["type"] == "text":
textual_output.append(block["text"])
diff --git a/src/agentscope/mcp/_client_base.py b/src/agentscope/mcp/_client_base.py
index 3bdcfad02..368887034 100644
--- a/src/agentscope/mcp/_client_base.py
+++ b/src/agentscope/mcp/_client_base.py
@@ -6,7 +6,13 @@ from typing import Callable, List
import mcp.types
from .._logging import logger
-from ..message import ImageBlock, Base64Source, AudioBlock, TextBlock
+from ..message import (
+ ImageBlock,
+ Base64Source,
+ AudioBlock,
+ TextBlock,
+ VideoBlock,
+)
class MCPClientBase:
@@ -33,7 +39,7 @@ class MCPClientBase:
@staticmethod
def _convert_mcp_content_to_as_blocks(
mcp_content_blocks: list,
- ) -> List[TextBlock | ImageBlock | AudioBlock]:
+ ) -> List[TextBlock | ImageBlock | AudioBlock | VideoBlock]:
"""Convert MCP content to AgentScope blocks."""
as_content: list = []
diff --git a/src/agentscope/message/_message_block.py b/src/agentscope/message/_message_block.py
index 792688565..3b3de65d2 100644
--- a/src/agentscope/message/_message_block.py
+++ b/src/agentscope/message/_message_block.py
@@ -96,7 +96,9 @@ class ToolResultBlock(TypedDict, total=False):
"""The type of the block"""
id: Required[str]
"""The identity of the tool call result"""
- output: Required[str | List[TextBlock | ImageBlock | AudioBlock]]
+ output: Required[
+ str | List[TextBlock | ImageBlock | AudioBlock | VideoBlock]
+ ]
"""The output of the tool function"""
name: Required[str]
"""The name of the tool function"""
diff --git a/src/agentscope/tool/_response.py b/src/agentscope/tool/_response.py
index c27c799bc..bb939199b 100644
--- a/src/agentscope/tool/_response.py
+++ b/src/agentscope/tool/_response.py
@@ -5,14 +5,14 @@ from dataclasses import dataclass, field
from typing import Optional, List
from .._utils._common import _get_timestamp
-from ..message import AudioBlock, ImageBlock, TextBlock
+from ..message import AudioBlock, ImageBlock, TextBlock, VideoBlock
@dataclass
class ToolResponse:
"""The result chunk of a tool call."""
- content: List[TextBlock | ImageBlock | AudioBlock]
+ content: List[TextBlock | ImageBlock | AudioBlock | VideoBlock]
"""The execution output of the tool function."""
metadata: Optional[dict] = None
diff --git a/tests/formatter_dashscope_test.py b/tests/formatter_dashscope_test.py
index cdd0a8720..61bb56c92 100644
--- a/tests/formatter_dashscope_test.py
+++ b/tests/formatter_dashscope_test.py
@@ -15,6 +15,7 @@ from agentscope.message import (
TextBlock,
ImageBlock,
AudioBlock,
+ VideoBlock,
URLSource,
Base64Source,
)
@@ -32,6 +33,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
self.mock_audio_path = (
"/var/folders/gf/krg8x_ws409cpw_46b2s6rjc0000gn/T/tmpfymnv2w9.wav"
)
+ self.mock_video_path = (
+ "/var/folders/gf/krg8x_ws409cpw_46b2s6rjc0000gn/T/tmpfymnv2w9.mp4"
+ )
self.msgs_system = [
Msg(
@@ -132,6 +136,14 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
data="ZmFrZSBhdWRpbyBjb250ZW50",
),
),
+ VideoBlock(
+ type="video",
+ source=Base64Source(
+ type="base64",
+ media_type="video/mp4",
+ data="ZmFrZSB2aWRlbyBjb250ZW50",
+ ),
+ ),
],
),
],
@@ -263,7 +275,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"content": "- The capital of Japan is Tokyo.\n"
"- The returned image can be found at: ./image.png"
"\n- The returned audio can be found at: "
- f"{self.mock_audio_path}",
+ f"{self.mock_audio_path}"
+ f"\n- The returned video can be found at: "
+ f"{self.mock_video_path}",
"name": "get_capital",
},
{
@@ -327,7 +341,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"content": "- The capital of Japan is Tokyo.\n- The returned"
" image can be found at: ./image.png\n- The"
" returned audio can be found at: "
- f"{self.mock_audio_path}",
+ f"{self.mock_audio_path}\n"
+ f"- The returned video can be found at: "
+ f"{self.mock_video_path}",
"name": "get_capital",
},
{
@@ -366,7 +382,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"content": "- The capital of Japan is Tokyo.\n- The returned"
" image can be found at: ./image.png\n- The"
" returned audio can be found at: "
- f"{self.mock_audio_path}",
+ f"{self.mock_audio_path}\n"
+ f"- The returned video can be found at: "
+ f"{self.mock_video_path}",
"name": "get_capital",
},
{
@@ -432,7 +450,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"tool_call_id": "1",
"content": "- The capital of Japan is Tokyo.\n- The returned"
" image can be found at: ./image.png\n- The returned audio can"
- f" be found at: {self.mock_audio_path}",
+ f" be found at: {self.mock_audio_path}"
+ f"\n- The returned video can be found at: "
+ f"{self.mock_video_path}",
"name": "get_capital",
},
{
@@ -475,13 +495,36 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
},
]
+ def _mock_save_base64_data(
+ self,
+ media_type: str,
+ _base64_data: str,
+ ) -> str:
+ """Mock function for _save_base64_data that returns different paths
+ based on media_type.
+
+ Args:
+ media_type: The MIME type of the media (e.g., "audio/wav",
+ "video/mp4")
+ _base64_data: The base64 encoded data (not used in mock)
+
+ Returns:
+ The mock file path for the media type
+ """
+ if "audio" in media_type:
+ return self.mock_audio_path
+ elif "video" in media_type:
+ return self.mock_video_path
+ else:
+ return self.mock_audio_path # fallback
+
@patch("agentscope.formatter._formatter_base._save_base64_data")
async def test_chat_formatter(
self,
mock_save_base64_data: MagicMock,
) -> None:
"""Test the chat formatter."""
- mock_save_base64_data.return_value = self.mock_audio_path
+ mock_save_base64_data.side_effect = self._mock_save_base64_data
formatter = DashScopeChatFormatter()
@@ -524,14 +567,18 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
self.assertListEqual(res, [])
@patch("agentscope.formatter._formatter_base._save_base64_data")
- async def test_chat_formatter_with_extract_image_blocks(
+ async def test_chat_formatter_with_extract_media_blocks(
self,
mock_save_base64_data: MagicMock,
) -> None:
"""Test the chat formatter with promote_tool_result_images=True."""
- mock_save_base64_data.return_value = self.mock_audio_path
+ mock_save_base64_data.side_effect = self._mock_save_base64_data
- formatter = DashScopeChatFormatter(promote_tool_result_images=True)
+ formatter = DashScopeChatFormatter(
+ promote_tool_result_images=True,
+ promote_tool_result_audios=True,
+ promote_tool_result_videos=True,
+ )
# Test with tool result containing image blocks
res = await formatter.format(
@@ -599,7 +646,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"content": "- The capital of Japan is Tokyo.\n"
"- The returned image can be found at: ./image.png"
"\n- The returned audio can be found at: "
- f"{self.mock_audio_path}",
+ f"{self.mock_audio_path}"
+ f"\n- The returned video can be found at: "
+ f"{self.mock_video_path}",
"name": "get_capital",
},
{
@@ -607,7 +656,7 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"content": [
{
"text": "<system-info>The following are "
- "the image contents from the tool "
+ "the media contents from the tool "
"result of 'get_capital':",
},
{
@@ -616,6 +665,20 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
{
"image": f"file://{os.path.abspath(self.image_path)}",
},
+ {
+ "text": "\n- The audio from "
+ f"'{self.mock_audio_path}': ",
+ },
+ {
+ "audio": self.mock_audio_path,
+ },
+ {
+ "text": "\n- The video from "
+ f"'{self.mock_video_path}': ",
+ },
+ {
+ "video": self.mock_video_path,
+ },
{
"text": "</system-info>",
},
@@ -635,7 +698,7 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
mock_save_base64_data: MagicMock,
) -> None:
"""Test the multi-agent formatter."""
- mock_save_base64_data.return_value = self.mock_audio_path
+ mock_save_base64_data.side_effect = self._mock_save_base64_data
formatter = DashScopeMultiAgentFormatter()
@@ -731,16 +794,18 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
)
@patch("agentscope.formatter._formatter_base._save_base64_data")
- async def test_multiagent_formatter_with_promote_tool_result_images(
+ async def test_multiagent_formatter_with_promote_media_tool_result(
self,
mock_save_base64_data: MagicMock,
) -> None:
"""Test the multi-agent formatter with
promote_tool_result_images=True."""
- mock_save_base64_data.return_value = self.mock_audio_path
+ mock_save_base64_data.side_effect = self._mock_save_base64_data
formatter = DashScopeMultiAgentFormatter(
promote_tool_result_images=True,
+ promote_tool_result_audios=True,
+ promote_tool_result_videos=True,
)
# Test with tool result containing image blocks
@@ -809,7 +874,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"content": "- The capital of Japan is Tokyo.\n- The returned"
" image can be found at: ./image.png\n- The"
" returned audio can be found at: "
- f"{self.mock_audio_path}",
+ f"{self.mock_audio_path}"
+ f"\n- The returned video can be found at: "
+ f"{self.mock_video_path}",
"name": "get_capital",
},
{
@@ -817,7 +884,7 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"content": [
{
"text": "<system-info>The following are "
- "the image contents from the tool "
+ "the media contents from the tool "
"result of 'get_capital':",
},
{
@@ -826,6 +893,20 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
{
"image": f"file://{os.path.abspath(self.image_path)}",
},
+ {
+ "text": "\n- The audio from "
+ f"'{self.mock_audio_path}': ",
+ },
+ {
+ "audio": self.mock_audio_path,
+ },
+ {
+ "text": "\n- The video from "
+ f"'{self.mock_video_path}': ",
+ },
+ {
+ "video": self.mock_video_path,
+ },
{
"text": "</system-info>",
},