Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(formatter): support video block promoting in DashScope formatters (#964)
qbcbd5d926b678f
diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.pyindex 904cb74ed..ed33a99ee 100644--- a/src/agentscope/formatter/_dashscope_formatter.py+++ b/src/agentscope/formatter/_dashscope_formatter.py@@ -14,6 +14,7 @@ from ..message import (TextBlock,ImageBlock,AudioBlock,+ VideoBlock,ToolUseBlock,ToolResultBlock,URLSource,@@ -162,6 +163,7 @@ class DashScopeChatFormatter(TruncatedFormatterBase):TextBlock,ImageBlock,AudioBlock,+ VideoBlock,ToolUseBlock,ToolResultBlock,]@@ -169,6 +171,8 @@ class DashScopeChatFormatter(TruncatedFormatterBase):def __init__(self,promote_tool_result_images: bool = False,+ promote_tool_result_audios: bool = False,+ promote_tool_result_videos: bool = False,token_counter: TokenCounterBase | None = None,max_tokens: int | None = None,) -> None:@@ -181,6 +185,18 @@ class DashScopeChatFormatter(TruncatedFormatterBase):do support them in user message blocks. When `True`, images areextracted and appended as a separate user message withexplanatory text indicating their source.+ promote_tool_result_audios (`bool`, defaults to `False`):+ Whether to promote audios from tool results to user messages.+ Most LLM APIs don't support audios in tool result blocks, but+ do support them in user message blocks. When `True`, audios are+ extracted and appended as a separate user message with+ explanatory text indicating their source.+ promote_tool_result_videos (`bool`, defaults to `False`):+ Whether to promote videos from tool results to user messages.+ Most LLM APIs don't support videos in tool result blocks, but+ do support them in user message blocks. When `True`, videos are+ extracted and appended as a separate user message with+ explanatory text indicating their source.token_counter (`TokenCounterBase | None`, optional):A token counter instance used to count tokens in the messages.If not provided, the formatter will format the messages@@ -192,6 +208,8 @@ class DashScopeChatFormatter(TruncatedFormatterBase):"""super().__init__(token_counter, max_tokens)self.promote_tool_result_images = promote_tool_result_images+ self.promote_tool_result_audios = promote_tool_result_audios+ self.promote_tool_result_videos = promote_tool_result_videosasync def _format(self,@@ -227,7 +245,7 @@ class DashScopeChatFormatter(TruncatedFormatterBase):},)- elif typ in ["image", "audio"]:+ elif typ in ["image", "audio", "video"]:content_blocks.append(_format_dashscope_media_block(block, # type: ignore[arg-type]@@ -287,6 +305,44 @@ class DashScopeChatFormatter(TruncatedFormatterBase):),],)+ elif (+ multimodal_block["type"] == "audio"+ and self.promote_tool_result_audios+ ):+ promoted_blocks.extend(+ [+ TextBlock(+ type="text",+ text=f"\n- The audio from '{url}': ",+ ),+ AudioBlock(+ type="audio",+ source=URLSource(+ type="url",+ url=url,+ ),+ ),+ ],+ )+ elif (+ multimodal_block["type"] == "video"+ and self.promote_tool_result_videos+ ):+ promoted_blocks.extend(+ [+ TextBlock(+ type="text",+ text=f"\n- The video from '{url}': ",+ ),+ VideoBlock(+ type="video",+ source=URLSource(+ type="url",+ url=url,+ ),+ ),+ ],+ )if promoted_blocks:# Insert promoted blocks as new user message(s)@@ -294,7 +350,7 @@ class DashScopeChatFormatter(TruncatedFormatterBase):TextBlock(type="text",text="<system-info>The following are "- "the image contents from the tool "+ f"the media contents from the tool "f"result of '{block['name']}':",),*promoted_blocks,@@ -373,6 +429,7 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):# MultimodalImageBlock,AudioBlock,+ VideoBlock,# Tool useToolUseBlock,ToolResultBlock,@@ -387,6 +444,8 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):"your conversation history\n"),promote_tool_result_images: bool = False,+ promote_tool_result_audios: bool = False,+ promote_tool_result_videos: bool = False,token_counter: TokenCounterBase | None = None,max_tokens: int | None = None,) -> None:@@ -401,6 +460,18 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):do support them in user message blocks. When `True`, images areextracted and appended as a separate user message withexplanatory text indicating their source.+ promote_tool_result_audios (`bool`, defaults to `False`):+ Whether to promote audios from tool results to user messages.+ Most LLM APIs don't support audios in tool result blocks, but+ do support them in user message blocks. When `True`, audios are+ extracted and appended as a separate user message with+ explanatory text indicating their source.+ promote_tool_result_videos (`bool`, defaults to `False`):+ Whether to promote videos from tool results to user messages.+ Most LLM APIs don't support videos in tool result blocks, but+ do support them in user message blocks. When `True`, videos are+ extracted and appended as a separate user message with+ explanatory text indicating their source.token_counter (`TokenCounterBase | None`, optional):The token counter used for truncation.max_tokens (`int | None`, optional):@@ -410,6 +481,8 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):super().__init__(token_counter=token_counter, max_tokens=max_tokens)self.conversation_history_prompt = conversation_history_promptself.promote_tool_result_images = promote_tool_result_images+ self.promote_tool_result_audios = promote_tool_result_audios+ self.promote_tool_result_videos = promote_tool_result_videosasync def _format_tool_sequence(self,@@ -428,6 +501,8 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):"""return await DashScopeChatFormatter(promote_tool_result_images=self.promote_tool_result_images,+ promote_tool_result_audios=self.promote_tool_result_audios,+ promote_tool_result_videos=self.promote_tool_result_videos,).format(msgs)async def _format_agent_message(@@ -467,7 +542,7 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):if block["type"] == "text":accumulated_text.append(f"{msg.name}: {block['text']}")- elif block["type"] in ["image", "audio"]:+ elif block["type"] in ["image", "audio", "video"]:# Handle the accumulated text as a single blockif accumulated_text:conversation_blocks.append(diff --git a/src/agentscope/formatter/_formatter_base.py b/src/agentscope/formatter/_formatter_base.pyindex ba28c3032..6abb05a99 100644--- a/src/agentscope/formatter/_formatter_base.py+++ b/src/agentscope/formatter/_formatter_base.py@@ -5,7 +5,7 @@ from abc import abstractmethodfrom typing import Any, List, Tuple, Sequencefrom .._utils._common import _save_base64_data-from ..message import Msg, AudioBlock, ImageBlock, TextBlock+from ..message import Msg, AudioBlock, ImageBlock, TextBlock, VideoBlockclass FormatterBase:@@ -35,8 +35,16 @@ class FormatterBase:@staticmethoddef convert_tool_result_to_string(- output: str | List[TextBlock | ImageBlock | AudioBlock],- ) -> tuple[str, Sequence[Tuple[str, ImageBlock | AudioBlock | TextBlock]]]:+ output: str | List[TextBlock | ImageBlock | AudioBlock | VideoBlock],+ ) -> tuple[+ str,+ Sequence[+ Tuple[+ str,+ ImageBlock | AudioBlock | TextBlock | VideoBlock,+ ]+ ],+ ]:"""Turn the tool result list into a textual output to be compatiblewith the LLM API that doesn't support multimodal data in the toolresult.@@ -46,12 +54,13 @@ class FormatterBase:is included in the returned list.Args:- output (`str | List[TextBlock | ImageBlock | AudioBlock]`):+ output (`str | List[TextBlock | ImageBlock | AudioBlock | \+ VideoBlock]`):The output of the tool response, including text and multimodaldata like images and audio.Returns:- `tuple[str, list[Tuple[str, ImageBlock | AudioBlock | \+ `tuple[str, list[Tuple[str, ImageBlock | AudioBlock | VideoBlock \TextBlock]]]`:A tuple containing the textual representation of the toolresult and a list of tuples. The first element of each tuple@@ -66,8 +75,8 @@ class FormatterBase:multimodal_data = []for block in output:assert isinstance(block, dict) and "type" in block, (- f"Invalid block: {block}, a TextBlock, ImageBlock, or "- f"AudioBlock is expected."+ f"Invalid block: {block}, a TextBlock, ImageBlock, "+ f"AudioBlock, or VideoBlock is expected.")if block["type"] == "text":textual_output.append(block["text"])diff --git a/src/agentscope/mcp/_client_base.py b/src/agentscope/mcp/_client_base.pyindex 3bdcfad02..368887034 100644--- a/src/agentscope/mcp/_client_base.py+++ b/src/agentscope/mcp/_client_base.py@@ -6,7 +6,13 @@ from typing import Callable, Listimport mcp.typesfrom .._logging import logger-from ..message import ImageBlock, Base64Source, AudioBlock, TextBlock+from ..message import (+ ImageBlock,+ Base64Source,+ AudioBlock,+ TextBlock,+ VideoBlock,+)class MCPClientBase:@@ -33,7 +39,7 @@ class MCPClientBase:@staticmethoddef _convert_mcp_content_to_as_blocks(mcp_content_blocks: list,- ) -> List[TextBlock | ImageBlock | AudioBlock]:+ ) -> List[TextBlock | ImageBlock | AudioBlock | VideoBlock]:"""Convert MCP content to AgentScope blocks."""as_content: list = []diff --git a/src/agentscope/message/_message_block.py b/src/agentscope/message/_message_block.pyindex 792688565..3b3de65d2 100644--- a/src/agentscope/message/_message_block.py+++ b/src/agentscope/message/_message_block.py@@ -96,7 +96,9 @@ class ToolResultBlock(TypedDict, total=False):"""The type of the block"""id: Required[str]"""The identity of the tool call result"""- output: Required[str | List[TextBlock | ImageBlock | AudioBlock]]+ output: Required[+ str | List[TextBlock | ImageBlock | AudioBlock | VideoBlock]+ ]"""The output of the tool function"""name: Required[str]"""The name of the tool function"""diff --git a/src/agentscope/tool/_response.py b/src/agentscope/tool/_response.pyindex c27c799bc..bb939199b 100644--- a/src/agentscope/tool/_response.py+++ b/src/agentscope/tool/_response.py@@ -5,14 +5,14 @@ from dataclasses import dataclass, fieldfrom typing import Optional, Listfrom .._utils._common import _get_timestamp-from ..message import AudioBlock, ImageBlock, TextBlock+from ..message import AudioBlock, ImageBlock, TextBlock, VideoBlock@dataclassclass ToolResponse:"""The result chunk of a tool call."""- content: List[TextBlock | ImageBlock | AudioBlock]+ content: List[TextBlock | ImageBlock | AudioBlock | VideoBlock]"""The execution output of the tool function."""metadata: Optional[dict] = Nonediff --git a/tests/formatter_dashscope_test.py b/tests/formatter_dashscope_test.pyindex cdd0a8720..61bb56c92 100644--- a/tests/formatter_dashscope_test.py+++ b/tests/formatter_dashscope_test.py@@ -15,6 +15,7 @@ from agentscope.message import (TextBlock,ImageBlock,AudioBlock,+ VideoBlock,URLSource,Base64Source,)@@ -32,6 +33,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):self.mock_audio_path = ("/var/folders/gf/krg8x_ws409cpw_46b2s6rjc0000gn/T/tmpfymnv2w9.wav")+ self.mock_video_path = (+ "/var/folders/gf/krg8x_ws409cpw_46b2s6rjc0000gn/T/tmpfymnv2w9.mp4"+ )self.msgs_system = [Msg(@@ -132,6 +136,14 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):data="ZmFrZSBhdWRpbyBjb250ZW50",),),+ VideoBlock(+ type="video",+ source=Base64Source(+ type="base64",+ media_type="video/mp4",+ data="ZmFrZSB2aWRlbyBjb250ZW50",+ ),+ ),],),],@@ -263,7 +275,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):"content": "- The capital of Japan is Tokyo.\n""- The returned image can be found at: ./image.png""\n- The returned audio can be found at: "- f"{self.mock_audio_path}",+ f"{self.mock_audio_path}"+ f"\n- The returned video can be found at: "+ f"{self.mock_video_path}","name": "get_capital",},{@@ -327,7 +341,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):"content": "- The capital of Japan is Tokyo.\n- The returned"" image can be found at: ./image.png\n- The"" returned audio can be found at: "- f"{self.mock_audio_path}",+ f"{self.mock_audio_path}\n"+ f"- The returned video can be found at: "+ f"{self.mock_video_path}","name": "get_capital",},{@@ -366,7 +382,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):"content": "- The capital of Japan is Tokyo.\n- The returned"" image can be found at: ./image.png\n- The"" returned audio can be found at: "- f"{self.mock_audio_path}",+ f"{self.mock_audio_path}\n"+ f"- The returned video can be found at: "+ f"{self.mock_video_path}","name": "get_capital",},{@@ -432,7 +450,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):"tool_call_id": "1","content": "- The capital of Japan is Tokyo.\n- The returned"" image can be found at: ./image.png\n- The returned audio can"- f" be found at: {self.mock_audio_path}",+ f" be found at: {self.mock_audio_path}"+ f"\n- The returned video can be found at: "+ f"{self.mock_video_path}","name": "get_capital",},{@@ -475,13 +495,36 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):},]+ def _mock_save_base64_data(+ self,+ media_type: str,+ _base64_data: str,+ ) -> str:+ """Mock function for _save_base64_data that returns different paths+ based on media_type.++ Args:+ media_type: The MIME type of the media (e.g., "audio/wav",+ "video/mp4")+ _base64_data: The base64 encoded data (not used in mock)++ Returns:+ The mock file path for the media type+ """+ if "audio" in media_type:+ return self.mock_audio_path+ elif "video" in media_type:+ return self.mock_video_path+ else:+ return self.mock_audio_path # fallback+@patch("agentscope.formatter._formatter_base._save_base64_data")async def test_chat_formatter(self,mock_save_base64_data: MagicMock,) -> None:"""Test the chat formatter."""- mock_save_base64_data.return_value = self.mock_audio_path+ mock_save_base64_data.side_effect = self._mock_save_base64_dataformatter = DashScopeChatFormatter()@@ -524,14 +567,18 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):self.assertListEqual(res, [])@patch("agentscope.formatter._formatter_base._save_base64_data")- async def test_chat_formatter_with_extract_image_blocks(+ async def test_chat_formatter_with_extract_media_blocks(self,mock_save_base64_data: MagicMock,) -> None:"""Test the chat formatter with promote_tool_result_images=True."""- mock_save_base64_data.return_value = self.mock_audio_path+ mock_save_base64_data.side_effect = self._mock_save_base64_data- formatter = DashScopeChatFormatter(promote_tool_result_images=True)+ formatter = DashScopeChatFormatter(+ promote_tool_result_images=True,+ promote_tool_result_audios=True,+ promote_tool_result_videos=True,+ )# Test with tool result containing image blocksres = await formatter.format(@@ -599,7 +646,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):"content": "- The capital of Japan is Tokyo.\n""- The returned image can be found at: ./image.png""\n- The returned audio can be found at: "- f"{self.mock_audio_path}",+ f"{self.mock_audio_path}"+ f"\n- The returned video can be found at: "+ f"{self.mock_video_path}","name": "get_capital",},{@@ -607,7 +656,7 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):"content": [{"text": "<system-info>The following are "- "the image contents from the tool "+ "the media contents from the tool ""result of 'get_capital':",},{@@ -616,6 +665,20 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):{"image": f"file://{os.path.abspath(self.image_path)}",},+ {+ "text": "\n- The audio from "+ f"'{self.mock_audio_path}': ",+ },+ {+ "audio": self.mock_audio_path,+ },+ {+ "text": "\n- The video from "+ f"'{self.mock_video_path}': ",+ },+ {+ "video": self.mock_video_path,+ },{"text": "</system-info>",},@@ -635,7 +698,7 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):mock_save_base64_data: MagicMock,) -> None:"""Test the multi-agent formatter."""- mock_save_base64_data.return_value = self.mock_audio_path+ mock_save_base64_data.side_effect = self._mock_save_base64_dataformatter = DashScopeMultiAgentFormatter()@@ -731,16 +794,18 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):)@patch("agentscope.formatter._formatter_base._save_base64_data")- async def test_multiagent_formatter_with_promote_tool_result_images(+ async def test_multiagent_formatter_with_promote_media_tool_result(self,mock_save_base64_data: MagicMock,) -> None:"""Test the multi-agent formatter withpromote_tool_result_images=True."""- mock_save_base64_data.return_value = self.mock_audio_path+ mock_save_base64_data.side_effect = self._mock_save_base64_dataformatter = DashScopeMultiAgentFormatter(promote_tool_result_images=True,+ promote_tool_result_audios=True,+ promote_tool_result_videos=True,)# Test with tool result containing image blocks@@ -809,7 +874,9 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):"content": "- The capital of Japan is Tokyo.\n- The returned"" image can be found at: ./image.png\n- The"" returned audio can be found at: "- f"{self.mock_audio_path}",+ f"{self.mock_audio_path}"+ f"\n- The returned video can be found at: "+ f"{self.mock_video_path}","name": "get_capital",},{@@ -817,7 +884,7 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):"content": [{"text": "<system-info>The following are "- "the image contents from the tool "+ "the media contents from the tool ""result of 'get_capital':",},{@@ -826,6 +893,20 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):{"image": f"file://{os.path.abspath(self.image_path)}",},+ {+ "text": "\n- The audio from "+ f"'{self.mock_audio_path}': ",+ },+ {+ "audio": self.mock_audio_path,+ },+ {+ "text": "\n- The video from "+ f"'{self.mock_video_path}': ",+ },+ {+ "video": self.mock_video_path,+ },{"text": "</system-info>",},