Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(formatter): add audio input support for Volcengine (#3706)
qbc2cb553840102
diff --git a/scripts/model_examples/README.md b/scripts/model_examples/README.mdindex b07d02203..2c423e26d 100644--- a/scripts/model_examples/README.md+++ b/scripts/model_examples/README.md@@ -53,6 +53,7 @@ scripts/model_examples/│├── volcengine_call.py # Volcengine Ark / Doubao├── volcengine_multiagent.py+├── volcengine_multimodal.py # Image, video, and audio input│├── xai_call.py # xAI Grok├── xai_multiagent.pydiff --git a/scripts/model_examples/run_tests.py b/scripts/model_examples/run_tests.pyindex ba4023e7e..5cdf459af 100644--- a/scripts/model_examples/run_tests.py+++ b/scripts/model_examples/run_tests.py@@ -184,7 +184,7 @@ ALL_PROVIDERS: list[Provider] = [name="volcengine",env_var="VOLCENGINE_API_KEY",file_prefix="volcengine",- supported_tests=["call", "multiagent"],+ supported_tests=["call", "multiagent", "multimodal"],description="Volcengine Ark / Doubao models",),Provider(diff --git a/scripts/model_examples/volcengine_multimodal.py b/scripts/model_examples/volcengine_multimodal.pynew file mode 100644index 000000000..731e67946--- /dev/null+++ b/scripts/model_examples/volcengine_multimodal.py@@ -0,0 +1,166 @@+# -*- coding: utf-8 -*-+"""Examples of Volcengine image, video, and audio input using DataBlock.++Set VOLCENGINE_API_KEY and run this script without arguments, or use the+example runner with --providers volcengine --tests multimodal.+The examples use doubao-seed-2-1-lite-260915. For a local audio file, call+``await example_audio("./sample.wav", "audio/wav")``.+"""++import asyncio+import base64+import os+from pathlib import Path++from _utils import stream_and_collect+from agentscope.credential import VolcengineCredential+from agentscope.message import (+ Base64Source,+ DataBlock,+ TextBlock,+ URLSource,+ UserMsg,+)+from agentscope.model import VolcengineChatModel+++TEST_IMAGE_URL = (+ "https://help-static-aliyun-doc.aliyuncs.com/file-manage"+ "-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg"+)+TEST_VIDEO_URL = (+ "https://help-static-aliyun-doc.aliyuncs.com/file-manage"+ "-files/zh-CN/20241115/cqqkru/1.mp4"+)+TEST_AUDIO_URL = (+ "https://ark-project.tos-cn-beijing.volces.com/doc_audio/"+ "ark_demo_audio.mp3"+)+++def _build_model() -> VolcengineChatModel:+ """Build a Doubao model supporting image, video, and audio input."""+ return VolcengineChatModel(+ credential=VolcengineCredential(+ api_key=os.environ["VOLCENGINE_API_KEY"],+ ),+ model="doubao-seed-2-1-lite-260915",+ stream=True,+ context_size=1_024_000,+ )+++async def example_image_url() -> None:+ """Describe an image provided as a public URL."""+ model = _build_model()+ messages = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="What is in this image? Describe it briefly."),+ DataBlock(+ source=URLSource(+ url=TEST_IMAGE_URL,+ media_type="image/jpeg",+ ),+ ),+ ],+ ),+ ]+ print("=== Multimodal Call (Image URL) ===")+ await stream_and_collect(await model(messages))+++async def example_image_local_path() -> None:+ """Describe a local image using a portable file URI."""+ model = _build_model()+ image_path = Path(__file__).resolve().parent / "test.jpeg"+ messages = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="What is in this image? Describe it briefly."),+ DataBlock(+ source=URLSource(+ url=image_path.as_uri(),+ media_type="image/jpeg",+ ),+ ),+ ],+ ),+ ]+ print("=== Local Path Call (file://) ===")+ await stream_and_collect(await model(messages))+++async def example_image_base64() -> None:+ """Describe a local image using explicit Base64 encoding."""+ model = _build_model()+ image_path = Path(__file__).resolve().parent / "test.jpeg"+ data = base64.b64encode(image_path.read_bytes()).decode("ascii")+ messages = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="What is in this image? Describe it briefly."),+ DataBlock(+ source=Base64Source(data=data, media_type="image/jpeg"),+ ),+ ],+ ),+ ]+ print("=== Explicit Base64 Call ===")+ await stream_and_collect(await model(messages))+++async def example_video() -> None:+ """Describe a video provided as a public URL."""+ model = _build_model()+ messages = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="What is happening in this video?"),+ DataBlock(+ source=URLSource(+ url=TEST_VIDEO_URL,+ media_type="video/mp4",+ ),+ ),+ ],+ ),+ ]+ print("=== Multimodal Call (Video URL) ===")+ await stream_and_collect(await model(messages))+++async def example_audio(+ audio: str = TEST_AUDIO_URL,+ media_type: str = "audio/mpeg",+) -> None:+ """Transcribe a public audio URL or a local MP3, WAV, AAC, or M4A file."""+ model = _build_model()+ url = (+ audio+ if audio.startswith(("http://", "https://", "file://"))+ else Path(audio).expanduser().resolve().as_uri()+ )+ messages = [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Transcribe this audio."),+ DataBlock(source=URLSource(url=url, media_type=media_type)),+ ],+ ),+ ]+ print("=== Multimodal Call (Audio) ===")+ await stream_and_collect(await model(messages))+++if __name__ == "__main__":+ asyncio.run(example_image_url())+ asyncio.run(example_image_local_path())+ asyncio.run(example_image_base64())+ asyncio.run(example_video())+ asyncio.run(example_audio())diff --git a/src/agentscope/formatter/_volcengine_formatter.py b/src/agentscope/formatter/_volcengine_formatter.pyindex 3349ad540..310935491 100644--- a/src/agentscope/formatter/_volcengine_formatter.py+++ b/src/agentscope/formatter/_volcengine_formatter.py@@ -1,8 +1,12 @@# -*- coding: utf-8 -*-"""The Volcengine Ark formatter module."""+import base64from fnmatch import fnmatch+from pathlib import Pathfrom typing import Any+from urllib.parse import urlsplit+from urllib.request import url2pathnamefrom pydantic import Field@@ -21,6 +25,15 @@ from ..message import ()+_AUDIO_FORMATS = {+ "audio/mpeg": "mp3",+ "audio/mp3": "mp3",+ "audio/wav": "wav",+ "audio/aac": "aac",+ "audio/mp4": "m4a",+}++class _VolcengineFormatterBase(_OpenAIFormatterBase):"""Base formatter with shared Ark multimodal conversion logic."""@@ -28,7 +41,7 @@ class _VolcengineFormatterBase(_OpenAIFormatterBase):self,block: DataBlock,) -> dict[str, Any] | None:- """Format supported image and video data for the Ark Chat API.+ """Format supported image, video, and audio for the Ark Chat API.Args:block (`DataBlock`):@@ -55,6 +68,8 @@ class _VolcengineFormatterBase(_OpenAIFormatterBase):return self._format_image_source(block.source)if main_type == "video":return self._format_video_source(block.source)+ if main_type == "audio":+ return self._format_ark_audio_source(block.source)logger.warning("Unsupported main media type %s for Volcengine Ark API. "@@ -63,6 +78,38 @@ class _VolcengineFormatterBase(_OpenAIFormatterBase):)return None+ @staticmethod+ def _format_ark_audio_source(+ source: URLSource | Base64Source,+ ) -> dict[str, Any]:+ """Convert audio to Ark's URL or raw base64 input_audio payload.++ Remote URLs are passed through. Local file URIs are decoded using+ the platform's path conversion, including Windows drive/UNC paths.+ """+ if source.media_type not in _AUDIO_FORMATS:+ raise TypeError(+ f"Unsupported audio media type: {source.media_type}. "+ "Ark supports MP3, WAV, AAC, and M4A audio.",+ )++ payload = {"format": _AUDIO_FORMATS[source.media_type]}+ if isinstance(source, Base64Source):+ payload["data"] = source.data+ else:+ url = str(source.url)+ parsed = urlsplit(url)+ if parsed.scheme == "file":+ path = parsed.path+ if parsed.netloc and parsed.netloc != "localhost":+ path = "//" + parsed.netloc + path+ data = Path(url2pathname(path)).read_bytes()+ payload["data"] = base64.b64encode(data).decode("ascii")+ else:+ payload["url"] = url++ return {"type": "input_audio", "input_audio": payload}+def _format_video_source(self,source: URLSource | Base64Source,@@ -91,10 +138,15 @@ class VolcengineChatFormatter(_VolcengineFormatterBase):"""input_types: list[str] = Field(- default_factory=lambda: ["text/plain", "image/*", "video/*"],+ default_factory=lambda: [+ "text/plain",+ "image/*",+ "video/*",+ *_AUDIO_FORMATS,+ ],description=(- "The supported input types. Defaults to "- '``["text/plain", "image/*", "video/*"]``.'+ "The supported input types: text, images, video, and "+ "MP3, WAV, AAC, or M4A audio."),)@@ -311,10 +363,15 @@ class VolcengineMultiAgentFormatter(_VolcengineFormatterBase):)input_types: list[str] = Field(- default_factory=lambda: ["text/plain", "image/*", "video/*"],+ default_factory=lambda: [+ "text/plain",+ "image/*",+ "video/*",+ *_AUDIO_FORMATS,+ ],description=(- "The supported input types. Defaults to "- '``["text/plain", "image/*", "video/*"]``.'+ "The supported input types: text, images, video, and "+ "MP3, WAV, AAC, or M4A audio."),)diff --git a/src/agentscope/model/_volcengine/_models/doubao-seed-2-1-lite-260915.yaml b/src/agentscope/model/_volcengine/_models/doubao-seed-2-1-lite-260915.yamlindex dcd451100..27568e2f9 100644--- a/src/agentscope/model/_volcengine/_models/doubao-seed-2-1-lite-260915.yaml+++ b/src/agentscope/model/_volcengine/_models/doubao-seed-2-1-lite-260915.yaml@@ -9,6 +9,11 @@ input_types:- application/x-thinking- image/*- video/*+ - audio/mpeg+ - audio/mp3+ - audio/wav+ - audio/aac+ - audio/mp4output_types:- text/plaindiff --git a/tests/formatter_volcengine_test.py b/tests/formatter_volcengine_test.pyindex 4eb2cc4c0..85b1072fd 100644--- a/tests/formatter_volcengine_test.py+++ b/tests/formatter_volcengine_test.py@@ -3,7 +3,12 @@VolcengineMultiAgentFormatter, with exact ground-truth comparisons."""+from pathlib import Path+from tempfile import TemporaryDirectoryfrom unittest import IsolatedAsyncioTestCase+from unittest.mock import patch++from nturl2path import url2pathname as windows_url2pathnamefrom agentscope.formatter import (VolcengineChatFormatter,@@ -822,3 +827,300 @@ class TestVolcengineFormatter(IsolatedAsyncioTestCase):],res,)+++class TestVolcengineAudio(IsolatedAsyncioTestCase):+ """Compare complete formatted requests without network access."""++ async def test_audio_sources_and_formats(self) -> None:+ """Both formatters preserve audio-only URL and base64 messages."""+ for media_type, audio_format in (+ ("audio/mpeg", "mp3"),+ ("audio/mp3", "mp3"),+ ("audio/wav", "wav"),+ ("audio/aac", "aac"),+ ("audio/mp4", "m4a"),+ ):+ for source, payload in (+ (+ Base64Source(data="YXVkaW8=", media_type=media_type),+ {"data": "YXVkaW8=", "format": audio_format},+ ),+ (+ URLSource(+ url="https://example.com/audio?token=abc%2B123",+ media_type=media_type,+ ),+ {+ "url": "https://example.com/audio?token=abc%2B123",+ "format": audio_format,+ },+ ),+ ):+ msg = UserMsg(+ name="user",+ content=[DataBlock(source=source)],+ )+ for formatter in (+ VolcengineChatFormatter(),+ VolcengineMultiAgentFormatter(),+ ):+ with self.subTest(+ media_type=media_type,+ source=source.type,+ formatter=type(formatter).__name__,+ ):+ self.assertEqual(+ await formatter.format([msg]),+ [+ {+ "role": "user",+ "content": [+ {+ "type": "input_audio",+ "input_audio": payload,+ },+ ],+ },+ ],+ )++ async def test_local_audio_files(self) -> None:+ """Native file URIs decode special characters exactly once."""+ with TemporaryDirectory() as directory:+ for filename in (+ "audio #1.wav",+ "café.wav",+ "audio%20.wav",+ "audio+1.wav",+ ):+ path = Path(directory) / filename+ path.write_bytes(b"audio")+ msg = UserMsg(+ name="user",+ content=[+ DataBlock(+ source=URLSource(+ url=path.as_uri(),+ media_type="audio/wav",+ ),+ ),+ ],+ )+ for formatter in (+ VolcengineChatFormatter(),+ VolcengineMultiAgentFormatter(),+ ):+ with self.subTest(filename=filename):+ self.assertEqual(+ await formatter.format([msg]),+ [+ {+ "role": "user",+ "content": [+ {+ "type": "input_audio",+ "input_audio": {+ "data": "YXVkaW8=",+ "format": "wav",+ },+ },+ ],+ },+ ],+ )++ async def test_windows_file_uris(self) -> None:+ """Windows drive and UNC URIs use the native path converter."""+ module = "agentscope.formatter._volcengine_formatter"+ for uri, expected_path in (+ ("file:///C:/audio%20clip.wav", "C:\\audio clip.wav"),+ (+ "file://server/share/audio%20clip.wav",+ "\\\\server\\share\\audio clip.wav",+ ),+ ):+ with (+ self.subTest(uri=uri),+ patch(+ f"{module}.url2pathname",+ side_effect=windows_url2pathname,+ ),+ patch(f"{module}.Path") as mock_path,+ ):+ mock_path.return_value.read_bytes.return_value = b"audio"+ result = await VolcengineChatFormatter().format(+ [+ UserMsg(+ name="user",+ content=[+ DataBlock(+ source=URLSource(+ url=uri,+ media_type="audio/wav",+ ),+ ),+ ],+ ),+ ],+ )+ mock_path.assert_called_once_with(expected_path)+ self.assertEqual(+ result,+ [+ {+ "role": "user",+ "content": [+ {+ "type": "input_audio",+ "input_audio": {+ "data": "YXVkaW8=",+ "format": "wav",+ },+ },+ ],+ },+ ],+ )++ async def test_hint_audio_and_text(self) -> None:+ """Hints remain standalone user messages with text and audio."""+ msg = AssistantMsg(+ name="assistant",+ content=[+ HintBlock(hint="Listen carefully."),+ HintBlock(+ hint=[+ TextBlock(text="Transcribe this."),+ DataBlock(+ source=Base64Source(+ data="YXVkaW8=",+ media_type="audio/aac",+ ),+ ),+ ],+ ),+ ],+ )+ audio = {+ "type": "input_audio",+ "input_audio": {"data": "YXVkaW8=", "format": "aac"},+ }+ expected = [+ {+ "role": "user",+ "content": [{"type": "text", "text": "Listen carefully."}],+ },+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Transcribe this."},+ audio,+ ],+ },+ ]+ for formatter in (+ VolcengineChatFormatter(),+ VolcengineMultiAgentFormatter(),+ ):+ with self.subTest(formatter=type(formatter).__name__):+ self.assertEqual(await formatter.format([msg]), expected)++ async def test_tool_audio(self) -> None:+ """Both formatters promote tool audio after the tool response."""+ msg = AssistantMsg(+ name="assistant",+ content=[+ ToolCallBlock(id="call_1", name="record", input="{}"),+ ToolResultBlock(+ id="call_1",+ name="record",+ state=ToolResultState.SUCCESS,+ output=[+ DataBlock(+ id="audio_1",+ source=Base64Source(+ data="YXVkaW8=",+ media_type="audio/mp4",+ ),+ ),+ ],+ ),+ ],+ )+ expected = [+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_1",+ "type": "function",+ "function": {"name": "record", "arguments": "{}"},+ },+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "name": "record",+ "content": (+ "<system-reminder>A(n) audio file is returned and will "+ "be presented to you with the identifier "+ "[audio_1].</system-reminder>"+ ),+ },+ {+ "role": "user",+ "name": "system-reminder",+ "content": [+ {+ "type": "text",+ "text": "<system-reminder>The multimodal data and "+ "their identifiers are listed as follows:",+ },+ {"type": "text", "text": "- audio_1 (audio file): "},+ {+ "type": "input_audio",+ "input_audio": {"data": "YXVkaW8=", "format": "m4a"},+ },+ {"type": "text", "text": "</system-reminder>"},+ ],+ },+ ]+ for formatter in (+ VolcengineChatFormatter(),+ VolcengineMultiAgentFormatter(),+ ):+ self.assertEqual(await formatter.format([msg]), expected)++ async def test_audio_filtering(self) -> None:+ """Configured input types still filter audio; unknown formats fail."""+ for formatter_class in (+ VolcengineChatFormatter,+ VolcengineMultiAgentFormatter,+ ):+ for media_type, input_types in (+ ("audio/wav", ["text/plain"]),+ ("audio/ogg", None),+ ):+ formatter = (+ formatter_class(input_types=input_types)+ if input_types is not None+ else formatter_class()+ )+ msg = UserMsg(+ name="user",+ content=[+ DataBlock(+ source=Base64Source(+ data="YXVkaW8=",+ media_type=media_type,+ ),+ ),+ ],+ )+ self.assertEqual(await formatter.format([msg]), [])++ with self.assertRaisesRegex(TypeError, "audio/ogg"):+ await formatter_class(input_types=["audio/*"]).format([msg])diff --git a/tests/model_volcengine_test.py b/tests/model_volcengine_test.pyindex 79edbfddb..faf9e66cc 100644--- a/tests/model_volcengine_test.py+++ b/tests/model_volcengine_test.py@@ -15,7 +15,14 @@ from unittest.mock import AsyncMock, MagicMockfrom utils import AnyString-from agentscope.message import TextBlock, ToolCallBlock, ThinkingBlock+from agentscope.message import (+ Base64Source,+ DataBlock,+ TextBlock,+ ToolCallBlock,+ ThinkingBlock,+ UserMsg,+)from agentscope.model import VolcengineChatModelfrom agentscope.credential import CredentialFactory, VolcengineCredentialfrom agentscope.tool import ToolChoice@@ -692,6 +699,78 @@ class TestVolcengineModelParameters(unittest.TestCase):class TestVolcengineRequestParameters(IsolatedAsyncioTestCase):"""Tests for Ark-specific request parameters."""+ async def test_audio_request_matches_model_card(self) -> None:+ """A card-configured formatter sends audio through the SDK."""+ model_name = "doubao-seed-2-1-lite-260915"+ card = next(+ c+ for c in VolcengineChatModel.list_models()+ if c.name == model_name+ )+ self.assertEqual(+ card.input_types,+ [+ "text/plain",+ "application/x-thinking",+ "image/*",+ "video/*",+ "audio/mpeg",+ "audio/mp3",+ "audio/wav",+ "audio/aac",+ "audio/mp4",+ ],+ )+ model = VolcengineChatModel(+ credential=VolcengineCredential(api_key="test"),+ model=model_name,+ stream=False,+ )+ model.formatter.input_types = card.input_types+ mock_create = AsyncMock(return_value=_mock_completion(text="Audio."))+ model.client = MagicMock()+ model.client.chat.completions.create = mock_create++ await model(+ [+ UserMsg(+ name="user",+ content=[+ TextBlock(text="Transcribe this."),+ DataBlock(+ source=Base64Source(+ data="YXVkaW8=",+ media_type="audio/wav",+ ),+ ),+ ],+ ),+ ],+ )++ self.assertEqual(+ mock_create.await_args.kwargs,+ {+ "model": model_name,+ "stream": False,+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Transcribe this."},+ {+ "type": "input_audio",+ "input_audio": {+ "data": "YXVkaW8=",+ "format": "wav",+ },+ },+ ],+ },+ ],+ },+ )+async def test_thinking_and_reasoning_effort_are_forwarded(self) -> None:"""Thinking controls use the request shape accepted by Ark."""model = VolcengineChatModel(