Skip to content

Build and run agents you can see, understand and trust.

feat(formatter): add audio input support for Volcengine (#3706)

qbc2cb553840102
diff --git a/scripts/model_examples/README.md b/scripts/model_examples/README.md
index b07d02203..2c423e26d 100644
--- a/scripts/model_examples/README.md
+++ b/scripts/model_examples/README.md
@@ -53,6 +53,7 @@ scripts/model_examples/
│
├── volcengine_call.py # Volcengine Ark / Doubao
├── volcengine_multiagent.py
+├── volcengine_multimodal.py # Image, video, and audio input
│
├── xai_call.py # xAI Grok
├── xai_multiagent.py
diff --git a/scripts/model_examples/run_tests.py b/scripts/model_examples/run_tests.py
index ba4023e7e..5cdf459af 100644
--- a/scripts/model_examples/run_tests.py
+++ b/scripts/model_examples/run_tests.py
@@ -184,7 +184,7 @@ ALL_PROVIDERS: list[Provider] = [
name="volcengine",
env_var="VOLCENGINE_API_KEY",
file_prefix="volcengine",
- supported_tests=["call", "multiagent"],
+ supported_tests=["call", "multiagent", "multimodal"],
description="Volcengine Ark / Doubao models",
),
Provider(
diff --git a/scripts/model_examples/volcengine_multimodal.py b/scripts/model_examples/volcengine_multimodal.py
new file mode 100644
index 000000000..731e67946
--- /dev/null
+++ b/scripts/model_examples/volcengine_multimodal.py
@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+"""Examples of Volcengine image, video, and audio input using DataBlock.
+
+Set VOLCENGINE_API_KEY and run this script without arguments, or use the
+example runner with --providers volcengine --tests multimodal.
+The examples use doubao-seed-2-1-lite-260915. For a local audio file, call
+``await example_audio("./sample.wav", "audio/wav")``.
+"""
+
+import asyncio
+import base64
+import os
+from pathlib import Path
+
+from _utils import stream_and_collect
+from agentscope.credential import VolcengineCredential
+from agentscope.message import (
+ Base64Source,
+ DataBlock,
+ TextBlock,
+ URLSource,
+ UserMsg,
+)
+from agentscope.model import VolcengineChatModel
+
+
+TEST_IMAGE_URL = (
+ "https://help-static-aliyun-doc.aliyuncs.com/file-manage"
+ "-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg"
+)
+TEST_VIDEO_URL = (
+ "https://help-static-aliyun-doc.aliyuncs.com/file-manage"
+ "-files/zh-CN/20241115/cqqkru/1.mp4"
+)
+TEST_AUDIO_URL = (
+ "https://ark-project.tos-cn-beijing.volces.com/doc_audio/"
+ "ark_demo_audio.mp3"
+)
+
+
+def _build_model() -> VolcengineChatModel:
+ """Build a Doubao model supporting image, video, and audio input."""
+ return VolcengineChatModel(
+ credential=VolcengineCredential(
+ api_key=os.environ["VOLCENGINE_API_KEY"],
+ ),
+ model="doubao-seed-2-1-lite-260915",
+ stream=True,
+ context_size=1_024_000,
+ )
+
+
+async def example_image_url() -> None:
+ """Describe an image provided as a public URL."""
+ model = _build_model()
+ messages = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="What is in this image? Describe it briefly."),
+ DataBlock(
+ source=URLSource(
+ url=TEST_IMAGE_URL,
+ media_type="image/jpeg",
+ ),
+ ),
+ ],
+ ),
+ ]
+ print("=== Multimodal Call (Image URL) ===")
+ await stream_and_collect(await model(messages))
+
+
+async def example_image_local_path() -> None:
+ """Describe a local image using a portable file URI."""
+ model = _build_model()
+ image_path = Path(__file__).resolve().parent / "test.jpeg"
+ messages = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="What is in this image? Describe it briefly."),
+ DataBlock(
+ source=URLSource(
+ url=image_path.as_uri(),
+ media_type="image/jpeg",
+ ),
+ ),
+ ],
+ ),
+ ]
+ print("=== Local Path Call (file://) ===")
+ await stream_and_collect(await model(messages))
+
+
+async def example_image_base64() -> None:
+ """Describe a local image using explicit Base64 encoding."""
+ model = _build_model()
+ image_path = Path(__file__).resolve().parent / "test.jpeg"
+ data = base64.b64encode(image_path.read_bytes()).decode("ascii")
+ messages = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="What is in this image? Describe it briefly."),
+ DataBlock(
+ source=Base64Source(data=data, media_type="image/jpeg"),
+ ),
+ ],
+ ),
+ ]
+ print("=== Explicit Base64 Call ===")
+ await stream_and_collect(await model(messages))
+
+
+async def example_video() -> None:
+ """Describe a video provided as a public URL."""
+ model = _build_model()
+ messages = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="What is happening in this video?"),
+ DataBlock(
+ source=URLSource(
+ url=TEST_VIDEO_URL,
+ media_type="video/mp4",
+ ),
+ ),
+ ],
+ ),
+ ]
+ print("=== Multimodal Call (Video URL) ===")
+ await stream_and_collect(await model(messages))
+
+
+async def example_audio(
+ audio: str = TEST_AUDIO_URL,
+ media_type: str = "audio/mpeg",
+) -> None:
+ """Transcribe a public audio URL or a local MP3, WAV, AAC, or M4A file."""
+ model = _build_model()
+ url = (
+ audio
+ if audio.startswith(("http://", "https://", "file://"))
+ else Path(audio).expanduser().resolve().as_uri()
+ )
+ messages = [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Transcribe this audio."),
+ DataBlock(source=URLSource(url=url, media_type=media_type)),
+ ],
+ ),
+ ]
+ print("=== Multimodal Call (Audio) ===")
+ await stream_and_collect(await model(messages))
+
+
+if __name__ == "__main__":
+ asyncio.run(example_image_url())
+ asyncio.run(example_image_local_path())
+ asyncio.run(example_image_base64())
+ asyncio.run(example_video())
+ asyncio.run(example_audio())
diff --git a/src/agentscope/formatter/_volcengine_formatter.py b/src/agentscope/formatter/_volcengine_formatter.py
index 3349ad540..310935491 100644
--- a/src/agentscope/formatter/_volcengine_formatter.py
+++ b/src/agentscope/formatter/_volcengine_formatter.py
@@ -1,8 +1,12 @@
# -*- coding: utf-8 -*-
"""The Volcengine Ark formatter module."""
+import base64
from fnmatch import fnmatch
+from pathlib import Path
from typing import Any
+from urllib.parse import urlsplit
+from urllib.request import url2pathname
from pydantic import Field
@@ -21,6 +25,15 @@ from ..message import (
)
+_AUDIO_FORMATS = {
+ "audio/mpeg": "mp3",
+ "audio/mp3": "mp3",
+ "audio/wav": "wav",
+ "audio/aac": "aac",
+ "audio/mp4": "m4a",
+}
+
+
class _VolcengineFormatterBase(_OpenAIFormatterBase):
"""Base formatter with shared Ark multimodal conversion logic."""
@@ -28,7 +41,7 @@ class _VolcengineFormatterBase(_OpenAIFormatterBase):
self,
block: DataBlock,
) -> dict[str, Any] | None:
- """Format supported image and video data for the Ark Chat API.
+ """Format supported image, video, and audio for the Ark Chat API.
Args:
block (`DataBlock`):
@@ -55,6 +68,8 @@ class _VolcengineFormatterBase(_OpenAIFormatterBase):
return self._format_image_source(block.source)
if main_type == "video":
return self._format_video_source(block.source)
+ if main_type == "audio":
+ return self._format_ark_audio_source(block.source)
logger.warning(
"Unsupported main media type %s for Volcengine Ark API. "
@@ -63,6 +78,38 @@ class _VolcengineFormatterBase(_OpenAIFormatterBase):
)
return None
+ @staticmethod
+ def _format_ark_audio_source(
+ source: URLSource | Base64Source,
+ ) -> dict[str, Any]:
+ """Convert audio to Ark's URL or raw base64 input_audio payload.
+
+ Remote URLs are passed through. Local file URIs are decoded using
+ the platform's path conversion, including Windows drive/UNC paths.
+ """
+ if source.media_type not in _AUDIO_FORMATS:
+ raise TypeError(
+ f"Unsupported audio media type: {source.media_type}. "
+ "Ark supports MP3, WAV, AAC, and M4A audio.",
+ )
+
+ payload = {"format": _AUDIO_FORMATS[source.media_type]}
+ if isinstance(source, Base64Source):
+ payload["data"] = source.data
+ else:
+ url = str(source.url)
+ parsed = urlsplit(url)
+ if parsed.scheme == "file":
+ path = parsed.path
+ if parsed.netloc and parsed.netloc != "localhost":
+ path = "//" + parsed.netloc + path
+ data = Path(url2pathname(path)).read_bytes()
+ payload["data"] = base64.b64encode(data).decode("ascii")
+ else:
+ payload["url"] = url
+
+ return {"type": "input_audio", "input_audio": payload}
+
def _format_video_source(
self,
source: URLSource | Base64Source,
@@ -91,10 +138,15 @@ class VolcengineChatFormatter(_VolcengineFormatterBase):
"""
input_types: list[str] = Field(
- default_factory=lambda: ["text/plain", "image/*", "video/*"],
+ default_factory=lambda: [
+ "text/plain",
+ "image/*",
+ "video/*",
+ *_AUDIO_FORMATS,
+ ],
description=(
- "The supported input types. Defaults to "
- '``["text/plain", "image/*", "video/*"]``.'
+ "The supported input types: text, images, video, and "
+ "MP3, WAV, AAC, or M4A audio."
),
)
@@ -311,10 +363,15 @@ class VolcengineMultiAgentFormatter(_VolcengineFormatterBase):
)
input_types: list[str] = Field(
- default_factory=lambda: ["text/plain", "image/*", "video/*"],
+ default_factory=lambda: [
+ "text/plain",
+ "image/*",
+ "video/*",
+ *_AUDIO_FORMATS,
+ ],
description=(
- "The supported input types. Defaults to "
- '``["text/plain", "image/*", "video/*"]``.'
+ "The supported input types: text, images, video, and "
+ "MP3, WAV, AAC, or M4A audio."
),
)
diff --git a/src/agentscope/model/_volcengine/_models/doubao-seed-2-1-lite-260915.yaml b/src/agentscope/model/_volcengine/_models/doubao-seed-2-1-lite-260915.yaml
index dcd451100..27568e2f9 100644
--- a/src/agentscope/model/_volcengine/_models/doubao-seed-2-1-lite-260915.yaml
+++ b/src/agentscope/model/_volcengine/_models/doubao-seed-2-1-lite-260915.yaml
@@ -9,6 +9,11 @@ input_types:
- application/x-thinking
- image/*
- video/*
+ - audio/mpeg
+ - audio/mp3
+ - audio/wav
+ - audio/aac
+ - audio/mp4
output_types:
- text/plain
diff --git a/tests/formatter_volcengine_test.py b/tests/formatter_volcengine_test.py
index 4eb2cc4c0..85b1072fd 100644
--- a/tests/formatter_volcengine_test.py
+++ b/tests/formatter_volcengine_test.py
@@ -3,7 +3,12 @@
VolcengineMultiAgentFormatter, with exact ground-truth comparisons.
"""
+from pathlib import Path
+from tempfile import TemporaryDirectory
from unittest import IsolatedAsyncioTestCase
+from unittest.mock import patch
+
+from nturl2path import url2pathname as windows_url2pathname
from agentscope.formatter import (
VolcengineChatFormatter,
@@ -822,3 +827,300 @@ class TestVolcengineFormatter(IsolatedAsyncioTestCase):
],
res,
)
+
+
+class TestVolcengineAudio(IsolatedAsyncioTestCase):
+ """Compare complete formatted requests without network access."""
+
+ async def test_audio_sources_and_formats(self) -> None:
+ """Both formatters preserve audio-only URL and base64 messages."""
+ for media_type, audio_format in (
+ ("audio/mpeg", "mp3"),
+ ("audio/mp3", "mp3"),
+ ("audio/wav", "wav"),
+ ("audio/aac", "aac"),
+ ("audio/mp4", "m4a"),
+ ):
+ for source, payload in (
+ (
+ Base64Source(data="YXVkaW8=", media_type=media_type),
+ {"data": "YXVkaW8=", "format": audio_format},
+ ),
+ (
+ URLSource(
+ url="https://example.com/audio?token=abc%2B123",
+ media_type=media_type,
+ ),
+ {
+ "url": "https://example.com/audio?token=abc%2B123",
+ "format": audio_format,
+ },
+ ),
+ ):
+ msg = UserMsg(
+ name="user",
+ content=[DataBlock(source=source)],
+ )
+ for formatter in (
+ VolcengineChatFormatter(),
+ VolcengineMultiAgentFormatter(),
+ ):
+ with self.subTest(
+ media_type=media_type,
+ source=source.type,
+ formatter=type(formatter).__name__,
+ ):
+ self.assertEqual(
+ await formatter.format([msg]),
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_audio",
+ "input_audio": payload,
+ },
+ ],
+ },
+ ],
+ )
+
+ async def test_local_audio_files(self) -> None:
+ """Native file URIs decode special characters exactly once."""
+ with TemporaryDirectory() as directory:
+ for filename in (
+ "audio #1.wav",
+ "café.wav",
+ "audio%20.wav",
+ "audio+1.wav",
+ ):
+ path = Path(directory) / filename
+ path.write_bytes(b"audio")
+ msg = UserMsg(
+ name="user",
+ content=[
+ DataBlock(
+ source=URLSource(
+ url=path.as_uri(),
+ media_type="audio/wav",
+ ),
+ ),
+ ],
+ )
+ for formatter in (
+ VolcengineChatFormatter(),
+ VolcengineMultiAgentFormatter(),
+ ):
+ with self.subTest(filename=filename):
+ self.assertEqual(
+ await formatter.format([msg]),
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_audio",
+ "input_audio": {
+ "data": "YXVkaW8=",
+ "format": "wav",
+ },
+ },
+ ],
+ },
+ ],
+ )
+
+ async def test_windows_file_uris(self) -> None:
+ """Windows drive and UNC URIs use the native path converter."""
+ module = "agentscope.formatter._volcengine_formatter"
+ for uri, expected_path in (
+ ("file:///C:/audio%20clip.wav", "C:\\audio clip.wav"),
+ (
+ "file://server/share/audio%20clip.wav",
+ "\\\\server\\share\\audio clip.wav",
+ ),
+ ):
+ with (
+ self.subTest(uri=uri),
+ patch(
+ f"{module}.url2pathname",
+ side_effect=windows_url2pathname,
+ ),
+ patch(f"{module}.Path") as mock_path,
+ ):
+ mock_path.return_value.read_bytes.return_value = b"audio"
+ result = await VolcengineChatFormatter().format(
+ [
+ UserMsg(
+ name="user",
+ content=[
+ DataBlock(
+ source=URLSource(
+ url=uri,
+ media_type="audio/wav",
+ ),
+ ),
+ ],
+ ),
+ ],
+ )
+ mock_path.assert_called_once_with(expected_path)
+ self.assertEqual(
+ result,
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_audio",
+ "input_audio": {
+ "data": "YXVkaW8=",
+ "format": "wav",
+ },
+ },
+ ],
+ },
+ ],
+ )
+
+ async def test_hint_audio_and_text(self) -> None:
+ """Hints remain standalone user messages with text and audio."""
+ msg = AssistantMsg(
+ name="assistant",
+ content=[
+ HintBlock(hint="Listen carefully."),
+ HintBlock(
+ hint=[
+ TextBlock(text="Transcribe this."),
+ DataBlock(
+ source=Base64Source(
+ data="YXVkaW8=",
+ media_type="audio/aac",
+ ),
+ ),
+ ],
+ ),
+ ],
+ )
+ audio = {
+ "type": "input_audio",
+ "input_audio": {"data": "YXVkaW8=", "format": "aac"},
+ }
+ expected = [
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "Listen carefully."}],
+ },
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Transcribe this."},
+ audio,
+ ],
+ },
+ ]
+ for formatter in (
+ VolcengineChatFormatter(),
+ VolcengineMultiAgentFormatter(),
+ ):
+ with self.subTest(formatter=type(formatter).__name__):
+ self.assertEqual(await formatter.format([msg]), expected)
+
+ async def test_tool_audio(self) -> None:
+ """Both formatters promote tool audio after the tool response."""
+ msg = AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(id="call_1", name="record", input="{}"),
+ ToolResultBlock(
+ id="call_1",
+ name="record",
+ state=ToolResultState.SUCCESS,
+ output=[
+ DataBlock(
+ id="audio_1",
+ source=Base64Source(
+ data="YXVkaW8=",
+ media_type="audio/mp4",
+ ),
+ ),
+ ],
+ ),
+ ],
+ )
+ expected = [
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "record", "arguments": "{}"},
+ },
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "name": "record",
+ "content": (
+ "<system-reminder>A(n) audio file is returned and will "
+ "be presented to you with the identifier "
+ "[audio_1].</system-reminder>"
+ ),
+ },
+ {
+ "role": "user",
+ "name": "system-reminder",
+ "content": [
+ {
+ "type": "text",
+ "text": "<system-reminder>The multimodal data and "
+ "their identifiers are listed as follows:",
+ },
+ {"type": "text", "text": "- audio_1 (audio file): "},
+ {
+ "type": "input_audio",
+ "input_audio": {"data": "YXVkaW8=", "format": "m4a"},
+ },
+ {"type": "text", "text": "</system-reminder>"},
+ ],
+ },
+ ]
+ for formatter in (
+ VolcengineChatFormatter(),
+ VolcengineMultiAgentFormatter(),
+ ):
+ self.assertEqual(await formatter.format([msg]), expected)
+
+ async def test_audio_filtering(self) -> None:
+ """Configured input types still filter audio; unknown formats fail."""
+ for formatter_class in (
+ VolcengineChatFormatter,
+ VolcengineMultiAgentFormatter,
+ ):
+ for media_type, input_types in (
+ ("audio/wav", ["text/plain"]),
+ ("audio/ogg", None),
+ ):
+ formatter = (
+ formatter_class(input_types=input_types)
+ if input_types is not None
+ else formatter_class()
+ )
+ msg = UserMsg(
+ name="user",
+ content=[
+ DataBlock(
+ source=Base64Source(
+ data="YXVkaW8=",
+ media_type=media_type,
+ ),
+ ),
+ ],
+ )
+ self.assertEqual(await formatter.format([msg]), [])
+
+ with self.assertRaisesRegex(TypeError, "audio/ogg"):
+ await formatter_class(input_types=["audio/*"]).format([msg])
diff --git a/tests/model_volcengine_test.py b/tests/model_volcengine_test.py
index 79edbfddb..faf9e66cc 100644
--- a/tests/model_volcengine_test.py
+++ b/tests/model_volcengine_test.py
@@ -15,7 +15,14 @@ from unittest.mock import AsyncMock, MagicMock
from utils import AnyString
-from agentscope.message import TextBlock, ToolCallBlock, ThinkingBlock
+from agentscope.message import (
+ Base64Source,
+ DataBlock,
+ TextBlock,
+ ToolCallBlock,
+ ThinkingBlock,
+ UserMsg,
+)
from agentscope.model import VolcengineChatModel
from agentscope.credential import CredentialFactory, VolcengineCredential
from agentscope.tool import ToolChoice
@@ -692,6 +699,78 @@ class TestVolcengineModelParameters(unittest.TestCase):
class TestVolcengineRequestParameters(IsolatedAsyncioTestCase):
"""Tests for Ark-specific request parameters."""
+ async def test_audio_request_matches_model_card(self) -> None:
+ """A card-configured formatter sends audio through the SDK."""
+ model_name = "doubao-seed-2-1-lite-260915"
+ card = next(
+ c
+ for c in VolcengineChatModel.list_models()
+ if c.name == model_name
+ )
+ self.assertEqual(
+ card.input_types,
+ [
+ "text/plain",
+ "application/x-thinking",
+ "image/*",
+ "video/*",
+ "audio/mpeg",
+ "audio/mp3",
+ "audio/wav",
+ "audio/aac",
+ "audio/mp4",
+ ],
+ )
+ model = VolcengineChatModel(
+ credential=VolcengineCredential(api_key="test"),
+ model=model_name,
+ stream=False,
+ )
+ model.formatter.input_types = card.input_types
+ mock_create = AsyncMock(return_value=_mock_completion(text="Audio."))
+ model.client = MagicMock()
+ model.client.chat.completions.create = mock_create
+
+ await model(
+ [
+ UserMsg(
+ name="user",
+ content=[
+ TextBlock(text="Transcribe this."),
+ DataBlock(
+ source=Base64Source(
+ data="YXVkaW8=",
+ media_type="audio/wav",
+ ),
+ ),
+ ],
+ ),
+ ],
+ )
+
+ self.assertEqual(
+ mock_create.await_args.kwargs,
+ {
+ "model": model_name,
+ "stream": False,
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Transcribe this."},
+ {
+ "type": "input_audio",
+ "input_audio": {
+ "data": "YXVkaW8=",
+ "format": "wav",
+ },
+ },
+ ],
+ },
+ ],
+ },
+ )
+
async def test_thinking_and_reasoning_effort_are_forwarded(self) -> None:
"""Thinking controls use the request shape accepted by Ark."""
model = VolcengineChatModel(