Skip to content

Build and run agents you can see, understand and trust.

fix(formatter): use data URLs for DashScope base64 audio (#2315)

qbc3a4e2ae3eec0
diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.py
index dcd0a1be4..7040068f5 100644
--- a/src/agentscope/formatter/_dashscope_formatter.py
+++ b/src/agentscope/formatter/_dashscope_formatter.py
@@ -175,30 +175,32 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
"""Convert an audio source to DashScope ``input_audio`` format.
DashScope's compatible API accepts URLs directly in the ``data``
- field (unlike standard OpenAI which requires base64). Local files
- are still read and base64-encoded.
+ field. Base64-encoded audio must be wrapped in a data URL. Local
+ files are read, base64-encoded, and wrapped in the same form.
"""
+ fmt = source.media_type.split("/")[-1]
+ if fmt == "mpeg":
+ fmt = "mp3"
+
if isinstance(source, Base64Source):
- fmt = source.media_type.split("/")[-1]
return {
"type": "input_audio",
"input_audio": {
- "data": source.data,
+ "data": f"data:;base64,{source.data}",
"format": fmt,
},
}
if isinstance(source, URLSource):
url_str = str(source.url)
- fmt = source.media_type.split("/")[-1]
if url_str.startswith("file://"):
local_path = url_str.removeprefix("file://")
with open(local_path, "rb") as f:
- data = base64.b64encode(f.read()).decode("utf-8")
+ encoded = base64.b64encode(f.read()).decode("utf-8")
return {
"type": "input_audio",
"input_audio": {
- "data": data,
+ "data": f"data:;base64,{encoded}",
"format": fmt,
},
}
diff --git a/tests/formatter_dashscope_test.py b/tests/formatter_dashscope_test.py
index 8d64914ef..5fe6dbeca 100644
--- a/tests/formatter_dashscope_test.py
+++ b/tests/formatter_dashscope_test.py
@@ -4,7 +4,7 @@ DashScopeMultiAgentFormatter, following the reference test style with exact
ground-truth comparisons.
"""
from unittest import IsolatedAsyncioTestCase
-from unittest.mock import patch
+from unittest.mock import mock_open, patch
from agentscope.formatter import (
DashScopeChatFormatter,
@@ -173,7 +173,7 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"type": "input_audio",
"input_audio": {
"data": self.audio_url,
- "format": "mpeg",
+ "format": "mp3",
},
},
],
@@ -295,7 +295,7 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
"type": "input_audio",
"input_audio": {
"data": self.audio_url,
- "format": "mpeg",
+ "format": "mp3",
},
},
],
@@ -373,6 +373,122 @@ class TestDashScopeFormatter(IsolatedAsyncioTestCase):
res,
)
+ async def test_chat_formatter_base64_audio(self) -> None:
+ """Base64-encoded audio is inlined as a DashScope data URL."""
+ fmt = DashScopeChatFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ DataBlock(
+ source=Base64Source(
+ data="UklGRg==",
+ media_type="audio/wav",
+ ),
+ ),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_audio",
+ "input_audio": {
+ "data": "data:;base64,UklGRg==",
+ "format": "wav",
+ },
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
+ async def test_chat_formatter_mpeg_audio_uses_mp3_format(self) -> None:
+ """The standard audio/mpeg MIME type maps to DashScope's mp3."""
+ fmt = DashScopeChatFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ DataBlock(
+ source=Base64Source(
+ data="SUQz",
+ media_type="audio/mpeg",
+ ),
+ ),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_audio",
+ "input_audio": {
+ "data": "data:;base64,SUQz",
+ "format": "mp3",
+ },
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
+ @patch("builtins.open", new_callable=mock_open, read_data=b"RIFF")
+ async def test_chat_formatter_local_audio(
+ self,
+ mocked_open: object,
+ ) -> None:
+ """Local audio is read and inlined as a DashScope data URL."""
+ fmt = DashScopeChatFormatter()
+ msgs = [
+ UserMsg(
+ name="user",
+ content=[
+ DataBlock(
+ source=URLSource(
+ url="file:///tmp/audio.wav",
+ media_type="audio/wav",
+ ),
+ ),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_audio",
+ "input_audio": {
+ "data": "data:;base64,UklGRg==",
+ "format": "wav",
+ },
+ },
+ ],
+ },
+ ],
+ res,
+ )
+ mocked_open.assert_called_once_with("/tmp/audio.wav", "rb")
+
@patch(
"agentscope.formatter._formatter_base.shortuuid.uuid",
return_value=_FIXED_ID,