Skip to content

Build and run agents you can see, understand and trust.

fix(model): wrap non-streaming chat audio in a real WAV header (#3028)

ANIRUDDHA ADAK48c4b9ce0f9a
diff --git a/src/agentscope/_utils/_audio.py b/src/agentscope/_utils/_audio.py
index b93b5cbf6..c232006c7 100644
--- a/src/agentscope/_utils/_audio.py
+++ b/src/agentscope/_utils/_audio.py
@@ -3,28 +3,30 @@
import struct
-def _build_streaming_wav_header(
+def _wav_header(
+ riff_size: int,
+ data_size: int,
sample_rate: int = 24000,
channels: int = 1,
bits_per_sample: int = 16,
) -> bytes:
- """Build a 44-byte WAV/RIFF header for streaming PCM.
+ """Build a 44-byte WAV/RIFF header with explicit chunk sizes.
- The RIFF and ``data`` chunk sizes are set to ``0xFFFFFFFF`` since the
- total length isn't known yet. Decoders that only need sample-rate,
- channel count and bit depth (e.g. the web ``WavStreamPlayer``) treat
- everything after the ``data`` chunk header as PCM, so this is
- sufficient for live decoding of an open-ended stream.
+ Args:
+ riff_size (`int`):
+ The size to declare for the RIFF chunk.
+ data_size (`int`):
+ The size to declare for the ``data`` chunk.
- Both DashScope omni and OpenAI streaming deliver raw PCM upstream;
- prefixing the first chunk with this header lets the frontend start
- playback immediately without buffering the whole response.
+ Returns:
+ `bytes`:
+ A 44-byte WAV header.
"""
byte_rate = sample_rate * channels * bits_per_sample // 8
block_align = channels * bits_per_sample // 8
return (
b"RIFF"
- + struct.pack("<I", 0xFFFFFFFF)
+ + struct.pack("<I", riff_size)
+ b"WAVE"
+ b"fmt "
+ struct.pack("<I", 16)
@@ -38,5 +40,55 @@ def _build_streaming_wav_header(
bits_per_sample,
)
+ b"data"
- + struct.pack("<I", 0xFFFFFFFF)
+ + struct.pack("<I", data_size)
+ )
+
+
+def _build_streaming_wav_header(
+ sample_rate: int = 24000,
+ channels: int = 1,
+ bits_per_sample: int = 16,
+) -> bytes:
+ """Build a 44-byte WAV/RIFF header for streaming PCM.
+
+ The RIFF and ``data`` chunk sizes are set to ``0xFFFFFFFF`` since the
+ total length isn't known yet. Decoders that only need sample-rate,
+ channel count and bit depth (e.g. the web ``WavStreamPlayer``) treat
+ everything after the ``data`` chunk header as PCM, so this is
+ sufficient for live decoding of an open-ended stream.
+
+ Both DashScope omni and OpenAI streaming deliver raw PCM upstream;
+ prefixing the first chunk with this header lets the frontend start
+ playback immediately without buffering the whole response.
+
+ For a *complete* buffer use :func:`_build_wav_header` instead, which
+ declares real chunk sizes.
+ """
+ return _wav_header(
+ 0xFFFFFFFF,
+ 0xFFFFFFFF,
+ sample_rate,
+ channels,
+ bits_per_sample,
+ )
+
+
+def _build_wav_header(
+ data_len: int,
+ sample_rate: int = 24000,
+ channels: int = 1,
+ bits_per_sample: int = 16,
+) -> bytes:
+ """Build a 44-byte WAV/RIFF header for a complete PCM buffer.
+
+ Unlike :func:`_build_streaming_wav_header`, the RIFF and ``data`` chunk
+ sizes are real, so the result is a well-formed WAV that ordinary decoders
+ and file-based players accept, not only a live-stream player.
+ """
+ return _wav_header(
+ 36 + data_len,
+ data_len,
+ sample_rate,
+ channels,
+ bits_per_sample,
)
diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.py
index 57de22a23..b08e200ff 100644
--- a/src/agentscope/model/_openai_chat/_model.py
+++ b/src/agentscope/model/_openai_chat/_model.py
@@ -7,7 +7,10 @@ from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
from pydantic import BaseModel, Field
-from ..._utils._audio import _build_streaming_wav_header
+from ..._utils._audio import (
+ _build_streaming_wav_header,
+ _build_wav_header,
+)
from ..._utils._common import _generate_id, _flatten_json_schema
from .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODES
from .._model_response import ChatResponse
@@ -505,11 +508,22 @@ class OpenAIChatModel(ChatModelBase):
if not choice.message.content and audio_transcript:
content_blocks.append(TextBlock(text=audio_transcript))
if audio_data:
+ payload = base64.b64decode(audio_data)
+ media_type = f"audio/{audio_format}"
+ if audio_format == "pcm16":
+ # The request forces ``format: pcm16`` (see the
+ # ``audio`` request block), which comes back as
+ # headerless raw PCM. Wrap it so the block is a
+ # playable WAV, matching the streaming path and the
+ # documented intent.
+ payload = _build_wav_header(len(payload)) + payload
+ media_type = "audio/wav"
+ encoded = base64.b64encode(payload).decode("ascii")
content_blocks.append(
DataBlock(
source=Base64Source(
- data=audio_data,
- media_type=f"audio/{audio_format}",
+ data=encoded,
+ media_type=media_type,
),
),
)
diff --git a/tests/model_openai_chat_test.py b/tests/model_openai_chat_test.py
index d4a48402c..004f7282a 100644
--- a/tests/model_openai_chat_test.py
+++ b/tests/model_openai_chat_test.py
@@ -9,6 +9,7 @@ Tests cover both non-streaming and streaming modes, verifying that:
"""
import base64
import io
+import struct
import wave
from typing import Any
import unittest
@@ -333,6 +334,97 @@ class TestOpenAIChatNonStream(IsolatedAsyncioTestCase):
),
)
+ async def test_audio_response_with_voice_is_a_playable_wav(self) -> None:
+ """``voice`` forces ``pcm16``, and the result must still be a WAV.
+
+ ``voice`` is the only way to make the request carry
+ ``format: pcm16``, so it is the only way to reach the forced wire
+ format. The streaming path already re-wraps those bytes in a WAV
+ header; the non-streaming path must produce the same playable block
+ rather than headerless PCM.
+ """
+ model = OpenAIChatModel(
+ credential=OpenAICredential(api_key="test"),
+ model="gpt-4o-audio-preview",
+ stream=False,
+ context_size=128_000,
+ parameters=OpenAIChatModel.Parameters(voice="alloy"),
+ )
+ mock_client = MagicMock()
+ model.client = mock_client
+
+ pcm = b"\x01\x02" * 480
+ expected_wav = io.BytesIO()
+ with wave.open(expected_wav, "wb") as wav_writer:
+ wav_writer.setnchannels(1)
+ wav_writer.setsampwidth(2)
+ wav_writer.setframerate(24000)
+ wav_writer.writeframes(pcm)
+ expected_payload = expected_wav.getvalue()
+ expected_audio_data = base64.b64encode(expected_payload).decode()
+
+ mock_client.chat.completions.create = AsyncMock(
+ return_value=_mock_completion(
+ text=None,
+ audio={
+ "data": base64.b64encode(pcm).decode(),
+ "transcript": "Hello from audio.",
+ },
+ ),
+ )
+
+ result = await model([])
+
+ # The trigger: asking for audio pins the wire format to raw PCM.
+ sent = mock_client.chat.completions.create.call_args.kwargs
+ self.assertEqual(sent["audio"]["format"], "pcm16")
+
+ self.assertEqual(
+ (result.is_last, result.content),
+ (
+ True,
+ [
+ TextBlock.model_construct(
+ id=A,
+ created_at=A,
+ text="Hello from audio.",
+ ),
+ DataBlock.model_construct(
+ id=A,
+ created_at=A,
+ source=Base64Source.model_construct(
+ type="base64",
+ media_type="audio/wav",
+ data=expected_audio_data,
+ ),
+ ),
+ ],
+ ),
+ )
+
+ # A well-formed file, not just something a live-stream player will
+ # tolerate: the stdlib ``wave`` module reads it and the samples
+ # round-trip unchanged.
+ blocks = [b for b in result.content if isinstance(b, DataBlock)]
+ source = blocks[0].source
+ self.assertIsInstance(source, Base64Source)
+ payload = base64.b64decode(source.data)
+ self.assertEqual(payload[:4], b"RIFF")
+ self.assertEqual(
+ struct.unpack("<I", payload[4:8])[0],
+ 36 + len(pcm),
+ )
+ self.assertEqual(
+ struct.unpack("<I", payload[40:44])[0],
+ len(pcm),
+ )
+ with wave.open(io.BytesIO(payload), "rb") as wav:
+ self.assertEqual(wav.getnchannels(), 1)
+ self.assertEqual(wav.getsampwidth(), 2)
+ self.assertEqual(wav.getframerate(), 24000)
+ self.assertEqual(wav.getnframes(), len(pcm) // 2)
+ self.assertEqual(wav.readframes(wav.getnframes()), pcm)
+
async def test_thinking_response(
self,
) -> None: