Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(model): wrap non-streaming chat audio in a real WAV header (#3028)
ANIRUDDHA ADAK48c4b9ce0f9a
diff --git a/src/agentscope/_utils/_audio.py b/src/agentscope/_utils/_audio.pyindex b93b5cbf6..c232006c7 100644--- a/src/agentscope/_utils/_audio.py+++ b/src/agentscope/_utils/_audio.py@@ -3,28 +3,30 @@import struct-def _build_streaming_wav_header(+def _wav_header(+ riff_size: int,+ data_size: int,sample_rate: int = 24000,channels: int = 1,bits_per_sample: int = 16,) -> bytes:- """Build a 44-byte WAV/RIFF header for streaming PCM.+ """Build a 44-byte WAV/RIFF header with explicit chunk sizes.- The RIFF and ``data`` chunk sizes are set to ``0xFFFFFFFF`` since the- total length isn't known yet. Decoders that only need sample-rate,- channel count and bit depth (e.g. the web ``WavStreamPlayer``) treat- everything after the ``data`` chunk header as PCM, so this is- sufficient for live decoding of an open-ended stream.+ Args:+ riff_size (`int`):+ The size to declare for the RIFF chunk.+ data_size (`int`):+ The size to declare for the ``data`` chunk.- Both DashScope omni and OpenAI streaming deliver raw PCM upstream;- prefixing the first chunk with this header lets the frontend start- playback immediately without buffering the whole response.+ Returns:+ `bytes`:+ A 44-byte WAV header."""byte_rate = sample_rate * channels * bits_per_sample // 8block_align = channels * bits_per_sample // 8return (b"RIFF"- + struct.pack("<I", 0xFFFFFFFF)+ + struct.pack("<I", riff_size)+ b"WAVE"+ b"fmt "+ struct.pack("<I", 16)@@ -38,5 +40,55 @@ def _build_streaming_wav_header(bits_per_sample,)+ b"data"- + struct.pack("<I", 0xFFFFFFFF)+ + struct.pack("<I", data_size)+ )+++def _build_streaming_wav_header(+ sample_rate: int = 24000,+ channels: int = 1,+ bits_per_sample: int = 16,+) -> bytes:+ """Build a 44-byte WAV/RIFF header for streaming PCM.++ The RIFF and ``data`` chunk sizes are set to ``0xFFFFFFFF`` since the+ total length isn't known yet. Decoders that only need sample-rate,+ channel count and bit depth (e.g. the web ``WavStreamPlayer``) treat+ everything after the ``data`` chunk header as PCM, so this is+ sufficient for live decoding of an open-ended stream.++ Both DashScope omni and OpenAI streaming deliver raw PCM upstream;+ prefixing the first chunk with this header lets the frontend start+ playback immediately without buffering the whole response.++ For a *complete* buffer use :func:`_build_wav_header` instead, which+ declares real chunk sizes.+ """+ return _wav_header(+ 0xFFFFFFFF,+ 0xFFFFFFFF,+ sample_rate,+ channels,+ bits_per_sample,+ )+++def _build_wav_header(+ data_len: int,+ sample_rate: int = 24000,+ channels: int = 1,+ bits_per_sample: int = 16,+) -> bytes:+ """Build a 44-byte WAV/RIFF header for a complete PCM buffer.++ Unlike :func:`_build_streaming_wav_header`, the RIFF and ``data`` chunk+ sizes are real, so the result is a well-formed WAV that ordinary decoders+ and file-based players accept, not only a live-stream player.+ """+ return _wav_header(+ 36 + data_len,+ data_len,+ sample_rate,+ channels,+ bits_per_sample,)diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.pyindex 57de22a23..b08e200ff 100644--- a/src/agentscope/model/_openai_chat/_model.py+++ b/src/agentscope/model/_openai_chat/_model.py@@ -7,7 +7,10 @@ from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Typefrom pydantic import BaseModel, Field-from ..._utils._audio import _build_streaming_wav_header+from ..._utils._audio import (+ _build_streaming_wav_header,+ _build_wav_header,+)from ..._utils._common import _generate_id, _flatten_json_schemafrom .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODESfrom .._model_response import ChatResponse@@ -505,11 +508,22 @@ class OpenAIChatModel(ChatModelBase):if not choice.message.content and audio_transcript:content_blocks.append(TextBlock(text=audio_transcript))if audio_data:+ payload = base64.b64decode(audio_data)+ media_type = f"audio/{audio_format}"+ if audio_format == "pcm16":+ # The request forces ``format: pcm16`` (see the+ # ``audio`` request block), which comes back as+ # headerless raw PCM. Wrap it so the block is a+ # playable WAV, matching the streaming path and the+ # documented intent.+ payload = _build_wav_header(len(payload)) + payload+ media_type = "audio/wav"+ encoded = base64.b64encode(payload).decode("ascii")content_blocks.append(DataBlock(source=Base64Source(- data=audio_data,- media_type=f"audio/{audio_format}",+ data=encoded,+ media_type=media_type,),),)diff --git a/tests/model_openai_chat_test.py b/tests/model_openai_chat_test.pyindex d4a48402c..004f7282a 100644--- a/tests/model_openai_chat_test.py+++ b/tests/model_openai_chat_test.py@@ -9,6 +9,7 @@ Tests cover both non-streaming and streaming modes, verifying that:"""import base64import io+import structimport wavefrom typing import Anyimport unittest@@ -333,6 +334,97 @@ class TestOpenAIChatNonStream(IsolatedAsyncioTestCase):),)+ async def test_audio_response_with_voice_is_a_playable_wav(self) -> None:+ """``voice`` forces ``pcm16``, and the result must still be a WAV.++ ``voice`` is the only way to make the request carry+ ``format: pcm16``, so it is the only way to reach the forced wire+ format. The streaming path already re-wraps those bytes in a WAV+ header; the non-streaming path must produce the same playable block+ rather than headerless PCM.+ """+ model = OpenAIChatModel(+ credential=OpenAICredential(api_key="test"),+ model="gpt-4o-audio-preview",+ stream=False,+ context_size=128_000,+ parameters=OpenAIChatModel.Parameters(voice="alloy"),+ )+ mock_client = MagicMock()+ model.client = mock_client++ pcm = b"\x01\x02" * 480+ expected_wav = io.BytesIO()+ with wave.open(expected_wav, "wb") as wav_writer:+ wav_writer.setnchannels(1)+ wav_writer.setsampwidth(2)+ wav_writer.setframerate(24000)+ wav_writer.writeframes(pcm)+ expected_payload = expected_wav.getvalue()+ expected_audio_data = base64.b64encode(expected_payload).decode()++ mock_client.chat.completions.create = AsyncMock(+ return_value=_mock_completion(+ text=None,+ audio={+ "data": base64.b64encode(pcm).decode(),+ "transcript": "Hello from audio.",+ },+ ),+ )++ result = await model([])++ # The trigger: asking for audio pins the wire format to raw PCM.+ sent = mock_client.chat.completions.create.call_args.kwargs+ self.assertEqual(sent["audio"]["format"], "pcm16")++ self.assertEqual(+ (result.is_last, result.content),+ (+ True,+ [+ TextBlock.model_construct(+ id=A,+ created_at=A,+ text="Hello from audio.",+ ),+ DataBlock.model_construct(+ id=A,+ created_at=A,+ source=Base64Source.model_construct(+ type="base64",+ media_type="audio/wav",+ data=expected_audio_data,+ ),+ ),+ ],+ ),+ )++ # A well-formed file, not just something a live-stream player will+ # tolerate: the stdlib ``wave`` module reads it and the samples+ # round-trip unchanged.+ blocks = [b for b in result.content if isinstance(b, DataBlock)]+ source = blocks[0].source+ self.assertIsInstance(source, Base64Source)+ payload = base64.b64decode(source.data)+ self.assertEqual(payload[:4], b"RIFF")+ self.assertEqual(+ struct.unpack("<I", payload[4:8])[0],+ 36 + len(pcm),+ )+ self.assertEqual(+ struct.unpack("<I", payload[40:44])[0],+ len(pcm),+ )+ with wave.open(io.BytesIO(payload), "rb") as wav:+ self.assertEqual(wav.getnchannels(), 1)+ self.assertEqual(wav.getsampwidth(), 2)+ self.assertEqual(wav.getframerate(), 24000)+ self.assertEqual(wav.getnframes(), len(pcm) // 2)+ self.assertEqual(wav.readframes(wav.getnframes()), pcm)+async def test_thinking_response(self,) -> None: