Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(model): decode the effective openai audio format (#3907)
Wang guanyu897d90949ef8
diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.pyindex f6d12b330..111462425 100644--- a/src/agentscope/model/_openai_chat/_model.py+++ b/src/agentscope/model/_openai_chat/_model.py@@ -2,6 +2,7 @@"""The OpenAI Chat Completions model implementation."""import base64from collections import OrderedDict+from collections.abc import Mappingfrom datetime import datetimefrom typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type@@ -284,9 +285,14 @@ class OpenAIChatModel(ChatModelBase):response = await self.client.chat.completions.create(**kwargs)audio_cfg = kwargs.get("audio")+ extra_body = kwargs.get("extra_body")+ if isinstance(extra_body, Mapping):+ # The SDK merges extra_body over the request at the top level.+ # Decode the format actually sent, including per-call overrides.+ audio_cfg = extra_body.get("audio", audio_cfg)audio_fmt = (audio_cfg.get("format", "wav")- if isinstance(audio_cfg, dict)+ if isinstance(audio_cfg, Mapping)else "wav")diff --git a/tests/model_openai_chat_test.py b/tests/model_openai_chat_test.pyindex 9aeb8453d..045e98b70 100644--- a/tests/model_openai_chat_test.py+++ b/tests/model_openai_chat_test.py@@ -8,14 +8,19 @@ Tests cover both non-streaming and streaming modes, verifying that:1 final ChatResponse (is_last=True) with the full accumulated content."""import base64+import copyimport io+import jsonimport structimport wave+from functools import partialfrom typing import Anyimport unittestfrom unittest import IsolatedAsyncioTestCasefrom unittest.mock import AsyncMock, MagicMock+import httpx+from utils import AnyStringfrom agentscope.message import (@@ -37,6 +42,22 @@ A = AnyString()# ---------------------------------------------------------------------------+def _wav() -> bytes:+ """Return a valid silent WAV instead of a header-shaped byte fixture."""+ output = io.BytesIO()+ with wave.open(output, "wb") as audio:+ audio.setnchannels(1)+ audio.setsampwidth(2)+ audio.setframerate(24000)+ audio.writeframes(b"\x00\x00" * 24)+ return output.getvalue()+++def _audio(format_name: str) -> dict:+ """Build a complete audio request accepted by the provider."""+ return {"voice": "alloy", "format": format_name}++def _make_model(stream: bool = False) -> Any:return OpenAIChatModel(credential=OpenAICredential(api_key="test"),@@ -489,6 +510,126 @@ class TestOpenAIChatNonStream(IsolatedAsyncioTestCase):)+class TestOpenAIChatAudioFormat(IsolatedAsyncioTestCase):+ """The parser and transport agree on the effective non-stream format."""++ async def test_effective_audio_format_and_override_precedence(+ self,+ ) -> None:+ """Instance and per-call extra_body overrides follow SDK precedence."""+ cases = [+ ("default_pcm", None, {}, "pcm16"),+ ("direct_wav", None, {"audio": _audio("wav")}, "wav"),+ ("instance_wav", {"audio": _audio("wav")}, {}, "wav"),+ ("instance_mp3", {"audio": _audio("mp3")}, {}, "mp3"),+ (+ "extra_beats_direct",+ {"audio": _audio("wav")},+ {"audio": _audio("pcm16")},+ "wav",+ ),+ (+ "per_call_extra",+ {"audio": _audio("mp3")},+ {"extra_body": {"audio": _audio("wav")}},+ "wav",+ ),+ (+ "unrelated_extra",+ {"offline_option": True},+ {"audio": _audio("wav")},+ "wav",+ ),+ ]+ original_wav = _wav()+ for name, extra_body, generate_kwargs, expected_format in cases:+ with self.subTest(case=name):+ before = copy.deepcopy((extra_body, generate_kwargs))+ captured = []++ def handle(+ request: httpx.Request,+ recorded: list,+ ) -> httpx.Response:+ data = json.loads(request.content)+ recorded.append(data)+ wire_format = data["audio"]["format"]+ payload = {+ "wav": original_wav,+ "pcm16": b"\x00\x00" * 24,+ "mp3": b"ID3-offline-fixture",+ }[wire_format]+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl-offline",+ "object": "chat.completion",+ "created": 1,+ "model": "gpt-audio",+ "choices": [+ {+ "index": 0,+ "message": {+ "role": "assistant",+ "content": None,+ "audio": {+ "id": "audio_1",+ "data": base64.b64encode(+ payload,+ ).decode(),+ "transcript": "hello",+ "expires_at": 9999999999,+ },+ },+ "finish_reason": "stop",+ },+ ],+ "usage": {+ "prompt_tokens": 1,+ "completion_tokens": 1,+ "total_tokens": 2,+ },+ },+ )++ async with httpx.AsyncClient(+ transport=httpx.MockTransport(+ partial(handle, recorded=captured),+ ),+ ) as client:+ model = OpenAIChatModel(+ credential=OpenAICredential(api_key="offline-test"),+ model="gpt-audio",+ stream=False,+ parameters=OpenAIChatModel.Parameters(voice="alloy"),+ extra_body=extra_body,+ client_kwargs={"http_client": client},+ )+ response = await model([], **generate_kwargs)+ self.assertEqual(+ captured[0]["audio"]["format"],+ expected_format,+ )+ source = next(+ block.source+ for block in response.content+ if block.type == "data"+ )+ payload = base64.b64decode(source.data)+ if expected_format == "mp3":+ self.assertEqual(source.media_type, "audio/mp3")+ self.assertEqual(payload, b"ID3-offline-fixture")+ else:+ self.assertEqual(source.media_type, "audio/wav")+ self.assertEqual(payload, original_wav)+ with wave.open(io.BytesIO(payload), "rb") as audio:+ self.assertEqual(+ audio.readframes(24),+ b"\x00\x00" * 24,+ )+ self.assertEqual((extra_body, generate_kwargs), before)++# ---------------------------------------------------------------------------# Streaming tests# ---------------------------------------------------------------------------