Skip to content

Build and run agents you can see, understand and trust.

fix(model): decode the effective openai audio format (#3907)

Wang guanyu897d90949ef8
diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.py
index f6d12b330..111462425 100644
--- a/src/agentscope/model/_openai_chat/_model.py
+++ b/src/agentscope/model/_openai_chat/_model.py
@@ -2,6 +2,7 @@
"""The OpenAI Chat Completions model implementation."""
import base64
from collections import OrderedDict
+from collections.abc import Mapping
from datetime import datetime
from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
@@ -284,9 +285,14 @@ class OpenAIChatModel(ChatModelBase):
response = await self.client.chat.completions.create(**kwargs)
audio_cfg = kwargs.get("audio")
+ extra_body = kwargs.get("extra_body")
+ if isinstance(extra_body, Mapping):
+ # The SDK merges extra_body over the request at the top level.
+ # Decode the format actually sent, including per-call overrides.
+ audio_cfg = extra_body.get("audio", audio_cfg)
audio_fmt = (
audio_cfg.get("format", "wav")
- if isinstance(audio_cfg, dict)
+ if isinstance(audio_cfg, Mapping)
else "wav"
)
diff --git a/tests/model_openai_chat_test.py b/tests/model_openai_chat_test.py
index 9aeb8453d..045e98b70 100644
--- a/tests/model_openai_chat_test.py
+++ b/tests/model_openai_chat_test.py
@@ -8,14 +8,19 @@ Tests cover both non-streaming and streaming modes, verifying that:
1 final ChatResponse (is_last=True) with the full accumulated content.
"""
import base64
+import copy
import io
+import json
import struct
import wave
+from functools import partial
from typing import Any
import unittest
from unittest import IsolatedAsyncioTestCase
from unittest.mock import AsyncMock, MagicMock
+import httpx
+
from utils import AnyString
from agentscope.message import (
@@ -37,6 +42,22 @@ A = AnyString()
# ---------------------------------------------------------------------------
+def _wav() -> bytes:
+ """Return a valid silent WAV instead of a header-shaped byte fixture."""
+ output = io.BytesIO()
+ with wave.open(output, "wb") as audio:
+ audio.setnchannels(1)
+ audio.setsampwidth(2)
+ audio.setframerate(24000)
+ audio.writeframes(b"\x00\x00" * 24)
+ return output.getvalue()
+
+
+def _audio(format_name: str) -> dict:
+ """Build a complete audio request accepted by the provider."""
+ return {"voice": "alloy", "format": format_name}
+
+
def _make_model(stream: bool = False) -> Any:
return OpenAIChatModel(
credential=OpenAICredential(api_key="test"),
@@ -489,6 +510,126 @@ class TestOpenAIChatNonStream(IsolatedAsyncioTestCase):
)
+class TestOpenAIChatAudioFormat(IsolatedAsyncioTestCase):
+ """The parser and transport agree on the effective non-stream format."""
+
+ async def test_effective_audio_format_and_override_precedence(
+ self,
+ ) -> None:
+ """Instance and per-call extra_body overrides follow SDK precedence."""
+ cases = [
+ ("default_pcm", None, {}, "pcm16"),
+ ("direct_wav", None, {"audio": _audio("wav")}, "wav"),
+ ("instance_wav", {"audio": _audio("wav")}, {}, "wav"),
+ ("instance_mp3", {"audio": _audio("mp3")}, {}, "mp3"),
+ (
+ "extra_beats_direct",
+ {"audio": _audio("wav")},
+ {"audio": _audio("pcm16")},
+ "wav",
+ ),
+ (
+ "per_call_extra",
+ {"audio": _audio("mp3")},
+ {"extra_body": {"audio": _audio("wav")}},
+ "wav",
+ ),
+ (
+ "unrelated_extra",
+ {"offline_option": True},
+ {"audio": _audio("wav")},
+ "wav",
+ ),
+ ]
+ original_wav = _wav()
+ for name, extra_body, generate_kwargs, expected_format in cases:
+ with self.subTest(case=name):
+ before = copy.deepcopy((extra_body, generate_kwargs))
+ captured = []
+
+ def handle(
+ request: httpx.Request,
+ recorded: list,
+ ) -> httpx.Response:
+ data = json.loads(request.content)
+ recorded.append(data)
+ wire_format = data["audio"]["format"]
+ payload = {
+ "wav": original_wav,
+ "pcm16": b"\x00\x00" * 24,
+ "mp3": b"ID3-offline-fixture",
+ }[wire_format]
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl-offline",
+ "object": "chat.completion",
+ "created": 1,
+ "model": "gpt-audio",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": None,
+ "audio": {
+ "id": "audio_1",
+ "data": base64.b64encode(
+ payload,
+ ).decode(),
+ "transcript": "hello",
+ "expires_at": 9999999999,
+ },
+ },
+ "finish_reason": "stop",
+ },
+ ],
+ "usage": {
+ "prompt_tokens": 1,
+ "completion_tokens": 1,
+ "total_tokens": 2,
+ },
+ },
+ )
+
+ async with httpx.AsyncClient(
+ transport=httpx.MockTransport(
+ partial(handle, recorded=captured),
+ ),
+ ) as client:
+ model = OpenAIChatModel(
+ credential=OpenAICredential(api_key="offline-test"),
+ model="gpt-audio",
+ stream=False,
+ parameters=OpenAIChatModel.Parameters(voice="alloy"),
+ extra_body=extra_body,
+ client_kwargs={"http_client": client},
+ )
+ response = await model([], **generate_kwargs)
+ self.assertEqual(
+ captured[0]["audio"]["format"],
+ expected_format,
+ )
+ source = next(
+ block.source
+ for block in response.content
+ if block.type == "data"
+ )
+ payload = base64.b64decode(source.data)
+ if expected_format == "mp3":
+ self.assertEqual(source.media_type, "audio/mp3")
+ self.assertEqual(payload, b"ID3-offline-fixture")
+ else:
+ self.assertEqual(source.media_type, "audio/wav")
+ self.assertEqual(payload, original_wav)
+ with wave.open(io.BytesIO(payload), "rb") as audio:
+ self.assertEqual(
+ audio.readframes(24),
+ b"\x00\x00" * 24,
+ )
+ self.assertEqual((extra_body, generate_kwargs), before)
+
+
# ---------------------------------------------------------------------------
# Streaming tests
# ---------------------------------------------------------------------------