Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(tts): implement OpenAI TTS API (#1878)
nuthalapativarun22576ebb3fad
diff --git a/src/agentscope/credential/_openai.py b/src/agentscope/credential/_openai.pyindex 6cc132e7b..ec746ceec 100644--- a/src/agentscope/credential/_openai.py+++ b/src/agentscope/credential/_openai.py@@ -9,6 +9,7 @@ from ._base import CredentialBaseif TYPE_CHECKING:from ..embedding import EmbeddingModelBasefrom ..model import ChatModelBase+ from ..tts import TTSModelBaseclass OpenAICredential(CredentialBase):@@ -54,3 +55,10 @@ class OpenAICredential(CredentialBase):from ..embedding import OpenAIEmbeddingModelreturn OpenAIEmbeddingModel++ @classmethod+ def get_tts_model_classes(cls) -> list[Type["TTSModelBase"]]:+ """Return the OpenAI TTS model classes."""+ from ..tts import OpenAITTSModel++ return [OpenAITTSModel]diff --git a/src/agentscope/tts/__init__.py b/src/agentscope/tts/__init__.pyindex 7a3e4ded0..43180fee7 100644--- a/src/agentscope/tts/__init__.py+++ b/src/agentscope/tts/__init__.py@@ -9,6 +9,7 @@ from ._dashscope import (DashScopeTTSModel,DashScopeRealtimeTTSModel,)+from ._openai import OpenAITTSModel__all__ = ["TTSModelBase",@@ -18,4 +19,5 @@ __all__ = ["DashScopeCosyVoiceTTSModel","DashScopeTTSModel","DashScopeRealtimeTTSModel",+ "OpenAITTSModel",]diff --git a/src/agentscope/tts/_openai/__init__.py b/src/agentscope/tts/_openai/__init__.pynew file mode 100644index 000000000..153f72178--- /dev/null+++ b/src/agentscope/tts/_openai/__init__.py@@ -0,0 +1,8 @@+# -*- coding: utf-8 -*-+"""The OpenAI TTS module."""++from ._model import OpenAITTSModel++__all__ = [+ "OpenAITTSModel",+]diff --git a/src/agentscope/tts/_openai/_model.py b/src/agentscope/tts/_openai/_model.pynew file mode 100644index 000000000..125333a74--- /dev/null+++ b/src/agentscope/tts/_openai/_model.py@@ -0,0 +1,252 @@+# -*- coding: utf-8 -*-+"""OpenAI TTS model implementation using the Audio Speech API."""+import base64+from datetime import datetime+from typing import (+ Any,+ AsyncGenerator,+ Literal,+ TYPE_CHECKING,+)++from pydantic import BaseModel, Field++from .._tts_base import TTSModelBase+from .._tts_response import TTSResponse, TTSUsage+from ...credential import OpenAICredential+from ...message import DataBlock, Base64Source++if TYPE_CHECKING:+ from openai import AsyncOpenAI+++# Map the OpenAI ``response_format`` values to their MIME media types.+_MEDIA_TYPES = {+ "mp3": "audio/mpeg",+ "opus": "audio/opus",+ "aac": "audio/aac",+ "flac": "audio/flac",+ "wav": "audio/wav",+ "pcm": "audio/pcm",+}+_DEFAULT_RESPONSE_FORMAT = "mp3"+++def _parse_usage(usage: Any, elapsed: float) -> TTSUsage | None:+ """Extract a TTSUsage from the OpenAI usage object, or None."""+ if usage is None:+ return None+ return TTSUsage(+ input_tokens=getattr(usage, "input_tokens", 0) or 0,+ output_tokens=getattr(usage, "output_tokens", 0) or 0,+ time=elapsed,+ )+++class OpenAITTSModel(TTSModelBase):+ """OpenAI TTS model implementation using the Audio Speech API. For more+ details please see the `official document+ <https://platform.openai.com/docs/guides/text-to-speech>`_.+ """++ class Parameters(BaseModel):+ """Frontend-exposed parameters for OpenAI TTS models."""++ voice: str = Field(+ default="alloy",+ title="Voice",+ description="The voice to use for synthesis.",+ )++ response_format: Literal[+ "mp3",+ "opus",+ "aac",+ "flac",+ "wav",+ "pcm",+ ] = Field(+ default=_DEFAULT_RESPONSE_FORMAT,+ title="Response Format",+ description="The audio format of the synthesized speech.",+ )++ instructions: str | None = Field(+ default=None,+ title="Instructions",+ description=(+ "Additional instructions for controlling the voice "+ "(only supported by some models, e.g. gpt-4o-mini-tts)."+ ),+ )++ type: Literal["openai_tts"] = "openai_tts"+ """The type of the TTS model."""++ realtime: bool = False++ def __init__(+ self,+ credential: OpenAICredential,+ model: str = "tts-1",+ parameters: "OpenAITTSModel.Parameters | None" = None,+ stream: bool = True,+ ) -> None:+ """Initialize the OpenAI TTS model.++ .. note:: More details about the parameters, such as ``model``+ and ``voice``, can be found in the `official document+ <https://platform.openai.com/docs/guides/text-to-speech>`_.++ Args:+ credential (`OpenAICredential`):+ The OpenAI credential used to authenticate the API call.+ model (`str`, defaults to ``"tts-1"``):+ The TTS model name. Supported models include ``tts-1``,+ ``tts-1-hd`` and ``gpt-4o-mini-tts``.+ parameters (`OpenAITTSModel.Parameters | None`, defaults to \+ `None`):+ The TTS parameters (voice, response format, etc.). When+ ``None``, the default parameters will be used.+ stream (`bool`, defaults to `True`):+ Whether to use streaming output. When `True`,+ :meth:`synthesize` returns an async generator yielding+ ``TTSResponse`` chunks; when `False`, it returns a single+ aggregated ``TTSResponse``.+ """+ super().__init__(+ credential=credential,+ model=model,+ parameters=parameters,+ stream=stream,+ )++ async def synthesize(+ self,+ text: str | None = None,+ **kwargs: Any,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Call the OpenAI Audio Speech API to synthesize speech from text.++ Args:+ text (`str | None`, optional):+ The text to be synthesized.+ **kwargs (`Any`):+ Additional keyword arguments to pass to the TTS API call.++ Returns:+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:+ A single ``TTSResponse`` when ``stream=False``, or an async+ generator yielding ``TTSResponse`` chunks when ``stream=True``.+ """+ if not text:+ return TTSResponse(content=None)++ import openai++ client = openai.AsyncClient(+ api_key=self.credential.api_key.get_secret_value(),+ organization=self.credential.organization,+ base_url=self.credential.base_url,+ )++ media_type = _MEDIA_TYPES.get(+ self.parameters.response_format,+ _MEDIA_TYPES[_DEFAULT_RESPONSE_FORMAT],+ )++ request_kwargs: dict[str, Any] = {+ "model": self.model,+ "voice": self.parameters.voice,+ "input": text,+ "response_format": self.parameters.response_format,+ **kwargs,+ }+ if self.parameters.instructions:+ request_kwargs["instructions"] = self.parameters.instructions++ if self.stream:+ return self._stream(client, media_type, **request_kwargs)++ return await self._aggregate(client, media_type, **request_kwargs)++ @staticmethod+ async def _aggregate(+ client: "AsyncOpenAI",+ media_type: str,+ **request_kwargs: Any,+ ) -> TTSResponse:+ """Call the API and aggregate the full audio into a single+ ``TTSResponse``."""+ start_datetime = datetime.now()+ response = await client.audio.speech.create(**request_kwargs)+ audio_bytes = response.content+ elapsed = (datetime.now() - start_datetime).total_seconds()++ return TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(audio_bytes).decode("ascii"),+ media_type=media_type,+ ),+ ),+ usage=_parse_usage(None, elapsed),+ )++ @staticmethod+ async def _stream(+ client: "AsyncOpenAI",+ media_type: str,+ **request_kwargs: Any,+ ) -> AsyncGenerator[TTSResponse, None]:+ """Call the API and yield incremental audio chunks as+ ``TTSResponse`` objects.++ Args:+ client (`AsyncOpenAI`):+ The async OpenAI client.+ media_type (`str`):+ The media type of the synthesized audio.+ **request_kwargs (`Any`):+ Keyword arguments forwarded to the Audio Speech API.++ Yields:+ `TTSResponse`:+ A ``TTSResponse`` for each incremental audio chunk; the+ final response has ``is_last=True``.+ """+ start_datetime = datetime.now()+ pending: TTSResponse | None = None++ async with client.audio.speech.with_streaming_response.create(+ **request_kwargs,+ ) as response:+ async for delta_bytes in response.iter_bytes():+ if not delta_bytes:+ continue+ if pending is not None:+ yield pending+ pending = TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(delta_bytes).decode(+ "ascii",+ ),+ media_type=media_type,+ ),+ ),+ is_last=False,+ )++ elapsed = (datetime.now() - start_datetime).total_seconds()++ if pending is not None:+ pending.is_last = True+ pending.usage = _parse_usage(None, elapsed)+ yield pending+ else:+ yield TTSResponse(+ content=None,+ is_last=True,+ usage=_parse_usage(None, elapsed),+ )diff --git a/src/agentscope/tts/_openai/_models/gpt-4o-mini-tts.yaml b/src/agentscope/tts/_openai/_models/gpt-4o-mini-tts.yamlnew file mode 100644index 000000000..49171714c--- /dev/null+++ b/src/agentscope/tts/_openai/_models/gpt-4o-mini-tts.yaml@@ -0,0 +1,30 @@+name: gpt-4o-mini-tts+label: GPT-4o Mini TTS+status: active++input_types:+ - text/plain++output_types:+ - audio/mpeg+ - audio/opus+ - audio/aac+ - audio/flac+ - audio/wav+ - audio/pcm++voices:+ # Source: https://platform.openai.com/docs/guides/text-to-speech+ - alloy+ - ash+ - ballad+ - coral+ - echo+ - fable+ - onyx+ - nova+ - sage+ - shimmer+ - verse++parameter_overrides: {}diff --git a/src/agentscope/tts/_openai/_models/tts-1-hd.yaml b/src/agentscope/tts/_openai/_models/tts-1-hd.yamlnew file mode 100644index 000000000..744cbba03--- /dev/null+++ b/src/agentscope/tts/_openai/_models/tts-1-hd.yaml@@ -0,0 +1,34 @@+name: tts-1-hd+label: TTS-1 HD+status: active++input_types:+ - text/plain++output_types:+ - audio/mpeg+ - audio/opus+ - audio/aac+ - audio/flac+ - audio/wav+ - audio/pcm++voices:+ # Source: https://platform.openai.com/docs/guides/text-to-speech+ - alloy+ - ash+ - ballad+ - coral+ - echo+ - fable+ - onyx+ - nova+ - sage+ - shimmer+ - verse++parameter_overrides:+ # The instructions parameter is not supported by tts-1-hd, hide it from+ # the frontend to avoid confusing users.+ instructions:+ hidden: truediff --git a/src/agentscope/tts/_openai/_models/tts-1.yaml b/src/agentscope/tts/_openai/_models/tts-1.yamlnew file mode 100644index 000000000..755d21274--- /dev/null+++ b/src/agentscope/tts/_openai/_models/tts-1.yaml@@ -0,0 +1,34 @@+name: tts-1+label: TTS-1+status: active++input_types:+ - text/plain++output_types:+ - audio/mpeg+ - audio/opus+ - audio/aac+ - audio/flac+ - audio/wav+ - audio/pcm++voices:+ # Source: https://platform.openai.com/docs/guides/text-to-speech+ - alloy+ - ash+ - ballad+ - coral+ - echo+ - fable+ - onyx+ - nova+ - sage+ - shimmer+ - verse++parameter_overrides:+ # The instructions parameter is not supported by tts-1, hide it from+ # the frontend to avoid confusing users.+ instructions:+ hidden: truediff --git a/tests/tts_openai_test.py b/tests/tts_openai_test.pynew file mode 100644index 000000000..a181fbb09--- /dev/null+++ b/tests/tts_openai_test.py@@ -0,0 +1,167 @@+# -*- coding: utf-8 -*-+# pylint: disable=protected-access+"""Unit tests for the OpenAI TTS model.++Covers:+ * ``OpenAITTSModel`` non-streaming aggregation.+ * ``OpenAITTSModel`` streaming: incremental chunks and ``is_last``+ placement at the final chunk only.+ * Edge cases: empty/None input short-circuits without calling the API.+"""+import base64+from typing import Any+from unittest import IsolatedAsyncioTestCase+from unittest.mock import AsyncMock, MagicMock, patch++from agentscope.credential import OpenAICredential+from agentscope.tts import OpenAITTSModel, TTSResponse+++_MEDIA_TYPE_MP3 = "audio/mpeg"+_MEDIA_TYPE_WAV = "audio/wav"+++def _make_mock_client(audio_bytes: bytes, chunks: list[bytes]) -> MagicMock:+ """Build a mock ``openai.AsyncClient`` shaped like what+ ``audio.speech.create`` / ``with_streaming_response.create`` return."""+ client = MagicMock()++ # Non-streaming: client.audio.speech.create(...) -> response with+ # .content bytes.+ create_response = MagicMock()+ create_response.content = audio_bytes+ client.audio.speech.create = AsyncMock(return_value=create_response)++ # Streaming: client.audio.speech.with_streaming_response.create(...)+ # is an async context manager whose value exposes .iter_bytes().+ stream_response = MagicMock()++ async def _iter_bytes() -> Any:+ for chunk in chunks:+ yield chunk++ stream_response.iter_bytes = _iter_bytes++ stream_ctx = MagicMock()+ stream_ctx.__aenter__ = AsyncMock(return_value=stream_response)+ stream_ctx.__aexit__ = AsyncMock(return_value=None)++ client.audio.speech.with_streaming_response.create = MagicMock(+ return_value=stream_ctx,+ )++ return client+++class TestOpenAITTSModel(IsolatedAsyncioTestCase):+ """The unittests for the OpenAI TTS model (non-realtime)."""++ def _make_model(self, stream: bool = False, **kwargs: Any) -> Any:+ """Create an OpenAITTSModel with test credentials."""+ return OpenAITTSModel(+ credential=OpenAICredential(api_key="test"),+ model="tts-1",+ stream=stream,+ **kwargs,+ )++ async def test_aggregates_response(self) -> None:+ """Non-streaming returns a single TTSResponse with the full audio."""+ client = _make_mock_client(b"AAAABBBBCCCC", [])+ model = self._make_model(stream=False)++ with patch("openai.AsyncClient", return_value=client):+ result = await model.synthesize("Hello world")++ self.assertIsInstance(result, TTSResponse)+ self.assertEqual(result.content.source.media_type, _MEDIA_TYPE_MP3)+ self.assertEqual(+ base64.b64decode(result.content.source.data),+ b"AAAABBBBCCCC",+ )+ self.assertTrue(result.is_last)++ async def test_none_short_circuits(self) -> None:+ """``synthesize(None)`` returns an empty response without touching+ the API."""+ model = self._make_model(stream=False)++ with patch("openai.AsyncClient") as mock_client_cls:+ result = await model.synthesize(None)++ self.assertIsNone(result.content)+ mock_client_cls.assert_not_called()++ async def test_empty_string_short_circuits(self) -> None:+ """``synthesize("")`` returns an empty response without touching+ the API."""+ model = self._make_model(stream=False)++ with patch("openai.AsyncClient") as mock_client_cls:+ result = await model.synthesize("")++ self.assertIsNone(result.content)+ mock_client_cls.assert_not_called()++ async def test_wav_media_type(self) -> None:+ """The media type follows the ``response_format`` parameter."""+ client = _make_mock_client(b"AAAA", [])+ model = self._make_model(+ stream=False,+ parameters=OpenAITTSModel.Parameters(response_format="wav"),+ )++ with patch("openai.AsyncClient", return_value=client):+ result = await model.synthesize("Hello world")++ self.assertEqual(result.content.source.media_type, _MEDIA_TYPE_WAV)++ async def test_incremental_chunks(self) -> None:+ """Each streamed byte chunk yields one TTSResponse."""+ client = _make_mock_client(b"", [b"AAAA", b"BBBB", b"CCCC"])+ model = self._make_model(stream=True)++ with patch("openai.AsyncClient", return_value=client):+ gen = await model.synthesize("Hello world")+ chunks = [c async for c in gen]++ payloads = [base64.b64decode(c.content.source.data) for c in chunks]+ self.assertEqual(payloads, [b"AAAA", b"BBBB", b"CCCC"])+ self.assertEqual(+ [c.is_last for c in chunks],+ [False, False, True],+ )+ self.assertEqual(+ [c.content.source.media_type for c in chunks],+ [_MEDIA_TYPE_MP3] * 3,+ )++ async def test_single_chunk_marked_last(self) -> None:+ """A lone audio chunk is flagged ``is_last=True``."""+ client = _make_mock_client(b"", [b"ONLYCHUNK"])+ model = self._make_model(stream=True)++ with patch("openai.AsyncClient", return_value=client):+ gen = await model.synthesize("Hello world")+ chunks = [c async for c in gen]++ self.assertEqual(len(chunks), 1)+ self.assertTrue(chunks[0].is_last)+ self.assertEqual(+ base64.b64decode(chunks[0].content.source.data),+ b"ONLYCHUNK",+ )++ async def test_empty_stream_yields_terminal(self) -> None:+ """When the API yields no audio, the generator emits a terminal+ sentinel so consumers can detect EOS."""+ client = _make_mock_client(b"", [])+ model = self._make_model(stream=True)++ with patch("openai.AsyncClient", return_value=client):+ gen = await model.synthesize("Hello world")+ chunks = [c async for c in gen]++ self.assertEqual(len(chunks), 1)+ self.assertIsNone(chunks[0].content)+ self.assertTrue(chunks[0].is_last)