Skip to content

Build and run agents you can see, understand and trust.

feat(tts): implement OpenAI TTS API (#1878)

nuthalapativarun22576ebb3fad
diff --git a/src/agentscope/credential/_openai.py b/src/agentscope/credential/_openai.py
index 6cc132e7b..ec746ceec 100644
--- a/src/agentscope/credential/_openai.py
+++ b/src/agentscope/credential/_openai.py
@@ -9,6 +9,7 @@ from ._base import CredentialBase
if TYPE_CHECKING:
from ..embedding import EmbeddingModelBase
from ..model import ChatModelBase
+ from ..tts import TTSModelBase
class OpenAICredential(CredentialBase):
@@ -54,3 +55,10 @@ class OpenAICredential(CredentialBase):
from ..embedding import OpenAIEmbeddingModel
return OpenAIEmbeddingModel
+
+ @classmethod
+ def get_tts_model_classes(cls) -> list[Type["TTSModelBase"]]:
+ """Return the OpenAI TTS model classes."""
+ from ..tts import OpenAITTSModel
+
+ return [OpenAITTSModel]
diff --git a/src/agentscope/tts/__init__.py b/src/agentscope/tts/__init__.py
index 7a3e4ded0..43180fee7 100644
--- a/src/agentscope/tts/__init__.py
+++ b/src/agentscope/tts/__init__.py
@@ -9,6 +9,7 @@ from ._dashscope import (
DashScopeTTSModel,
DashScopeRealtimeTTSModel,
)
+from ._openai import OpenAITTSModel
__all__ = [
"TTSModelBase",
@@ -18,4 +19,5 @@ __all__ = [
"DashScopeCosyVoiceTTSModel",
"DashScopeTTSModel",
"DashScopeRealtimeTTSModel",
+ "OpenAITTSModel",
]
diff --git a/src/agentscope/tts/_openai/__init__.py b/src/agentscope/tts/_openai/__init__.py
new file mode 100644
index 000000000..153f72178
--- /dev/null
+++ b/src/agentscope/tts/_openai/__init__.py
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+"""The OpenAI TTS module."""
+
+from ._model import OpenAITTSModel
+
+__all__ = [
+ "OpenAITTSModel",
+]
diff --git a/src/agentscope/tts/_openai/_model.py b/src/agentscope/tts/_openai/_model.py
new file mode 100644
index 000000000..125333a74
--- /dev/null
+++ b/src/agentscope/tts/_openai/_model.py
@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+"""OpenAI TTS model implementation using the Audio Speech API."""
+import base64
+from datetime import datetime
+from typing import (
+ Any,
+ AsyncGenerator,
+ Literal,
+ TYPE_CHECKING,
+)
+
+from pydantic import BaseModel, Field
+
+from .._tts_base import TTSModelBase
+from .._tts_response import TTSResponse, TTSUsage
+from ...credential import OpenAICredential
+from ...message import DataBlock, Base64Source
+
+if TYPE_CHECKING:
+ from openai import AsyncOpenAI
+
+
+# Map the OpenAI ``response_format`` values to their MIME media types.
+_MEDIA_TYPES = {
+ "mp3": "audio/mpeg",
+ "opus": "audio/opus",
+ "aac": "audio/aac",
+ "flac": "audio/flac",
+ "wav": "audio/wav",
+ "pcm": "audio/pcm",
+}
+_DEFAULT_RESPONSE_FORMAT = "mp3"
+
+
+def _parse_usage(usage: Any, elapsed: float) -> TTSUsage | None:
+ """Extract a TTSUsage from the OpenAI usage object, or None."""
+ if usage is None:
+ return None
+ return TTSUsage(
+ input_tokens=getattr(usage, "input_tokens", 0) or 0,
+ output_tokens=getattr(usage, "output_tokens", 0) or 0,
+ time=elapsed,
+ )
+
+
+class OpenAITTSModel(TTSModelBase):
+ """OpenAI TTS model implementation using the Audio Speech API. For more
+ details please see the `official document
+ <https://platform.openai.com/docs/guides/text-to-speech>`_.
+ """
+
+ class Parameters(BaseModel):
+ """Frontend-exposed parameters for OpenAI TTS models."""
+
+ voice: str = Field(
+ default="alloy",
+ title="Voice",
+ description="The voice to use for synthesis.",
+ )
+
+ response_format: Literal[
+ "mp3",
+ "opus",
+ "aac",
+ "flac",
+ "wav",
+ "pcm",
+ ] = Field(
+ default=_DEFAULT_RESPONSE_FORMAT,
+ title="Response Format",
+ description="The audio format of the synthesized speech.",
+ )
+
+ instructions: str | None = Field(
+ default=None,
+ title="Instructions",
+ description=(
+ "Additional instructions for controlling the voice "
+ "(only supported by some models, e.g. gpt-4o-mini-tts)."
+ ),
+ )
+
+ type: Literal["openai_tts"] = "openai_tts"
+ """The type of the TTS model."""
+
+ realtime: bool = False
+
+ def __init__(
+ self,
+ credential: OpenAICredential,
+ model: str = "tts-1",
+ parameters: "OpenAITTSModel.Parameters | None" = None,
+ stream: bool = True,
+ ) -> None:
+ """Initialize the OpenAI TTS model.
+
+ .. note:: More details about the parameters, such as ``model``
+ and ``voice``, can be found in the `official document
+ <https://platform.openai.com/docs/guides/text-to-speech>`_.
+
+ Args:
+ credential (`OpenAICredential`):
+ The OpenAI credential used to authenticate the API call.
+ model (`str`, defaults to ``"tts-1"``):
+ The TTS model name. Supported models include ``tts-1``,
+ ``tts-1-hd`` and ``gpt-4o-mini-tts``.
+ parameters (`OpenAITTSModel.Parameters | None`, defaults to \
+ `None`):
+ The TTS parameters (voice, response format, etc.). When
+ ``None``, the default parameters will be used.
+ stream (`bool`, defaults to `True`):
+ Whether to use streaming output. When `True`,
+ :meth:`synthesize` returns an async generator yielding
+ ``TTSResponse`` chunks; when `False`, it returns a single
+ aggregated ``TTSResponse``.
+ """
+ super().__init__(
+ credential=credential,
+ model=model,
+ parameters=parameters,
+ stream=stream,
+ )
+
+ async def synthesize(
+ self,
+ text: str | None = None,
+ **kwargs: Any,
+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:
+ """Call the OpenAI Audio Speech API to synthesize speech from text.
+
+ Args:
+ text (`str | None`, optional):
+ The text to be synthesized.
+ **kwargs (`Any`):
+ Additional keyword arguments to pass to the TTS API call.
+
+ Returns:
+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:
+ A single ``TTSResponse`` when ``stream=False``, or an async
+ generator yielding ``TTSResponse`` chunks when ``stream=True``.
+ """
+ if not text:
+ return TTSResponse(content=None)
+
+ import openai
+
+ client = openai.AsyncClient(
+ api_key=self.credential.api_key.get_secret_value(),
+ organization=self.credential.organization,
+ base_url=self.credential.base_url,
+ )
+
+ media_type = _MEDIA_TYPES.get(
+ self.parameters.response_format,
+ _MEDIA_TYPES[_DEFAULT_RESPONSE_FORMAT],
+ )
+
+ request_kwargs: dict[str, Any] = {
+ "model": self.model,
+ "voice": self.parameters.voice,
+ "input": text,
+ "response_format": self.parameters.response_format,
+ **kwargs,
+ }
+ if self.parameters.instructions:
+ request_kwargs["instructions"] = self.parameters.instructions
+
+ if self.stream:
+ return self._stream(client, media_type, **request_kwargs)
+
+ return await self._aggregate(client, media_type, **request_kwargs)
+
+ @staticmethod
+ async def _aggregate(
+ client: "AsyncOpenAI",
+ media_type: str,
+ **request_kwargs: Any,
+ ) -> TTSResponse:
+ """Call the API and aggregate the full audio into a single
+ ``TTSResponse``."""
+ start_datetime = datetime.now()
+ response = await client.audio.speech.create(**request_kwargs)
+ audio_bytes = response.content
+ elapsed = (datetime.now() - start_datetime).total_seconds()
+
+ return TTSResponse(
+ content=DataBlock(
+ source=Base64Source(
+ data=base64.b64encode(audio_bytes).decode("ascii"),
+ media_type=media_type,
+ ),
+ ),
+ usage=_parse_usage(None, elapsed),
+ )
+
+ @staticmethod
+ async def _stream(
+ client: "AsyncOpenAI",
+ media_type: str,
+ **request_kwargs: Any,
+ ) -> AsyncGenerator[TTSResponse, None]:
+ """Call the API and yield incremental audio chunks as
+ ``TTSResponse`` objects.
+
+ Args:
+ client (`AsyncOpenAI`):
+ The async OpenAI client.
+ media_type (`str`):
+ The media type of the synthesized audio.
+ **request_kwargs (`Any`):
+ Keyword arguments forwarded to the Audio Speech API.
+
+ Yields:
+ `TTSResponse`:
+ A ``TTSResponse`` for each incremental audio chunk; the
+ final response has ``is_last=True``.
+ """
+ start_datetime = datetime.now()
+ pending: TTSResponse | None = None
+
+ async with client.audio.speech.with_streaming_response.create(
+ **request_kwargs,
+ ) as response:
+ async for delta_bytes in response.iter_bytes():
+ if not delta_bytes:
+ continue
+ if pending is not None:
+ yield pending
+ pending = TTSResponse(
+ content=DataBlock(
+ source=Base64Source(
+ data=base64.b64encode(delta_bytes).decode(
+ "ascii",
+ ),
+ media_type=media_type,
+ ),
+ ),
+ is_last=False,
+ )
+
+ elapsed = (datetime.now() - start_datetime).total_seconds()
+
+ if pending is not None:
+ pending.is_last = True
+ pending.usage = _parse_usage(None, elapsed)
+ yield pending
+ else:
+ yield TTSResponse(
+ content=None,
+ is_last=True,
+ usage=_parse_usage(None, elapsed),
+ )
diff --git a/src/agentscope/tts/_openai/_models/gpt-4o-mini-tts.yaml b/src/agentscope/tts/_openai/_models/gpt-4o-mini-tts.yaml
new file mode 100644
index 000000000..49171714c
--- /dev/null
+++ b/src/agentscope/tts/_openai/_models/gpt-4o-mini-tts.yaml
@@ -0,0 +1,30 @@
+name: gpt-4o-mini-tts
+label: GPT-4o Mini TTS
+status: active
+
+input_types:
+ - text/plain
+
+output_types:
+ - audio/mpeg
+ - audio/opus
+ - audio/aac
+ - audio/flac
+ - audio/wav
+ - audio/pcm
+
+voices:
+ # Source: https://platform.openai.com/docs/guides/text-to-speech
+ - alloy
+ - ash
+ - ballad
+ - coral
+ - echo
+ - fable
+ - onyx
+ - nova
+ - sage
+ - shimmer
+ - verse
+
+parameter_overrides: {}
diff --git a/src/agentscope/tts/_openai/_models/tts-1-hd.yaml b/src/agentscope/tts/_openai/_models/tts-1-hd.yaml
new file mode 100644
index 000000000..744cbba03
--- /dev/null
+++ b/src/agentscope/tts/_openai/_models/tts-1-hd.yaml
@@ -0,0 +1,34 @@
+name: tts-1-hd
+label: TTS-1 HD
+status: active
+
+input_types:
+ - text/plain
+
+output_types:
+ - audio/mpeg
+ - audio/opus
+ - audio/aac
+ - audio/flac
+ - audio/wav
+ - audio/pcm
+
+voices:
+ # Source: https://platform.openai.com/docs/guides/text-to-speech
+ - alloy
+ - ash
+ - ballad
+ - coral
+ - echo
+ - fable
+ - onyx
+ - nova
+ - sage
+ - shimmer
+ - verse
+
+parameter_overrides:
+ # The instructions parameter is not supported by tts-1-hd, hide it from
+ # the frontend to avoid confusing users.
+ instructions:
+ hidden: true
diff --git a/src/agentscope/tts/_openai/_models/tts-1.yaml b/src/agentscope/tts/_openai/_models/tts-1.yaml
new file mode 100644
index 000000000..755d21274
--- /dev/null
+++ b/src/agentscope/tts/_openai/_models/tts-1.yaml
@@ -0,0 +1,34 @@
+name: tts-1
+label: TTS-1
+status: active
+
+input_types:
+ - text/plain
+
+output_types:
+ - audio/mpeg
+ - audio/opus
+ - audio/aac
+ - audio/flac
+ - audio/wav
+ - audio/pcm
+
+voices:
+ # Source: https://platform.openai.com/docs/guides/text-to-speech
+ - alloy
+ - ash
+ - ballad
+ - coral
+ - echo
+ - fable
+ - onyx
+ - nova
+ - sage
+ - shimmer
+ - verse
+
+parameter_overrides:
+ # The instructions parameter is not supported by tts-1, hide it from
+ # the frontend to avoid confusing users.
+ instructions:
+ hidden: true
diff --git a/tests/tts_openai_test.py b/tests/tts_openai_test.py
new file mode 100644
index 000000000..a181fbb09
--- /dev/null
+++ b/tests/tts_openai_test.py
@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=protected-access
+"""Unit tests for the OpenAI TTS model.
+
+Covers:
+ * ``OpenAITTSModel`` non-streaming aggregation.
+ * ``OpenAITTSModel`` streaming: incremental chunks and ``is_last``
+ placement at the final chunk only.
+ * Edge cases: empty/None input short-circuits without calling the API.
+"""
+import base64
+from typing import Any
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from agentscope.credential import OpenAICredential
+from agentscope.tts import OpenAITTSModel, TTSResponse
+
+
+_MEDIA_TYPE_MP3 = "audio/mpeg"
+_MEDIA_TYPE_WAV = "audio/wav"
+
+
+def _make_mock_client(audio_bytes: bytes, chunks: list[bytes]) -> MagicMock:
+ """Build a mock ``openai.AsyncClient`` shaped like what
+ ``audio.speech.create`` / ``with_streaming_response.create`` return."""
+ client = MagicMock()
+
+ # Non-streaming: client.audio.speech.create(...) -> response with
+ # .content bytes.
+ create_response = MagicMock()
+ create_response.content = audio_bytes
+ client.audio.speech.create = AsyncMock(return_value=create_response)
+
+ # Streaming: client.audio.speech.with_streaming_response.create(...)
+ # is an async context manager whose value exposes .iter_bytes().
+ stream_response = MagicMock()
+
+ async def _iter_bytes() -> Any:
+ for chunk in chunks:
+ yield chunk
+
+ stream_response.iter_bytes = _iter_bytes
+
+ stream_ctx = MagicMock()
+ stream_ctx.__aenter__ = AsyncMock(return_value=stream_response)
+ stream_ctx.__aexit__ = AsyncMock(return_value=None)
+
+ client.audio.speech.with_streaming_response.create = MagicMock(
+ return_value=stream_ctx,
+ )
+
+ return client
+
+
+class TestOpenAITTSModel(IsolatedAsyncioTestCase):
+ """The unittests for the OpenAI TTS model (non-realtime)."""
+
+ def _make_model(self, stream: bool = False, **kwargs: Any) -> Any:
+ """Create an OpenAITTSModel with test credentials."""
+ return OpenAITTSModel(
+ credential=OpenAICredential(api_key="test"),
+ model="tts-1",
+ stream=stream,
+ **kwargs,
+ )
+
+ async def test_aggregates_response(self) -> None:
+ """Non-streaming returns a single TTSResponse with the full audio."""
+ client = _make_mock_client(b"AAAABBBBCCCC", [])
+ model = self._make_model(stream=False)
+
+ with patch("openai.AsyncClient", return_value=client):
+ result = await model.synthesize("Hello world")
+
+ self.assertIsInstance(result, TTSResponse)
+ self.assertEqual(result.content.source.media_type, _MEDIA_TYPE_MP3)
+ self.assertEqual(
+ base64.b64decode(result.content.source.data),
+ b"AAAABBBBCCCC",
+ )
+ self.assertTrue(result.is_last)
+
+ async def test_none_short_circuits(self) -> None:
+ """``synthesize(None)`` returns an empty response without touching
+ the API."""
+ model = self._make_model(stream=False)
+
+ with patch("openai.AsyncClient") as mock_client_cls:
+ result = await model.synthesize(None)
+
+ self.assertIsNone(result.content)
+ mock_client_cls.assert_not_called()
+
+ async def test_empty_string_short_circuits(self) -> None:
+ """``synthesize("")`` returns an empty response without touching
+ the API."""
+ model = self._make_model(stream=False)
+
+ with patch("openai.AsyncClient") as mock_client_cls:
+ result = await model.synthesize("")
+
+ self.assertIsNone(result.content)
+ mock_client_cls.assert_not_called()
+
+ async def test_wav_media_type(self) -> None:
+ """The media type follows the ``response_format`` parameter."""
+ client = _make_mock_client(b"AAAA", [])
+ model = self._make_model(
+ stream=False,
+ parameters=OpenAITTSModel.Parameters(response_format="wav"),
+ )
+
+ with patch("openai.AsyncClient", return_value=client):
+ result = await model.synthesize("Hello world")
+
+ self.assertEqual(result.content.source.media_type, _MEDIA_TYPE_WAV)
+
+ async def test_incremental_chunks(self) -> None:
+ """Each streamed byte chunk yields one TTSResponse."""
+ client = _make_mock_client(b"", [b"AAAA", b"BBBB", b"CCCC"])
+ model = self._make_model(stream=True)
+
+ with patch("openai.AsyncClient", return_value=client):
+ gen = await model.synthesize("Hello world")
+ chunks = [c async for c in gen]
+
+ payloads = [base64.b64decode(c.content.source.data) for c in chunks]
+ self.assertEqual(payloads, [b"AAAA", b"BBBB", b"CCCC"])
+ self.assertEqual(
+ [c.is_last for c in chunks],
+ [False, False, True],
+ )
+ self.assertEqual(
+ [c.content.source.media_type for c in chunks],
+ [_MEDIA_TYPE_MP3] * 3,
+ )
+
+ async def test_single_chunk_marked_last(self) -> None:
+ """A lone audio chunk is flagged ``is_last=True``."""
+ client = _make_mock_client(b"", [b"ONLYCHUNK"])
+ model = self._make_model(stream=True)
+
+ with patch("openai.AsyncClient", return_value=client):
+ gen = await model.synthesize("Hello world")
+ chunks = [c async for c in gen]
+
+ self.assertEqual(len(chunks), 1)
+ self.assertTrue(chunks[0].is_last)
+ self.assertEqual(
+ base64.b64decode(chunks[0].content.source.data),
+ b"ONLYCHUNK",
+ )
+
+ async def test_empty_stream_yields_terminal(self) -> None:
+ """When the API yields no audio, the generator emits a terminal
+ sentinel so consumers can detect EOS."""
+ client = _make_mock_client(b"", [])
+ model = self._make_model(stream=True)
+
+ with patch("openai.AsyncClient", return_value=client):
+ gen = await model.synthesize("Hello world")
+ chunks = [c async for c in gen]
+
+ self.assertEqual(len(chunks), 1)
+ self.assertIsNone(chunks[0].content)
+ self.assertTrue(chunks[0].is_last)