Skip to content

Build and run agents you can see, understand and trust.

feat(tts): support dashscope cosyvoice tts model (#1040)

qbc93d04b67af39
diff --git a/docs/tutorial/en/src/task_tts.py b/docs/tutorial/en/src/task_tts.py
index cdd909d82..4d0e18f51 100644
--- a/docs/tutorial/en/src/task_tts.py
+++ b/docs/tutorial/en/src/task_tts.py
@@ -25,12 +25,24 @@ AgentScope supports the following TTS APIs:
- ✅
- ✅
- ✅
+ * - DashScope CosyVoice Realtime API
+ - ``DashScopeCosyVoiceRealtimeTTSModel``
+ - ✅
+ - ✅
+ - ✅
+ - ✅
* - DashScope API
- ``DashScopeTTSModel``
- ❌
- ✅
- ✅
- ✅
+ * - DashScope CosyVoice API
+ - ``DashScopeCosyVoiceTTSModel``
+ - ❌
+ - ✅
+ - ✅
+ - ✅
* - OpenAI API
- ``OpenAITTSModel``
- ❌
diff --git a/docs/tutorial/zh_CN/src/task_tts.py b/docs/tutorial/zh_CN/src/task_tts.py
index 35c9eed93..313a60ced 100644
--- a/docs/tutorial/zh_CN/src/task_tts.py
+++ b/docs/tutorial/zh_CN/src/task_tts.py
@@ -25,12 +25,24 @@ AgentScope 支持以下 TTS API:
- ✅
- ✅
- ✅
+ * - DashScope CosyVoice 实时 API
+ - ``DashScopeCosyVoiceRealtimeTTSModel``
+ - ✅
+ - ✅
+ - ✅
+ - ✅
* - DashScope API
- ``DashScopeTTSModel``
- ❌
- ✅
- ✅
- ✅
+ * - DashScope CosyVoice API
+ - ``DashScopeCosyVoiceTTSModel``
+ - ❌
+ - ✅
+ - ✅
+ - ✅
* - OpenAI API
- ``OpenAITTSModel``
- ❌
diff --git a/src/agentscope/tts/__init__.py b/src/agentscope/tts/__init__.py
index b54a1f3ed..325ef5885 100644
--- a/src/agentscope/tts/__init__.py
+++ b/src/agentscope/tts/__init__.py
@@ -7,6 +7,10 @@ from ._dashscope_tts_model import DashScopeTTSModel
from ._dashscope_realtime_tts_model import DashScopeRealtimeTTSModel
from ._gemini_tts_model import GeminiTTSModel
from ._openai_tts_model import OpenAITTSModel
+from ._dashscope_cosyvoice_tts_model import DashScopeCosyVoiceTTSModel
+from ._dashscope_cosyvoice_realtime_tts_model import (
+ DashScopeCosyVoiceRealtimeTTSModel,
+)
__all__ = [
"TTSModelBase",
@@ -16,4 +20,6 @@ __all__ = [
"DashScopeRealtimeTTSModel",
"GeminiTTSModel",
"OpenAITTSModel",
+ "DashScopeCosyVoiceTTSModel",
+ "DashScopeCosyVoiceRealtimeTTSModel",
]
diff --git a/src/agentscope/tts/_dashscope_cosyvoice_realtime_tts_model.py b/src/agentscope/tts/_dashscope_cosyvoice_realtime_tts_model.py
new file mode 100644
index 000000000..5b80f464d
--- /dev/null
+++ b/src/agentscope/tts/_dashscope_cosyvoice_realtime_tts_model.py
@@ -0,0 +1,279 @@
+# -*- coding: utf-8 -*-
+"""DashScope CosyVoice Realtime TTS model implementation."""
+
+from typing import Any, Literal, AsyncGenerator
+
+from ._tts_base import TTSModelBase
+from ._tts_response import TTSResponse
+from ._utils import _get_cosyvoice_callback_class
+from ..message import Msg
+from ..types import JSONSerializableObject
+
+
+class DashScopeCosyVoiceRealtimeTTSModel(TTSModelBase):
+ """TTS implementation for DashScope CosyVoice Realtime TTS API,
+ which supports streaming input. The supported models include
+ "cosyvoice-v3-plus", "cosyvoice-v3-flash", "sambert" etc.
+
+ For more details, please see the `official document
+ <https://help.aliyun.com/zh/model-studio/text-to-speech>`_.
+
+ .. note:: The DashScopeCosyVoiceRealtimeTTSModel can only handle one
+ streaming input request at a time, and cannot process multiple
+ streaming input requests concurrently. For example, it cannot handle
+ input sequences like `[msg_1_chunk0, msg_1_chunk1, msg_2_chunk0]`,
+ where the prefixes "msg_x" indicate different streaming input requests.
+ """
+
+ supports_streaming_input: bool = True
+ """Whether the model supports streaming input."""
+
+ def __init__(
+ self,
+ api_key: str,
+ model_name: str = "cosyvoice-v3-plus",
+ voice: Literal[
+ "longanyang",
+ "longanhuan",
+ "longhuhu_v3",
+ "longyingmu_v3",
+ ]
+ | str = "longanyang",
+ stream: bool = True,
+ cold_start_length: int | None = None,
+ cold_start_words: int | None = None,
+ client_kwargs: dict[str, JSONSerializableObject] | None = None,
+ generate_kwargs: dict[str, JSONSerializableObject] | None = None,
+ max_retries: int = 3,
+ retry_delay: float = 5.0,
+ ) -> None:
+ """Initialize the DashScope CosyVoice Realtime TTS model by
+ specifying the model, voice, and other parameters.
+
+ .. note:: More details about the parameters, such as `model_name`,
+ `voice`, and `mode` can be found in the `official document
+ <https://help.aliyun.com/zh/model-studio/cosyvoice-voice-list>`_.
+
+ .. note:: You can use `cold_start_length` and `cold_start_words`
+ simultaneously to set both character and word thresholds for the first
+ TTS request. For Chinese text, word segmentation (based on spaces) may
+ not be effective.
+
+ Args:
+ api_key (`str`):
+ The DashScope API key.
+ model_name (`str`, defaults to "cosyvoice-v3-plus"):
+ The TTS model name, e.g. "cosyvoice-v3-plus",
+ "cosyvoice-v3-flash", etc.
+ voice (`Literal["longanyang", "longanhuan", "longhuhu_v3", \
+ "longyingmu_v3"] | str`, defaults to "longanyang".):
+ The voice to use for synthesis. Refer to `official document
+ <https://help.aliyun.com/zh/model-studio/cosyvoice-voice-list>`_
+ for the supported voices for each model.
+ stream (`bool`, defaults to `True`):
+ Whether to use streaming synthesis.
+ cold_start_length (`int | None`, optional):
+ The minimum length send threshold for the first TTS request,
+ ensuring there is no pause in the synthesized speech for too
+ short input text. The length is measured in number of
+ characters.
+ cold_start_words (`int | None`, optional):
+ The minimum words send threshold for the first TTS request,
+ ensuring there is no pause in the synthesized speech for too
+ short input text. The words are identified by spaces in the
+ text.
+ client_kwargs (`dict[str, JSONSerializableObject] | None`, \
+ optional):
+ The extra keyword arguments to initialize the DashScope
+ CosyVoice Realtime tts client.
+ generate_kwargs (`dict[str, JSONSerializableObject] | None`, \
+ optional):
+ The extra keyword arguments used in DashScope CosyVoice
+ Realtime tts API generation.
+ max_retries (`int`, defaults to 3):
+ The maximum number of retry attempts when TTS synthesis fails.
+ retry_delay (`float`, defaults to 5.0):
+ The delay in seconds before retrying. Uses exponential backoff.
+ """
+ super().__init__(model_name=model_name, stream=stream)
+
+ import dashscope
+ from dashscope.audio.tts_v2 import SpeechSynthesizer
+
+ dashscope.api_key = api_key
+
+ # Store configuration
+ self.voice = voice
+ self.cold_start_length = cold_start_length
+ self.cold_start_words = cold_start_words
+ self.client_kwargs = client_kwargs or {}
+ self.generate_kwargs = generate_kwargs or {}
+ self.max_retries = max_retries
+ self.retry_delay = retry_delay
+
+ # Initialize TTS client
+ # Save callback reference (for DashScope SDK)
+ self._dashscope_callback = _get_cosyvoice_callback_class()()
+
+ # The variables for tracking streaming input messages
+ # If we have sent text for the current message
+ self._first_send: bool = True
+ # The current message ID being processed
+ self._current_msg_id: str | None = None
+ # The current prefix text already sent
+ self._current_prefix: str = ""
+ self._synthesizer: SpeechSynthesizer | None = None
+
+ async def connect(self) -> None:
+ """Connect to the TTS model and initialize resources."""
+ from dashscope.audio.tts_v2 import SpeechSynthesizer, AudioFormat
+
+ self._synthesizer = SpeechSynthesizer(
+ model=self.model_name,
+ voice=self.voice,
+ format=AudioFormat.PCM_24000HZ_MONO_16BIT,
+ callback=self._dashscope_callback,
+ **self.client_kwargs,
+ **self.generate_kwargs,
+ )
+
+ async def close(self) -> None:
+ """Close the TTS model and release resources."""
+ self._synthesizer.close()
+
+ async def push(
+ self,
+ msg: Msg,
+ **kwargs: Any,
+ ) -> TTSResponse:
+ """Append text to be synthesized and return the received TTS response.
+ Note this method is non-blocking, and maybe return an empty response
+ if no audio is received yet.
+
+ To receive all the synthesized speech, call the `synthesize` method
+ after pushing all the text chunks.
+
+ Args:
+ msg (`Msg`):
+ The message to be synthesized. The `msg.id` identifies the
+ streaming input request.
+ **kwargs (`Any`):
+ Additional keyword arguments to pass to the TTS API call.
+
+ Returns:
+ `TTSResponse`:
+ The TTSResponse containing audio blocks.
+ """
+
+ if self._current_msg_id is not None and self._current_msg_id != msg.id:
+ raise RuntimeError(
+ "DashScopeCosyVoiceRealtimeTTSModel can only handle one "
+ "streaming input request at a time. Please ensure that all "
+ "chunks belong to the same message ID.",
+ )
+
+ # Record current message ID
+ self._current_msg_id = msg.id
+
+ text = msg.get_text_content()
+
+ # Determine if we should send text based on cold start settings only
+ # for the first input chunk and not the last chunk
+ if text:
+ if self._first_send:
+ # If we have cold start settings
+ if self.cold_start_length:
+ if len(text) < self.cold_start_length:
+ delta_to_send = ""
+ else:
+ delta_to_send = text
+ else:
+ delta_to_send = text
+
+ if delta_to_send and self.cold_start_words:
+ if len(delta_to_send.split()) < self.cold_start_words:
+ delta_to_send = ""
+ else:
+ # Remove the already sent prefix if not the first send
+ delta_to_send = text.removeprefix(self._current_prefix)
+
+ if delta_to_send:
+ self._synthesizer.streaming_call(delta_to_send)
+
+ # Record sent prefix
+ self._current_prefix += delta_to_send
+ self._first_send = False
+
+ # Wait for the audio data to be available
+ res = await self._dashscope_callback.get_audio_data(block=False)
+
+ return res
+
+ # Return empty response if no text to send
+ return TTSResponse(content=None)
+
+ async def synthesize(
+ self,
+ msg: Msg | None = None,
+ **kwargs: Any,
+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:
+ """Append text to be synthesized and return TTS response.
+
+ Args:
+ msg (`Msg | None`, optional):
+ The message to be synthesized.
+ **kwargs (`Any`):
+ Additional keyword arguments to pass to the TTS API call.
+
+ Returns:
+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:
+ The TTSResponse object in non-streaming mode, or an async
+ generator yielding TTSResponse objects in streaming mode.
+ """
+ if (
+ self._current_msg_id is not None
+ and msg
+ and self._current_msg_id != msg.id
+ ):
+ raise RuntimeError(
+ "DashScopeCosyVoiceRealtimeTTSModel can only handle one "
+ "streaming input request at a time. Please ensure that all "
+ "chunks belong to the same message ID.",
+ )
+
+ if msg is None:
+ delta_to_send = ""
+
+ else:
+ # Record current message ID
+ self._current_msg_id = msg.id
+ delta_to_send = (msg.get_text_content() or "").removeprefix(
+ self._current_prefix,
+ )
+
+ # Determine if we should send text based on cold start settings only
+ # for the first input chunk and not the last chunk
+ if delta_to_send:
+ self._synthesizer.streaming_call(delta_to_send)
+
+ # To keep correct prefix tracking
+ self._current_prefix += delta_to_send
+ self._first_send = False
+
+ # We need to block until synthesis is complete to get all audio
+ self._synthesizer.streaming_complete()
+
+ if self.stream:
+ # Return an async generator for audio chunks
+ res = self._dashscope_callback.get_audio_chunk()
+
+ else:
+ # Block and wait for all audio data to be available
+ res = await self._dashscope_callback.get_audio_data(block=True)
+
+ # Update state for next message
+ self._current_msg_id = None
+ self._first_send = True
+ self._current_prefix = ""
+
+ return res
diff --git a/src/agentscope/tts/_dashscope_cosyvoice_tts_model.py b/src/agentscope/tts/_dashscope_cosyvoice_tts_model.py
new file mode 100644
index 000000000..bf2085859
--- /dev/null
+++ b/src/agentscope/tts/_dashscope_cosyvoice_tts_model.py
@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+"""DashScope CosyVoice TTS model implementation."""
+
+import base64
+from typing import Any, Literal, AsyncGenerator
+
+from ._tts_base import TTSModelBase
+from ._tts_response import TTSResponse
+from ._utils import _get_cosyvoice_callback_class
+from ..message import Msg, AudioBlock, Base64Source
+from ..types import JSONSerializableObject
+
+
+class DashScopeCosyVoiceTTSModel(TTSModelBase):
+ """TTS implementation for DashScope CosyVoice TTS API.
+ The supported models include "cosyvoice-v3-plus",
+ "cosyvoice-v3-flash", "sambert" etc.
+
+ This model does NOT support streaming text input. For streaming input,
+ use `DashScopeCosyVoiceRealtimeTTSModel` instead.
+
+ For more details, please see the `official document
+ <https://help.aliyun.com/zh/model-studio/text-to-speech>`_.
+ """
+
+ supports_streaming_input: bool = False
+ """Whether the model supports streaming input."""
+
+ def __init__(
+ self,
+ api_key: str,
+ model_name: str = "cosyvoice-v3-plus",
+ voice: Literal[
+ "longanyang",
+ "longanhuan",
+ "longhuhu_v3",
+ "longyingmu_v3",
+ ]
+ | str = "longanyang",
+ stream: bool = False,
+ client_kwargs: dict[str, JSONSerializableObject] | None = None,
+ generate_kwargs: dict[str, JSONSerializableObject] | None = None,
+ ) -> None:
+ """Initialize the DashScope CosyVoice TTS model by
+ specifying the model, voice, and other parameters.
+
+ .. note:: More details about the parameters, such as `model_name`,
+ `voice`, and `mode` can be found in the `official document
+ <https://help.aliyun.com/zh/model-studio/cosyvoice-voice-list>`_.
+
+ Args:
+ api_key (`str`):
+ The DashScope API key.
+ model_name (`str`, defaults to "cosyvoice-v3-plus"):
+ The TTS model name, e.g. "cosyvoice-v3-plus",
+ "cosyvoice-v3-flash", etc.
+ voice (`Literal["longanyang", "longanhuan", "longhuhu_v3", \
+ "longyingmu_v3"] | str`, defaults to "longanyang".):
+ The voice to use for synthesis. Refer to `official document
+ <https://help.aliyun.com/zh/model-studio/cosyvoice-voice-list>`_
+ for the supported voices for each model.
+ stream (`bool`, defaults to `False`):
+ Whether to use streaming audio output.
+ client_kwargs (`dict[str, JSONSerializableObject] | None`, \
+ optional):
+ The extra keyword arguments to initialize the DashScope
+ CosyVoice tts client.
+ generate_kwargs (`dict[str, JSONSerializableObject] | None`, \
+ optional):
+ The extra keyword arguments used in DashScope CosyVoice
+ tts API generation.
+ """
+ super().__init__(model_name=model_name, stream=stream)
+
+ import dashscope
+
+ dashscope.api_key = api_key
+
+ # Store configuration
+ self.voice = voice
+ self.client_kwargs = client_kwargs or {}
+ self.generate_kwargs = generate_kwargs or {}
+
+ def _create_synthesizer(self) -> tuple:
+ """Create a new SpeechSynthesizer instance for each request."""
+ from dashscope.audio.tts_v2 import SpeechSynthesizer, AudioFormat
+
+ callback = _get_cosyvoice_callback_class()() if self.stream else None
+
+ synthesizer = SpeechSynthesizer(
+ model=self.model_name,
+ voice=self.voice,
+ format=AudioFormat.PCM_24000HZ_MONO_16BIT,
+ callback=callback,
+ **self.client_kwargs,
+ **self.generate_kwargs,
+ )
+ return synthesizer, callback
+
+ async def push(self, msg: Msg, **kwargs: Any) -> TTSResponse:
+ """Push a message to the TTS model and return TTS response.
+
+ Args:
+ msg (`Msg`):
+ The message to be synthesized.
+ **kwargs (`Any`):
+ Additional keyword arguments to pass to the TTS API call.
+
+ Returns:
+ `TTSResponse`:
+ The TTSResponse object.
+ """
+ return TTSResponse(content=None)
+
+ async def synthesize(
+ self,
+ msg: Msg | None = None,
+ **kwargs: Any,
+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:
+ """Synthesize text to speech and return TTS response.
+
+ Args:
+ msg (`Msg | None`, optional):
+ The message to be synthesized.
+ **kwargs (`Any`):
+ Additional keyword arguments to pass to the TTS API call.
+
+ Returns:
+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:
+ The TTSResponse object in non-streaming mode, or an async
+ generator yielding TTSResponse objects in streaming mode.
+ """
+ if msg is None:
+ return TTSResponse(content=None)
+
+ text = msg.get_text_content()
+ if not text:
+ return TTSResponse(content=None)
+
+ # Create a new synthesizer for each request to avoid connection issues
+ synthesizer, callback = self._create_synthesizer()
+
+ if self.stream:
+ # Streaming output mode: use callback to get audio chunks
+ synthesizer.call(text=text)
+ return callback.get_audio_chunk()
+ else:
+ # Non-streaming mode: call directly returns audio bytes
+ audio = synthesizer.call(text=text)
+
+ if not audio:
+ return TTSResponse(content=None)
+
+ encoded_data = base64.b64encode(audio).decode()
+
+ return TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=encoded_data,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ )
diff --git a/src/agentscope/tts/_utils.py b/src/agentscope/tts/_utils.py
new file mode 100644
index 000000000..90540eaad
--- /dev/null
+++ b/src/agentscope/tts/_utils.py
@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+"""Utility classes for DashScope CosyVoice TTS models."""
+
+import base64
+import threading
+from typing import TYPE_CHECKING, AsyncGenerator, Any
+
+from ._tts_response import TTSResponse
+from .._logging import logger
+from ..message import AudioBlock, Base64Source
+
+if TYPE_CHECKING:
+ from dashscope.audio.tts_v2 import ResultCallback
+else:
+ ResultCallback = "dashscope.audio.tts_v2.ResultCallback"
+
+
+def _get_cosyvoice_callback_class() -> type["ResultCallback"]:
+ """Get the callback class for CosyVoice TTS streaming audio output.
+
+ This callback handles audio data accumulation with proper PCM and base64
+ alignment. It encodes audio in chunks of 6 bytes (LCM of 2 and 3) to
+ ensure both PCM alignment (2 bytes per sample) and base64 alignment
+ (3 bytes per encoded unit).
+
+ Returns:
+ The callback class for CosyVoice TTS.
+ """
+ from dashscope.audio.tts_v2 import ResultCallback
+
+ class _CosyVoiceTTSCallback(ResultCallback):
+ """CosyVoice TTS callback for streaming audio output."""
+
+ def __init__(self) -> None:
+ """Initialize the CosyVoice TTS callback."""
+ super().__init__()
+
+ # The event that will be set when a new audio chunk is received
+ self.chunk_event = threading.Event()
+ # The event that will be set when the TTS synthesis is finished
+ self.finish_event = threading.Event()
+ # Accumulate raw bytes
+ self._audio_bytes: bytes = b""
+ # Accumulated base64 string (boundary-aligned)
+ self._audio_base64: str = ""
+ # Last encoded byte position (must be multiple of 6 for alignment)
+ self._last_encoded_pos: int = 0
+
+ def on_open(self) -> None:
+ """Called when the WebSocket connection is opened."""
+ self._audio_bytes = b""
+ self._audio_base64 = ""
+ self._last_encoded_pos = 0
+ self.finish_event.clear()
+
+ def on_data(self, data: bytes) -> None:
+ """Called when data is received from the WebSocket connection.
+
+ Args:
+ data (`bytes`):
+ The data received from the WebSocket connection.
+ """
+ if data:
+ self._audio_bytes += data
+ # Encode in chunks of 6 bytes (LCM of 2 and 3)
+ # This ensures both PCM alignment (2 bytes) and
+ # base64 alignment (3 bytes)
+ aligned_len = (len(self._audio_bytes) // 6) * 6
+ if aligned_len > self._last_encoded_pos:
+ new_chunk = self._audio_bytes[
+ self._last_encoded_pos : aligned_len
+ ]
+ self._audio_base64 += base64.b64encode(new_chunk).decode()
+ self._last_encoded_pos = aligned_len
+
+ # Signal that a new audio chunk is available
+ if not self.chunk_event.is_set():
+ self.chunk_event.set()
+
+ def on_close(self) -> None:
+ """Called when the WebSocket connection is closed."""
+ # Encode any remaining bytes
+ if len(self._audio_bytes) > self._last_encoded_pos:
+ remaining = self._audio_bytes[self._last_encoded_pos :]
+ self._audio_base64 += base64.b64encode(remaining).decode()
+ self._last_encoded_pos = len(self._audio_bytes)
+
+ # Unblock waiting operations to prevent deadlock
+ self.finish_event.set()
+ self.chunk_event.set()
+
+ def on_error(self, message: Any) -> None:
+ """Called when an error occurs."""
+ logger.error(message)
+
+ # Unblock waiting operations to prevent deadlock
+ self.finish_event.set()
+ self.chunk_event.set()
+
+ async def get_audio_data(self, block: bool = True) -> TTSResponse:
+ """Get the current accumulated audio data as base64 string.
+
+ Args:
+ block (`bool`, defaults to `True`):
+ Whether to block until synthesis is finished.
+
+ Returns:
+ `TTSResponse`:
+ The TTSResponse containing base64-encoded audio data.
+ """
+ # Block until synthesis is finished
+ if block:
+ self.finish_event.wait()
+
+ # Return the accumulated audio data
+ if self._audio_base64:
+ return TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self._audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ )
+
+ # Reset for next tts request
+ await self._reset()
+
+ # Return empty response if no audio data
+ return TTSResponse(content=None)
+
+ async def get_audio_chunk(self) -> AsyncGenerator[TTSResponse, None]:
+ """Get the audio data chunk as an async generator of TTSResponse.
+
+ Returns accumulated base64 string. The agent code uses string
+ slicing to get the delta, so we must ensure boundary alignment.
+
+ Returns:
+ `AsyncGenerator[TTSResponse, None]`:
+ The async generator yielding TTSResponse with audio chunks.
+ """
+ while True:
+ if self.finish_event.is_set():
+ # Yield final chunk with all accumulated data
+ yield TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self._audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ is_last=True,
+ )
+
+ # Reset for next tts request
+ await self._reset()
+
+ break
+
+ if self.chunk_event.is_set():
+ # Clear the event for next chunk
+ self.chunk_event.clear()
+ else:
+ # Wait for the next chunk
+ self.chunk_event.wait()
+
+ # Yield current accumulated data
+ if self._audio_base64:
+ yield TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self._audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ is_last=False,
+ )
+
+ async def _reset(self) -> None:
+ """Reset the callback state for a new TTS request."""
+ self.finish_event.clear()
+ self.chunk_event.clear()
+ self._audio_bytes = b""
+ self._audio_base64 = ""
+ self._last_encoded_pos = 0
+
+ def has_audio_data(self) -> bool:
+ """Check if audio data has been received."""
+ return bool(self._audio_bytes)
+
+ return _CosyVoiceTTSCallback
diff --git a/tests/tts_dashscope_cosyvoice_test.py b/tests/tts_dashscope_cosyvoice_test.py
new file mode 100644
index 000000000..93f98e8b6
--- /dev/null
+++ b/tests/tts_dashscope_cosyvoice_test.py
@@ -0,0 +1,402 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=protected-access
+"""The unittests for DashScope CosyVoice TTS models."""
+import base64
+from typing import AsyncGenerator
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import Mock, patch, AsyncMock, MagicMock
+
+from agentscope.message import Msg, AudioBlock, Base64Source
+from agentscope.tts import (
+ DashScopeCosyVoiceRealtimeTTSModel,
+ DashScopeCosyVoiceTTSModel,
+ TTSResponse,
+)
+
+
+class DashScopeCosyVoiceTTSModelTest(IsolatedAsyncioTestCase):
+ """The unittests for DashScope CosyVoice TTS model (non-realtime)."""
+
+ def setUp(self) -> None:
+ """Set up the test case."""
+ self.api_key = "test_api_key"
+ self.mock_audio_bytes = b"fake_audio_data_bytes"
+ self.mock_audio_base64 = base64.b64encode(
+ self.mock_audio_bytes,
+ ).decode("utf-8")
+
+ def _create_mock_dashscope_modules(self) -> dict:
+ """Create mock dashscope modules for patching."""
+ mock_audio_format = MagicMock()
+ mock_audio_format.PCM_24000HZ_MONO_16BIT = "pcm_24000hz_mono_16bit"
+
+ mock_speech_synthesizer = MagicMock()
+
+ mock_tts_v2 = MagicMock()
+ mock_tts_v2.SpeechSynthesizer = mock_speech_synthesizer
+ mock_tts_v2.AudioFormat = mock_audio_format
+ mock_tts_v2.ResultCallback = Mock
+
+ mock_audio = MagicMock()
+ mock_audio.tts_v2 = mock_tts_v2
+
+ mock_dashscope = MagicMock()
+ mock_dashscope.api_key = None
+ mock_dashscope.audio = mock_audio
+
+ return {
+ "dashscope": mock_dashscope,
+ "dashscope.audio": mock_audio,
+ "dashscope.audio.tts_v2": mock_tts_v2,
+ }
+
+ def test_init(self) -> None:
+ """Test initialization of DashScopeCosyVoiceTTSModel."""
+ mock_modules = self._create_mock_dashscope_modules()
+
+ with patch.dict("sys.modules", mock_modules):
+ model = DashScopeCosyVoiceTTSModel(
+ api_key=self.api_key,
+ model_name="cosyvoice-v3-plus",
+ voice="longanyang",
+ stream=False,
+ )
+ self.assertEqual(model.model_name, "cosyvoice-v3-plus")
+ self.assertEqual(model.voice, "longanyang")
+ self.assertFalse(model.stream)
+ self.assertFalse(model.supports_streaming_input)
+
+ async def test_synthesize_non_streaming(self) -> None:
+ """Test synthesize method in non-streaming mode."""
+ mock_modules = self._create_mock_dashscope_modules()
+
+ with patch.dict("sys.modules", mock_modules):
+ model = DashScopeCosyVoiceTTSModel(
+ api_key=self.api_key,
+ stream=False,
+ )
+
+ # Mock _create_synthesizer to return a mock synthesizer
+ mock_synthesizer = MagicMock()
+ mock_synthesizer.call.return_value = self.mock_audio_bytes
+ model._create_synthesizer = Mock(
+ return_value=(mock_synthesizer, None),
+ )
+
+ msg = Msg(name="user", content="Hello! Test message.", role="user")
+ response = await model.synthesize(msg)
+
+ self.assertIsInstance(response, TTSResponse)
+ self.assertEqual(response.content["type"], "audio")
+ self.assertEqual(
+ response.content["source"]["data"],
+ self.mock_audio_base64,
+ )
+ mock_synthesizer.call.assert_called_once_with(
+ text="Hello! Test message.",
+ )
+
+ async def test_synthesize_streaming(self) -> None:
+ """Test synthesize method in streaming mode."""
+ mock_modules = self._create_mock_dashscope_modules()
+
+ with patch.dict("sys.modules", mock_modules):
+ model = DashScopeCosyVoiceTTSModel(
+ api_key=self.api_key,
+ stream=True,
+ )
+
+ # Create mock callback with proper async generator
+ mock_callback = MagicMock()
+
+ async def mock_generator() -> AsyncGenerator[TTSResponse, None]:
+ yield TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self.mock_audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ is_last=False,
+ )
+ yield TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self.mock_audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ is_last=True,
+ )
+
+ # Directly return the generator object
+ mock_callback.get_audio_chunk = Mock(return_value=mock_generator())
+
+ mock_synthesizer = MagicMock()
+ model._create_synthesizer = Mock(
+ return_value=(mock_synthesizer, mock_callback),
+ )
+
+ msg = Msg(name="user", content="Test streaming.", role="user")
+ response = await model.synthesize(msg)
+
+ # Verify response is an async generator
+ self.assertIsInstance(response, AsyncGenerator)
+ chunks = [chunk async for chunk in response]
+
+ # Verify we got some chunks
+ self.assertGreater(len(chunks), 0)
+ # Verify each chunk is a TTSResponse
+ for chunk in chunks:
+ self.assertIsInstance(chunk, TTSResponse)
+ mock_synthesizer.call.assert_called_once_with(
+ text="Test streaming.",
+ )
+
+
+class DashScopeCosyVoiceRealtimeTTSModelTest(IsolatedAsyncioTestCase):
+ """The unittests for DashScope CosyVoice Realtime TTS model."""
+
+ def setUp(self) -> None:
+ """Set up the test case."""
+ self.api_key = "test_api_key"
+ self.mock_audio_bytes = b"fake_audio_data_bytes"
+ self.mock_audio_base64 = base64.b64encode(
+ self.mock_audio_bytes,
+ ).decode("utf-8")
+
+ def _create_mock_dashscope_modules(self) -> dict:
+ """Create mock dashscope modules for patching."""
+ mock_audio_format = MagicMock()
+ mock_audio_format.PCM_24000HZ_MONO_16BIT = "pcm_24000hz_mono_16bit"
+
+ mock_speech_synthesizer = MagicMock()
+
+ mock_tts_v2 = MagicMock()
+ mock_tts_v2.SpeechSynthesizer = mock_speech_synthesizer
+ mock_tts_v2.AudioFormat = mock_audio_format
+ mock_tts_v2.ResultCallback = Mock
+
+ mock_audio = MagicMock()
+ mock_audio.tts_v2 = mock_tts_v2
+
+ mock_dashscope = MagicMock()
+ mock_dashscope.api_key = None
+ mock_dashscope.audio = mock_audio
+
+ return {
+ "dashscope": mock_dashscope,
+ "dashscope.audio": mock_audio,
+ "dashscope.audio.tts_v2": mock_tts_v2,
+ }
+
+ def _create_mock_callback(self) -> MagicMock:
+ """Create a mock callback for testing."""
+ mock_callback = MagicMock()
+ mock_callback.chunk_event = MagicMock()
+ mock_callback.finish_event = MagicMock()
+ mock_callback._audio_bytes = b""
+ mock_callback._audio_base64 = ""
+ mock_callback._last_encoded_pos = 0
+ return mock_callback
+
+ def test_init(self) -> None:
+ """Test initialization of DashScopeCosyVoiceRealtimeTTSModel."""
+ mock_modules = self._create_mock_dashscope_modules()
+
+ with patch.dict("sys.modules", mock_modules):
+ # Mock _get_cosyvoice_callback_class
+ with patch(
+ "agentscope.tts._dashscope_cosyvoice_realtime_tts_model"
+ "._get_cosyvoice_callback_class",
+ ) as mock_get_callback:
+ mock_callback_class = MagicMock()
+ mock_callback_class.return_value = self._create_mock_callback()
+ mock_get_callback.return_value = mock_callback_class
+
+ model = DashScopeCosyVoiceRealtimeTTSModel(
+ api_key=self.api_key,
+ model_name="cosyvoice-v3-plus",
+ voice="longanyang",
+ stream=True,
+ )
+ self.assertEqual(model.model_name, "cosyvoice-v3-plus")
+ self.assertEqual(model.voice, "longanyang")
+ self.assertTrue(model.stream)
+ self.assertTrue(model.supports_streaming_input)
+
+ async def test_push_incremental_text(self) -> None:
+ """Test push method with incremental text chunks."""
+ mock_modules = self._create_mock_dashscope_modules()
+ mock_synthesizer_instance = MagicMock()
+ mock_modules[
+ "dashscope.audio.tts_v2"
+ ].SpeechSynthesizer.return_value = mock_synthesizer_instance
+
+ with patch.dict("sys.modules", mock_modules):
+ with patch(
+ "agentscope.tts._dashscope_cosyvoice_realtime_tts_model"
+ "._get_cosyvoice_callback_class",
+ ) as mock_get_callback:
+ mock_callback_class = MagicMock()
+ mock_callback = self._create_mock_callback()
+ mock_callback.get_audio_data = AsyncMock(
+ return_value=TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self.mock_audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ ),
+ )
+ mock_callback_class.return_value = mock_callback
+ mock_get_callback.return_value = mock_callback_class
+
+ model = DashScopeCosyVoiceRealtimeTTSModel(
+ api_key=self.api_key,
+ )
+ await model.connect()
+
+ msg_id = "test_msg_001"
+ text_chunks = ["Hello there!\n\n", "This is a test message."]
+
+ accumulated_text = ""
+ for chunk in text_chunks:
+ accumulated_text += chunk
+ msg = Msg(
+ name="user",
+ content=accumulated_text,
+ role="user",
+ )
+ msg.id = msg_id
+
+ response = await model.push(msg)
+ self.assertIsInstance(response, TTSResponse)
+
+ # Verify streaming_call was called
+ self.assertGreater(
+ mock_synthesizer_instance.streaming_call.call_count,
+ 0,
+ )
+
+ async def test_synthesize_non_streaming(self) -> None:
+ """Test synthesize method in non-streaming mode."""
+ mock_modules = self._create_mock_dashscope_modules()
+ mock_synthesizer_instance = MagicMock()
+ mock_modules[
+ "dashscope.audio.tts_v2"
+ ].SpeechSynthesizer.return_value = mock_synthesizer_instance
+
+ with patch.dict("sys.modules", mock_modules):
+ with patch(
+ "agentscope.tts._dashscope_cosyvoice_realtime_tts_model"
+ "._get_cosyvoice_callback_class",
+ ) as mock_get_callback:
+ mock_callback_class = MagicMock()
+ mock_callback = self._create_mock_callback()
+ mock_callback.get_audio_data = AsyncMock(
+ return_value=TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self.mock_audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ ),
+ )
+ mock_callback_class.return_value = mock_callback
+ mock_get_callback.return_value = mock_callback_class
+
+ model = DashScopeCosyVoiceRealtimeTTSModel(
+ api_key=self.api_key,
+ stream=False,
+ )
+ await model.connect()
+
+ msg = Msg(
+ name="user",
+ content="Hello! Test message.",
+ role="user",
+ )
+ response = await model.synthesize(msg)
+
+ self.assertIsInstance(response, TTSResponse)
+ self.assertEqual(response.content["type"], "audio")
+ mock_synthesizer_instance.streaming_complete.assert_called_once() # noqa
+
+ async def test_synthesize_streaming(self) -> None:
+ """Test synthesize method in streaming mode."""
+ mock_modules = self._create_mock_dashscope_modules()
+ mock_synthesizer_instance = MagicMock()
+ mock_modules[
+ "dashscope.audio.tts_v2"
+ ].SpeechSynthesizer.return_value = mock_synthesizer_instance
+
+ with patch.dict("sys.modules", mock_modules):
+ with patch(
+ "agentscope.tts._dashscope_cosyvoice_realtime_tts_model"
+ "._get_cosyvoice_callback_class",
+ ) as mock_get_callback:
+ mock_callback_class = MagicMock()
+ mock_callback = self._create_mock_callback()
+
+ async def mock_generator() -> AsyncGenerator[
+ TTSResponse,
+ None,
+ ]:
+ yield TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self.mock_audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ is_last=False,
+ )
+ yield TTSResponse(
+ content=AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ data=self.mock_audio_base64,
+ media_type="audio/pcm;rate=24000",
+ ),
+ ),
+ is_last=True,
+ )
+
+ mock_callback.get_audio_chunk = Mock(
+ return_value=mock_generator(),
+ )
+ mock_callback_class.return_value = mock_callback
+ mock_get_callback.return_value = mock_callback_class
+
+ model = DashScopeCosyVoiceRealtimeTTSModel(
+ api_key=self.api_key,
+ stream=True,
+ )
+ await model.connect()
+
+ msg = Msg(name="user", content="Test streaming.", role="user")
+ response = await model.synthesize(msg)
+
+ # Verify response is an async generator
+ self.assertIsInstance(response, AsyncGenerator)
+ chunks = [chunk async for chunk in response]
+
+ # Verify we got some chunks
+ self.assertGreater(len(chunks), 0)
+ # Verify each chunk is a TTSResponse
+ for chunk in chunks:
+ self.assertIsInstance(chunk, TTSResponse)