Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(tts): support dashscope cosyvoice tts model (#1040)
qbc93d04b67af39
diff --git a/docs/tutorial/en/src/task_tts.py b/docs/tutorial/en/src/task_tts.pyindex cdd909d82..4d0e18f51 100644--- a/docs/tutorial/en/src/task_tts.py+++ b/docs/tutorial/en/src/task_tts.py@@ -25,12 +25,24 @@ AgentScope supports the following TTS APIs:- ✅- ✅- ✅+ * - DashScope CosyVoice Realtime API+ - ``DashScopeCosyVoiceRealtimeTTSModel``+ - ✅+ - ✅+ - ✅+ - ✅* - DashScope API- ``DashScopeTTSModel``- ❌- ✅- ✅- ✅+ * - DashScope CosyVoice API+ - ``DashScopeCosyVoiceTTSModel``+ - ❌+ - ✅+ - ✅+ - ✅* - OpenAI API- ``OpenAITTSModel``- ❌diff --git a/docs/tutorial/zh_CN/src/task_tts.py b/docs/tutorial/zh_CN/src/task_tts.pyindex 35c9eed93..313a60ced 100644--- a/docs/tutorial/zh_CN/src/task_tts.py+++ b/docs/tutorial/zh_CN/src/task_tts.py@@ -25,12 +25,24 @@ AgentScope 支持以下 TTS API:- ✅- ✅- ✅+ * - DashScope CosyVoice 实时 API+ - ``DashScopeCosyVoiceRealtimeTTSModel``+ - ✅+ - ✅+ - ✅+ - ✅* - DashScope API- ``DashScopeTTSModel``- ❌- ✅- ✅- ✅+ * - DashScope CosyVoice API+ - ``DashScopeCosyVoiceTTSModel``+ - ❌+ - ✅+ - ✅+ - ✅* - OpenAI API- ``OpenAITTSModel``- ❌diff --git a/src/agentscope/tts/__init__.py b/src/agentscope/tts/__init__.pyindex b54a1f3ed..325ef5885 100644--- a/src/agentscope/tts/__init__.py+++ b/src/agentscope/tts/__init__.py@@ -7,6 +7,10 @@ from ._dashscope_tts_model import DashScopeTTSModelfrom ._dashscope_realtime_tts_model import DashScopeRealtimeTTSModelfrom ._gemini_tts_model import GeminiTTSModelfrom ._openai_tts_model import OpenAITTSModel+from ._dashscope_cosyvoice_tts_model import DashScopeCosyVoiceTTSModel+from ._dashscope_cosyvoice_realtime_tts_model import (+ DashScopeCosyVoiceRealtimeTTSModel,+)__all__ = ["TTSModelBase",@@ -16,4 +20,6 @@ __all__ = ["DashScopeRealtimeTTSModel","GeminiTTSModel","OpenAITTSModel",+ "DashScopeCosyVoiceTTSModel",+ "DashScopeCosyVoiceRealtimeTTSModel",]diff --git a/src/agentscope/tts/_dashscope_cosyvoice_realtime_tts_model.py b/src/agentscope/tts/_dashscope_cosyvoice_realtime_tts_model.pynew file mode 100644index 000000000..5b80f464d--- /dev/null+++ b/src/agentscope/tts/_dashscope_cosyvoice_realtime_tts_model.py@@ -0,0 +1,279 @@+# -*- coding: utf-8 -*-+"""DashScope CosyVoice Realtime TTS model implementation."""++from typing import Any, Literal, AsyncGenerator++from ._tts_base import TTSModelBase+from ._tts_response import TTSResponse+from ._utils import _get_cosyvoice_callback_class+from ..message import Msg+from ..types import JSONSerializableObject+++class DashScopeCosyVoiceRealtimeTTSModel(TTSModelBase):+ """TTS implementation for DashScope CosyVoice Realtime TTS API,+ which supports streaming input. The supported models include+ "cosyvoice-v3-plus", "cosyvoice-v3-flash", "sambert" etc.++ For more details, please see the `official document+ <https://help.aliyun.com/zh/model-studio/text-to-speech>`_.++ .. note:: The DashScopeCosyVoiceRealtimeTTSModel can only handle one+ streaming input request at a time, and cannot process multiple+ streaming input requests concurrently. For example, it cannot handle+ input sequences like `[msg_1_chunk0, msg_1_chunk1, msg_2_chunk0]`,+ where the prefixes "msg_x" indicate different streaming input requests.+ """++ supports_streaming_input: bool = True+ """Whether the model supports streaming input."""++ def __init__(+ self,+ api_key: str,+ model_name: str = "cosyvoice-v3-plus",+ voice: Literal[+ "longanyang",+ "longanhuan",+ "longhuhu_v3",+ "longyingmu_v3",+ ]+ | str = "longanyang",+ stream: bool = True,+ cold_start_length: int | None = None,+ cold_start_words: int | None = None,+ client_kwargs: dict[str, JSONSerializableObject] | None = None,+ generate_kwargs: dict[str, JSONSerializableObject] | None = None,+ max_retries: int = 3,+ retry_delay: float = 5.0,+ ) -> None:+ """Initialize the DashScope CosyVoice Realtime TTS model by+ specifying the model, voice, and other parameters.++ .. note:: More details about the parameters, such as `model_name`,+ `voice`, and `mode` can be found in the `official document+ <https://help.aliyun.com/zh/model-studio/cosyvoice-voice-list>`_.++ .. note:: You can use `cold_start_length` and `cold_start_words`+ simultaneously to set both character and word thresholds for the first+ TTS request. For Chinese text, word segmentation (based on spaces) may+ not be effective.++ Args:+ api_key (`str`):+ The DashScope API key.+ model_name (`str`, defaults to "cosyvoice-v3-plus"):+ The TTS model name, e.g. "cosyvoice-v3-plus",+ "cosyvoice-v3-flash", etc.+ voice (`Literal["longanyang", "longanhuan", "longhuhu_v3", \+ "longyingmu_v3"] | str`, defaults to "longanyang".):+ The voice to use for synthesis. Refer to `official document+ <https://help.aliyun.com/zh/model-studio/cosyvoice-voice-list>`_+ for the supported voices for each model.+ stream (`bool`, defaults to `True`):+ Whether to use streaming synthesis.+ cold_start_length (`int | None`, optional):+ The minimum length send threshold for the first TTS request,+ ensuring there is no pause in the synthesized speech for too+ short input text. The length is measured in number of+ characters.+ cold_start_words (`int | None`, optional):+ The minimum words send threshold for the first TTS request,+ ensuring there is no pause in the synthesized speech for too+ short input text. The words are identified by spaces in the+ text.+ client_kwargs (`dict[str, JSONSerializableObject] | None`, \+ optional):+ The extra keyword arguments to initialize the DashScope+ CosyVoice Realtime tts client.+ generate_kwargs (`dict[str, JSONSerializableObject] | None`, \+ optional):+ The extra keyword arguments used in DashScope CosyVoice+ Realtime tts API generation.+ max_retries (`int`, defaults to 3):+ The maximum number of retry attempts when TTS synthesis fails.+ retry_delay (`float`, defaults to 5.0):+ The delay in seconds before retrying. Uses exponential backoff.+ """+ super().__init__(model_name=model_name, stream=stream)++ import dashscope+ from dashscope.audio.tts_v2 import SpeechSynthesizer++ dashscope.api_key = api_key++ # Store configuration+ self.voice = voice+ self.cold_start_length = cold_start_length+ self.cold_start_words = cold_start_words+ self.client_kwargs = client_kwargs or {}+ self.generate_kwargs = generate_kwargs or {}+ self.max_retries = max_retries+ self.retry_delay = retry_delay++ # Initialize TTS client+ # Save callback reference (for DashScope SDK)+ self._dashscope_callback = _get_cosyvoice_callback_class()()++ # The variables for tracking streaming input messages+ # If we have sent text for the current message+ self._first_send: bool = True+ # The current message ID being processed+ self._current_msg_id: str | None = None+ # The current prefix text already sent+ self._current_prefix: str = ""+ self._synthesizer: SpeechSynthesizer | None = None++ async def connect(self) -> None:+ """Connect to the TTS model and initialize resources."""+ from dashscope.audio.tts_v2 import SpeechSynthesizer, AudioFormat++ self._synthesizer = SpeechSynthesizer(+ model=self.model_name,+ voice=self.voice,+ format=AudioFormat.PCM_24000HZ_MONO_16BIT,+ callback=self._dashscope_callback,+ **self.client_kwargs,+ **self.generate_kwargs,+ )++ async def close(self) -> None:+ """Close the TTS model and release resources."""+ self._synthesizer.close()++ async def push(+ self,+ msg: Msg,+ **kwargs: Any,+ ) -> TTSResponse:+ """Append text to be synthesized and return the received TTS response.+ Note this method is non-blocking, and maybe return an empty response+ if no audio is received yet.++ To receive all the synthesized speech, call the `synthesize` method+ after pushing all the text chunks.++ Args:+ msg (`Msg`):+ The message to be synthesized. The `msg.id` identifies the+ streaming input request.+ **kwargs (`Any`):+ Additional keyword arguments to pass to the TTS API call.++ Returns:+ `TTSResponse`:+ The TTSResponse containing audio blocks.+ """++ if self._current_msg_id is not None and self._current_msg_id != msg.id:+ raise RuntimeError(+ "DashScopeCosyVoiceRealtimeTTSModel can only handle one "+ "streaming input request at a time. Please ensure that all "+ "chunks belong to the same message ID.",+ )++ # Record current message ID+ self._current_msg_id = msg.id++ text = msg.get_text_content()++ # Determine if we should send text based on cold start settings only+ # for the first input chunk and not the last chunk+ if text:+ if self._first_send:+ # If we have cold start settings+ if self.cold_start_length:+ if len(text) < self.cold_start_length:+ delta_to_send = ""+ else:+ delta_to_send = text+ else:+ delta_to_send = text++ if delta_to_send and self.cold_start_words:+ if len(delta_to_send.split()) < self.cold_start_words:+ delta_to_send = ""+ else:+ # Remove the already sent prefix if not the first send+ delta_to_send = text.removeprefix(self._current_prefix)++ if delta_to_send:+ self._synthesizer.streaming_call(delta_to_send)++ # Record sent prefix+ self._current_prefix += delta_to_send+ self._first_send = False++ # Wait for the audio data to be available+ res = await self._dashscope_callback.get_audio_data(block=False)++ return res++ # Return empty response if no text to send+ return TTSResponse(content=None)++ async def synthesize(+ self,+ msg: Msg | None = None,+ **kwargs: Any,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Append text to be synthesized and return TTS response.++ Args:+ msg (`Msg | None`, optional):+ The message to be synthesized.+ **kwargs (`Any`):+ Additional keyword arguments to pass to the TTS API call.++ Returns:+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:+ The TTSResponse object in non-streaming mode, or an async+ generator yielding TTSResponse objects in streaming mode.+ """+ if (+ self._current_msg_id is not None+ and msg+ and self._current_msg_id != msg.id+ ):+ raise RuntimeError(+ "DashScopeCosyVoiceRealtimeTTSModel can only handle one "+ "streaming input request at a time. Please ensure that all "+ "chunks belong to the same message ID.",+ )++ if msg is None:+ delta_to_send = ""++ else:+ # Record current message ID+ self._current_msg_id = msg.id+ delta_to_send = (msg.get_text_content() or "").removeprefix(+ self._current_prefix,+ )++ # Determine if we should send text based on cold start settings only+ # for the first input chunk and not the last chunk+ if delta_to_send:+ self._synthesizer.streaming_call(delta_to_send)++ # To keep correct prefix tracking+ self._current_prefix += delta_to_send+ self._first_send = False++ # We need to block until synthesis is complete to get all audio+ self._synthesizer.streaming_complete()++ if self.stream:+ # Return an async generator for audio chunks+ res = self._dashscope_callback.get_audio_chunk()++ else:+ # Block and wait for all audio data to be available+ res = await self._dashscope_callback.get_audio_data(block=True)++ # Update state for next message+ self._current_msg_id = None+ self._first_send = True+ self._current_prefix = ""++ return resdiff --git a/src/agentscope/tts/_dashscope_cosyvoice_tts_model.py b/src/agentscope/tts/_dashscope_cosyvoice_tts_model.pynew file mode 100644index 000000000..bf2085859--- /dev/null+++ b/src/agentscope/tts/_dashscope_cosyvoice_tts_model.py@@ -0,0 +1,165 @@+# -*- coding: utf-8 -*-+"""DashScope CosyVoice TTS model implementation."""++import base64+from typing import Any, Literal, AsyncGenerator++from ._tts_base import TTSModelBase+from ._tts_response import TTSResponse+from ._utils import _get_cosyvoice_callback_class+from ..message import Msg, AudioBlock, Base64Source+from ..types import JSONSerializableObject+++class DashScopeCosyVoiceTTSModel(TTSModelBase):+ """TTS implementation for DashScope CosyVoice TTS API.+ The supported models include "cosyvoice-v3-plus",+ "cosyvoice-v3-flash", "sambert" etc.++ This model does NOT support streaming text input. For streaming input,+ use `DashScopeCosyVoiceRealtimeTTSModel` instead.++ For more details, please see the `official document+ <https://help.aliyun.com/zh/model-studio/text-to-speech>`_.+ """++ supports_streaming_input: bool = False+ """Whether the model supports streaming input."""++ def __init__(+ self,+ api_key: str,+ model_name: str = "cosyvoice-v3-plus",+ voice: Literal[+ "longanyang",+ "longanhuan",+ "longhuhu_v3",+ "longyingmu_v3",+ ]+ | str = "longanyang",+ stream: bool = False,+ client_kwargs: dict[str, JSONSerializableObject] | None = None,+ generate_kwargs: dict[str, JSONSerializableObject] | None = None,+ ) -> None:+ """Initialize the DashScope CosyVoice TTS model by+ specifying the model, voice, and other parameters.++ .. note:: More details about the parameters, such as `model_name`,+ `voice`, and `mode` can be found in the `official document+ <https://help.aliyun.com/zh/model-studio/cosyvoice-voice-list>`_.++ Args:+ api_key (`str`):+ The DashScope API key.+ model_name (`str`, defaults to "cosyvoice-v3-plus"):+ The TTS model name, e.g. "cosyvoice-v3-plus",+ "cosyvoice-v3-flash", etc.+ voice (`Literal["longanyang", "longanhuan", "longhuhu_v3", \+ "longyingmu_v3"] | str`, defaults to "longanyang".):+ The voice to use for synthesis. Refer to `official document+ <https://help.aliyun.com/zh/model-studio/cosyvoice-voice-list>`_+ for the supported voices for each model.+ stream (`bool`, defaults to `False`):+ Whether to use streaming audio output.+ client_kwargs (`dict[str, JSONSerializableObject] | None`, \+ optional):+ The extra keyword arguments to initialize the DashScope+ CosyVoice tts client.+ generate_kwargs (`dict[str, JSONSerializableObject] | None`, \+ optional):+ The extra keyword arguments used in DashScope CosyVoice+ tts API generation.+ """+ super().__init__(model_name=model_name, stream=stream)++ import dashscope++ dashscope.api_key = api_key++ # Store configuration+ self.voice = voice+ self.client_kwargs = client_kwargs or {}+ self.generate_kwargs = generate_kwargs or {}++ def _create_synthesizer(self) -> tuple:+ """Create a new SpeechSynthesizer instance for each request."""+ from dashscope.audio.tts_v2 import SpeechSynthesizer, AudioFormat++ callback = _get_cosyvoice_callback_class()() if self.stream else None++ synthesizer = SpeechSynthesizer(+ model=self.model_name,+ voice=self.voice,+ format=AudioFormat.PCM_24000HZ_MONO_16BIT,+ callback=callback,+ **self.client_kwargs,+ **self.generate_kwargs,+ )+ return synthesizer, callback++ async def push(self, msg: Msg, **kwargs: Any) -> TTSResponse:+ """Push a message to the TTS model and return TTS response.++ Args:+ msg (`Msg`):+ The message to be synthesized.+ **kwargs (`Any`):+ Additional keyword arguments to pass to the TTS API call.++ Returns:+ `TTSResponse`:+ The TTSResponse object.+ """+ return TTSResponse(content=None)++ async def synthesize(+ self,+ msg: Msg | None = None,+ **kwargs: Any,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Synthesize text to speech and return TTS response.++ Args:+ msg (`Msg | None`, optional):+ The message to be synthesized.+ **kwargs (`Any`):+ Additional keyword arguments to pass to the TTS API call.++ Returns:+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:+ The TTSResponse object in non-streaming mode, or an async+ generator yielding TTSResponse objects in streaming mode.+ """+ if msg is None:+ return TTSResponse(content=None)++ text = msg.get_text_content()+ if not text:+ return TTSResponse(content=None)++ # Create a new synthesizer for each request to avoid connection issues+ synthesizer, callback = self._create_synthesizer()++ if self.stream:+ # Streaming output mode: use callback to get audio chunks+ synthesizer.call(text=text)+ return callback.get_audio_chunk()+ else:+ # Non-streaming mode: call directly returns audio bytes+ audio = synthesizer.call(text=text)++ if not audio:+ return TTSResponse(content=None)++ encoded_data = base64.b64encode(audio).decode()++ return TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=encoded_data,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ )diff --git a/src/agentscope/tts/_utils.py b/src/agentscope/tts/_utils.pynew file mode 100644index 000000000..90540eaad--- /dev/null+++ b/src/agentscope/tts/_utils.py@@ -0,0 +1,197 @@+# -*- coding: utf-8 -*-+"""Utility classes for DashScope CosyVoice TTS models."""++import base64+import threading+from typing import TYPE_CHECKING, AsyncGenerator, Any++from ._tts_response import TTSResponse+from .._logging import logger+from ..message import AudioBlock, Base64Source++if TYPE_CHECKING:+ from dashscope.audio.tts_v2 import ResultCallback+else:+ ResultCallback = "dashscope.audio.tts_v2.ResultCallback"+++def _get_cosyvoice_callback_class() -> type["ResultCallback"]:+ """Get the callback class for CosyVoice TTS streaming audio output.++ This callback handles audio data accumulation with proper PCM and base64+ alignment. It encodes audio in chunks of 6 bytes (LCM of 2 and 3) to+ ensure both PCM alignment (2 bytes per sample) and base64 alignment+ (3 bytes per encoded unit).++ Returns:+ The callback class for CosyVoice TTS.+ """+ from dashscope.audio.tts_v2 import ResultCallback++ class _CosyVoiceTTSCallback(ResultCallback):+ """CosyVoice TTS callback for streaming audio output."""++ def __init__(self) -> None:+ """Initialize the CosyVoice TTS callback."""+ super().__init__()++ # The event that will be set when a new audio chunk is received+ self.chunk_event = threading.Event()+ # The event that will be set when the TTS synthesis is finished+ self.finish_event = threading.Event()+ # Accumulate raw bytes+ self._audio_bytes: bytes = b""+ # Accumulated base64 string (boundary-aligned)+ self._audio_base64: str = ""+ # Last encoded byte position (must be multiple of 6 for alignment)+ self._last_encoded_pos: int = 0++ def on_open(self) -> None:+ """Called when the WebSocket connection is opened."""+ self._audio_bytes = b""+ self._audio_base64 = ""+ self._last_encoded_pos = 0+ self.finish_event.clear()++ def on_data(self, data: bytes) -> None:+ """Called when data is received from the WebSocket connection.++ Args:+ data (`bytes`):+ The data received from the WebSocket connection.+ """+ if data:+ self._audio_bytes += data+ # Encode in chunks of 6 bytes (LCM of 2 and 3)+ # This ensures both PCM alignment (2 bytes) and+ # base64 alignment (3 bytes)+ aligned_len = (len(self._audio_bytes) // 6) * 6+ if aligned_len > self._last_encoded_pos:+ new_chunk = self._audio_bytes[+ self._last_encoded_pos : aligned_len+ ]+ self._audio_base64 += base64.b64encode(new_chunk).decode()+ self._last_encoded_pos = aligned_len++ # Signal that a new audio chunk is available+ if not self.chunk_event.is_set():+ self.chunk_event.set()++ def on_close(self) -> None:+ """Called when the WebSocket connection is closed."""+ # Encode any remaining bytes+ if len(self._audio_bytes) > self._last_encoded_pos:+ remaining = self._audio_bytes[self._last_encoded_pos :]+ self._audio_base64 += base64.b64encode(remaining).decode()+ self._last_encoded_pos = len(self._audio_bytes)++ # Unblock waiting operations to prevent deadlock+ self.finish_event.set()+ self.chunk_event.set()++ def on_error(self, message: Any) -> None:+ """Called when an error occurs."""+ logger.error(message)++ # Unblock waiting operations to prevent deadlock+ self.finish_event.set()+ self.chunk_event.set()++ async def get_audio_data(self, block: bool = True) -> TTSResponse:+ """Get the current accumulated audio data as base64 string.++ Args:+ block (`bool`, defaults to `True`):+ Whether to block until synthesis is finished.++ Returns:+ `TTSResponse`:+ The TTSResponse containing base64-encoded audio data.+ """+ # Block until synthesis is finished+ if block:+ self.finish_event.wait()++ # Return the accumulated audio data+ if self._audio_base64:+ return TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self._audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ )++ # Reset for next tts request+ await self._reset()++ # Return empty response if no audio data+ return TTSResponse(content=None)++ async def get_audio_chunk(self) -> AsyncGenerator[TTSResponse, None]:+ """Get the audio data chunk as an async generator of TTSResponse.++ Returns accumulated base64 string. The agent code uses string+ slicing to get the delta, so we must ensure boundary alignment.++ Returns:+ `AsyncGenerator[TTSResponse, None]`:+ The async generator yielding TTSResponse with audio chunks.+ """+ while True:+ if self.finish_event.is_set():+ # Yield final chunk with all accumulated data+ yield TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self._audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ is_last=True,+ )++ # Reset for next tts request+ await self._reset()++ break++ if self.chunk_event.is_set():+ # Clear the event for next chunk+ self.chunk_event.clear()+ else:+ # Wait for the next chunk+ self.chunk_event.wait()++ # Yield current accumulated data+ if self._audio_base64:+ yield TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self._audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ is_last=False,+ )++ async def _reset(self) -> None:+ """Reset the callback state for a new TTS request."""+ self.finish_event.clear()+ self.chunk_event.clear()+ self._audio_bytes = b""+ self._audio_base64 = ""+ self._last_encoded_pos = 0++ def has_audio_data(self) -> bool:+ """Check if audio data has been received."""+ return bool(self._audio_bytes)++ return _CosyVoiceTTSCallbackdiff --git a/tests/tts_dashscope_cosyvoice_test.py b/tests/tts_dashscope_cosyvoice_test.pynew file mode 100644index 000000000..93f98e8b6--- /dev/null+++ b/tests/tts_dashscope_cosyvoice_test.py@@ -0,0 +1,402 @@+# -*- coding: utf-8 -*-+# pylint: disable=protected-access+"""The unittests for DashScope CosyVoice TTS models."""+import base64+from typing import AsyncGenerator+from unittest import IsolatedAsyncioTestCase+from unittest.mock import Mock, patch, AsyncMock, MagicMock++from agentscope.message import Msg, AudioBlock, Base64Source+from agentscope.tts import (+ DashScopeCosyVoiceRealtimeTTSModel,+ DashScopeCosyVoiceTTSModel,+ TTSResponse,+)+++class DashScopeCosyVoiceTTSModelTest(IsolatedAsyncioTestCase):+ """The unittests for DashScope CosyVoice TTS model (non-realtime)."""++ def setUp(self) -> None:+ """Set up the test case."""+ self.api_key = "test_api_key"+ self.mock_audio_bytes = b"fake_audio_data_bytes"+ self.mock_audio_base64 = base64.b64encode(+ self.mock_audio_bytes,+ ).decode("utf-8")++ def _create_mock_dashscope_modules(self) -> dict:+ """Create mock dashscope modules for patching."""+ mock_audio_format = MagicMock()+ mock_audio_format.PCM_24000HZ_MONO_16BIT = "pcm_24000hz_mono_16bit"++ mock_speech_synthesizer = MagicMock()++ mock_tts_v2 = MagicMock()+ mock_tts_v2.SpeechSynthesizer = mock_speech_synthesizer+ mock_tts_v2.AudioFormat = mock_audio_format+ mock_tts_v2.ResultCallback = Mock++ mock_audio = MagicMock()+ mock_audio.tts_v2 = mock_tts_v2++ mock_dashscope = MagicMock()+ mock_dashscope.api_key = None+ mock_dashscope.audio = mock_audio++ return {+ "dashscope": mock_dashscope,+ "dashscope.audio": mock_audio,+ "dashscope.audio.tts_v2": mock_tts_v2,+ }++ def test_init(self) -> None:+ """Test initialization of DashScopeCosyVoiceTTSModel."""+ mock_modules = self._create_mock_dashscope_modules()++ with patch.dict("sys.modules", mock_modules):+ model = DashScopeCosyVoiceTTSModel(+ api_key=self.api_key,+ model_name="cosyvoice-v3-plus",+ voice="longanyang",+ stream=False,+ )+ self.assertEqual(model.model_name, "cosyvoice-v3-plus")+ self.assertEqual(model.voice, "longanyang")+ self.assertFalse(model.stream)+ self.assertFalse(model.supports_streaming_input)++ async def test_synthesize_non_streaming(self) -> None:+ """Test synthesize method in non-streaming mode."""+ mock_modules = self._create_mock_dashscope_modules()++ with patch.dict("sys.modules", mock_modules):+ model = DashScopeCosyVoiceTTSModel(+ api_key=self.api_key,+ stream=False,+ )++ # Mock _create_synthesizer to return a mock synthesizer+ mock_synthesizer = MagicMock()+ mock_synthesizer.call.return_value = self.mock_audio_bytes+ model._create_synthesizer = Mock(+ return_value=(mock_synthesizer, None),+ )++ msg = Msg(name="user", content="Hello! Test message.", role="user")+ response = await model.synthesize(msg)++ self.assertIsInstance(response, TTSResponse)+ self.assertEqual(response.content["type"], "audio")+ self.assertEqual(+ response.content["source"]["data"],+ self.mock_audio_base64,+ )+ mock_synthesizer.call.assert_called_once_with(+ text="Hello! Test message.",+ )++ async def test_synthesize_streaming(self) -> None:+ """Test synthesize method in streaming mode."""+ mock_modules = self._create_mock_dashscope_modules()++ with patch.dict("sys.modules", mock_modules):+ model = DashScopeCosyVoiceTTSModel(+ api_key=self.api_key,+ stream=True,+ )++ # Create mock callback with proper async generator+ mock_callback = MagicMock()++ async def mock_generator() -> AsyncGenerator[TTSResponse, None]:+ yield TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self.mock_audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ is_last=False,+ )+ yield TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self.mock_audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ is_last=True,+ )++ # Directly return the generator object+ mock_callback.get_audio_chunk = Mock(return_value=mock_generator())++ mock_synthesizer = MagicMock()+ model._create_synthesizer = Mock(+ return_value=(mock_synthesizer, mock_callback),+ )++ msg = Msg(name="user", content="Test streaming.", role="user")+ response = await model.synthesize(msg)++ # Verify response is an async generator+ self.assertIsInstance(response, AsyncGenerator)+ chunks = [chunk async for chunk in response]++ # Verify we got some chunks+ self.assertGreater(len(chunks), 0)+ # Verify each chunk is a TTSResponse+ for chunk in chunks:+ self.assertIsInstance(chunk, TTSResponse)+ mock_synthesizer.call.assert_called_once_with(+ text="Test streaming.",+ )+++class DashScopeCosyVoiceRealtimeTTSModelTest(IsolatedAsyncioTestCase):+ """The unittests for DashScope CosyVoice Realtime TTS model."""++ def setUp(self) -> None:+ """Set up the test case."""+ self.api_key = "test_api_key"+ self.mock_audio_bytes = b"fake_audio_data_bytes"+ self.mock_audio_base64 = base64.b64encode(+ self.mock_audio_bytes,+ ).decode("utf-8")++ def _create_mock_dashscope_modules(self) -> dict:+ """Create mock dashscope modules for patching."""+ mock_audio_format = MagicMock()+ mock_audio_format.PCM_24000HZ_MONO_16BIT = "pcm_24000hz_mono_16bit"++ mock_speech_synthesizer = MagicMock()++ mock_tts_v2 = MagicMock()+ mock_tts_v2.SpeechSynthesizer = mock_speech_synthesizer+ mock_tts_v2.AudioFormat = mock_audio_format+ mock_tts_v2.ResultCallback = Mock++ mock_audio = MagicMock()+ mock_audio.tts_v2 = mock_tts_v2++ mock_dashscope = MagicMock()+ mock_dashscope.api_key = None+ mock_dashscope.audio = mock_audio++ return {+ "dashscope": mock_dashscope,+ "dashscope.audio": mock_audio,+ "dashscope.audio.tts_v2": mock_tts_v2,+ }++ def _create_mock_callback(self) -> MagicMock:+ """Create a mock callback for testing."""+ mock_callback = MagicMock()+ mock_callback.chunk_event = MagicMock()+ mock_callback.finish_event = MagicMock()+ mock_callback._audio_bytes = b""+ mock_callback._audio_base64 = ""+ mock_callback._last_encoded_pos = 0+ return mock_callback++ def test_init(self) -> None:+ """Test initialization of DashScopeCosyVoiceRealtimeTTSModel."""+ mock_modules = self._create_mock_dashscope_modules()++ with patch.dict("sys.modules", mock_modules):+ # Mock _get_cosyvoice_callback_class+ with patch(+ "agentscope.tts._dashscope_cosyvoice_realtime_tts_model"+ "._get_cosyvoice_callback_class",+ ) as mock_get_callback:+ mock_callback_class = MagicMock()+ mock_callback_class.return_value = self._create_mock_callback()+ mock_get_callback.return_value = mock_callback_class++ model = DashScopeCosyVoiceRealtimeTTSModel(+ api_key=self.api_key,+ model_name="cosyvoice-v3-plus",+ voice="longanyang",+ stream=True,+ )+ self.assertEqual(model.model_name, "cosyvoice-v3-plus")+ self.assertEqual(model.voice, "longanyang")+ self.assertTrue(model.stream)+ self.assertTrue(model.supports_streaming_input)++ async def test_push_incremental_text(self) -> None:+ """Test push method with incremental text chunks."""+ mock_modules = self._create_mock_dashscope_modules()+ mock_synthesizer_instance = MagicMock()+ mock_modules[+ "dashscope.audio.tts_v2"+ ].SpeechSynthesizer.return_value = mock_synthesizer_instance++ with patch.dict("sys.modules", mock_modules):+ with patch(+ "agentscope.tts._dashscope_cosyvoice_realtime_tts_model"+ "._get_cosyvoice_callback_class",+ ) as mock_get_callback:+ mock_callback_class = MagicMock()+ mock_callback = self._create_mock_callback()+ mock_callback.get_audio_data = AsyncMock(+ return_value=TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self.mock_audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ ),+ )+ mock_callback_class.return_value = mock_callback+ mock_get_callback.return_value = mock_callback_class++ model = DashScopeCosyVoiceRealtimeTTSModel(+ api_key=self.api_key,+ )+ await model.connect()++ msg_id = "test_msg_001"+ text_chunks = ["Hello there!\n\n", "This is a test message."]++ accumulated_text = ""+ for chunk in text_chunks:+ accumulated_text += chunk+ msg = Msg(+ name="user",+ content=accumulated_text,+ role="user",+ )+ msg.id = msg_id++ response = await model.push(msg)+ self.assertIsInstance(response, TTSResponse)++ # Verify streaming_call was called+ self.assertGreater(+ mock_synthesizer_instance.streaming_call.call_count,+ 0,+ )++ async def test_synthesize_non_streaming(self) -> None:+ """Test synthesize method in non-streaming mode."""+ mock_modules = self._create_mock_dashscope_modules()+ mock_synthesizer_instance = MagicMock()+ mock_modules[+ "dashscope.audio.tts_v2"+ ].SpeechSynthesizer.return_value = mock_synthesizer_instance++ with patch.dict("sys.modules", mock_modules):+ with patch(+ "agentscope.tts._dashscope_cosyvoice_realtime_tts_model"+ "._get_cosyvoice_callback_class",+ ) as mock_get_callback:+ mock_callback_class = MagicMock()+ mock_callback = self._create_mock_callback()+ mock_callback.get_audio_data = AsyncMock(+ return_value=TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self.mock_audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ ),+ )+ mock_callback_class.return_value = mock_callback+ mock_get_callback.return_value = mock_callback_class++ model = DashScopeCosyVoiceRealtimeTTSModel(+ api_key=self.api_key,+ stream=False,+ )+ await model.connect()++ msg = Msg(+ name="user",+ content="Hello! Test message.",+ role="user",+ )+ response = await model.synthesize(msg)++ self.assertIsInstance(response, TTSResponse)+ self.assertEqual(response.content["type"], "audio")+ mock_synthesizer_instance.streaming_complete.assert_called_once() # noqa++ async def test_synthesize_streaming(self) -> None:+ """Test synthesize method in streaming mode."""+ mock_modules = self._create_mock_dashscope_modules()+ mock_synthesizer_instance = MagicMock()+ mock_modules[+ "dashscope.audio.tts_v2"+ ].SpeechSynthesizer.return_value = mock_synthesizer_instance++ with patch.dict("sys.modules", mock_modules):+ with patch(+ "agentscope.tts._dashscope_cosyvoice_realtime_tts_model"+ "._get_cosyvoice_callback_class",+ ) as mock_get_callback:+ mock_callback_class = MagicMock()+ mock_callback = self._create_mock_callback()++ async def mock_generator() -> AsyncGenerator[+ TTSResponse,+ None,+ ]:+ yield TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self.mock_audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ is_last=False,+ )+ yield TTSResponse(+ content=AudioBlock(+ type="audio",+ source=Base64Source(+ type="base64",+ data=self.mock_audio_base64,+ media_type="audio/pcm;rate=24000",+ ),+ ),+ is_last=True,+ )++ mock_callback.get_audio_chunk = Mock(+ return_value=mock_generator(),+ )+ mock_callback_class.return_value = mock_callback+ mock_get_callback.return_value = mock_callback_class++ model = DashScopeCosyVoiceRealtimeTTSModel(+ api_key=self.api_key,+ stream=True,+ )+ await model.connect()++ msg = Msg(name="user", content="Test streaming.", role="user")+ response = await model.synthesize(msg)++ # Verify response is an async generator+ self.assertIsInstance(response, AsyncGenerator)+ chunks = [chunk async for chunk in response]++ # Verify we got some chunks+ self.assertGreater(len(chunks), 0)+ # Verify each chunk is a TTSResponse+ for chunk in chunks:+ self.assertIsInstance(chunk, TTSResponse)