Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(tts): add DashScope CosyVoice TTS model (#1866)
YingchaoXb6698c5dbaa1
diff --git a/pyproject.toml b/pyproject.tomlindex daac05461..b431d424a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -146,7 +146,11 @@ include-package-data = true# subdirs that ship with each model provider (they have no# ``__init__.py`` and are not packages, but the files still# live alongside the parent package's source on disk).-"*" = ["py.typed", "_models/*.yaml", "_cosyvoice_models/*.yaml"]+"*" = [+ "py.typed",+ "_models/*.yaml",+ "_cosyvoice_models/*.yaml",+]"agentscope.workspace._docker" = ["Dockerfile*.template"][build-system]diff --git a/src/agentscope/credential/_dashscope.py b/src/agentscope/credential/_dashscope.pyindex 0eb41bcae..11e6ab59f 100644--- a/src/agentscope/credential/_dashscope.py+++ b/src/agentscope/credential/_dashscope.py@@ -48,7 +48,7 @@ class DashScopeCredential(CredentialBase):def get_tts_model_classes(cls) -> list[Type["TTSModelBase"]]:"""Return the DashScope TTS model classes."""from ..tts import (- DashScopeCosyVoiceRealtimeTTSModel,+ DashScopeCosyVoiceTTSModel,DashScopeRealtimeTTSModel,DashScopeTTSModel,)@@ -56,7 +56,7 @@ class DashScopeCredential(CredentialBase):return [DashScopeTTSModel,DashScopeRealtimeTTSModel,- DashScopeCosyVoiceRealtimeTTSModel,+ DashScopeCosyVoiceTTSModel,]@classmethoddiff --git a/src/agentscope/tts/__init__.py b/src/agentscope/tts/__init__.pyindex 4448f5d0e..7a3e4ded0 100644--- a/src/agentscope/tts/__init__.py+++ b/src/agentscope/tts/__init__.py@@ -5,9 +5,9 @@ from ._tts_base import TTSModelBasefrom ._tts_model_card import TTSModelCardfrom ._tts_response import TTSResponse, TTSUsagefrom ._dashscope import (+ DashScopeCosyVoiceTTSModel,DashScopeTTSModel,DashScopeRealtimeTTSModel,- DashScopeCosyVoiceRealtimeTTSModel,)__all__ = [@@ -15,7 +15,7 @@ __all__ = ["TTSModelCard","TTSResponse","TTSUsage",+ "DashScopeCosyVoiceTTSModel","DashScopeTTSModel","DashScopeRealtimeTTSModel",- "DashScopeCosyVoiceRealtimeTTSModel",]diff --git a/src/agentscope/tts/_dashscope/__init__.py b/src/agentscope/tts/_dashscope/__init__.pyindex b34b47880..03599ae96 100644--- a/src/agentscope/tts/_dashscope/__init__.py+++ b/src/agentscope/tts/_dashscope/__init__.py@@ -1,12 +1,12 @@# -*- coding: utf-8 -*-"""The DashScope TTS module."""+from ._cosyvoice_model import DashScopeCosyVoiceTTSModelfrom ._model import DashScopeTTSModelfrom ._realtime_model import DashScopeRealtimeTTSModel-from ._cosyvoice_realtime_model import DashScopeCosyVoiceRealtimeTTSModel__all__ = [+ "DashScopeCosyVoiceTTSModel","DashScopeTTSModel","DashScopeRealtimeTTSModel",- "DashScopeCosyVoiceRealtimeTTSModel",]diff --git a/src/agentscope/tts/_dashscope/_cosyvoice_model.py b/src/agentscope/tts/_dashscope/_cosyvoice_model.pynew file mode 100644index 000000000..2b1207107--- /dev/null+++ b/src/agentscope/tts/_dashscope/_cosyvoice_model.py@@ -0,0 +1,397 @@+# -*- coding: utf-8 -*-+"""DashScope CosyVoice TTS model implementation."""+import asyncio+import base64+import io+import os+from typing import Any, AsyncGenerator, Literal, TYPE_CHECKING+import wave++from pydantic import BaseModel, Field++from .._tts_base import TTSModelBase+from .._tts_model_card import TTSModelCard+from .._tts_response import TTSResponse+from ._cosyvoice_utils import (+ _BITS_PER_SAMPLE,+ _CHANNELS,+ _make_cosyvoice_callback_class,+ _MEDIA_TYPE,+ _SAMPLE_RATE,+)+from ..._logging import logger+from ...credential import DashScopeCredential+from ...message import DataBlock, Base64Source++if TYPE_CHECKING:+ from dashscope.audio.tts_v2 import SpeechSynthesizer, ResultCallback+++class DashScopeCosyVoiceTTSModel(TTSModelBase):+ """DashScope CosyVoice TTS model using the WebSocket SpeechSynthesizer."""++ class Parameters(BaseModel):+ """Frontend-exposed parameters for DashScope CosyVoice TTS models."""++ voice: str = Field(+ default="longanhuan",+ title="Voice",+ description="The voice to use for synthesis.",+ )+ realtime: bool = Field(+ default=False,+ title="Realtime",+ description="Whether to enable streaming input mode.",+ )++ type: Literal["dashscope_cosyvoice_tts"] = "dashscope_cosyvoice_tts"+ """The type of the TTS model."""++ _MODELS_DIR = os.path.join(os.path.dirname(__file__), "_cosyvoice_models")++ def __init__(+ self,+ credential: DashScopeCredential,+ model: str = "cosyvoice-v3-flash",+ parameters: "DashScopeCosyVoiceTTSModel.Parameters | None" = None,+ stream: bool = True,+ cold_start_length: int | None = None,+ cold_start_words: int | None = None,+ max_retries: int = 3,+ retry_delay: float = 5.0,+ ) -> None:+ """Initialize the DashScope CosyVoice TTS model.++ Args:+ credential (`DashScopeCredential`):+ The DashScope credential used to authenticate the API call.+ model (`str`, defaults to ``"cosyvoice-v3-flash"``):+ The CosyVoice model name.+ parameters (`DashScopeCosyVoiceTTSModel.Parameters | None`, \+ defaults to `None`):+ The TTS parameters. When ``None``, the default voice is used.+ stream (`bool`, defaults to `True`):+ Whether :meth:`synthesize` returns an async generator yielding+ ``TTSResponse`` chunks. When ``False``, audio is aggregated.+ cold_start_length (`int | None`, defaults to `None`):+ Minimum character count before the first realtime text chunk is+ sent to the synthesizer.+ cold_start_words (`int | None`, defaults to `None`):+ Minimum word count before the first realtime text chunk is+ sent.+ max_retries (`int`, defaults to `3`):+ Max retry attempts on realtime synthesis failure.+ retry_delay (`float`, defaults to `5.0`):+ Initial retry delay in seconds (exponential backoff).+ """+ super().__init__(+ credential=credential,+ model=model,+ parameters=parameters,+ stream=stream,+ )+ self.realtime = self.parameters.realtime+ self.cold_start_length = cold_start_length+ self.cold_start_words = cold_start_words+ self.max_retries = max_retries+ self.retry_delay = retry_delay++ self._synthesizer: Any = None+ self._callback: Any = None+ self._connected = False+ self._cold_start_buffer: str = ""+ self._cold_start_done: bool = False+ self._accumulated_text: str = ""++ @classmethod+ def list_models(+ cls,+ custom_yaml_dir: str | None = None,+ ) -> list[TTSModelCard]:+ """List CosyVoice model cards from the dedicated card directory."""+ return super().list_models(+ custom_yaml_dir=custom_yaml_dir or cls._MODELS_DIR,+ )++ async def synthesize(+ self,+ text: str | None = None,+ **kwargs: Any,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Call the DashScope CosyVoice TTS API to synthesize speech.++ Args:+ text (`str | None`, optional):+ The text to be synthesized.+ **kwargs (`Any`):+ Reserved for API compatibility. Runtime synthesis options+ are configured when the model is initialized.++ Returns:+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:+ A single aggregated response when ``stream=False``, or an+ async generator yielding incremental chunks when+ ``stream=True``.+ """+ if self.realtime:+ return await self._synthesize_realtime(text)++ return self._synthesize_once(text)++ def _synthesize_once(+ self,+ text: str | None = None,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Synthesize text with the non-realtime one-shot SDK path."""+ if not text:+ return TTSResponse(content=None)++ synthesizer, callback = self._create_synthesizer(+ callback=self.stream,+ )++ if self.stream:+ assert callback is not None+ synthesizer.call(text=text)+ return callback.get_audio_chunks()++ audio_data = synthesizer.call(text=text)+ if not audio_data:+ return TTSResponse(content=None)+ return self._build_wav_response(audio_data)++ def _create_synthesizer(+ self,+ *,+ callback: bool,+ ) -> tuple["SpeechSynthesizer", "ResultCallback | None"]:+ """Create a fresh SpeechSynthesizer and optional callback."""+ import dashscope+ from dashscope.audio.tts_v2 import AudioFormat, SpeechSynthesizer++ dashscope.api_key = self.credential.api_key.get_secret_value()++ callback_instance = None+ if callback:+ callback_cls = _make_cosyvoice_callback_class()+ callback_instance = callback_cls()++ synthesizer = SpeechSynthesizer(+ model=self.model,+ voice=self.parameters.voice,+ format=AudioFormat.PCM_24000HZ_MONO_16BIT,+ callback=callback_instance,+ )+ return synthesizer, callback_instance++ async def connect(self) -> None:+ """Initialize the SpeechSynthesizer for realtime mode."""+ if self._connected:+ return+ self._synthesizer, self._callback = self._create_synthesizer(+ callback=True,+ )+ self._connected = True++ async def close(self) -> None:+ """Close the realtime SpeechSynthesizer."""+ if not self._connected:+ return+ self._connected = False+ try:+ if self._synthesizer is not None:+ self._synthesizer.close()+ except Exception:+ pass++ async def _reconnect(self) -> None:+ """Reconnect by recreating the realtime synthesizer."""+ try:+ if self._synthesizer is not None:+ self._synthesizer.close()+ except Exception:+ pass+ self._connected = False+ self._cold_start_buffer = ""+ self._cold_start_done = False+ await self.connect()++ async def push(+ self,+ text: str,+ **kwargs: Any,+ ) -> TTSResponse:+ """Push an incremental text delta for realtime synthesis."""+ del kwargs+ if not self.realtime:+ return await super().push(text)++ if not self._connected:+ raise RuntimeError(+ "TTS model is not connected. Call `connect()` first.",+ )++ if not text:+ return TTSResponse(content=None)++ self._accumulated_text += text++ if self._cold_start_done:+ text_to_send = text+ else:+ self._cold_start_buffer += text+ if (+ self.cold_start_length+ and len(self._cold_start_buffer) < self.cold_start_length+ ) or (+ self.cold_start_words+ and len(self._cold_start_buffer.split())+ < self.cold_start_words+ ):+ return self._callback.get_audio_response(block=False)+ text_to_send = self._cold_start_buffer+ self._cold_start_buffer = ""+ self._cold_start_done = True++ try:+ self._synthesizer.streaming_call(text_to_send)+ except Exception:+ return TTSResponse(content=None)++ return self._callback.get_audio_response(block=False)++ async def _synthesize_realtime(+ self,+ text: str | None = None,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Finalize synthesis for the current realtime utterance."""+ if not self._connected:+ raise RuntimeError(+ "TTS model is not connected. Call `connect()` first.",+ )++ if text is not None:+ self._accumulated_text += text++ unsent = self._cold_start_buffer+ if text is not None:+ unsent += text+ self._cold_start_buffer = ""++ full_text = self._accumulated_text+ delay = self.retry_delay++ try:+ if not full_text and not unsent:+ if self.stream:++ async def _empty_gen() -> AsyncGenerator[+ TTSResponse,+ None,+ ]:+ yield TTSResponse(content=None)++ return _empty_gen()+ return TTSResponse(content=None)++ for attempt in range(self.max_retries):+ try:+ if unsent:+ self._synthesizer.streaming_call(unsent)++ self._synthesizer.streaming_complete()++ finished = await asyncio.to_thread(+ self._callback.finish_event.wait,+ 30,+ )++ if not finished:+ logger.warning(+ "CosyVoice TTS: timed out waiting for synthesis "+ "completion (30s)",+ )+ if attempt < self.max_retries - 1:+ await asyncio.sleep(delay)+ await self._reconnect()+ unsent = full_text+ delay *= 2+ continue+ raise RuntimeError(+ "CosyVoice TTS synthesis timed out after 30s",+ )++ if full_text and not self._callback.has_audio_data():+ if attempt < self.max_retries - 1:+ logger.warning(+ "CosyVoice TTS: no audio received, retrying "+ "(%d/%d) in %.1fs...",+ attempt + 1,+ self.max_retries,+ delay,+ )+ await asyncio.sleep(delay)+ await self._reconnect()+ unsent = full_text+ delay *= 2+ continue+ raise RuntimeError(+ f"CosyVoice TTS synthesis failed: no audio after "+ f"{self.max_retries} attempts",+ )+ break++ except RuntimeError:+ raise+ except Exception as e:+ if attempt < self.max_retries - 1:+ logger.warning(+ "CosyVoice TTS error, retrying (%d/%d) in "+ "%.1fs: %s",+ attempt + 1,+ self.max_retries,+ delay,+ e,+ )+ await asyncio.sleep(delay)+ await self._reconnect()+ unsent = full_text+ delay *= 2+ else:+ raise++ if self.stream:+ return self._callback.get_audio_chunks()++ response = self._callback.get_audio_response(block=True)+ self._callback.reset()+ return response+ finally:+ self._reset_state()++ def _reset_state(self) -> None:+ """Reset per-utterance realtime tracking state."""+ self._cold_start_buffer = ""+ self._cold_start_done = False+ self._accumulated_text = ""++ @staticmethod+ def _build_wav_response(+ audio_data: bytes,+ ) -> TTSResponse:+ """Build a self-contained WAV response from PCM audio bytes."""+ buf = io.BytesIO()+ with wave.open(buf, "wb") as wav:+ wav.setnchannels(_CHANNELS)+ wav.setsampwidth(_BITS_PER_SAMPLE // 8)+ wav.setframerate(_SAMPLE_RATE)+ wav.writeframes(audio_data)++ return TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(buf.getvalue()).decode("ascii"),+ media_type=_MEDIA_TYPE,+ ),+ ),+ )diff --git a/src/agentscope/tts/_dashscope/_cosyvoice_models/cosyvoice-v3-flash.yaml b/src/agentscope/tts/_dashscope/_cosyvoice_models/cosyvoice-v3-flash.yamlindex 608cd198b..5810bfd1d 100644--- a/src/agentscope/tts/_dashscope/_cosyvoice_models/cosyvoice-v3-flash.yaml+++ b/src/agentscope/tts/_dashscope/_cosyvoice_models/cosyvoice-v3-flash.yaml@@ -1,7 +1,6 @@name: cosyvoice-v3-flashlabel: CosyVoice-v3-Flashstatus: active-realtime: trueinput_types:- text/plain@@ -15,11 +14,10 @@ voices:- longanhuan- longhuhu_v3- longyingmu_v3- - longxiaochun- - longxiaoxia- - longlaotie- - longshuo- - longjing- - longshu+ - longxiaochun_v3+ - longxiaoxia_v3+ - longlaotie_v3+ - longshuo_v3+ - longshu_v3parameter_overrides: {}diff --git a/src/agentscope/tts/_dashscope/_cosyvoice_models/cosyvoice-v3-plus.yaml b/src/agentscope/tts/_dashscope/_cosyvoice_models/cosyvoice-v3-plus.yamlindex d5f69d073..fa68a04b5 100644--- a/src/agentscope/tts/_dashscope/_cosyvoice_models/cosyvoice-v3-plus.yaml+++ b/src/agentscope/tts/_dashscope/_cosyvoice_models/cosyvoice-v3-plus.yaml@@ -1,7 +1,6 @@name: cosyvoice-v3-pluslabel: CosyVoice-v3-Plusstatus: active-realtime: trueinput_types:- text/plain@@ -13,13 +12,5 @@ voices:# Source: https://help.aliyun.com/en/model-studio/cosyvoice-voice-list- longanyang- longanhuan- - longhuhu_v3- - longyingmu_v3- - longxiaochun- - longxiaoxia- - longlaotie- - longshuo- - longjing- - longshuparameter_overrides: {}diff --git a/src/agentscope/tts/_dashscope/_cosyvoice_realtime_model.py b/src/agentscope/tts/_dashscope/_cosyvoice_realtime_model.pydeleted file mode 100644index 81ed34504..000000000--- a/src/agentscope/tts/_dashscope/_cosyvoice_realtime_model.py+++ /dev/null@@ -1,507 +0,0 @@-# -*- coding: utf-8 -*--"""DashScope CosyVoice Realtime TTS model implementation.--Uses the older ``dashscope.audio.tts_v2.SpeechSynthesizer`` SDK for models-such as ``cosyvoice-v3-plus``, ``cosyvoice-v3-flash``, ``sambert``, etc.-"""-import asyncio-import base64-import os-import threading-from typing import Any, AsyncGenerator, Literal, TYPE_CHECKING--from pydantic import BaseModel, Field--from .._tts_base import TTSModelBase-from .._tts_response import TTSResponse-from ..._logging import logger-from ..._utils._audio import _build_streaming_wav_header-from ...credential import DashScopeCredential-from ...message import DataBlock, Base64Source--if TYPE_CHECKING:- from dashscope.audio.tts_v2 import ResultCallback---_MEDIA_TYPE = "audio/wav"-_SAMPLE_RATE = 24000-_CHANNELS = 1-_BITS_PER_SAMPLE = 16---def _make_cosyvoice_callback_class() -> type["ResultCallback"]:- """Create the DashScope CosyVoice TTS callback class lazily."""- from dashscope.audio.tts_v2 import ResultCallback-- class _CosyVoiceCallback(ResultCallback):- """Internal callback that accumulates PCM audio from the WebSocket- and exposes incremental deltas."""-- def __init__(self) -> None:- """Initialize callback with audio buffer and synchronization- events."""- super().__init__()- self.chunk_event = threading.Event()- self.finish_event = threading.Event()- self._pcm_bytes: bytearray = bytearray()- self._consumed: int = 0-- def on_open(self) -> None:- """Handle WebSocket open — reset audio state."""- self._pcm_bytes = bytearray()- self._consumed = 0- self.finish_event.clear()- self.chunk_event.clear()-- def on_data(self, data: bytes) -> None:- """Handle incoming PCM audio data."""- if data:- self._pcm_bytes += data- if not self.chunk_event.is_set():- self.chunk_event.set()-- def on_complete(self) -> None:- """Handle synthesis completion."""- self.finish_event.set()- self.chunk_event.set()-- def on_close(self) -> None:- """Handle WebSocket close."""- self.finish_event.set()- self.chunk_event.set()-- def on_error(self, message: Any) -> None:- """Handle synthesis error."""- logger.error("CosyVoice TTS error: %s", message)- self.finish_event.set()- self.chunk_event.set()-- def _take_delta(self, header: bool = False) -> bytes | None:- """Return new PCM bytes since last call, or None if empty."""- new_data = self._pcm_bytes[self._consumed :]- if not new_data:- return None- self._consumed = len(self._pcm_bytes)- if header:- return _build_streaming_wav_header(- sample_rate=_SAMPLE_RATE,- channels=_CHANNELS,- bits_per_sample=_BITS_PER_SAMPLE,- ) + bytes(new_data)- return bytes(new_data)-- def get_audio_response(self, block: bool) -> TTSResponse:- """Return incremental audio delta."""- if block:- self.finish_event.wait()- delta = self._take_delta(header=self._consumed == 0)- if delta:- return TTSResponse(- content=DataBlock(- source=Base64Source(- data=base64.b64encode(delta).decode("ascii"),- media_type=_MEDIA_TYPE,- ),- ),- )- return TTSResponse(content=None)-- async def get_audio_chunks(self) -> AsyncGenerator[TTSResponse, None]:- """Yield incremental WAV audio chunks as they arrive."""- header_sent = self._consumed > 0- while True:- if self.finish_event.is_set():- delta = self._take_delta(header=not header_sent)- if delta:- yield TTSResponse(- content=DataBlock(- source=Base64Source(- data=base64.b64encode(delta).decode(- "ascii",- ),- media_type=_MEDIA_TYPE,- ),- ),- is_last=True,- )- else:- yield TTSResponse(content=None, is_last=True)- self.reset()- break-- if self.chunk_event.is_set():- self.chunk_event.clear()- else:- await asyncio.to_thread(self.chunk_event.wait, 30)-- if self.finish_event.is_set():- continue-- delta = self._take_delta(header=not header_sent)- if delta:- header_sent = True- yield TTSResponse(- content=DataBlock(- source=Base64Source(- data=base64.b64encode(delta).decode("ascii"),- media_type=_MEDIA_TYPE,- ),- ),- is_last=False,- )-- def reset(self) -> None:- """Reset internal state for the next utterance."""- self.finish_event.clear()- self.chunk_event.clear()- self._pcm_bytes = bytearray()- self._consumed = 0-- def has_audio_data(self) -> bool:- """Return whether any audio data has been received."""- return bool(self._pcm_bytes)-- return _CosyVoiceCallback---class DashScopeCosyVoiceRealtimeTTSModel(TTSModelBase):- """DashScope CosyVoice Realtime TTS model using the- ``SpeechSynthesizer`` streaming API.-- Supports streaming input: text can be pushed incrementally via- :meth:`push`, and :meth:`synthesize` finalizes the current utterance.-- Supported models include ``cosyvoice-v3-plus``, ``cosyvoice-v3-flash``,- ``sambert``, etc. For more details see the `official document- <https://help.aliyun.com/document_detail/2712523.html>`_.-- .. note:: Only one streaming input request can be active at a time.-- .. note:: Unlike ``DashScopeRealtimeTTSModel`` (Qwen3) which produces- audio at token-level granularity, CosyVoice server automatically- segments incoming text into sentences and synthesizes per-sentence.- Audio is returned via callback only after a complete sentence boundary- is detected. This means :meth:`push` may return empty responses until- enough text accumulates to form a sentence. Calling- :meth:`synthesize` forces synthesis of all remaining text (including- incomplete sentences). See `official docs- <https://help.aliyun.com/en/model-studio/cosyvoice-python-sdk>`_.- """-- class Parameters(BaseModel):- """Frontend-exposed parameters for CosyVoice Realtime TTS models."""-- voice: str = Field(- default="longanyang",- title="Voice",- description="The voice to use for synthesis.",- )-- type: Literal[- "dashscope_cosyvoice_realtime_tts"- ] = "dashscope_cosyvoice_realtime_tts"-- realtime: bool = True-- _MODELS_DIR = os.path.join(os.path.dirname(__file__), "_cosyvoice_models")-- @classmethod- def list_models(- cls,- custom_yaml_dir: str | None = None,- ) -> list:- """List CosyVoice model cards from the dedicated YAML directory."""- return super().list_models(- custom_yaml_dir=custom_yaml_dir or cls._MODELS_DIR,- )-- def __init__(- self,- credential: DashScopeCredential,- model: str = "cosyvoice-v3-plus",- parameters: "DashScopeCosyVoiceRealtimeTTSModel.Parameters | None" = (- None- ),- stream: bool = True,- cold_start_length: int | None = None,- cold_start_words: int | None = None,- max_retries: int = 3,- retry_delay: float = 5.0,- ) -> None:- """Initialize the DashScope CosyVoice Realtime TTS model.-- Args:- credential (`DashScopeCredential`):- The DashScope credential.- model (`str`, defaults to ``"cosyvoice-v3-plus"``):- The CosyVoice model name, e.g. ``"cosyvoice-v3-plus"``,- ``"cosyvoice-v3-flash"``, ``"sambert"``.- parameters (`Parameters | None`, defaults to `None`):- The TTS parameters (voice, etc.).- stream (`bool`, defaults to `True`):- Whether :meth:`synthesize` returns a streaming async generator.- cold_start_length (`int | None`, defaults to `None`):- Minimum character count before the first text chunk is sent- to the synthesizer.- cold_start_words (`int | None`, defaults to `None`):- Minimum word count (split by spaces) before the first text- chunk is sent.- max_retries (`int`, defaults to `3`):- Max retry attempts on synthesis failure.- retry_delay (`float`, defaults to `5.0`):- Initial retry delay in seconds (exponential backoff).- """- super().__init__(- credential=credential,- model=model,- parameters=parameters,- stream=stream,- )- self.cold_start_length = cold_start_length- self.cold_start_words = cold_start_words- self.max_retries = max_retries- self.retry_delay = retry_delay-- self._synthesizer: Any = None- self._callback: Any = None- self._connected = False- self._cold_start_buffer: str = ""- self._cold_start_done: bool = False- self._accumulated_text: str = ""-- def _create_synthesizer(self) -> None:- """Create a fresh SpeechSynthesizer and callback."""- import dashscope- from dashscope.audio.tts_v2 import SpeechSynthesizer, AudioFormat-- dashscope.api_key = self.credential.api_key.get_secret_value()-- callback_cls = _make_cosyvoice_callback_class()- self._callback = callback_cls()- self._synthesizer = SpeechSynthesizer(- model=self.model,- voice=self.parameters.voice,- format=AudioFormat.PCM_24000HZ_MONO_16BIT,- callback=self._callback,- )-- async def connect(self) -> None:- """Initialize the SpeechSynthesizer."""- if self._connected:- return- self._create_synthesizer()- self._connected = True-- async def close(self) -> None:- """Close the SpeechSynthesizer."""- if not self._connected:- return- self._connected = False- try:- if self._synthesizer is not None:- self._synthesizer.close()- except Exception:- pass-- async def _reconnect(self) -> None:- """Reconnect by recreating the synthesizer."""- try:- if self._synthesizer is not None:- self._synthesizer.close()- except Exception:- pass- self._connected = False- self._cold_start_buffer = ""- self._cold_start_done = False- await self.connect()-- async def push(- self,- text: str,- **kwargs: Any,- ) -> TTSResponse:- """Push an incremental text delta for realtime synthesis.-- .. note:: The CosyVoice server automatically segments text into- sentences before synthesizing. Audio is only produced after a- complete sentence is detected, so this method often returns an- empty response (``content=None``) for partial sentences. Remaining- audio is force-synthesized when :meth:`synthesize` is called.- See `CosyVoice Python SDK docs- <https://help.aliyun.com/en/model-studio/cosyvoice-python-sdk>`_.-- Args:- text (`str`):- An incremental text chunk (delta) to append.- **kwargs (`Any`):- Additional keyword arguments (unused).-- Returns:- `TTSResponse`:- Audio accumulated so far, or empty if not yet available.- """- if not self._connected:- raise RuntimeError(- "TTS model is not connected. Call `connect()` first.",- )-- if not text:- return TTSResponse(content=None)-- self._accumulated_text += text-- if self._cold_start_done:- text_to_send = text- else:- self._cold_start_buffer += text- if (- self.cold_start_length- and len(self._cold_start_buffer) < self.cold_start_length- ) or (- self.cold_start_words- and len(self._cold_start_buffer.split())- < self.cold_start_words- ):- return self._callback.get_audio_response(block=False)- text_to_send = self._cold_start_buffer- self._cold_start_buffer = ""- self._cold_start_done = True-- try:- self._synthesizer.streaming_call(text_to_send)- except Exception:- return TTSResponse(content=None)-- return self._callback.get_audio_response(block=False)-- async def synthesize(- self,- text: str | None = None,- **kwargs: Any,- ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:- """Finalize synthesis for the current utterance.-- If text was previously pushed via :meth:`push`, this flushes any- remaining buffered text, calls ``streaming_complete()``, and waits- for audio. If ``text`` is provided, it is appended before finalizing.-- Args:- text (`str | None`, defaults to `None`):- Optional additional text to append before finalizing.- **kwargs (`Any`):- Additional keyword arguments (unused).-- Returns:- `TTSResponse | AsyncGenerator[TTSResponse, None]`:- A single response when ``stream=False``, or an async generator- of incremental chunks when ``stream=True``.- """- if not self._connected:- raise RuntimeError(- "TTS model is not connected. Call `connect()` first.",- )-- if text is not None:- self._accumulated_text += text-- unsent = self._cold_start_buffer- if text is not None:- unsent += text- self._cold_start_buffer = ""-- full_text = self._accumulated_text- delay = self.retry_delay-- try:- if not full_text and not unsent:- if self.stream:-- async def _empty_gen() -> AsyncGenerator[- TTSResponse,- None,- ]:- yield TTSResponse(content=None)-- return _empty_gen()- return TTSResponse(content=None)-- for attempt in range(self.max_retries):- try:- if unsent:- self._synthesizer.streaming_call(unsent)-- self._synthesizer.streaming_complete()-- finished = await asyncio.to_thread(- self._callback.finish_event.wait,- 30,- )-- if not finished:- logger.warning(- "CosyVoice TTS: timed out waiting for synthesis "- "completion (30s)",- )- if attempt < self.max_retries - 1:- await asyncio.sleep(delay)- await self._reconnect()- unsent = full_text- delay *= 2- continue- raise RuntimeError(- "CosyVoice TTS synthesis timed out after 30s",- )-- if full_text and not self._callback.has_audio_data():- if attempt < self.max_retries - 1:- logger.warning(- "CosyVoice TTS: no audio received, retrying "- "(%d/%d) in %.1fs...",- attempt + 1,- self.max_retries,- delay,- )- await asyncio.sleep(delay)- await self._reconnect()- unsent = full_text- delay *= 2- continue- raise RuntimeError(- f"CosyVoice TTS synthesis failed: no audio after "- f"{self.max_retries} attempts",- )- break-- except RuntimeError:- raise- except Exception as e:- if attempt < self.max_retries - 1:- logger.warning(- "CosyVoice TTS error, retrying (%d/%d) in "- "%.1fs: %s",- attempt + 1,- self.max_retries,- delay,- e,- )- await asyncio.sleep(delay)- await self._reconnect()- unsent = full_text- delay *= 2- else:- raise-- if self.stream:- return self._callback.get_audio_chunks()-- response = self._callback.get_audio_response(block=True)- self._callback.reset()- return response- finally:- self._reset_state()-- def _reset_state(self) -> None:- """Reset per-utterance tracking state."""- self._cold_start_buffer = ""- self._cold_start_done = False- self._accumulated_text = ""diff --git a/src/agentscope/tts/_dashscope/_cosyvoice_utils.py b/src/agentscope/tts/_dashscope/_cosyvoice_utils.pynew file mode 100644index 000000000..b1573b597--- /dev/null+++ b/src/agentscope/tts/_dashscope/_cosyvoice_utils.py@@ -0,0 +1,157 @@+# -*- coding: utf-8 -*-+"""Shared helpers for DashScope CosyVoice TTS models."""+import asyncio+import base64+import threading+from typing import Any, AsyncGenerator, TYPE_CHECKING++from .._tts_response import TTSResponse+from ..._logging import logger+from ..._utils._audio import _build_streaming_wav_header+from ...message import DataBlock, Base64Source++if TYPE_CHECKING:+ from dashscope.audio.tts_v2 import ResultCallback+++_SAMPLE_RATE = 24000+_CHANNELS = 1+_BITS_PER_SAMPLE = 16+_MEDIA_TYPE = "audio/wav"+++def _build_audio_response(+ audio_data: bytes,+ media_type: str,+ *,+ is_last: bool = True,+) -> TTSResponse:+ """Build a TTSResponse containing base64-encoded audio data."""+ return TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(audio_data).decode("ascii"),+ media_type=media_type,+ ),+ ),+ is_last=is_last,+ )+++def _make_cosyvoice_callback_class() -> type["ResultCallback"]:+ """Create the shared CosyVoice callback class lazily."""+ from dashscope.audio.tts_v2 import ResultCallback++ class _CosyVoiceCallback(ResultCallback):+ """Accumulate PCM audio from the WebSocket and expose deltas."""++ def __init__(self) -> None:+ """Initialize callback with audio buffer and synchronization+ events."""+ super().__init__()+ self.chunk_event = threading.Event()+ self.finish_event = threading.Event()+ self._pcm_bytes: bytearray = bytearray()+ self._consumed: int = 0++ def on_open(self) -> None:+ """Handle WebSocket open by resetting audio state."""+ self._pcm_bytes = bytearray()+ self._consumed = 0+ self.finish_event.clear()+ self.chunk_event.clear()++ def on_data(self, data: bytes) -> None:+ """Handle incoming PCM audio data."""+ if data:+ self._pcm_bytes += data+ if not self.chunk_event.is_set():+ self.chunk_event.set()++ def on_complete(self) -> None:+ """Handle synthesis completion."""+ self.finish_event.set()+ self.chunk_event.set()++ def on_close(self) -> None:+ """Handle WebSocket close."""+ self.finish_event.set()+ self.chunk_event.set()++ def on_error(self, message: Any) -> None:+ """Handle synthesis error."""+ logger.error("CosyVoice TTS error: %s", message)+ self.finish_event.set()+ self.chunk_event.set()++ def _take_delta(self, header: bool = False) -> bytes | None:+ """Return new PCM bytes since last call, or None if empty."""+ new_data = self._pcm_bytes[self._consumed :]+ if not new_data:+ return None+ self._consumed = len(self._pcm_bytes)+ if header:+ return _build_streaming_wav_header(+ sample_rate=_SAMPLE_RATE,+ channels=_CHANNELS,+ bits_per_sample=_BITS_PER_SAMPLE,+ ) + bytes(new_data)+ return bytes(new_data)++ def get_audio_response(self, block: bool) -> TTSResponse:+ """Return incremental audio delta."""+ if block:+ self.finish_event.wait()+ delta = self._take_delta(header=self._consumed == 0)+ if delta:+ return _build_audio_response(delta, _MEDIA_TYPE)+ return TTSResponse(content=None)++ async def get_audio_chunks(+ self,+ ) -> AsyncGenerator[TTSResponse, None]:+ """Yield incremental audio chunks as they arrive."""+ header_sent = self._consumed > 0+ while True:+ if self.finish_event.is_set():+ delta = self._take_delta(header=not header_sent)+ if delta:+ yield _build_audio_response(+ delta,+ _MEDIA_TYPE,+ is_last=True,+ )+ else:+ yield TTSResponse(content=None, is_last=True)+ self.reset()+ break++ if self.chunk_event.is_set():+ self.chunk_event.clear()+ else:+ await asyncio.to_thread(self.chunk_event.wait, 30)++ if self.finish_event.is_set():+ continue++ delta = self._take_delta(header=not header_sent)+ if delta:+ header_sent = True+ yield _build_audio_response(+ delta,+ _MEDIA_TYPE,+ is_last=False,+ )++ def reset(self) -> None:+ """Reset internal state for the next utterance."""+ self.finish_event.clear()+ self.chunk_event.clear()+ self._pcm_bytes = bytearray()+ self._consumed = 0++ def has_audio_data(self) -> bool:+ """Return whether any audio data has been received."""+ return bool(self._pcm_bytes)++ return _CosyVoiceCallbackdiff --git a/tests/tts_dashscope_test.py b/tests/tts_dashscope_test.pyindex b0b0a6ec9..6c3fc2b47 100644--- a/tests/tts_dashscope_test.py+++ b/tests/tts_dashscope_test.py@@ -8,9 +8,11 @@ Covers:* ``DashScopeTTSModel`` non-streaming aggregation.* ``DashScopeTTSModel`` streaming: incremental deltas and ``is_last``placement at the final chunk only.+ * ``DashScopeCosyVoiceTTSModel`` WebSocket TTS streaming and non-streaming+ output over a mocked SDK.* ``DashScopeRealtimeTTSModel`` connect / close / push / synthesizelifecycle over a mocked WebSocket.- * ``DashScopeCosyVoiceRealtimeTTSModel`` connect / close / push /+ * ``DashScopeCosyVoiceTTSModel`` realtime connect / close / push /synthesize lifecycle over a mocked SpeechSynthesizer."""import base64@@ -20,9 +22,10 @@ from typing import Any, AsyncGeneratorfrom unittest import IsolatedAsyncioTestCasefrom unittest.mock import MagicMock, Mock, patch+from agentscope.app._service._tts_model import _resolve_tts_classfrom agentscope.credential import DashScopeCredentialfrom agentscope.tts import (- DashScopeCosyVoiceRealtimeTTSModel,+ DashScopeCosyVoiceTTSModel,DashScopeTTSModel,DashScopeRealtimeTTSModel,TTSModelBase,@@ -318,6 +321,164 @@ class TestDashScopeTTSModel(IsolatedAsyncioTestCase):self.assertTrue(chunks[0].is_last)+# ---------------------------------------------------------------------------+# DashScopeCosyVoiceTTSModel — WebSocket TTS streaming and aggregation+# ---------------------------------------------------------------------------+++class TestDashScopeCosyVoiceTTSModel(IsolatedAsyncioTestCase):+ """The unittests for DashScope CosyVoice TTS model."""++ def setUp(self) -> None:+ """Set up the test case."""+ self.patcher = patch(+ "dashscope.audio.tts_v2.SpeechSynthesizer",+ )+ self.mock_synthesizer_class = self.patcher.start()+ self.synthesizer_instances: list[Mock] = []+ self.next_audio = b"FULL_AUDIO"+ self.next_chunks = [b"CHUNK"]+ self.mock_synthesizer_class.side_effect = self._make_synthesizer++ def tearDown(self) -> None:+ """Tear down the test case."""+ self.patcher.stop()++ def _make_synthesizer(self, **init_kwargs: Any) -> Mock:+ """Create a mocked tts_v2 SpeechSynthesizer instance."""+ synth = Mock()+ synth.init_kwargs = init_kwargs+ callback = init_kwargs.get("callback")++ if callback is None:+ synth.call = Mock(return_value=self.next_audio)+ else:++ def _call(text: str, **kwargs: Any) -> None:+ del text, kwargs+ for chunk in self.next_chunks:+ callback.on_data(chunk)+ callback.on_complete()+ callback.on_close()++ synth.call = Mock(side_effect=_call)++ self.synthesizer_instances.append(synth)+ return synth++ def _make_model(+ self,+ stream: bool = False,+ ) -> DashScopeCosyVoiceTTSModel:+ """Create a DashScopeCosyVoiceTTSModel with test credentials."""+ return DashScopeCosyVoiceTTSModel(+ credential=DashScopeCredential(api_key="test"),+ model="cosyvoice-v3-flash",+ parameters=DashScopeCosyVoiceTTSModel.Parameters(+ voice="longanhuan",+ ),+ stream=stream,+ )++ async def test_empty_string_short_circuits(self) -> None:+ """``synthesize("")`` returns empty without touching the SDK."""+ model = self._make_model(stream=False)++ result = await model.synthesize("")++ self.assertIsNone(result.content)+ self.mock_synthesizer_class.assert_not_called()++ async def test_non_stream_returns_full_audio(+ self,+ ) -> None:+ """stream=False returns one full audio payload from the SDK."""+ self.next_audio = b"AAAABBBB"+ model = self._make_model(stream=False)++ result = await model.synthesize("Hello")++ self.assertIsInstance(result, TTSResponse)+ self.assertEqual(result.content.source.media_type, _MEDIA_TYPE)+ wav_payload = base64.b64decode(result.content.source.data)+ with wave.open(io.BytesIO(wav_payload), "rb") as wav:+ self.assertEqual(wav.getnchannels(), 1)+ self.assertEqual(wav.getframerate(), 24000)+ self.assertEqual(wav.readframes(wav.getnframes()), b"AAAABBBB")++ synth = self.synthesizer_instances[-1]+ init_kwargs = synth.init_kwargs+ self.assertEqual(init_kwargs["model"], "cosyvoice-v3-flash")+ self.assertEqual(init_kwargs["voice"], "longanhuan")+ self.assertEqual(init_kwargs["format"].format, "pcm")+ self.assertEqual(init_kwargs["format"].sample_rate, 24000)+ synth.call.assert_called_once_with(text="Hello")++ async def test_streaming_returns_callback_audio_chunks(+ self,+ ) -> None:+ """stream=True returns callback audio chunks."""+ self.next_chunks = [b"AAAA", b"BBBB"]+ model = self._make_model(stream=True)++ gen = await model.synthesize("Hello")+ chunks = [c async for c in gen]++ self.assertEqual(len(chunks), 1)+ payload = base64.b64decode(chunks[0].content.source.data)+ self.assertTrue(payload.startswith(b"RIFF"))+ self.assertTrue(payload.endswith(b"AAAABBBB"))+ self.assertEqual([c.is_last for c in chunks], [True])++ synth = self.synthesizer_instances[-1]+ self.assertIsNotNone(synth.init_kwargs["callback"])+ synth.call.assert_called_once_with(text="Hello")++ async def test_model_cards_and_credential_wiring(self) -> None:+ """CosyVoice model cards are discovered only by the CosyVoice class,+ and DashScope credentials expose the class."""+ cosyvoice_cards = DashScopeCosyVoiceTTSModel.list_models()+ cosyvoice_names = {card.name for card in cosyvoice_cards}+ self.assertEqual(len(cosyvoice_cards), 2)+ self.assertIn("cosyvoice-v3-flash", cosyvoice_names)+ self.assertIn("cosyvoice-v3-plus", cosyvoice_names)+ self.assertTrue(all(not card.realtime for card in cosyvoice_cards))+ for card in cosyvoice_cards:+ properties = card.parameter_schema["properties"]+ self.assertIn("voice", properties)+ self.assertIn("realtime", properties)+ plus_card = next(+ card+ for card in cosyvoice_cards+ if card.name == "cosyvoice-v3-plus"+ )+ self.assertIn(+ "longanhuan",+ plus_card.parameter_schema["properties"]["voice"]["enum"],+ )++ qwen_names = {card.name for card in DashScopeTTSModel.list_models()}+ self.assertNotIn("cosyvoice-v3-flash", qwen_names)+ self.assertNotIn("cosyvoice-v3-plus", qwen_names)++ credential_classes = DashScopeCredential.get_tts_model_classes()+ self.assertIn(DashScopeCosyVoiceTTSModel, credential_classes)+ credential_names = {+ card.name for card in DashScopeCredential.list_tts_models()+ }+ self.assertIn("cosyvoice-v3-flash", credential_names)+ self.assertIn("cosyvoice-v3-plus", credential_names)++ resolved_without_realtime = _resolve_tts_class(+ credential_classes,+ "cosyvoice-v3-plus",+ )+ self.assertIs(+ resolved_without_realtime,+ DashScopeCosyVoiceTTSModel,+ )++# ---------------------------------------------------------------------------# DashScopeRealtimeTTSModel — realtime push / synthesize lifecycle# ---------------------------------------------------------------------------@@ -701,15 +862,15 @@ class TestDashScopeRealtimeTTSModel( # pylint: disable=too-many-public-methods# ----------------------------------------------------------------------------# DashScopeCosyVoiceRealtimeTTSModel — realtime push / synthesize lifecycle+# DashScopeCosyVoiceTTSModel — realtime push / synthesize lifecycle# ---------------------------------------------------------------------------# pylint: disable=too-many-public-methods-class TestDashScopeCosyVoiceRealtimeTTSModel(+class TestDashScopeCosyVoiceRealtimeMode(IsolatedAsyncioTestCase,):- """Unit tests for the CosyVoice Realtime TTS model."""+ """Unit tests for CosyVoice realtime mode."""def setUp(self) -> None:self.mock_modules = self._create_mock_cosyvoice_modules()@@ -753,16 +914,20 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(def _make_model(self,**kwargs: Any,- ) -> DashScopeCosyVoiceRealtimeTTSModel:+ ) -> DashScopeCosyVoiceTTSModel:defaults: dict[str, Any] = {"credential": DashScopeCredential(api_key="test"),"model": "cosyvoice-v3-plus",+ "parameters": DashScopeCosyVoiceTTSModel.Parameters(+ voice="longanhuan",+ realtime=True,+ ),"stream": True,"max_retries": 1,"retry_delay": 0.0,}defaults.update(kwargs)- return DashScopeCosyVoiceRealtimeTTSModel(**defaults)+ return DashScopeCosyVoiceTTSModel(**defaults)def _mock_synthesize_callback(self, model: Any) -> None:"""Set up callback mocks so synthesize() doesn't block."""@@ -783,6 +948,21 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(self.assertTrue(model._connected)self.assertFalse(model._connected)+ async def test_async_context_skips_connect_when_not_realtime(self) -> None:+ """async with does not connect when the realtime parameter is off."""+ with patch.dict("sys.modules", self.mock_modules):+ model = self._make_model(+ parameters=DashScopeCosyVoiceTTSModel.Parameters(+ voice="longanhuan",+ realtime=False,+ ),+ )+ async with model:+ self.assertFalse(model._connected)+ self.mock_modules[+ "dashscope.audio.tts_v2"+ ].SpeechSynthesizer.assert_not_called()+# -- push --async def test_push_incremental_deltas(self) -> None:@@ -1043,7 +1223,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(async def test_callback_on_complete_sets_finish_event(self) -> None:"""on_complete() correctly sets finish_event and chunk_event."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)@@ -1061,7 +1241,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(async def test_callback_on_data_accumulates(self) -> None:"""on_data() accumulates PCM bytes and signals chunk_event."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)@@ -1077,7 +1257,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(async def test_callback_take_delta_incremental(self) -> None:"""_take_delta() returns only new bytes since last call."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)@@ -1098,7 +1278,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(async def test_callback_take_delta_with_header(self) -> None:"""_take_delta(header=True) prepends WAV header to first chunk."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)@@ -1118,7 +1298,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel("""get_audio_chunks() prepends a WAV header when push() has notyet consumed any bytes (_consumed == 0)."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)@@ -1146,7 +1326,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(push() already consumed and sent the first chunk with a header(_consumed > 0)."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)@@ -1181,7 +1361,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel("""get_audio_response() prepends a WAV header when no data has beenconsumed yet (_consumed == 0)."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)@@ -1204,7 +1384,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(push() already consumed and sent the first chunk with a header(_consumed > 0)."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)@@ -1228,7 +1408,7 @@ class TestDashScopeCosyVoiceRealtimeTTSModel(async def test_callback_reset(self) -> None:"""reset() clears all state."""with patch.dict("sys.modules", self.mock_modules):- from agentscope.tts._dashscope._cosyvoice_realtime_model import (+ from agentscope.tts._dashscope._cosyvoice_utils import (_make_cosyvoice_callback_class,)