Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(tts): add TTS module with DashScope backend and streaming middleware (#1832)
qbc680626b463bf
diff --git a/examples/web_ui/frontend/src/api/index.ts b/examples/web_ui/frontend/src/api/index.tsindex 810b20143..78512276d 100644--- a/examples/web_ui/frontend/src/api/index.ts+++ b/examples/web_ui/frontend/src/api/index.ts@@ -5,4 +5,4 @@ export { credentialApi } from './credential';export { chatApi } from './chat';export { workspaceApi } from './workspace';export { scheduleApi } from './schedule';-export { modelApi } from './model';+export { modelApi, ttsModelApi } from './model';diff --git a/examples/web_ui/frontend/src/api/model.ts b/examples/web_ui/frontend/src/api/model.tsindex cf5abb15d..3ebe9484a 100644--- a/examples/web_ui/frontend/src/api/model.ts+++ b/examples/web_ui/frontend/src/api/model.ts@@ -1,6 +1,10 @@import { client } from './client';-import type { ListModelResponse } from './types';+import type { ListModelResponse, ListTTSModelResponse } from './types';export const modelApi = {list: (provider: string) => client.get<ListModelResponse>('/model/', { provider }),};++export const ttsModelApi = {+ list: (provider: string) => client.get<ListTTSModelResponse>('/tts-model/', { provider }),+};diff --git a/examples/web_ui/frontend/src/api/types.ts b/examples/web_ui/frontend/src/api/types.tsindex 400832cd6..b3429754d 100644--- a/examples/web_ui/frontend/src/api/types.ts+++ b/examples/web_ui/frontend/src/api/types.ts@@ -13,6 +13,13 @@ export interface ChatModelConfig {parameters: Record<string, unknown>;}+export interface TTSModelConfig {+ type: string;+ credential_id: string;+ model: string;+ parameters: Record<string, unknown>;+}+export interface ContextConfig {trigger_ratio?: number;reserve_ratio?: number;@@ -84,6 +91,8 @@ export interface SessionConfig {chat_model_config: ChatModelConfig;/** Fallback model used when the primary model fails. */fallback_chat_model_config: ChatModelConfig | null;+ /** TTS model configuration. null means TTS is not enabled. */+ tts_model_config: TTSModelConfig | null;workspace_id: string;}@@ -112,6 +121,8 @@ export interface CreateSessionRequest {chat_model_config?: ChatModelConfig | null;/** Optional fallback model. Omit (or pass null) for no fallback. */fallback_chat_model_config?: ChatModelConfig | null;+ /** Optional TTS model. Omit (or pass null) for no TTS. */+ tts_model_config?: TTSModelConfig | null;}export interface CreateSessionResponse {@@ -128,6 +139,13 @@ export interface UpdateSessionRequest {* - set to a value → replace the existing fallback*/fallback_chat_model_config?: ChatModelConfig | null;+ /**+ * New TTS model. PATCH semantics:+ * - omit the field → leave unchanged+ * - set to `null` → disable TTS+ * - set to a value → replace the existing TTS config+ */+ tts_model_config?: TTSModelConfig | null;permission_mode?: PermissionMode;}@@ -422,3 +440,21 @@ export interface ListModelResponse {models: ModelCard[];total: number;}++export interface TTSModelCard {+ type: 'tts_model';+ name: string;+ label: string;+ status: 'active' | 'deprecated' | 'sunset';+ deprecated_at: string | null;+ input_types: string[];+ output_types: string[];+ realtime: boolean;+ parameter_schema: Record<string, unknown>;+ parameters_overrides: Record<string, Record<string, unknown>>;+}++export interface ListTTSModelResponse {+ models: TTSModelCard[];+ total: number;+}diff --git a/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx b/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsxindex 22f3783d3..847317f58 100644--- a/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx+++ b/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx@@ -1,7 +1,8 @@import { ChevronDown, SlidersHorizontal } from 'lucide-react';import { useCallback, useEffect, useState } from 'react';-import type { ChatModelConfig, ModelCard } from '@/api';+import type { ChatModelConfig, ModelCard, TTSModelCard, TTSModelConfig } from '@/api';+import { Badge } from '@/components/ui/badge';import { Button } from '@/components/ui/button';import {DropdownMenu,@@ -21,6 +22,7 @@ import { Label } from '@/components/ui/label';import { Switch } from '@/components/ui/switch';import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';import { useAvailableModels } from '@/hooks/useAvailableModels';+import { useAvailableTTSModels } from '@/hooks/useAvailableTTSModels';import { useTranslation } from '@/i18n/useI18n';interface ParameterProperty {@@ -204,6 +206,10 @@ interface Props {selectedFallbackModel: ChatModelConfig | null;/** Called when the user picks a fallback model or clears the selection. */onFallbackChange: (config: ChatModelConfig | null) => void;+ /** Currently selected TTS model. `null` means TTS is disabled. */+ selectedTTSModel: TTSModelConfig | null;+ /** Called when the user picks a TTS model+voice or disables TTS. */+ onTTSChange: (config: TTSModelConfig | null) => void;}/**@@ -221,10 +227,13 @@ export function ModelParametersPopover({onChange,selectedFallbackModel,onFallbackChange,+ selectedTTSModel,+ onTTSChange,}: Props) {const [values, setValues] = useState<Record<string, unknown>>({});const { t } = useTranslation();const { groups } = useAvailableModels();+ const { groups: ttsGroups } = useAvailableTTSModels();const schema = modelCard?.parameter_schema as ParameterSchema | undefined;const properties = schema?.properties ?? {};@@ -400,6 +409,207 @@ export function ModelParametersPopover({)}</DropdownMenuSubContent></DropdownMenuSub>++ {/* ----- TTS ----- */}+ <DropdownMenuSub>+ <DropdownMenuSubTrigger>+ <span className="truncate">{t('model-parameters.ttsLabel')}</span>+ </DropdownMenuSubTrigger>+ <DropdownMenuSubContent className="max-h-96 overflow-y-auto">+ {Object.keys(ttsGroups).length === 0 ? (+ <div className="px-2 py-3 text-center text-sm text-muted-foreground">+ <p>{t('model-parameters.ttsEmpty')}</p>+ </div>+ ) : (+ Object.entries(ttsGroups).map(([type, items], idx) => (+ <div key={type}>+ {idx > 0 && <DropdownMenuSeparator />}+ <DropdownMenuLabel>+ {type.replace(/_credential$/, '')}+ </DropdownMenuLabel>+ {items.flatMap(({ credential, models }) =>+ models.map((m) => {+ const isSelected =+ selectedTTSModel?.credential_id === credential.id &&+ selectedTTSModel?.model === m.name;+ return (+ <DropdownMenuCheckboxItem+ key={`${credential.id}-${m.name}`}+ checked={isSelected}+ onSelect={(e) => e.preventDefault()}+ onCheckedChange={(checked) => {+ if (!checked) return;+ const schema = m.parameter_schema as+ | ParameterSchema+ | undefined;+ const defaults: Record<string, unknown> =+ {};+ if (schema?.properties) {+ for (const [k, p] of Object.entries(+ schema.properties,+ )) {+ if (p.default !== undefined) {+ defaults[k] = p.default;+ }+ }+ }+ onTTSChange({+ type,+ credential_id: credential.id,+ model: m.name,+ parameters: defaults,+ });+ }}+ >+ {m.label}+ {m.realtime && (+ <Badge+ variant="outline"+ className="ml-1.5 text-[10px] px-1 py-0"+ >+ Realtime+ </Badge>+ )}+ </DropdownMenuCheckboxItem>+ );+ }),+ )}+ </div>+ ))+ )}++ {/* TTS parameters sub-panel (hover to expand right) */}+ {selectedTTSModel && (+ <>+ <DropdownMenuSeparator />+ <DropdownMenuSub>+ <DropdownMenuSubTrigger>+ {t('model-parameters.ttsParameters')}+ </DropdownMenuSubTrigger>+ <DropdownMenuSubContent className="w-72 max-h-96 overflow-y-auto p-3">+ <div className="mb-3">+ <p className="text-sm font-medium">+ {t('model-parameters.title')}+ </p>+ <p className="text-muted-foreground text-xs">+ {t('model-parameters.ttsParametersDescription')}+ </p>+ </div>+ {(() => {+ if (!selectedTTSModel) return null;+ const selType = selectedTTSModel.type;+ const selItems = ttsGroups[selType];+ if (!selItems) return null;+ let selModel: TTSModelCard | undefined;+ for (const { credential, models } of selItems) {+ if (+ credential.id !== selectedTTSModel.credential_id+ )+ continue;+ selModel = models.find(+ (m) => m.name === selectedTTSModel.model,+ );+ if (selModel) break;+ }+ if (!selModel) return null;+ const mSchema = selModel.parameter_schema as+ | ParameterSchema+ | undefined;+ const mProps = mSchema?.properties ?? {};+ const mRequired = mSchema?.required ?? [];+ const mEntries = Object.entries(mProps);+ if (mEntries.length === 0) {+ return (+ <p className="text-muted-foreground text-xs">+ {t('model-parameters.empty')}+ </p>+ );+ }+ const curParams = selectedTTSModel.parameters ?? {};++ return (+ <div+ className="grid grid-cols-[auto_1fr] items-center gap-x-3 gap-y-3"+ onPointerDown={(e) => e.stopPropagation()}+ onKeyDown={(e) => e.stopPropagation()}+ >+ {mEntries.map(([key, prop]) => {+ const { type: effectiveType, enumValues } =+ resolveType(prop);+ const label = prop.title ?? key;+ const isReq = mRequired.includes(key);+ const fieldProps: FieldProps = {+ id: `tts-${selModel!.name}-${key}`,+ label,+ required: isReq,+ prop,+ value: curParams[key],+ onChange: (v) => {+ const next = {+ ...curParams,+ [key]: v,+ };+ if (v === '' || v === undefined) {+ delete next[key];+ }+ onTTSChange({+ ...selectedTTSModel,+ parameters: next,+ });+ },+ };++ let field: React.ReactNode;+ if (effectiveType === 'boolean') {+ field = (+ <BooleanField {...fieldProps} />+ );+ } else if (enumValues) {+ field = <EnumField {...fieldProps} />;+ } else if (+ effectiveType === 'number' ||+ effectiveType === 'integer'+ ) {+ field = <NumberField {...fieldProps} />;+ } else {+ field = <StringField {...fieldProps} />;+ }++ return (+ <Tooltip key={key}>+ <TooltipTrigger asChild>+ <div className="col-span-2 grid grid-cols-subgrid items-center">+ {field}+ </div>+ </TooltipTrigger>+ {prop.description && (+ <TooltipContent side="left">+ {prop.description}+ </TooltipContent>+ )}+ </Tooltip>+ );+ })}+ </div>+ );+ })()}+ </DropdownMenuSubContent>+ </DropdownMenuSub>+ </>+ )}++ <DropdownMenuSeparator />+ <DropdownMenuCheckboxItem+ checked={!selectedTTSModel}+ onSelect={(e) => e.preventDefault()}+ onCheckedChange={(checked) => {+ if (checked) onTTSChange(null);+ }}+ >+ {t('model-parameters.noTts')}+ </DropdownMenuCheckboxItem>+ </DropdownMenuSubContent>+ </DropdownMenuSub></DropdownMenuContent></DropdownMenu>);diff --git a/examples/web_ui/frontend/src/context/AudioContext.tsx b/examples/web_ui/frontend/src/context/AudioContext.tsxindex 041a4cddc..6f9036a3a 100644--- a/examples/web_ui/frontend/src/context/AudioContext.tsx+++ b/examples/web_ui/frontend/src/context/AudioContext.tsx@@ -28,6 +28,7 @@ export function AudioProvider({ children }: { children: ReactNode }) {const replay: ReplayController = useMemo(() => ({play(el: HTMLAudioElement) {+ manager.stopLivePlayback();if (currentRef.current && currentRef.current !== el) {currentRef.current.pause();currentRef.current.currentTime = 0;@@ -38,7 +39,7 @@ export function AudioProvider({ children }: { children: ReactNode }) {currentRef.current = null;},}),- [],+ [manager],);return (diff --git a/examples/web_ui/frontend/src/hooks/useAvailableTTSModels.ts b/examples/web_ui/frontend/src/hooks/useAvailableTTSModels.tsnew file mode 100644index 000000000..2c5147ab1--- /dev/null+++ b/examples/web_ui/frontend/src/hooks/useAvailableTTSModels.ts@@ -0,0 +1,61 @@+import { useState, useEffect, useCallback } from 'react';++import { credentialApi, ttsModelApi } from '@/api';+import type { CredentialRecord, TTSModelCard } from '@/api';++export interface CredentialWithTTSModels {+ credential: CredentialRecord;+ models: TTSModelCard[];+}++/**+ * Fetches all credentials and their available TTS models, grouped by provider type.+ * Credentials/providers that expose no TTS models are omitted.+ */+export function useAvailableTTSModels() {+ const [groups, setGroups] = useState<Record<string, CredentialWithTTSModels[]>>({});+ const [loading, setLoading] = useState(false);+ const [error, setError] = useState<Error | null>(null);++ const refetch = useCallback(async () => {+ setLoading(true);+ setError(null);+ try {+ const { credentials } = await credentialApi.list();+ const result: Record<string, CredentialWithTTSModels[]> = {};++ await Promise.all(+ credentials.map(async (credential) => {+ const type = credential.data.type as string | undefined;+ if (!type) return;+ if (!result[type]) result[type] = [];+ try {+ const { models } = await ttsModelApi.list(type);+ if (models.length > 0) {+ result[type].push({ credential, models });+ }+ } catch {+ // Provider doesn't support TTS — skip silently+ }+ }),+ );++ // Remove provider groups with no TTS models+ for (const key of Object.keys(result)) {+ if (result[key].length === 0) delete result[key];+ }++ setGroups(result);+ } catch (e) {+ setError(e as Error);+ } finally {+ setLoading(false);+ }+ }, []);++ useEffect(() => {+ refetch();+ }, [refetch]);++ return { groups, loading, error, refetch };+}diff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.jsonindex ba9899624..3d54547d6 100644--- a/examples/web_ui/frontend/src/i18n/locales/en.json+++ b/examples/web_ui/frontend/src/i18n/locales/en.json@@ -182,7 +182,8 @@"maxContext": "Max. Context","maxOutput": "Max. Output","inputTypes": "Input",- "outputTypes": "Output"+ "outputTypes": "Output",+ "availableTTSModels": "TTS Models"},"llm-select": {"placeholder": "Select a model",@@ -285,7 +286,12 @@"empty": "No configurable parameters.","fallbackLabel": "Fallback model","fallbackLabelWithModel": "Fallback model ({{model}})",- "parametersLabel": "Parameters"+ "parametersLabel": "Parameters",+ "ttsLabel": "TTS",+ "ttsEmpty": "No TTS models available.",+ "ttsParameters": "Parameters",+ "ttsParametersDescription": "Adjust parameters for the selected TTS model.",+ "noTts": "No TTS"},"setup": {"title": "Connect to Service",diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.jsonindex 58da0cbaf..9d230e92f 100644--- a/examples/web_ui/frontend/src/i18n/locales/zh.json+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json@@ -182,7 +182,8 @@"maxContext": "最大上下文","maxOutput": "最大输出","inputTypes": "输入模态",- "outputTypes": "输出模态"+ "outputTypes": "输出模态",+ "availableTTSModels": "语音合成模型"},"llm-select": {"placeholder": "选择模型",@@ -285,7 +286,12 @@"empty": "暂无可配置参数。","fallbackLabel": "兜底模型","fallbackLabelWithModel": "兜底模型({{model}})",- "parametersLabel": "参数"+ "parametersLabel": "参数",+ "ttsLabel": "语音合成",+ "ttsEmpty": "暂无可用的语音合成模型。",+ "ttsParameters": "参数",+ "ttsParametersDescription": "调整所选语音合成模型的参数。",+ "noTts": "不使用语音合成"},"setup": {"title": "连接到服务器",diff --git a/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx b/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsxindex af22962f8..5ed4efbcb 100644--- a/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx+++ b/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx@@ -2,7 +2,7 @@ import type { TaskContext } from '@agentscope-ai/agentscope/state';import { Toolbox } from 'lucide-react';import { useCallback, useEffect, useMemo, useState } from 'react';-import type { ChatModelConfig } from '@/api';+import type { ChatModelConfig, TTSModelConfig } from '@/api';import { sessionApi } from '@/api';import { ChatContent } from '@/components/chat/ChatContent.tsx';import { TaskPanel } from '@/components/chat/TaskPanel';@@ -76,6 +76,7 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewportconst [selectedFallbackModel, setSelectedFallbackModel] = useState<ChatModelConfig | null>(null,);+ const [selectedTTSModel, setSelectedTTSModel] = useState<TTSModelConfig | null>(null);const [selectedPermissionMode, setSelectedPermissionMode] = useState<string>('default');const [credentialOpen, setCredentialOpen] = useState(false);const [credentialRefetchTrigger, setCredentialRefetchTrigger] = useState(0);@@ -127,6 +128,7 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewportuseEffect(() => {setSelectedModel(null);setSelectedFallbackModel(null);+ setSelectedTTSModel(null);}, [sessionId]);const selectedModelCard = useMemo(() => {@@ -215,6 +217,7 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewport}setSelectedFallbackModel(view.session.config.fallback_chat_model_config ?? null);+ setSelectedTTSModel(view.session.config.tts_model_config ?? null);}, [view, groups, sessionId, agentId]);// Sync selectedPermissionMode when the session changes. Same@@ -266,6 +269,18 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewportawait refetchSessions();};+ /**+ * Persist a TTS model change. `null` disables TTS.+ *+ * @param config - New TTS config or `null` to disable.+ */+ const handleTTSChange = async (config: TTSModelConfig | null) => {+ if (!sessionId || !agentId) return;+ setSelectedTTSModel(config);+ await sessionApi.update(sessionId, agentId, { tts_model_config: config });+ await refetchSessions();+ };+/*** Persist a permission-mode change.*@@ -296,6 +311,8 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewportonChange={handleParametersChange}selectedFallbackModel={selectedFallbackModel}onFallbackChange={handleFallbackChange}+ selectedTTSModel={selectedTTSModel}+ onTTSChange={handleTTSChange}/></div><div id="tour-permission-mode" className="flex flex-row gap-x-2">diff --git a/examples/web_ui/frontend/src/pages/credential/index.tsx b/examples/web_ui/frontend/src/pages/credential/index.tsxindex e951d3a45..c1bcba5bd 100644--- a/examples/web_ui/frontend/src/pages/credential/index.tsx+++ b/examples/web_ui/frontend/src/pages/credential/index.tsx@@ -1,8 +1,8 @@import { Eye, EyeOff, Plus, Pencil, Trash2 } from 'lucide-react';import { useState, useEffect, useCallback } from 'react';-import { credentialApi, modelApi } from '@/api';-import type { CredentialRecord, CredentialSchema, ModelCard } from '@/api';+import { credentialApi, modelApi, ttsModelApi } from '@/api';+import type { CredentialRecord, CredentialSchema, ModelCard, TTSModelCard } from '@/api';import { InputTypeBadges } from '@/components/badge/InputTypeBadges';import { CreateCredentialDialog } from '@/components/dialog/CreateCredentialDialog';import { DeleteDialog } from '@/components/dialog/DeleteDialog';@@ -104,6 +104,52 @@ function ModelCardItem({ model }: { model: ModelCard }) {);}+// ─── TTS Model Card ──────────────────────────────────────────────────────────++function TTSModelCardItem({ model }: { model: TTSModelCard }) {+ const { t } = useTranslation();++ const statusVariant =+ model.status === 'active'+ ? 'default'+ : model.status === 'deprecated'+ ? 'secondary'+ : 'outline';++ return (+ <Card className="shadow">+ <CardHeader>+ <CardTitle+ className="text-sm font-semibold leading-tight truncate"+ title={model.name}+ >+ {model.label || model.name}+ </CardTitle>+ {model.realtime && (+ <CardAction>+ <Badge variant="outline">Realtime</Badge>+ </CardAction>+ )}+ </CardHeader>+ <CardContent className="flex flex-col">+ {model.status !== 'active' && (+ <Badge variant={statusVariant} className="text-xs">+ {model.status}+ </Badge>+ )}+ <div className="flex justify-between items-center text-[14px]">+ <span className="text-muted-foreground">{t('credential.inputTypes')}</span>+ <InputTypeBadges inputTypes={model.input_types} />+ </div>+ <div className="flex justify-between items-center text-[14px]">+ <span className="text-muted-foreground">{t('credential.outputTypes')}</span>+ <InputTypeBadges inputTypes={model.output_types} />+ </div>+ </CardContent>+ </Card>+ );+}+// ─── Detail panel ─────────────────────────────────────────────────────────────interface DetailPanelProps {@@ -116,6 +162,7 @@ interface DetailPanelProps {function DetailPanel({ credential, schema, onEdit, onDelete }: DetailPanelProps) {const { t } = useTranslation();const [models, setModels] = useState<ModelCard[]>([]);+ const [ttsModels, setTtsModels] = useState<TTSModelCard[]>([]);const [modelsLoading, setModelsLoading] = useState(false);const type = credential.data.type as string | undefined;@@ -123,10 +170,20 @@ function DetailPanel({ credential, schema, onEdit, onDelete }: DetailPanelProps)useEffect(() => {if (!type) return;setModelsLoading(true);- modelApi- .list(type)- .then((res) => setModels(res.models))- .catch(() => setModels([]))+ Promise.all([+ modelApi+ .list(type)+ .then((res) => res.models)+ .catch(() => [] as ModelCard[]),+ ttsModelApi+ .list(type)+ .then((res) => res.models)+ .catch(() => [] as TTSModelCard[]),+ ])+ .then(([chatModels, tts]) => {+ setModels(chatModels);+ setTtsModels(tts);+ }).finally(() => setModelsLoading(false));}, [credential.id, type]);@@ -223,6 +280,23 @@ function DetailPanel({ credential, schema, onEdit, onDelete }: DetailPanelProps)</div>)}</div>++ {/* Available TTS Models */}+ {ttsModels.length > 0 && (+ <>+ <Separator />+ <div className="flex flex-col gap-y-4">+ <h3 className="text-sm font-semibold">+ {t('credential.availableTTSModels')}({ttsModels.length})+ </h3>+ <div className="grid grid-cols-2 xl:grid-cols-3 gap-3">+ {ttsModels.map((m) => (+ <TTSModelCardItem key={m.name} model={m} />+ ))}+ </div>+ </div>+ </>+ )}</div>);}diff --git a/examples/web_ui/frontend/src/utils/streamingAudio.ts b/examples/web_ui/frontend/src/utils/streamingAudio.tsindex 41deb2cb0..f27b0384c 100644--- a/examples/web_ui/frontend/src/utils/streamingAudio.ts+++ b/examples/web_ui/frontend/src/utils/streamingAudio.ts@@ -225,6 +225,8 @@ interface Session {chunks: Bytes[];totalBytes: number;livePlayer: WavStreamPlayer | null;+ /** True if a live player was created at start — survives dispose. */+ hadLivePlayer: boolean;state: StreamingAudioState;}@@ -267,6 +269,7 @@ export class StreamingAudioManager {chunks: [],totalBytes: 0,livePlayer,+ hadLivePlayer: livePlayer !== null,state: { status: 'streaming', mediaType, url: null, interruptCount: 0 },});this.emit(blockId);@@ -300,7 +303,7 @@ export class StreamingAudioManager {const session = this.sessions.get(blockId);if (!session) return;- const hadLivePlayback = session.livePlayer !== null;+ const hadLivePlayback = session.hadLivePlayer;if (session.livePlayer) {// Let queued PCM finish playing; AudioContext closes via a// deferred timer. The livePlayer reference is kept so that@@ -332,6 +335,22 @@ export class StreamingAudioManager {}}+ /**+ * Stop live streaming playback only (WavStreamPlayers). Does not+ * touch replay ``<audio>`` elements or bump ``interruptCount``.+ * Called by the replay controller so that clicking play on a past+ * audio block silences any in-progress streaming audio without+ * interfering with the element that's about to start playing.+ */+ stopLivePlayback(): void {+ for (const session of this.sessions.values()) {+ if (session.livePlayer) {+ session.livePlayer.dispose();+ session.livePlayer = null;+ }+ }+ }+/*** Stop all in-progress and replay audio. Called when a new reply* starts so previous audio doesn't overlap with the new one.diff --git a/src/agentscope/app/_app.py b/src/agentscope/app/_app.pyindex 3e7403e7a..20a61fbff 100644--- a/src/agentscope/app/_app.py+++ b/src/agentscope/app/_app.py@@ -9,6 +9,7 @@ from ._router import (chat_router,credential_router,model_router,+ tts_model_router,schedule_router,session_router,workspace_router,@@ -169,6 +170,7 @@ def create_app(session_router,workspace_router,model_router,+ tts_model_router,):app.include_router(router)diff --git a/src/agentscope/app/_router/__init__.py b/src/agentscope/app/_router/__init__.pyindex 73f2fb145..6511f3957 100644--- a/src/agentscope/app/_router/__init__.py+++ b/src/agentscope/app/_router/__init__.py@@ -6,11 +6,13 @@ from ._credential import credential_routerfrom ._schedule import schedule_routerfrom ._session import session_routerfrom ._model import model_router+from ._tts_model import tts_model_routerfrom ._workspace import workspace_router__all__ = ["agent_router","model_router",+ "tts_model_router","chat_router","credential_router","schedule_router",diff --git a/src/agentscope/app/_router/_schema/__init__.py b/src/agentscope/app/_router/_schema/__init__.pyindex e9132450c..e84deeaf9 100644--- a/src/agentscope/app/_router/_schema/__init__.py+++ b/src/agentscope/app/_router/_schema/__init__.py@@ -3,6 +3,7 @@from ._chat import ChatRequest, ChatTriggerResponsefrom ._model import ListModelsResponse, ListModelsRequest+from ._tts_model import ListTTSModelsResponse, ListTTSModelsRequestfrom ._schedule import (CreateScheduleRequest,CreateScheduleResponse,@@ -55,6 +56,9 @@ __all__ = [# Model"ListModelsRequest","ListModelsResponse",+ # TTS Model+ "ListTTSModelsRequest",+ "ListTTSModelsResponse",# Schedule"CreateScheduleRequest","CreateScheduleResponse",diff --git a/src/agentscope/app/_router/_schema/_session.py b/src/agentscope/app/_router/_schema/_session.pyindex 260af5561..3b4865c5b 100644--- a/src/agentscope/app/_router/_schema/_session.py+++ b/src/agentscope/app/_router/_schema/_session.py@@ -3,7 +3,13 @@from pydantic import BaseModel, Fieldfrom ....permission import PermissionMode-from ...storage import AgentRecord, ChatModelConfig, SessionRecord, TeamRecord+from ...storage import (+ AgentRecord,+ ChatModelConfig,+ TTSModelConfig,+ SessionRecord,+ TeamRecord,+)class TeamMemberView(BaseModel):@@ -68,6 +74,10 @@ class CreateSessionRequest(BaseModel):description="Fallback model used when the primary model fails. ""Can be set later via PATCH.",)+ tts_model_config: TTSModelConfig | None = Field(+ default=None,+ description="TTS model configuration. Can be set later via PATCH.",+ )class CreateSessionResponse(BaseModel):@@ -97,6 +107,11 @@ class UpdateSessionRequest(BaseModel):description="New fallback model configuration. ""Pass null to clear; omit to leave unchanged.",)+ tts_model_config: TTSModelConfig | None = Field(+ default=None,+ description="New TTS model configuration. "+ "Pass null to clear; omit to leave unchanged.",+ )permission_mode: PermissionMode | None = Field(default=None,description="New permission mode for the session.",diff --git a/src/agentscope/app/_router/_schema/_tts_model.py b/src/agentscope/app/_router/_schema/_tts_model.pynew file mode 100644index 000000000..d37998a57--- /dev/null+++ b/src/agentscope/app/_router/_schema/_tts_model.py@@ -0,0 +1,23 @@+# -*- coding: utf-8 -*-+"""The TTS model configuration, used as DTO layer."""++from pydantic import BaseModel, Field++from ....tts import TTSModelCard+++class ListTTSModelsResponse(BaseModel):+ """List the candidate TTS models response."""++ models: list[TTSModelCard] = Field(+ description="The candidate TTS models.",+ )+ total: int = Field(description="The total number of candidates.")+++class ListTTSModelsRequest(BaseModel):+ """List the candidate TTS models request."""++ provider: str = Field(+ description="The provider type, e.g. dashscope_credential.",+ )diff --git a/src/agentscope/app/_router/_session.py b/src/agentscope/app/_router/_session.pyindex 56a3e1c0c..c7e4c4de5 100644--- a/src/agentscope/app/_router/_session.py+++ b/src/agentscope/app/_router/_session.py@@ -29,6 +29,7 @@ from .._service import SessionServicefrom ..storage import (AgentRecord,ChatModelConfig,+ TTSModelConfig,SessionConfig,SessionRecord,StorageBase,@@ -92,7 +93,7 @@ session_router = APIRouter(async def _ensure_credential_exists(storage: StorageBase,user_id: str,- config: ChatModelConfig | None,+ config: ChatModelConfig | TTSModelConfig | None,) -> None:"""Validate that the credential referenced by ``config`` belongs to thegiven user. No-op when ``config`` is ``None``.@@ -100,8 +101,8 @@ async def _ensure_credential_exists(Args:storage (`StorageBase`): Injected storage backend.user_id (`str`): The authenticated user ID.- config (`ChatModelConfig | None`): Model config to validate. Pass- ``None`` to skip the check.+ config (`ChatModelConfig | TTSModelConfig | None`): Model config to+ validate. Pass ``None`` to skip the check.Raises:`HTTPException`: 404 if the credential does not exist or does not@@ -230,6 +231,7 @@ async def create_session(user_id,body.fallback_chat_model_config,)+ await _ensure_credential_exists(storage, user_id, body.tts_model_config)session_record = await storage.upsert_session(user_id=user_id,@@ -238,6 +240,7 @@ async def create_session(workspace_id=body.workspace_id or uuid.uuid4().hex,chat_model_config=body.chat_model_config,fallback_chat_model_config=body.fallback_chat_model_config,+ tts_model_config=body.tts_model_config,**({"name": body.name} if body.name is not None else {}),),)@@ -325,6 +328,7 @@ async def update_session(user_id,body.fallback_chat_model_config,)+ await _ensure_credential_exists(storage, user_id, body.tts_model_config)updated_state = existing.stateif body.permission_mode is not None:diff --git a/src/agentscope/app/_router/_tts_model.py b/src/agentscope/app/_router/_tts_model.pynew file mode 100644index 000000000..949388c5b--- /dev/null+++ b/src/agentscope/app/_router/_tts_model.py@@ -0,0 +1,40 @@+# -*- coding: utf-8 -*-+"""The TTS model router."""++from fastapi import APIRouter, Depends, HTTPException, status++from ._schema import ListTTSModelsResponse, ListTTSModelsRequest+from ...credential import CredentialFactory++tts_model_router = APIRouter(+ prefix="/tts-model",+ tags=["tts-model"],+ responses={404: {"description": "Not found"}},+)+++@tts_model_router.get(+ "/",+ response_model=ListTTSModelsResponse,+ summary="List all candidate TTS models under the given credential type",+)+async def list_tts_models(+ body: ListTTSModelsRequest = Depends(),+) -> ListTTSModelsResponse:+ """Return all candidate TTS models under the given credential type.++ Args:+ body (ListTTSModelsRequest): The request body.++ Returns:+ `ListTTSModelsResponse`: The response body.+ """+ credential_cls = CredentialFactory.get_credential_class(body.provider)+ if credential_cls is None:+ raise HTTPException(+ status_code=status.HTTP_404_NOT_FOUND,+ detail=f"Provider '{body.provider}' not found.",+ )++ models = credential_cls.list_tts_models()+ return ListTTSModelsResponse(models=models, total=len(models))diff --git a/src/agentscope/app/_service/__init__.py b/src/agentscope/app/_service/__init__.pyindex 77c65bb5c..1b22f7b06 100644--- a/src/agentscope/app/_service/__init__.py+++ b/src/agentscope/app/_service/__init__.py@@ -3,6 +3,7 @@from ._chat import ChatServicefrom ._embedding import get_embedding_modelfrom ._model import get_model+from ._tts_model import get_tts_modelfrom ._session import SessionServicefrom ._toolkit import get_toolkit@@ -11,5 +12,6 @@ __all__ = ["SessionService","get_embedding_model","get_model",+ "get_tts_model","get_toolkit",]diff --git a/src/agentscope/app/_service/_chat.py b/src/agentscope/app/_service/_chat.pyindex 74e812585..da7a82203 100644--- a/src/agentscope/app/_service/_chat.py+++ b/src/agentscope/app/_service/_chat.py@@ -22,12 +22,14 @@ from ..middleware import (StateChangeMiddleware,ToolOffloadMiddleware,)+from ...middleware import TTSMiddlewarefrom .._types import (AgentMiddlewareFactory,AgentToolFactory,SubAgentTemplate,)from ._model import get_model+from ._tts_model import get_tts_modelfrom ._toolkit import get_toolkitfrom ..._logging import logger@@ -277,6 +279,18 @@ class ChatService:),)+ # ----------------------------------------------------------------+ # 3b. TTS middleware — inject when the session has a TTS config.+ # ----------------------------------------------------------------+ tts_cfg = session_record.config.tts_model_config+ if tts_cfg is not None:+ tts_model = await get_tts_model(+ user_id,+ tts_cfg,+ self._storage,+ )+ middlewares.append(TTSMiddleware(tts_model))+# ----------------------------------------------------------------# 4. Model + fallback (resolved from session's config).# ----------------------------------------------------------------diff --git a/src/agentscope/app/_service/_tts_model.py b/src/agentscope/app/_service/_tts_model.pynew file mode 100644index 000000000..6f2a3dbe5--- /dev/null+++ b/src/agentscope/app/_service/_tts_model.py@@ -0,0 +1,68 @@+# -*- coding: utf-8 -*-+"""TTS model service: builds a TTSModelBase from stored credential + config."""+from typing import Type++from fastapi import HTTPException, status++from ..storage import StorageBase, TTSModelConfig+from ...credential import CredentialFactory+from ...tts import TTSModelBase+++async def get_tts_model(+ user_id: str,+ config: TTSModelConfig,+ storage: StorageBase,+) -> TTSModelBase:+ """Get the TTS model instance from the configuration and storage.++ Args:+ user_id (`str`):+ The user id.+ config (`TTSModelConfig`):+ The TTS model configuration.+ storage (`StorageBase`):+ The storage instance.++ Returns:+ `TTSModelBase`:+ The TTS model instance.+ """+ credential_record = await storage.get_credential(+ user_id,+ config.credential_id,+ )+ if credential_record is None:+ raise HTTPException(+ status_code=status.HTTP_404_NOT_FOUND,+ detail=f"Credential {config.credential_id!r} not found.",+ )++ credential = CredentialFactory.from_dict(credential_record.data)+ tts_classes = credential.get_tts_model_classes()+ if not tts_classes:+ raise HTTPException(+ status_code=status.HTTP_400_BAD_REQUEST,+ detail=f"Provider {config.type!r} does not support TTS models.",+ )++ tts_cls = _resolve_tts_class(tts_classes, config.model)+ parameters = (+ tts_cls.Parameters(**config.parameters) if config.parameters else None+ )+ return tts_cls(+ credential=credential,+ model=config.model,+ parameters=parameters,+ )+++def _resolve_tts_class(+ classes: list[Type[TTSModelBase]],+ model: str,+) -> Type[TTSModelBase]:+ """Pick the TTS class that lists the given model name."""+ for cls in classes:+ if any(card.name == model for card in cls.list_models()):+ return cls+ return classes[0]diff --git a/src/agentscope/app/storage/__init__.py b/src/agentscope/app/storage/__init__.pyindex 86da74093..4baf8f70f 100644--- a/src/agentscope/app/storage/__init__.py+++ b/src/agentscope/app/storage/__init__.py@@ -14,6 +14,7 @@ from ._model import (SessionRecord,SessionSource,ChatModelConfig,+ TTSModelConfig,EmbeddingModelConfig,TeamData,TeamRecord,@@ -31,6 +32,7 @@ __all__ = ["SessionRecord","SessionSource","ChatModelConfig",+ "TTSModelConfig","EmbeddingModelConfig","TeamData","TeamRecord",diff --git a/src/agentscope/app/storage/_model/__init__.py b/src/agentscope/app/storage/_model/__init__.pyindex 74f61d06b..b68673f6d 100644--- a/src/agentscope/app/storage/_model/__init__.py+++ b/src/agentscope/app/storage/_model/__init__.py@@ -8,6 +8,7 @@ from ._session import (SessionRecord,SessionConfig,ChatModelConfig,+ TTSModelConfig,EmbeddingModelConfig,SessionSource,)@@ -25,6 +26,7 @@ __all__ = ["SessionRecord","SessionSource","ChatModelConfig",+ "TTSModelConfig","EmbeddingModelConfig","TeamData","TeamRecord",diff --git a/src/agentscope/app/storage/_model/_session.py b/src/agentscope/app/storage/_model/_session.pyindex 19d97b82b..a43c0a181 100644--- a/src/agentscope/app/storage/_model/_session.py+++ b/src/agentscope/app/storage/_model/_session.py@@ -32,6 +32,22 @@ class ChatModelConfig(BaseModel):"""The model parameters."""+class TTSModelConfig(BaseModel):+ """The TTS model configuration class."""++ type: str+ """The provider type."""++ credential_id: str+ """The credential id."""++ model: str+ """The TTS model name."""++ parameters: dict+ """TTS parameters (voice, language, etc.)."""++class EmbeddingModelConfig(BaseModel):"""Configuration for constructing an embedding model from a credential.@@ -73,6 +89,9 @@ class SessionConfig(BaseModel):"""The fallback chat model config. Used as a backup when the primarymodel fails. None means no fallback configured."""+ tts_model_config: TTSModelConfig | None = None+ """The TTS model config. None means TTS is not enabled."""+class SessionRecord(_RecordBase):"""The session record."""diff --git a/src/agentscope/credential/_base.py b/src/agentscope/credential/_base.pyindex 72f89940f..c8566acc8 100644--- a/src/agentscope/credential/_base.py+++ b/src/agentscope/credential/_base.py@@ -8,6 +8,8 @@ from pydantic import BaseModel, Fieldif TYPE_CHECKING:from ..embedding import EmbeddingModelBasefrom ..model import ChatModelBase, ModelCard+ from ..tts import TTSModelBase+ from ..tts._tts_model_card import TTSModelCardclass CredentialBase(BaseModel):@@ -37,6 +39,33 @@ class CredentialBase(BaseModel):f"{cls.__name__} must implement ``get_chat_model_class``.",)+ @classmethod+ def get_tts_model_classes(cls) -> list[Type["TTSModelBase"]]:+ """Return the TTS model classes supported by this credential.++ Subclasses that support TTS should override this to return one or+ more :class:`TTSModelBase` subclasses. The default returns an empty+ list (provider does not support TTS).++ Returns:+ `list[Type[TTSModelBase]]`:+ The TTS model classes, or an empty list.+ """+ return []++ @classmethod+ def list_tts_models(cls) -> list["TTSModelCard"]:+ """List the candidate TTS models available under this credential.++ Returns:+ `list[TTSModelCard]`:+ A list of TTS model cards, or empty if TTS is not supported.+ """+ cards: list["TTSModelCard"] = []+ for tts_cls in cls.get_tts_model_classes():+ cards.extend(tts_cls.list_models())+ return cards+@classmethoddef list_models(cls) -> list["ModelCard"]:"""List the candidate chat models that are available under thisdiff --git a/src/agentscope/credential/_dashscope.py b/src/agentscope/credential/_dashscope.pyindex 8e9a85666..11e95fd1c 100644--- a/src/agentscope/credential/_dashscope.py+++ b/src/agentscope/credential/_dashscope.py@@ -9,6 +9,7 @@ from ._base import CredentialBaseif TYPE_CHECKING:from ..embedding import EmbeddingModelBasefrom ..model import ChatModelBase+ from ..tts import TTSModelBase_DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"@@ -43,6 +44,13 @@ class DashScopeCredential(CredentialBase):return DashScopeChatModel+ @classmethod+ def get_tts_model_classes(cls) -> list[Type["TTSModelBase"]]:+ """Return the DashScope TTS model classes."""+ from ..tts import DashScopeRealtimeTTSModel, DashScopeTTSModel++ return [DashScopeTTSModel, DashScopeRealtimeTTSModel]+@classmethoddef get_embedding_model_class(cls) -> Type["EmbeddingModelBase"]:"""Return the DashScopeEmbeddingModel class."""diff --git a/src/agentscope/middleware/__init__.py b/src/agentscope/middleware/__init__.pyindex 3b501bd79..a15f6b5d6 100644--- a/src/agentscope/middleware/__init__.py+++ b/src/agentscope/middleware/__init__.py@@ -3,8 +3,10 @@from ._base import MiddlewareBasefrom ._tracing import TracingMiddleware+from ._tts_middleware import TTSMiddleware__all__ = ["MiddlewareBase","TracingMiddleware",+ "TTSMiddleware",]diff --git a/src/agentscope/middleware/_tts_middleware.py b/src/agentscope/middleware/_tts_middleware.pynew file mode 100644index 000000000..cff4502ee--- /dev/null+++ b/src/agentscope/middleware/_tts_middleware.py@@ -0,0 +1,179 @@+# -*- coding: utf-8 -*-+"""Middleware that turns reasoning text into speech and injects it as+``DATA_BLOCK_*`` events into the agent's event stream."""+import uuid+from typing import TYPE_CHECKING, AsyncGenerator, Callable++from ._base import MiddlewareBase+from ..event import (+ DataBlockDeltaEvent,+ DataBlockEndEvent,+ DataBlockStartEvent,+ TextBlockDeltaEvent,+ TextBlockEndEvent,+)+from ..tts import TTSModelBase, TTSResponse++if TYPE_CHECKING:+ from ..agent import Agent+++class TTSMiddleware(MiddlewareBase):+ """Synthesize speech for every text block produced during reasoning and+ inject the audio as ``DATA_BLOCK_*`` events into the stream.++ - Non-realtime TTS (``realtime=False``): on each+ ``TextBlockEndEvent`` the accumulated text is sent to+ :meth:`TTSModelBase.synthesize`; the resulting audio chunks are+ emitted as one ``DATA_BLOCK_START`` + N ``DATA_BLOCK_DELTA`` ++ ``DATA_BLOCK_END``.+ - Realtime TTS (``realtime=True``): each+ ``TextBlockDeltaEvent`` is pushed into the model via+ :meth:`TTSModelBase.push`; any audio produced is emitted immediately.+ On ``TextBlockEndEvent`` :meth:`TTSModelBase.synthesize` is called+ to drain remaining audio, and the data block is closed.++ Each ``DataBlockDeltaEvent.data`` carries an **incremental** base64 PCM+ chunk; the full audio is the concatenation of every delta's decoded+ bytes (the data block is keyed by ``block_id``).+ """++ def __init__(self, tts_model: TTSModelBase) -> None:+ """Initialize the TTS middleware.++ Args:+ tts_model (`TTSModelBase`):+ The TTS model used to synthesize speech for assistant text+ blocks produced during reasoning.+ """+ self.tts = tts_model++ async def on_reply(+ self,+ agent: "Agent",+ input_kwargs: dict,+ next_handler: Callable[..., AsyncGenerator],+ ) -> AsyncGenerator:+ """Intercept the reply stream, synthesize speech for text blocks,+ and inject ``DATA_BLOCK_*`` audio events into the output."""+ text_buffer: str = ""+ audio_block_id: str | None = None+ audio_media_type: str | None = None++ async with self.tts:+ async for evt in next_handler(**input_kwargs):+ yield evt++ if isinstance(evt, TextBlockDeltaEvent):+ text_buffer += evt.delta+ if self.tts.realtime and evt.delta:+ tts_res = await self.tts.push(evt.delta)+ async for audio_evt in self._emit_chunk(+ agent,+ tts_res,+ audio_block_id,+ audio_media_type,+ ):+ if isinstance(audio_evt, DataBlockStartEvent):+ audio_block_id = audio_evt.block_id+ audio_media_type = audio_evt.media_type+ yield audio_evt++ elif isinstance(evt, TextBlockEndEvent):+ text = text_buffer+ text_buffer = ""++ if self.tts.realtime:+ res = await self.tts.synthesize()+ async for audio_evt in self._emit_synth_result(+ agent,+ res,+ audio_block_id,+ audio_media_type,+ ):+ if isinstance(audio_evt, DataBlockStartEvent):+ audio_block_id = audio_evt.block_id+ audio_media_type = audio_evt.media_type+ yield audio_evt+ elif text.strip():+ res = await self.tts.synthesize(text)+ async for audio_evt in self._emit_synth_result(+ agent,+ res,+ audio_block_id,+ audio_media_type,+ ):+ if isinstance(audio_evt, DataBlockStartEvent):+ audio_block_id = audio_evt.block_id+ audio_media_type = audio_evt.media_type+ yield audio_evt++ if audio_block_id is not None:+ yield DataBlockEndEvent(+ reply_id=agent.state.reply_id,+ block_id=audio_block_id,+ )+ audio_block_id = None+ audio_media_type = None++ async def _emit_synth_result(+ self,+ agent: "Agent",+ res: TTSResponse | AsyncGenerator[TTSResponse, None],+ audio_block_id: str | None,+ audio_media_type: str | None,+ ) -> AsyncGenerator:+ """Normalize ``synthesize()`` returns (single response or async+ generator) into a stream of ``DATA_BLOCK_*`` events."""+ if isinstance(res, AsyncGenerator):+ async for chunk in res:+ async for ae in self._emit_chunk(+ agent,+ chunk,+ audio_block_id,+ audio_media_type,+ ):+ if isinstance(ae, DataBlockStartEvent):+ audio_block_id = ae.block_id+ audio_media_type = ae.media_type+ yield ae+ else:+ async for ae in self._emit_chunk(+ agent,+ res,+ audio_block_id,+ audio_media_type,+ ):+ yield ae++ @staticmethod+ async def _emit_chunk(+ agent: "Agent",+ tts_res: TTSResponse | None,+ audio_block_id: str | None,+ audio_media_type: str | None,+ ) -> AsyncGenerator:+ """Emit one TTSResponse chunk as ``DATA_BLOCK_START`` (if needed)+ followed by ``DATA_BLOCK_DELTA``."""+ if tts_res is None or tts_res.content is None:+ return+ media_type = tts_res.content.source.media_type+ data = tts_res.content.source.data+ if not data:+ return++ if audio_block_id is None:+ audio_block_id = uuid.uuid4().hex+ audio_media_type = media_type+ yield DataBlockStartEvent(+ reply_id=agent.state.reply_id,+ block_id=audio_block_id,+ media_type=media_type,+ )++ yield DataBlockDeltaEvent(+ reply_id=agent.state.reply_id,+ block_id=audio_block_id,+ data=data,+ media_type=audio_media_type,+ )diff --git a/src/agentscope/tts/__init__.py b/src/agentscope/tts/__init__.pynew file mode 100644index 000000000..42179d92d--- /dev/null+++ b/src/agentscope/tts/__init__.py@@ -0,0 +1,16 @@+# -*- coding: utf-8 -*-+"""The TTS (Text-to-Speech) module in AgentScope."""++from ._tts_base import TTSModelBase+from ._tts_model_card import TTSModelCard+from ._tts_response import TTSResponse, TTSUsage+from ._dashscope import DashScopeTTSModel, DashScopeRealtimeTTSModel++__all__ = [+ "TTSModelBase",+ "TTSModelCard",+ "TTSResponse",+ "TTSUsage",+ "DashScopeTTSModel",+ "DashScopeRealtimeTTSModel",+]diff --git a/src/agentscope/tts/_dashscope/__init__.py b/src/agentscope/tts/_dashscope/__init__.pynew file mode 100644index 000000000..33cb3c420--- /dev/null+++ b/src/agentscope/tts/_dashscope/__init__.py@@ -0,0 +1,10 @@+# -*- coding: utf-8 -*-+"""The DashScope TTS module."""++from ._model import DashScopeTTSModel+from ._realtime_model import DashScopeRealtimeTTSModel++__all__ = [+ "DashScopeTTSModel",+ "DashScopeRealtimeTTSModel",+]diff --git a/src/agentscope/tts/_dashscope/_model.py b/src/agentscope/tts/_dashscope/_model.pynew file mode 100644index 000000000..8f63a60bb--- /dev/null+++ b/src/agentscope/tts/_dashscope/_model.py@@ -0,0 +1,252 @@+# -*- coding: utf-8 -*-+"""DashScope TTS model implementation using MultiModalConversation API."""+import base64+import io+import wave+from datetime import datetime+from typing import (+ Any,+ AsyncGenerator,+ Generator,+ Literal,+ TYPE_CHECKING,+)++from pydantic import BaseModel, Field++from .._tts_base import TTSModelBase+from .._tts_response import TTSResponse, TTSUsage+from ..._utils._audio import _build_streaming_wav_header+from ...credential import DashScopeCredential+from ...message import DataBlock, Base64Source++if TYPE_CHECKING:+ from dashscope.api_entities.dashscope_response import (+ MultiModalConversationResponse,+ )+++# DashScope TTS returns raw PCM (24kHz, mono, 16-bit). We wrap it as WAV+# on the way out so the frontend can play it: streaming deltas get a+# streaming WAV header on the first chunk; non-streaming returns a+# self-contained fixed-size WAV.+_TTS_SAMPLE_RATE = 24000+_TTS_CHANNELS = 1+_TTS_BITS_PER_SAMPLE = 16+_DEFAULT_MEDIA_TYPE = "audio/wav"+_SENTINEL = object()+++def _parse_usage(usage: Any, elapsed: float) -> TTSUsage | None:+ """Extract a TTSUsage from the DashScope usage object, or None."""+ if usage is None:+ return None+ return TTSUsage(+ input_tokens=getattr(usage, "input_tokens", 0) or 0,+ output_tokens=getattr(usage, "output_tokens", 0) or 0,+ time=elapsed,+ )+++class DashScopeTTSModel(TTSModelBase):+ """DashScope TTS model implementation using the MultiModalConversation+ API. For more details please see the `official document+ <https://bailian.console.aliyun.com/?tab=doc#/doc/?type=model&url=2879134>`_.+ """++ class Parameters(BaseModel):+ """Frontend-exposed parameters for DashScope TTS models."""++ voice: str = Field(+ default="Cherry",+ title="Voice",+ description="The voice to use for synthesis.",+ )++ type: Literal["dashscope_tts"] = "dashscope_tts"+ """The type of the TTS model."""++ realtime: bool = False++ def __init__(+ self,+ credential: DashScopeCredential,+ model: str = "qwen3-tts-flash",+ parameters: "DashScopeTTSModel.Parameters | None" = None,+ stream: bool = True,+ ) -> None:+ """Initialize the DashScope TTS model.++ .. note:: More details about the parameters, such as ``model``+ and ``voice``, can be found in the `official document+ <https://bailian.console.aliyun.com/?tab=doc#/doc/?type=model&url=2879134>`_.++ Args:+ credential (`DashScopeCredential`):+ The DashScope credential used to authenticate the API call.+ model (`str`, defaults to ``"qwen3-tts-flash"``):+ The TTS model name. Supported models include+ ``qwen3-tts-flash``, ``qwen-tts``, etc.+ parameters (`DashScopeTTSModel.Parameters | None`, defaults to \+ `None`):+ The TTS parameters (voice, language, etc.). When ``None``,+ the default parameters will be used.+ stream (`bool`, defaults to `True`):+ Whether to use streaming output. When `True`,+ :meth:`synthesize` returns an async generator yielding+ ``TTSResponse`` chunks; when `False`, it returns a single+ aggregated ``TTSResponse``.+ """+ super().__init__(+ credential=credential,+ model=model,+ parameters=parameters,+ stream=stream,+ )++ async def synthesize(+ self,+ text: str | None = None,+ **kwargs: Any,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Call the DashScope TTS API to synthesize speech from text.++ Args:+ text (`str | None`, optional):+ The text to be synthesized.+ **kwargs (`Any`):+ Additional keyword arguments to pass to the TTS API call.++ Returns:+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:+ A single ``TTSResponse`` when ``stream=False``, or an async+ generator yielding ``TTSResponse`` chunks when ``stream=True``.+ """+ if not text:+ return TTSResponse(content=None)++ import dashscope++ response = dashscope.MultiModalConversation.call(+ model=self.model,+ api_key=self.credential.api_key.get_secret_value(),+ text=text,+ voice=self.parameters.voice,+ stream=True,+ **kwargs,+ )++ if self.stream:+ return self._parse_into_async_generator(response)++ return self._aggregate_sync(response)++ @staticmethod+ def _aggregate_sync(+ response: Generator["MultiModalConversationResponse", None, None],+ ) -> TTSResponse:+ """Aggregate all streaming chunks into a single self-contained WAV."""+ start_datetime = datetime.now()+ audio_bytes = bytearray()+ usage = None+ for chunk in response:+ if chunk.usage is not None:+ usage = chunk.usage+ if chunk.output is not None:+ audio = chunk.output.audio+ if audio and audio.data:+ audio_bytes += base64.b64decode(audio.data)+ elapsed = (datetime.now() - start_datetime).total_seconds()++ buf = io.BytesIO()+ with wave.open(buf, "wb") as wav:+ wav.setnchannels(_TTS_CHANNELS)+ wav.setsampwidth(_TTS_BITS_PER_SAMPLE // 8)+ wav.setframerate(_TTS_SAMPLE_RATE)+ wav.writeframes(bytes(audio_bytes))++ return TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(buf.getvalue()).decode("ascii"),+ media_type=_DEFAULT_MEDIA_TYPE,+ ),+ ),+ usage=_parse_usage(usage, elapsed),+ )++ @staticmethod+ async def _parse_into_async_generator(+ response: Generator["MultiModalConversationResponse", None, None],+ ) -> AsyncGenerator[TTSResponse, None]:+ """Parse the streaming TTS response into an async generator.++ Each yielded ``TTSResponse`` carries an **incremental** WAV chunk:+ the first chunk is prefixed with a streaming WAV/RIFF header so the+ frontend can start playback immediately (without waiting for+ end-of-stream); subsequent chunks are raw PCM bytes appended to that+ open stream. The final response has ``is_last=True``.++ Args:+ response (`Generator[MultiModalConversationResponse, None, None]`):+ The streaming response from the DashScope TTS API.++ Yields:+ `TTSResponse`:+ A ``TTSResponse`` for each incremental audio chunk; the final+ response has ``is_last=True``.+ """+ pending: TTSResponse | None = None+ header_sent = False+ usage = None+ start_datetime = datetime.now()+ it = iter(response)+ while True:+ chunk = next(it, _SENTINEL)+ if chunk is _SENTINEL:+ break+ if chunk.usage is not None:+ usage = chunk.usage+ if chunk.output is None:+ continue+ audio = chunk.output.audio+ if not audio or not audio.data:+ continue+ delta_bytes = base64.b64decode(audio.data)+ if not delta_bytes:+ continue+ if not header_sent:+ payload = (+ _build_streaming_wav_header(+ sample_rate=_TTS_SAMPLE_RATE,+ channels=_TTS_CHANNELS,+ bits_per_sample=_TTS_BITS_PER_SAMPLE,+ )+ + delta_bytes+ )+ header_sent = True+ else:+ payload = delta_bytes+ if pending is not None:+ yield pending+ pending = TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(payload).decode("ascii"),+ media_type=_DEFAULT_MEDIA_TYPE,+ ),+ ),+ is_last=False,+ )+ elapsed = (datetime.now() - start_datetime).total_seconds()++ if pending is not None:+ pending.is_last = True+ pending.usage = _parse_usage(usage, elapsed)+ yield pending+ else:+ yield TTSResponse(+ content=None,+ is_last=True,+ usage=_parse_usage(usage, elapsed),+ )diff --git a/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash-realtime.yaml b/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash-realtime.yamlnew file mode 100644index 000000000..a0f4f0cac--- /dev/null+++ b/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash-realtime.yaml@@ -0,0 +1,19 @@+name: qwen3-tts-flash-realtime+label: Qwen3-TTS-Flash-Realtime+status: active+realtime: true++input_types:+ - text/plain++output_types:+ - audio/wav++voices:+ # Source: https://help.aliyun.com/zh/model-studio/qwen-tts-voice-list+ - Cherry+ - Serena+ - Ethan+ - Chelsie++parameter_overrides: {}diff --git a/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash.yaml b/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash.yamlnew file mode 100644index 000000000..6750a5026--- /dev/null+++ b/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash.yaml@@ -0,0 +1,18 @@+name: qwen3-tts-flash+label: Qwen3-TTS-Flash+status: active++input_types:+ - text/plain++output_types:+ - audio/wav++voices:+ # Source: https://help.aliyun.com/zh/model-studio/qwen-tts-voice-list+ - Cherry+ - Serena+ - Ethan+ - Chelsie++parameter_overrides: {}diff --git a/src/agentscope/tts/_dashscope/_realtime_model.py b/src/agentscope/tts/_dashscope/_realtime_model.pynew file mode 100644index 000000000..40767c20b--- /dev/null+++ b/src/agentscope/tts/_dashscope/_realtime_model.py@@ -0,0 +1,463 @@+# -*- coding: utf-8 -*-+"""DashScope Realtime TTS model implementation."""+import asyncio+import base64+import threading+from typing import Any, AsyncGenerator, Literal, TYPE_CHECKING++from pydantic import BaseModel, Field++from .._tts_base import TTSModelBase+from .._tts_response import TTSResponse+from ..._logging import logger+from ..._utils._audio import _build_streaming_wav_header+from ...credential import DashScopeCredential+from ...message import DataBlock, Base64Source++if TYPE_CHECKING:+ from dashscope.audio.qwen_tts_realtime import (+ QwenTtsRealtime,+ QwenTtsRealtimeCallback,+ )+++_MEDIA_TYPE = "audio/wav"+_SAMPLE_RATE = 24000+_CHANNELS = 1+_BITS_PER_SAMPLE = 16+++def _make_callback_class() -> type["QwenTtsRealtimeCallback"]:+ """Create the DashScope realtime TTS callback class lazily to avoid+ importing dashscope at module level."""+ from dashscope.audio.qwen_tts_realtime import QwenTtsRealtimeCallback++ class _Callback(QwenTtsRealtimeCallback):+ """Internal callback that accumulates PCM audio from the WebSocket.++ Audio data is stored as raw bytes (decoded from base64) to allow+ incremental delta extraction without base64 boundary issues.+ """++ def __init__(self) -> None:+ super().__init__()+ self.chunk_event = threading.Event()+ self.finish_event = threading.Event()+ self._pcm_bytes: bytearray = bytearray()+ self._consumed: int = 0++ def on_event(self, response: dict[str, Any]) -> None:+ """Handle incoming WebSocket events from the TTS service."""+ try:+ event_type = response.get("type")++ if event_type == "session.created":+ self._pcm_bytes = bytearray()+ self._consumed = 0+ self.finish_event.clear()++ elif event_type == "response.audio.delta":+ audio_data = response.get("delta")+ if audio_data:+ if isinstance(audio_data, bytes):+ self._pcm_bytes += audio_data+ else:+ self._pcm_bytes += base64.b64decode(audio_data)+ if not self.chunk_event.is_set():+ self.chunk_event.set()++ elif event_type == "session.finished":+ self.chunk_event.set()+ self.finish_event.set()++ except Exception:+ logger.exception("Error in TTS WebSocket callback")+ self.finish_event.set()++ def on_close(self, close_status_code: int, close_msg: str) -> None:+ """Handle WebSocket connection closure."""+ self.finish_event.set()+ self.chunk_event.set()+ if close_status_code:+ logger.warning(+ "TTS WebSocket closed with code %s: %s",+ close_status_code,+ close_msg,+ )++ def _take_delta(self, header: bool = False) -> bytes | None:+ """Return new PCM bytes since last call, or None if empty.++ Args:+ header: If True, prepend a streaming WAV header to the+ first returned chunk.+ """+ new_data = self._pcm_bytes[self._consumed :]+ if not new_data:+ return None+ self._consumed = len(self._pcm_bytes)+ if header:+ return _build_streaming_wav_header(+ sample_rate=_SAMPLE_RATE,+ channels=_CHANNELS,+ bits_per_sample=_BITS_PER_SAMPLE,+ ) + bytes(new_data)+ return bytes(new_data)++ def get_audio_response(self, block: bool) -> TTSResponse:+ """Return incremental audio delta (non-blocking or blocking)."""+ if block:+ self.finish_event.wait()+ delta = self._take_delta(header=self._consumed == 0)+ if delta:+ return TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(delta).decode("ascii"),+ media_type=_MEDIA_TYPE,+ ),+ ),+ )+ return TTSResponse(content=None)++ async def get_audio_chunks(self) -> AsyncGenerator[TTSResponse, None]:+ """Yield incremental WAV audio chunks as they arrive."""+ header_sent = False+ while True:+ if self.finish_event.is_set():+ delta = self._take_delta(header=not header_sent)+ if delta:+ yield TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(delta).decode(+ "ascii",+ ),+ media_type=_MEDIA_TYPE,+ ),+ ),+ is_last=True,+ )+ else:+ yield TTSResponse(content=None, is_last=True)+ self.reset()+ break++ if self.chunk_event.is_set():+ self.chunk_event.clear()+ else:+ await asyncio.to_thread(self.chunk_event.wait)++ if self.finish_event.is_set():+ continue++ delta = self._take_delta(header=not header_sent)+ if delta:+ header_sent = True+ yield TTSResponse(+ content=DataBlock(+ source=Base64Source(+ data=base64.b64encode(delta).decode("ascii"),+ media_type=_MEDIA_TYPE,+ ),+ ),+ is_last=False,+ )++ def reset(self) -> None:+ """Reset internal state for the next utterance."""+ self.finish_event.clear()+ self.chunk_event.clear()+ self._pcm_bytes = bytearray()+ self._consumed = 0++ def has_audio_data(self) -> bool:+ """Return whether any audio data has been received."""+ return bool(self._pcm_bytes)++ return _Callback+++class DashScopeRealtimeTTSModel(TTSModelBase):+ """DashScope Realtime TTS model using the QwenTtsRealtime WebSocket API.++ This model supports streaming input: text can be pushed incrementally+ via :meth:`push`, and :meth:`synthesize` finalizes the current utterance.++ For more details see the `official document+ <https://bailian.console.aliyun.com/?tab=doc#/doc/?type=model&url=2938790>`_.++ .. note:: Only one streaming input request can be active at a time.+ """++ class Parameters(BaseModel):+ """Frontend-exposed parameters for DashScope Realtime TTS models."""++ voice: str = Field(+ default="Cherry",+ title="Voice",+ description="The voice to use for synthesis.",+ )++ type: Literal["dashscope_realtime_tts"] = "dashscope_realtime_tts"++ realtime: bool = True++ def __init__(+ self,+ credential: DashScopeCredential,+ model: str = "qwen3-tts-flash-realtime",+ parameters: "DashScopeRealtimeTTSModel.Parameters | None" = None,+ stream: bool = True,+ cold_start_length: int | None = None,+ cold_start_words: int | None = None,+ max_retries: int = 3,+ retry_delay: float = 5.0,+ ) -> None:+ """Initialize the DashScope Realtime TTS model.++ Args:+ credential (`DashScopeCredential`):+ The DashScope credential.+ model (`str`, defaults to ``"qwen3-tts-flash-realtime"``):+ The realtime TTS model name.+ parameters (`Parameters | None`, defaults to `None`):+ The TTS parameters (voice, etc.).+ stream (`bool`, defaults to `True`):+ Whether :meth:`synthesize` returns a streaming async generator.+ cold_start_length (`int | None`, defaults to `None`):+ Minimum character count before the first text chunk is sent.+ cold_start_words (`int | None`, defaults to `None`):+ Minimum word count before the first text chunk is sent.+ max_retries (`int`, defaults to `3`):+ Max retry attempts on WebSocket failure.+ retry_delay (`float`, defaults to `5.0`):+ Initial retry delay in seconds (exponential backoff).+ """+ super().__init__(+ credential=credential,+ model=model,+ parameters=parameters,+ stream=stream,+ )+ self.cold_start_length = cold_start_length+ self.cold_start_words = cold_start_words+ self.max_retries = max_retries+ self.retry_delay = retry_delay++ self._tts_client: QwenTtsRealtime | None = None+ self._callback: Any = None+ self._connected = False+ self._cold_start_buffer: str = ""+ self._cold_start_done: bool = False+ self._accumulated_text: str = ""++ def _create_client(self) -> None:+ """Create a fresh TTS client and callback."""+ import dashscope+ from dashscope.audio.qwen_tts_realtime import QwenTtsRealtime++ dashscope.api_key = self.credential.api_key.get_secret_value()++ callback_cls = _make_callback_class()+ self._callback = callback_cls()+ self._tts_client = QwenTtsRealtime(+ model=self.model,+ callback=self._callback,+ )++ async def connect(self) -> None:+ """Establish the WebSocket connection."""+ if self._connected:+ return++ self._create_client()+ self._tts_client.connect()+ self._tts_client.update_session(+ voice=self.parameters.voice,+ mode="server_commit",+ )+ self._connected = True++ async def close(self) -> None:+ """Close the WebSocket connection."""+ if not self._connected:+ return+ self._connected = False+ try:+ self._tts_client.close()+ except Exception:+ pass++ async def _reconnect(self) -> None:+ """Reconnect by recreating the client."""+ try:+ self._tts_client.close()+ except Exception:+ pass+ self._connected = False+ self._cold_start_buffer = ""+ self._cold_start_done = False+ await self.connect()++ async def push(+ self,+ text: str,+ **kwargs: Any,+ ) -> TTSResponse:+ """Push an incremental text delta for realtime synthesis.++ Args:+ text (`str`):+ An incremental text chunk (delta) to append.+ **kwargs (`Any`):+ Additional keyword arguments (unused).++ Returns:+ `TTSResponse`:+ Audio accumulated so far, or empty if not yet available.+ """+ from websocket import WebSocketConnectionClosedException++ if not self._connected:+ raise RuntimeError(+ "TTS model is not connected. Call `connect()` first.",+ )++ if not text:+ return TTSResponse(content=None)++ self._accumulated_text += text++ if not self._cold_start_done:+ self._cold_start_buffer += text+ ready = True+ if (+ self.cold_start_length+ and len(self._cold_start_buffer) < self.cold_start_length+ ):+ ready = False+ if (+ ready+ and self.cold_start_words+ and len(self._cold_start_buffer.split())+ < self.cold_start_words+ ):+ ready = False+ if ready:+ try:+ self._tts_client.append_text(self._cold_start_buffer)+ except WebSocketConnectionClosedException:+ return TTSResponse(content=None)+ self._cold_start_buffer = ""+ self._cold_start_done = True+ else:+ try:+ self._tts_client.append_text(text)+ except WebSocketConnectionClosedException:+ return TTSResponse(content=None)++ return self._callback.get_audio_response(block=False)++ async def synthesize(+ self,+ text: str | None = None,+ **kwargs: Any,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Finalize synthesis for the current utterance.++ If text was previously pushed via :meth:`push`, this flushes any+ remaining buffered text, commits, and waits for audio. If ``text``+ is provided, it is appended before committing.++ Args:+ text (`str | None`, defaults to `None`):+ Optional additional text to append before finalizing.+ If ``None``, finalizes previously pushed text.+ **kwargs (`Any`):+ Additional keyword arguments (unused).++ Returns:+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:+ A single response when ``stream=False``, or an async generator+ of incremental chunks when ``stream=True``.+ """+ from websocket import WebSocketConnectionClosedException++ if not self._connected:+ raise RuntimeError(+ "TTS model is not connected. Call `connect()` first.",+ )++ if text is not None:+ self._accumulated_text += text++ unsent = self._cold_start_buffer+ if text is not None:+ unsent += text+ self._cold_start_buffer = ""++ full_text = self._accumulated_text+ delay = self.retry_delay++ for attempt in range(self.max_retries):+ try:+ if unsent:+ self._tts_client.append_text(unsent)++ self._tts_client.commit()+ self._tts_client.finish()++ await asyncio.to_thread(+ self._callback.finish_event.wait,+ )++ if full_text and not self._callback.has_audio_data():+ if attempt < self.max_retries - 1:+ logger.warning(+ "TTS: no audio received, retrying (%d/%d) in "+ "%.1fs...",+ attempt + 1,+ self.max_retries,+ delay,+ )+ await asyncio.sleep(delay)+ await self._reconnect()+ unsent = full_text+ delay *= 2+ continue+ self._reset_state()+ raise RuntimeError(+ f"TTS synthesis failed: no audio after "+ f"{self.max_retries} attempts",+ )+ break++ except WebSocketConnectionClosedException:+ if attempt < self.max_retries - 1:+ logger.warning(+ "TTS WebSocket closed, retrying (%d/%d) in %.1fs...",+ attempt + 1,+ self.max_retries,+ delay,+ )+ await asyncio.sleep(delay)+ await self._reconnect()+ unsent = full_text+ delay *= 2+ else:+ self._reset_state()+ raise++ self._reset_state()++ if self.stream:+ return self._callback.get_audio_chunks()++ return self._callback.get_audio_response(block=True)++ def _reset_state(self) -> None:+ """Reset per-utterance tracking state."""+ self._cold_start_buffer = ""+ self._cold_start_done = False+ self._accumulated_text = ""diff --git a/src/agentscope/tts/_tts_base.py b/src/agentscope/tts/_tts_base.pynew file mode 100644index 000000000..14401b9f7--- /dev/null+++ b/src/agentscope/tts/_tts_base.py@@ -0,0 +1,212 @@+# -*- coding: utf-8 -*-+"""The TTS model base class."""+import inspect+from abc import abstractmethod+from pathlib import Path+from typing import TYPE_CHECKING, Any, AsyncGenerator++from pydantic import BaseModel++from ._tts_response import TTSResponse+from .._logging import logger+from ..credential import CredentialBase++if TYPE_CHECKING:+ from ._tts_model_card import TTSModelCard+++class TTSModelBase:+ """Base class for TTS models in AgentScope.++ This base class provides a unified abstraction for both non-realtime and+ realtime (streaming-input) TTS models, governed by the+ ``realtime`` flag.++ For non-realtime TTS models, only :meth:`synthesize` needs to be+ implemented. For realtime TTS models, the lifecycle is managed via the+ async context manager (``async with model: ...``) or by calling+ :meth:`connect` / :meth:`close` manually; :meth:`push` appends text+ chunks and returns whatever audio is currently available, while+ :meth:`synthesize` blocks until the full speech has been synthesized.+ """++ class Parameters(BaseModel):+ """Base parameter schema for TTS models. Subclasses should override+ this with provider-specific parameters."""++ credential: CredentialBase+ """The credential used to authenticate against the TTS provider."""++ model: str+ """The name of the TTS model."""++ parameters: BaseModel+ """The TTS model parameters."""++ stream: bool+ """Whether to use streaming output if supported by the model."""++ realtime: bool = False+ """Whether the TTS model supports realtime (streaming-input) mode."""++ def __init__(+ self,+ credential: CredentialBase,+ model: str,+ parameters: BaseModel | None = None,+ stream: bool = True,+ ) -> None:+ """Initialize the TTS model base class.++ Args:+ credential (`CredentialBase`):+ The credential used to authenticate against the TTS provider.+ model (`str`):+ The name of the TTS model.+ parameters (`BaseModel | None`, defaults to `None`):+ The TTS model parameters.+ stream (`bool`, defaults to `True`):+ Whether to use streaming output if supported by the model.+ """+ self.credential = credential+ self.model = model+ self.parameters = parameters or self.Parameters()+ self.stream = stream++ async def __aenter__(self) -> "TTSModelBase":+ """Enter the async context manager and initialize resources if+ needed."""+ if self.realtime:+ await self.connect()+ return self++ async def __aexit__(+ self,+ exc_type: Any,+ exc_value: Any,+ traceback: Any,+ ) -> None:+ """Exit the async context manager and clean up resources if needed."""+ if self.realtime:+ await self.close()++ async def connect(self) -> None:+ """Connect to the TTS model and initialize resources.++ .. note:: Only relevant for realtime TTS models — realtime subclasses+ must override this. The default is a no-op so that non-realtime+ models can ignore the lifecycle hooks; :meth:`__aenter__` only+ calls it when ``realtime`` is True.+ """+ return++ async def close(self) -> None:+ """Close the connection to the TTS model and clean up resources.++ .. note:: Only relevant for realtime TTS models — realtime subclasses+ must override this. See :meth:`connect` for the rationale behind+ the no-op default.+ """+ return++ async def push( # pylint: disable=unused-argument+ self,+ text: str,+ **kwargs: Any,+ ) -> TTSResponse:+ """Append text to be synthesized and return the received TTS response.+ This method is non-blocking and may return an empty response if no+ audio is available yet.++ To receive all the synthesized speech, call :meth:`synthesize` after+ pushing all the text chunks.++ .. note:: Only relevant for realtime TTS models — realtime subclasses+ must override this. Non-realtime models should call+ :meth:`synthesize` directly and never reach this method.++ Args:+ text (`str`):+ The text chunk to be synthesized.+ **kwargs (`Any`):+ Additional keyword arguments to pass to the TTS API call.++ Returns:+ `TTSResponse`:+ The TTSResponse containing the audio block.+ """+ return TTSResponse(content=None)++ @classmethod+ def list_models(+ cls,+ custom_yaml_dir: str | None = None,+ ) -> list["TTSModelCard"]:+ """List candidate TTS models by scanning YAML model cards.++ Args:+ custom_yaml_dir (`str | None`):+ The custom YAML directory. If ``None``, uses the ``_models``+ directory next to the concrete subclass's source file.++ Returns:+ `list[TTSModelCard]`:+ A list of TTS model cards.+ """+ from ._tts_model_card import TTSModelCard++ if custom_yaml_dir is None:+ subclass_file = Path(inspect.getfile(cls))+ yaml_dir = subclass_file.parent / "_models"+ else:+ yaml_dir = Path(custom_yaml_dir)++ yaml_files = list(yaml_dir.glob("*.yaml"))++ model_cards = []+ for yaml_file in yaml_files:+ try:+ card = TTSModelCard.from_yaml(+ yaml_path=str(yaml_file),+ parameter_class=cls.Parameters,+ )+ if card.realtime != cls.realtime:+ continue+ model_cards.append(card)+ except Exception as e:+ logger.warning(+ "Warning: Failed to load %s: %s",+ yaml_file,+ str(e),+ )+ continue++ return model_cards++ @abstractmethod+ async def synthesize(+ self,+ text: str | None = None,+ **kwargs: Any,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ """Synthesize speech from the appended text. Different from+ :meth:`push`, this method blocks until the full speech has been+ synthesized.++ Args:+ text (`str | None`, defaults to `None`):+ The text to be synthesized. If `None`, this method will+ wait for all previously pushed text to be synthesized and+ return the last synthesized TTSResponse.+ **kwargs (`Any`):+ Additional keyword arguments to pass to the TTS API call.++ Returns:+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:+ A single TTSResponse containing the full audio when+ ``stream=False``. When ``stream=True``, an async generator+ yielding TTSResponse chunks where each chunk carries an+ **incremental** audio delta (not a cumulative buffer); the+ full audio is the concatenation of every chunk's decoded+ bytes. The final yielded chunk has ``is_last=True``.+ """diff --git a/src/agentscope/tts/_tts_model_card.py b/src/agentscope/tts/_tts_model_card.pynew file mode 100644index 000000000..55ad4f2a8--- /dev/null+++ b/src/agentscope/tts/_tts_model_card.py@@ -0,0 +1,134 @@+# -*- coding: utf-8 -*-+"""The TTS model card class."""+import copy+from datetime import datetime+from typing import Literal, Self, Type++import yaml+from pydantic import BaseModel, Field+++class TTSModelCard(BaseModel):+ """The model card class for TTS models."""++ type: Literal["tts_model"] = "tts_model"+ """The model card type discriminator."""++ name: str = Field(description="The name of the TTS model")+ """The model name."""++ label: str = Field(description="The model label.")+ """The model label used for frontend rendering."""++ status: Literal["active", "deprecated", "sunset"] = Field(+ default="active",+ title="Status",+ description="The model status",+ )+ """The model status."""++ deprecated_at: datetime | None = Field(+ default=None,+ description="The model deprecation date and time.",+ title="Deprecation date",+ )+ """The model deprecated at."""++ input_types: list[str] = Field(+ description="The supported model input types.",+ title="Input types",+ default=["text/plain"],+ )+ """The model supported input types."""++ output_types: list[str] = Field(+ description="The supported model output types.",+ title="Output types",+ default=["audio/wav"],+ )+ """The model supported output types."""++ realtime: bool = Field(+ default=False,+ description="Whether the model supports streaming input (realtime).",+ title="Realtime",+ )+ """Whether the model supports streaming input."""++ parameter_schema: dict+ """The parameters schema."""++ parameters_overrides: dict[str, dict]+ """The parameter overrides."""++ @classmethod+ def from_yaml(+ cls,+ yaml_path: str,+ parameter_class: Type[BaseModel],+ ) -> Self:+ """Read a TTS model card from a YAML file.++ Args:+ yaml_path (`str`):+ Path to the YAML file+ parameter_class (`Type[BaseModel]`):+ The parameter class (e.g., DashScopeTTSModel.Parameters)++ Returns:+ `TTSModelCard`:+ TTSModelCard instance with merged parameter schema+ """+ with open(yaml_path, "r", encoding="utf-8") as file:+ config = yaml.safe_load(file)++ base_schema = parameter_class.model_json_schema()+ properties = copy.deepcopy(base_schema.get("properties", {}))++ # Auto-inject: populate voice enum from YAML voices list+ voices = config.get("voices", [])+ if voices and "voice" in properties:+ properties["voice"] = {+ **properties["voice"],+ "default": voices[0],+ "enum": voices,+ }++ # Apply parameter_overrides+ overrides = config.get("parameter_overrides", {})+ for param_name, override in overrides.items():+ if override is None:+ properties.pop(param_name, None)+ continue++ if isinstance(override, dict):+ if override.get("hidden"):+ properties.pop(param_name, None)+ continue++ if param_name in properties:+ properties[param_name] = {+ **properties[param_name],+ **override,+ }++ required = [+ r for r in base_schema.get("required", []) if r in properties+ ]+ final_schema = {+ "type": "object",+ "properties": properties,+ "required": required,+ }++ return cls(+ name=config["name"],+ label=config["label"],+ status=config.get("status", "active"),+ deprecated_at=config.get("deprecated_at"),+ input_types=config.get("input_types", ["text/plain"]),+ output_types=config.get("output_types", ["audio/wav"]),+ realtime=config.get("realtime", False),+ parameter_schema=final_schema,+ parameters_overrides=config.get("parameter_overrides", {}),+ )diff --git a/src/agentscope/tts/_tts_response.py b/src/agentscope/tts/_tts_response.pynew file mode 100644index 000000000..0fe660582--- /dev/null+++ b/src/agentscope/tts/_tts_response.py@@ -0,0 +1,56 @@+# -*- coding: utf-8 -*-+"""The TTS response module."""+from dataclasses import dataclass, field+from typing import Literal++from .._utils._common import _get_timestamp+from .._utils._mixin import DictMixin+from ..message import DataBlock+from ..types import JSONSerializableObject+++@dataclass+class TTSUsage(DictMixin):+ """The usage of a TTS model API invocation."""++ input_tokens: int+ """The number of input tokens."""++ output_tokens: int+ """The number of output tokens."""++ time: float+ """The time used in seconds."""++ type: Literal["tts"] = field(default_factory=lambda: "tts")+ """The type of the usage, must be `tts`."""+++@dataclass+class TTSResponse(DictMixin):+ """The response of TTS models."""++ content: DataBlock | None+ """The audio chunk produced by the TTS model. The audio format is+ indicated by ``content.source.media_type`` (e.g.+ ``"audio/pcm;rate=24000"``, ``"audio/mpeg"``)."""++ id: str = field(default_factory=lambda: _get_timestamp(True))+ """The unique identifier of the response."""++ created_at: str = field(default_factory=_get_timestamp)+ """When the response was created."""++ type: Literal["tts"] = field(default_factory=lambda: "tts")+ """The type of the response, which is always 'tts'."""++ usage: TTSUsage | None = field(default_factory=lambda: None)+ """The usage information of the TTS response, if available."""++ metadata: dict[str, JSONSerializableObject] | None = field(+ default_factory=lambda: None,+ )+ """The metadata of the TTS response."""++ is_last: bool = field(default_factory=lambda: True)+ """Whether this is the last response in a stream of TTS responses."""diff --git a/tests/tts_dashscope_test.py b/tests/tts_dashscope_test.pynew file mode 100644index 000000000..adf27521f--- /dev/null+++ b/tests/tts_dashscope_test.py@@ -0,0 +1,683 @@+# -*- coding: utf-8 -*-+# pylint: disable=protected-access+"""Unit tests for the TTS module.++Covers:+ * ``TTSModelBase`` default no-op behaviour for ``connect`` / ``close`` /+ ``push`` (so non-realtime subclasses needn't override them).+ * ``DashScopeTTSModel`` non-streaming aggregation.+ * ``DashScopeTTSModel`` streaming: incremental deltas and ``is_last``+ placement at the final chunk only.+ * ``DashScopeRealtimeTTSModel`` connect / close / push / synthesize+ lifecycle over a mocked WebSocket.+"""+import base64+import io+import wave+from typing import Any, AsyncGenerator+from unittest import IsolatedAsyncioTestCase+from unittest.mock import MagicMock, Mock, patch++from agentscope.credential import DashScopeCredential+from agentscope.tts import (+ DashScopeTTSModel,+ DashScopeRealtimeTTSModel,+ TTSModelBase,+ TTSResponse,+)+++_MEDIA_TYPE = "audio/wav"+# DashScope TTS emits 24kHz / mono / 16-bit PCM; the WAV wrapping in the+# model layer uses the same parameters.+_TTS_SAMPLE_RATE = 24000+_TTS_CHANNELS = 1+_TTS_SAMPLE_WIDTH = 2 # bytes (= 16 bit)+_WAV_HEADER_LEN = 44+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _make_api_chunk(+ data_bytes: bytes | None,+ usage: Any = None,+) -> MagicMock:+ """Build a chunk shaped like what dashscope.MultiModalConversation+ yields. ``data_bytes=None`` represents a chunk with no output."""+ chunk = MagicMock()+ chunk.usage = usage+ if data_bytes is None:+ chunk.output = None+ return chunk+ chunk.output = MagicMock()+ chunk.output.audio = MagicMock()+ chunk.output.audio.data = base64.b64encode(data_bytes).decode("ascii")+ return chunk+++def _make_usage(+ input_tokens: int = 0,+ output_tokens: int = 0,+ characters: int = 0,+) -> MagicMock:+ """Build a usage object shaped like what the DashScope API returns."""+ usage = MagicMock()+ usage.input_tokens = input_tokens+ usage.output_tokens = output_tokens+ usage.characters = characters+ return usage+++def _make_api_generator(chunks: list[bytes | None]) -> Any:+ """Build a sync generator like ``MultiModalConversation.call`` returns.+ The last chunk carries a usage object."""++ def _gen() -> Any:+ for i, data in enumerate(chunks):+ is_last = i == len(chunks) - 1+ usage = _make_usage(characters=10) if is_last else None+ yield _make_api_chunk(data, usage=usage)++ return _gen()+++# ---------------------------------------------------------------------------+# TTSModelBase — default no-op surface for non-realtime subclasses+# ---------------------------------------------------------------------------+++class _DummyTTS(TTSModelBase):+ """Minimal subclass that implements only ``synthesize`` — exercises the+ base class's no-op ``connect`` / ``close`` / ``push`` defaults."""++ async def synthesize(+ self,+ text: str | None = None,+ **kwargs: Any,+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:+ del text, kwargs+ return TTSResponse(content=None)+++class _RealtimeDummyTTS(_DummyTTS):+ """Realtime-flavoured dummy to assert ``__aenter__`` drives the lifecycle+ hooks when ``realtime`` is True."""++ realtime = True++ def __init__(self, *args: Any, **kwargs: Any) -> None:+ super().__init__(*args, **kwargs)+ self.connect_calls = 0+ self.close_calls = 0++ async def connect(self) -> None:+ self.connect_calls += 1++ async def close(self) -> None:+ self.close_calls += 1+++def _make_dummy(cls: type = _DummyTTS) -> TTSModelBase:+ return cls(+ credential=DashScopeCredential(api_key="test"),+ model="x",+ stream=False,+ )+++class TestTTSModelBaseDefaults(IsolatedAsyncioTestCase):+ """The base class supplies safe no-op defaults so non-realtime subclasses+ don't need to implement realtime-only hooks."""++ async def test_default_connect_close_noop(self) -> None:+ """Default connect/close return without raising."""+ model = _make_dummy()+ await model.connect()+ await model.close()++ async def test_default_push_returns_empty(self) -> None:+ """Default push returns an empty TTSResponse rather than raising,+ so a misuse on a non-realtime model degrades gracefully."""+ model = _make_dummy()+ resp = await model.push("ignored")+ self.assertIsNone(resp.content)++ async def test_aenter_skips_hooks_for_non_realtime(self) -> None:+ """``async with`` on a non-realtime model must not invoke connect/+ close (gated by ``realtime``)."""+ model = _make_dummy(_DummyTTS)+ async with model as m:+ self.assertIs(m, model)++ async def test_aenter_invokes_hooks_for_realtime(self) -> None:+ """For ``realtime=True`` subclasses, connect/close fire on+ enter/exit exactly once."""+ model = _make_dummy(_RealtimeDummyTTS)+ async with model:+ self.assertEqual(model.connect_calls, 1)+ self.assertEqual(model.close_calls, 0)+ self.assertEqual(model.close_calls, 1)+++# ---------------------------------------------------------------------------+# DashScopeTTSModel — non-streaming and streaming+# ---------------------------------------------------------------------------+++def _parse_wav_payload(wav_bytes: bytes) -> bytes:+ """Decode a full WAV file and return its raw PCM frames."""+ with wave.open(io.BytesIO(wav_bytes), "rb") as wav:+ return wav.readframes(wav.getnframes())+++class TestDashScopeTTSModel(IsolatedAsyncioTestCase):+ """The unittests for DashScope TTS model (non-realtime)."""++ def setUp(self) -> None:+ """Set up the test case."""+ self.patcher = patch("dashscope.MultiModalConversation")+ self.mock_mmc = self.patcher.start()++ def tearDown(self) -> None:+ """Tear down the test case."""+ self.patcher.stop()++ def _make_model(self, stream: bool = False) -> DashScopeTTSModel:+ """Create a DashScopeTTSModel with test credentials."""+ return DashScopeTTSModel(+ credential=DashScopeCredential(api_key="test"),+ model="qwen3-tts-flash",+ parameters=DashScopeTTSModel.Parameters(voice="Cherry"),+ stream=stream,+ )++ # -- non-streaming --++ async def test_aggregates_chunks(self) -> None:+ """All API chunks are aggregated into one self-contained WAV."""+ self.mock_mmc.call.return_value = _make_api_generator(+ [b"AAAA", b"BBBB", b"CCCC"],+ )+ model = self._make_model(stream=False)++ result = await model.synthesize("Hello world")++ self.assertIsInstance(result, TTSResponse)+ self.assertEqual(result.content.source.media_type, _MEDIA_TYPE)+ wav_bytes = base64.b64decode(result.content.source.data)+ with wave.open(io.BytesIO(wav_bytes), "rb") as wav:+ self.assertEqual(+ {+ "framerate": wav.getframerate(),+ "channels": wav.getnchannels(),+ "sampwidth": wav.getsampwidth(),+ "frames": wav.readframes(wav.getnframes()),+ },+ {+ "framerate": _TTS_SAMPLE_RATE,+ "channels": _TTS_CHANNELS,+ "sampwidth": _TTS_SAMPLE_WIDTH,+ "frames": b"AAAABBBBCCCC",+ },+ )+ self.assertTrue(result.is_last)++ async def test_none_short_circuits(self) -> None:+ """``synthesize(None)`` returns an empty response without touching+ the API."""+ model = self._make_model(stream=False)++ result = await model.synthesize(None)++ self.assertIsNone(result.content)+ self.mock_mmc.call.assert_not_called()++ async def test_empty_string_short_circuits(self) -> None:+ """``synthesize("")`` returns an empty response without touching+ the API."""+ model = self._make_model(stream=False)++ result = await model.synthesize("")++ self.assertIsNone(result.content)+ self.mock_mmc.call.assert_not_called()++ async def test_skips_empty_chunks(self) -> None:+ """Chunks without ``output`` are ignored during aggregation."""+ self.mock_mmc.call.return_value = _make_api_generator(+ [None, b"AAAA", None, b"BBBB"],+ )+ model = self._make_model(stream=False)++ result = await model.synthesize("Hello world")++ wav_bytes = base64.b64decode(result.content.source.data)+ self.assertEqual(_parse_wav_payload(wav_bytes), b"AAAABBBB")++ # -- streaming --++ async def test_incremental_deltas(self) -> None:+ """Each API chunk yields one TTSResponse with incremental PCM."""+ self.mock_mmc.call.return_value = _make_api_generator(+ [b"AAAA", b"BBBB", b"CCCC"],+ )+ model = self._make_model(stream=True)++ gen = await model.synthesize("Hello world")+ chunks = [c async for c in gen]++ payloads = [base64.b64decode(c.content.source.data) for c in chunks]++ self.assertTrue(payloads[0].startswith(b"RIFF"))+ self.assertEqual(payloads[0][8:12], b"WAVE")+ self.assertEqual(payloads[0][_WAV_HEADER_LEN:], b"AAAA")+ self.assertEqual(payloads[1], b"BBBB")+ self.assertEqual(payloads[2], b"CCCC")++ self.assertEqual(+ [c.is_last for c in chunks],+ [False, False, True],+ )+ self.assertEqual(+ [c.content.source.media_type for c in chunks],+ [_MEDIA_TYPE, _MEDIA_TYPE, _MEDIA_TYPE],+ )++ async def test_single_chunk_marked_last(self) -> None:+ """A lone audio chunk is flagged ``is_last=True`` with a streaming+ WAV header."""+ self.mock_mmc.call.return_value = _make_api_generator([b"ONLYCHUNK"])+ model = self._make_model(stream=True)++ gen = await model.synthesize("Hello world")+ chunks = [c async for c in gen]++ self.assertEqual(len(chunks), 1)+ self.assertTrue(chunks[0].is_last)+ payload = base64.b64decode(chunks[0].content.source.data)+ self.assertTrue(payload.startswith(b"RIFF"))+ self.assertEqual(payload[_WAV_HEADER_LEN:], b"ONLYCHUNK")++ async def test_empty_stream_yields_terminal(self) -> None:+ """When the API yields no audio, the generator emits a terminal+ sentinel so consumers can detect EOS."""+ self.mock_mmc.call.return_value = _make_api_generator([None, None])+ model = self._make_model(stream=True)++ gen = await model.synthesize("Hello world")+ chunks = [c async for c in gen]++ self.assertEqual(len(chunks), 1)+ self.assertIsNone(chunks[0].content)+ self.assertTrue(chunks[0].is_last)+++# ---------------------------------------------------------------------------+# DashScopeRealtimeTTSModel — realtime push / synthesize lifecycle+# ---------------------------------------------------------------------------+++class TestDashScopeRealtimeTTSModel( # pylint: disable=too-many-public-methods+ IsolatedAsyncioTestCase,+):+ """The unittests for DashScope Realtime TTS model."""++ def setUp(self) -> None:+ self.mock_modules = self._create_mock_dashscope_modules()+ self.mock_client = self._create_mock_tts_client()+ mock_tts_class = Mock(return_value=self.mock_client)+ self.mock_modules[+ "dashscope.audio.qwen_tts_realtime"+ ].QwenTtsRealtime = mock_tts_class++ @staticmethod+ def _create_mock_dashscope_modules() -> dict:+ mock_qwen_tts_realtime = MagicMock()+ mock_qwen_tts_realtime.QwenTtsRealtime = Mock+ mock_qwen_tts_realtime.QwenTtsRealtimeCallback = Mock++ mock_audio = MagicMock()+ mock_audio.qwen_tts_realtime = mock_qwen_tts_realtime++ mock_dashscope = MagicMock()+ mock_dashscope.api_key = None+ mock_dashscope.audio = mock_audio++ return {+ "dashscope": mock_dashscope,+ "dashscope.audio": mock_audio,+ "dashscope.audio.qwen_tts_realtime": mock_qwen_tts_realtime,+ }++ @staticmethod+ def _create_mock_tts_client() -> Mock:+ client = Mock()+ client.connect = Mock()+ client.close = Mock()+ client.finish = Mock()+ client.commit = Mock()+ client.update_session = Mock()+ client.append_text = Mock()+ return client++ def _make_model(self, **kwargs: Any) -> DashScopeRealtimeTTSModel:+ defaults: dict[str, Any] = {+ "credential": DashScopeCredential(api_key="test"),+ "model": "qwen3-tts-flash-realtime",+ "stream": True,+ "max_retries": 1,+ "retry_delay": 0.0,+ }+ defaults.update(kwargs)+ return DashScopeRealtimeTTSModel(**defaults)++ def _mock_synthesize_callback(self, model: Any) -> None:+ """Set up callback mocks so ``synthesize()`` doesn't block."""+ model._callback.finish_event = Mock()+ model._callback.finish_event.wait = Mock()+ model._callback.has_audio_data = Mock(return_value=True)+ model._callback.get_audio_response = Mock(+ return_value=TTSResponse(content=None),+ )++ # -- connect / close --++ async def test_connect_creates_client(self) -> None:+ """connect() creates a WebSocket client and calls update_session."""+ with patch.dict("sys.modules", self.mock_modules):+ model = self._make_model()+ await model.connect()++ self.assertTrue(model._connected)+ self.mock_client.connect.assert_called_once()+ self.mock_client.update_session.assert_called_once()++ async def test_close_disconnects(self) -> None:+ """close() sets _connected to False and closes the client."""+ with patch.dict("sys.modules", self.mock_modules):+ model = self._make_model()+ await model.connect()+ await model.close()++ self.assertFalse(model._connected)+ self.mock_client.close.assert_called_once()++ async def test_close_idempotent(self) -> None:+ """Calling close() when already disconnected is a no-op."""+ model = self._make_model()+ model._connected = False+ await model.close()+ self.assertFalse(model._connected)++ async def test_async_context_manager(self) -> None:+ """``async with`` triggers connect on enter and close on exit."""+ with patch.dict("sys.modules", self.mock_modules):+ model = self._make_model()+ async with model:+ self.assertTrue(model._connected)+ self.assertFalse(model._connected)++ # -- push --++ async def test_push_appends_text(self) -> None:+ """A single push forwards the delta to append_text."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ await model.push("Hello")++ self.mock_client.append_text.assert_called_once_with("Hello")+ self.assertEqual(model._accumulated_text, "Hello")+ self.assertTrue(model._cold_start_done)++ async def test_push_incremental_deltas(self) -> None:+ """Consecutive deltas are each forwarded verbatim."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ await model.push("Hello")+ await model.push(" world")++ self.assertEqual(+ self.mock_client.append_text.call_count,+ 2,+ )+ self.mock_client.append_text.assert_any_call("Hello")+ self.mock_client.append_text.assert_any_call(" world")++ async def test_push_empty_text_no_call(self) -> None:+ """Empty string does not call append_text."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ res = await model.push("")++ self.mock_client.append_text.assert_not_called()+ self.assertIsNone(res.content)++ async def test_push_not_connected_raises(self) -> None:+ """push() raises RuntimeError when not connected."""+ model = self._make_model()+ model._connected = False++ with self.assertRaises(RuntimeError):+ await model.push("Hello")++ async def test_push_returns_audio_when_available(self) -> None:+ """push() returns audio data from the callback when available."""+ mock_audio_data = base64.b64encode(b"PCMDATA").decode("ascii")++ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ model._callback.get_audio_response = Mock(+ return_value=TTSResponse(+ content=MagicMock(+ source=MagicMock(+ data=mock_audio_data,+ media_type=_MEDIA_TYPE,+ ),+ ),+ ),+ )+ res = await model.push("Hello")++ self.assertIsNotNone(res.content)+ self.assertEqual(res.content.source.data, mock_audio_data)++ async def test_push_returns_empty_when_no_audio(self) -> None:+ """push() returns empty response when no audio is ready."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ model._callback.get_audio_response = Mock(+ return_value=TTSResponse(content=None),+ )+ res = await model.push("Hello")+ self.assertIsNone(res.content)++ async def test_push_cold_start_buffers_across_deltas(self) -> None:+ """Multiple small deltas are buffered until cold_start_length is+ met, then flushed as a single ``append_text`` call."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model(cold_start_length=10) as model:+ await model.push("Hi")+ await model.push(" there")+ self.mock_client.append_text.assert_not_called()+ self.assertFalse(model._cold_start_done)++ await model.push(" friend!")+ self.mock_client.append_text.assert_called_once_with(+ "Hi there friend!",+ )+ self.assertTrue(model._cold_start_done)++ async def test_push_cold_start_single_delta_meets_threshold(self) -> None:+ """A single delta exceeding cold_start_length sends immediately."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model(cold_start_length=5) as model:+ await model.push("Hello world")++ self.mock_client.append_text.assert_called_once_with(+ "Hello world",+ )++ async def test_push_after_cold_start_forwards_directly(self) -> None:+ """Once cold start is done, subsequent deltas bypass the buffer."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model(cold_start_length=3) as model:+ await model.push("Hello")+ self.mock_client.append_text.assert_called_with("Hello")++ await model.push(" world")+ self.mock_client.append_text.assert_called_with(" world")+ self.assertEqual(+ self.mock_client.append_text.call_count,+ 2,+ )++ async def test_push_cold_start_words_buffers(self) -> None:+ """Deltas below cold_start_words are buffered."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model(cold_start_words=3) as model:+ await model.push("Hello")+ await model.push(" world")++ self.mock_client.append_text.assert_not_called()+ self.assertFalse(model._cold_start_done)++ # -- synthesize --++ async def test_synthesize_commits_and_finishes(self) -> None:+ """synthesize(text=...) appends text, commits, and finishes."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ self._mock_synthesize_callback(model)++ await model.synthesize(text="Hello")++ self.mock_client.append_text.assert_called_once_with("Hello")+ self.mock_client.commit.assert_called_once()+ self.mock_client.finish.assert_called_once()++ async def test_synthesize_appends_extra_text(self) -> None:+ """synthesize(text=...) after push appends the extra text."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ model._callback.get_audio_response = Mock(+ return_value=TTSResponse(content=None),+ )+ await model.push("Hello")+ self.mock_client.append_text.reset_mock()++ self._mock_synthesize_callback(model)+ await model.synthesize(text=" world")++ self.mock_client.append_text.assert_called_once_with(+ " world",+ )+ self.mock_client.commit.assert_called_once()++ async def test_synthesize_no_text_drain(self) -> None:+ """synthesize(None) after push commits without extra append."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ model._callback.get_audio_response = Mock(+ return_value=TTSResponse(content=None),+ )+ await model.push("Hello")+ self.mock_client.append_text.reset_mock()++ self._mock_synthesize_callback(model)+ await model.synthesize()++ self.mock_client.append_text.assert_not_called()+ self.mock_client.commit.assert_called_once()++ async def test_synthesize_resets_state(self) -> None:+ """State is reset after synthesize completes."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model() as model:+ self._mock_synthesize_callback(model)+ await model.synthesize(text="Hello")++ self.assertFalse(model._cold_start_done)+ self.assertEqual(model._accumulated_text, "")+ self.assertEqual(model._cold_start_buffer, "")++ async def test_synthesize_stream_returns_generator(self) -> None:+ """stream=True returns an async generator with is_last."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model(stream=True) as model:+ self._mock_synthesize_callback(model)++ async def mock_chunks() -> AsyncGenerator[TTSResponse, None]:+ yield TTSResponse(content=None, is_last=True)++ model._callback.get_audio_chunks = mock_chunks++ gen = await model.synthesize(text="Hello")+ chunks = [c async for c in gen]++ self.assertTrue(len(chunks) >= 1)+ self.assertTrue(chunks[-1].is_last)++ async def test_synthesize_non_stream_returns_single(self) -> None:+ """stream=False returns a single TTSResponse."""+ mock_audio_data = base64.b64encode(b"PCMDATA").decode("ascii")++ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model(stream=False) as model:+ self._mock_synthesize_callback(model)+ model._callback.get_audio_response = Mock(+ return_value=TTSResponse(+ content=MagicMock(+ source=MagicMock(+ data=mock_audio_data,+ media_type=_MEDIA_TYPE,+ ),+ ),+ ),+ )++ res = await model.synthesize(text="Hello")++ self.assertIsInstance(res, TTSResponse)+ self.assertIsNotNone(res.content)++ async def test_synthesize_flushes_cold_start_buffer(self) -> None:+ """If cold start was never met during push, synthesize flushes the+ buffered text before committing."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model(cold_start_length=100) as model:+ await model.push("Hi")+ await model.push(" there")+ self.mock_client.append_text.assert_not_called()++ self._mock_synthesize_callback(model)+ await model.synthesize()++ self.mock_client.append_text.assert_called_once_with(+ "Hi there",+ )+ self.mock_client.commit.assert_called_once()++ async def test_synthesize_not_connected_raises(self) -> None:+ """synthesize() raises RuntimeError when not connected."""+ model = self._make_model()+ model._connected = False++ with self.assertRaises(RuntimeError):+ await model.synthesize(text="Hello")++ async def test_synthesize_no_audio_raises_after_retries(self) -> None:+ """RuntimeError after max_retries with no audio received."""+ with patch.dict("sys.modules", self.mock_modules):+ async with self._make_model(+ max_retries=1,+ retry_delay=0.0,+ ) as model:+ model._callback.finish_event = Mock()+ model._callback.finish_event.wait = Mock()+ model._callback.has_audio_data = Mock(return_value=False)++ with self.assertRaises(RuntimeError):+ await model.synthesize(text="Hello")diff --git a/tests/tts_middleware_test.py b/tests/tts_middleware_test.pynew file mode 100644index 000000000..3f296c631--- /dev/null+++ b/tests/tts_middleware_test.py@@ -0,0 +1,388 @@+# -*- coding: utf-8 -*-+"""Unit tests for TTSMiddleware."""+import base64+from typing import Any, AsyncGenerator+from unittest import IsolatedAsyncioTestCase+from unittest.mock import AsyncMock, MagicMock++from agentscope.event import (+ DataBlockDeltaEvent,+ DataBlockEndEvent,+ DataBlockStartEvent,+ TextBlockDeltaEvent,+ TextBlockEndEvent,+)+from agentscope.message import Base64Source, DataBlock+from agentscope.middleware import TTSMiddleware+from agentscope.tts import TTSModelBase, TTSResponse++_EXCLUDE = {"id", "created_at", "metadata"}+++def _dump(evt: Any) -> dict:+ """Dump an event to a dict excluding auto-generated fields."""+ return evt.model_dump(exclude=_EXCLUDE)+++def _make_tts_response(+ data: str,+ media_type: str = "audio/wav",+) -> TTSResponse:+ return TTSResponse(+ content=DataBlock(+ source=Base64Source(data=data, media_type=media_type),+ ),+ )+++def _make_agent_stub(+ reply_id: str = "reply-1",+ name: str = "agent",+) -> MagicMock:+ agent = MagicMock()+ agent.name = name+ agent.state.reply_id = reply_id+ return agent+++class TestTTSMiddlewareNonRealtime(IsolatedAsyncioTestCase):+ """Tests for non-realtime (non-streaming-input) TTS path."""++ async def test_synthesize_on_text_block_end(self) -> None:+ """After TextBlockEnd, synthesize() is called with accumulated text+ and DATA_BLOCK_START + DATA_BLOCK_DELTA + DATA_BLOCK_END are emitted.+ """+ audio_b64 = base64.b64encode(b"\x00\x01\x02").decode()++ tts = MagicMock(spec=TTSModelBase)+ tts.realtime = False+ tts.__aenter__ = AsyncMock(return_value=tts)+ tts.__aexit__ = AsyncMock(return_value=None)+ tts.synthesize = AsyncMock(+ return_value=_make_tts_response(audio_b64),+ )++ middleware = TTSMiddleware(tts_model=tts)+ agent = _make_agent_stub()++ upstream_events = [+ TextBlockDeltaEvent(+ reply_id="reply-1",+ block_id="blk-1",+ delta="Hello ",+ ),+ TextBlockDeltaEvent(+ reply_id="reply-1",+ block_id="blk-1",+ delta="world",+ ),+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),+ ]++ async def next_handler(**_kwargs: Any) -> AsyncGenerator:+ for evt in upstream_events:+ yield evt++ emitted = []+ async for evt in middleware.on_reply(agent, {}, next_handler):+ emitted.append(evt)++ # Upstream events are passed through+ self.assertEqual(+ [_dump(e) for e in emitted[:3]],+ [_dump(e) for e in upstream_events],+ )++ # After TextBlockEnd: START + DELTA + END+ block_id = emitted[3].block_id+ self.assertEqual(+ [_dump(e) for e in emitted[3:]],+ [+ {+ "type": "DATA_BLOCK_START",+ "reply_id": "reply-1",+ "block_id": block_id,+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_DELTA",+ "reply_id": "reply-1",+ "block_id": block_id,+ "data": audio_b64,+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_END",+ "reply_id": "reply-1",+ "block_id": block_id,+ },+ ],+ )++ # synthesize was called with the full text+ tts.synthesize.assert_called_once_with("Hello world")++ async def test_empty_text_skips_synthesize(self) -> None:+ """When accumulated text is whitespace-only, synthesize is not called+ and no DATA_BLOCK events are emitted."""+ tts = MagicMock(spec=TTSModelBase)+ tts.realtime = False+ tts.__aenter__ = AsyncMock(return_value=tts)+ tts.__aexit__ = AsyncMock(return_value=None)+ tts.synthesize = AsyncMock()++ middleware = TTSMiddleware(tts_model=tts)+ agent = _make_agent_stub()++ upstream_events = [+ TextBlockDeltaEvent(+ reply_id="reply-1",+ block_id="blk-1",+ delta=" ",+ ),+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),+ ]++ async def next_handler(**_kwargs: Any) -> AsyncGenerator:+ for evt in upstream_events:+ yield evt++ emitted = []+ async for evt in middleware.on_reply(agent, {}, next_handler):+ emitted.append(evt)++ # Only the upstream events are passed through+ self.assertEqual(+ [_dump(e) for e in emitted],+ [_dump(e) for e in upstream_events],+ )+ tts.synthesize.assert_not_called()++ async def test_streaming_output_multiple_chunks(self) -> None:+ """When synthesize() returns an async generator, each chunk produces+ a DATA_BLOCK_DELTA under the same block_id."""+ chunk1 = _make_tts_response("AAAA")+ chunk2 = _make_tts_response("BBBB")++ async def synth_gen(*_args: Any, **_kwargs: Any) -> AsyncGenerator:+ yield chunk1+ yield chunk2++ tts = MagicMock(spec=TTSModelBase)+ tts.realtime = False+ tts.__aenter__ = AsyncMock(return_value=tts)+ tts.__aexit__ = AsyncMock(return_value=None)+ tts.synthesize = AsyncMock(return_value=synth_gen())++ middleware = TTSMiddleware(tts_model=tts)+ agent = _make_agent_stub()++ upstream_events = [+ TextBlockDeltaEvent(+ reply_id="reply-1",+ block_id="blk-1",+ delta="hi",+ ),+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),+ ]++ async def next_handler(**_kwargs: Any) -> AsyncGenerator:+ for evt in upstream_events:+ yield evt++ emitted = []+ async for evt in middleware.on_reply(agent, {}, next_handler):+ emitted.append(evt)++ # Upstream pass-through + START + 2x DELTA + END+ data_events = emitted[2:]+ block_id = data_events[0].block_id+ self.assertEqual(+ [_dump(e) for e in data_events],+ [+ {+ "type": "DATA_BLOCK_START",+ "reply_id": "reply-1",+ "block_id": block_id,+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_DELTA",+ "reply_id": "reply-1",+ "block_id": block_id,+ "data": "AAAA",+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_DELTA",+ "reply_id": "reply-1",+ "block_id": block_id,+ "data": "BBBB",+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_END",+ "reply_id": "reply-1",+ "block_id": block_id,+ },+ ],+ )+++class TestTTSMiddlewareRealtime(IsolatedAsyncioTestCase):+ """Tests for realtime (streaming-input) TTS path."""++ async def test_push_on_delta_and_drain_on_end(self) -> None:+ """In realtime mode, push() is called on each TextBlockDelta and+ synthesize() drains on TextBlockEnd."""+ push_audio = _make_tts_response("PUSH1")+ drain_audio = _make_tts_response("DRAIN")++ tts = MagicMock(spec=TTSModelBase)+ tts.realtime = True+ tts.__aenter__ = AsyncMock(return_value=tts)+ tts.__aexit__ = AsyncMock(return_value=None)+ tts.push = AsyncMock(return_value=push_audio)+ tts.synthesize = AsyncMock(return_value=drain_audio)++ middleware = TTSMiddleware(tts_model=tts)+ agent = _make_agent_stub()++ upstream_events = [+ TextBlockDeltaEvent(+ reply_id="reply-1",+ block_id="blk-1",+ delta="Hello",+ ),+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),+ ]++ async def next_handler(**_kwargs: Any) -> AsyncGenerator:+ for evt in upstream_events:+ yield evt++ emitted = []+ async for evt in middleware.on_reply(agent, {}, next_handler):+ emitted.append(evt)++ # push() called with the delta text+ tts.push.assert_called_once_with("Hello")++ # synthesize() called to drain+ tts.synthesize.assert_called_once()++ data_events = [+ e+ for e in emitted+ if isinstance(+ e,+ (DataBlockStartEvent, DataBlockDeltaEvent, DataBlockEndEvent),+ )+ ]+ # START + DELTA(push) + DELTA(drain) + END+ block_id = data_events[0].block_id+ self.assertEqual(+ [_dump(e) for e in data_events],+ [+ {+ "type": "DATA_BLOCK_START",+ "reply_id": "reply-1",+ "block_id": block_id,+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_DELTA",+ "reply_id": "reply-1",+ "block_id": block_id,+ "data": "PUSH1",+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_DELTA",+ "reply_id": "reply-1",+ "block_id": block_id,+ "data": "DRAIN",+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_END",+ "reply_id": "reply-1",+ "block_id": block_id,+ },+ ],+ )++ async def test_push_returns_none_no_audio_emitted(self) -> None:+ """When push() returns empty content, no DATA_BLOCK events are emitted+ until synthesize() drains."""+ empty_response = TTSResponse(content=None)+ drain_audio = _make_tts_response("FINAL")++ tts = MagicMock(spec=TTSModelBase)+ tts.realtime = True+ tts.__aenter__ = AsyncMock(return_value=tts)+ tts.__aexit__ = AsyncMock(return_value=None)+ tts.push = AsyncMock(return_value=empty_response)+ tts.synthesize = AsyncMock(return_value=drain_audio)++ middleware = TTSMiddleware(tts_model=tts)+ agent = _make_agent_stub()++ upstream_events = [+ TextBlockDeltaEvent(+ reply_id="reply-1",+ block_id="blk-1",+ delta="Hi",+ ),+ TextBlockDeltaEvent(+ reply_id="reply-1",+ block_id="blk-1",+ delta=" there",+ ),+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),+ ]++ async def next_handler(**_kwargs: Any) -> AsyncGenerator:+ for evt in upstream_events:+ yield evt++ emitted = []+ async for evt in middleware.on_reply(agent, {}, next_handler):+ emitted.append(evt)++ # push called twice but produced no audio+ self.assertEqual(tts.push.call_count, 2)++ data_events = [+ e+ for e in emitted+ if isinstance(+ e,+ (DataBlockStartEvent, DataBlockDeltaEvent, DataBlockEndEvent),+ )+ ]+ # Only drain produces: START + DELTA + END+ block_id = data_events[0].block_id+ self.assertEqual(+ [_dump(e) for e in data_events],+ [+ {+ "type": "DATA_BLOCK_START",+ "reply_id": "reply-1",+ "block_id": block_id,+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_DELTA",+ "reply_id": "reply-1",+ "block_id": block_id,+ "data": "FINAL",+ "media_type": "audio/wav",+ },+ {+ "type": "DATA_BLOCK_END",+ "reply_id": "reply-1",+ "block_id": block_id,+ },+ ],+ )