Skip to content

Build and run agents you can see, understand and trust.

feat(tts): add TTS module with DashScope backend and streaming middleware (#1832)

qbc680626b463bf
diff --git a/examples/web_ui/frontend/src/api/index.ts b/examples/web_ui/frontend/src/api/index.ts
index 810b20143..78512276d 100644
--- a/examples/web_ui/frontend/src/api/index.ts
+++ b/examples/web_ui/frontend/src/api/index.ts
@@ -5,4 +5,4 @@ export { credentialApi } from './credential';
export { chatApi } from './chat';
export { workspaceApi } from './workspace';
export { scheduleApi } from './schedule';
-export { modelApi } from './model';
+export { modelApi, ttsModelApi } from './model';
diff --git a/examples/web_ui/frontend/src/api/model.ts b/examples/web_ui/frontend/src/api/model.ts
index cf5abb15d..3ebe9484a 100644
--- a/examples/web_ui/frontend/src/api/model.ts
+++ b/examples/web_ui/frontend/src/api/model.ts
@@ -1,6 +1,10 @@
import { client } from './client';
-import type { ListModelResponse } from './types';
+import type { ListModelResponse, ListTTSModelResponse } from './types';
export const modelApi = {
list: (provider: string) => client.get<ListModelResponse>('/model/', { provider }),
};
+
+export const ttsModelApi = {
+ list: (provider: string) => client.get<ListTTSModelResponse>('/tts-model/', { provider }),
+};
diff --git a/examples/web_ui/frontend/src/api/types.ts b/examples/web_ui/frontend/src/api/types.ts
index 400832cd6..b3429754d 100644
--- a/examples/web_ui/frontend/src/api/types.ts
+++ b/examples/web_ui/frontend/src/api/types.ts
@@ -13,6 +13,13 @@ export interface ChatModelConfig {
parameters: Record<string, unknown>;
}
+export interface TTSModelConfig {
+ type: string;
+ credential_id: string;
+ model: string;
+ parameters: Record<string, unknown>;
+}
+
export interface ContextConfig {
trigger_ratio?: number;
reserve_ratio?: number;
@@ -84,6 +91,8 @@ export interface SessionConfig {
chat_model_config: ChatModelConfig;
/** Fallback model used when the primary model fails. */
fallback_chat_model_config: ChatModelConfig | null;
+ /** TTS model configuration. null means TTS is not enabled. */
+ tts_model_config: TTSModelConfig | null;
workspace_id: string;
}
@@ -112,6 +121,8 @@ export interface CreateSessionRequest {
chat_model_config?: ChatModelConfig | null;
/** Optional fallback model. Omit (or pass null) for no fallback. */
fallback_chat_model_config?: ChatModelConfig | null;
+ /** Optional TTS model. Omit (or pass null) for no TTS. */
+ tts_model_config?: TTSModelConfig | null;
}
export interface CreateSessionResponse {
@@ -128,6 +139,13 @@ export interface UpdateSessionRequest {
* - set to a value → replace the existing fallback
*/
fallback_chat_model_config?: ChatModelConfig | null;
+ /**
+ * New TTS model. PATCH semantics:
+ * - omit the field → leave unchanged
+ * - set to `null` → disable TTS
+ * - set to a value → replace the existing TTS config
+ */
+ tts_model_config?: TTSModelConfig | null;
permission_mode?: PermissionMode;
}
@@ -422,3 +440,21 @@ export interface ListModelResponse {
models: ModelCard[];
total: number;
}
+
+export interface TTSModelCard {
+ type: 'tts_model';
+ name: string;
+ label: string;
+ status: 'active' | 'deprecated' | 'sunset';
+ deprecated_at: string | null;
+ input_types: string[];
+ output_types: string[];
+ realtime: boolean;
+ parameter_schema: Record<string, unknown>;
+ parameters_overrides: Record<string, Record<string, unknown>>;
+}
+
+export interface ListTTSModelResponse {
+ models: TTSModelCard[];
+ total: number;
+}
diff --git a/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx b/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx
index 22f3783d3..847317f58 100644
--- a/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx
+++ b/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx
@@ -1,7 +1,8 @@
import { ChevronDown, SlidersHorizontal } from 'lucide-react';
import { useCallback, useEffect, useState } from 'react';
-import type { ChatModelConfig, ModelCard } from '@/api';
+import type { ChatModelConfig, ModelCard, TTSModelCard, TTSModelConfig } from '@/api';
+import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import {
DropdownMenu,
@@ -21,6 +22,7 @@ import { Label } from '@/components/ui/label';
import { Switch } from '@/components/ui/switch';
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
import { useAvailableModels } from '@/hooks/useAvailableModels';
+import { useAvailableTTSModels } from '@/hooks/useAvailableTTSModels';
import { useTranslation } from '@/i18n/useI18n';
interface ParameterProperty {
@@ -204,6 +206,10 @@ interface Props {
selectedFallbackModel: ChatModelConfig | null;
/** Called when the user picks a fallback model or clears the selection. */
onFallbackChange: (config: ChatModelConfig | null) => void;
+ /** Currently selected TTS model. `null` means TTS is disabled. */
+ selectedTTSModel: TTSModelConfig | null;
+ /** Called when the user picks a TTS model+voice or disables TTS. */
+ onTTSChange: (config: TTSModelConfig | null) => void;
}
/**
@@ -221,10 +227,13 @@ export function ModelParametersPopover({
onChange,
selectedFallbackModel,
onFallbackChange,
+ selectedTTSModel,
+ onTTSChange,
}: Props) {
const [values, setValues] = useState<Record<string, unknown>>({});
const { t } = useTranslation();
const { groups } = useAvailableModels();
+ const { groups: ttsGroups } = useAvailableTTSModels();
const schema = modelCard?.parameter_schema as ParameterSchema | undefined;
const properties = schema?.properties ?? {};
@@ -400,6 +409,207 @@ export function ModelParametersPopover({
)}
</DropdownMenuSubContent>
</DropdownMenuSub>
+
+ {/* ----- TTS ----- */}
+ <DropdownMenuSub>
+ <DropdownMenuSubTrigger>
+ <span className="truncate">{t('model-parameters.ttsLabel')}</span>
+ </DropdownMenuSubTrigger>
+ <DropdownMenuSubContent className="max-h-96 overflow-y-auto">
+ {Object.keys(ttsGroups).length === 0 ? (
+ <div className="px-2 py-3 text-center text-sm text-muted-foreground">
+ <p>{t('model-parameters.ttsEmpty')}</p>
+ </div>
+ ) : (
+ Object.entries(ttsGroups).map(([type, items], idx) => (
+ <div key={type}>
+ {idx > 0 && <DropdownMenuSeparator />}
+ <DropdownMenuLabel>
+ {type.replace(/_credential$/, '')}
+ </DropdownMenuLabel>
+ {items.flatMap(({ credential, models }) =>
+ models.map((m) => {
+ const isSelected =
+ selectedTTSModel?.credential_id === credential.id &&
+ selectedTTSModel?.model === m.name;
+ return (
+ <DropdownMenuCheckboxItem
+ key={`${credential.id}-${m.name}`}
+ checked={isSelected}
+ onSelect={(e) => e.preventDefault()}
+ onCheckedChange={(checked) => {
+ if (!checked) return;
+ const schema = m.parameter_schema as
+ | ParameterSchema
+ | undefined;
+ const defaults: Record<string, unknown> =
+ {};
+ if (schema?.properties) {
+ for (const [k, p] of Object.entries(
+ schema.properties,
+ )) {
+ if (p.default !== undefined) {
+ defaults[k] = p.default;
+ }
+ }
+ }
+ onTTSChange({
+ type,
+ credential_id: credential.id,
+ model: m.name,
+ parameters: defaults,
+ });
+ }}
+ >
+ {m.label}
+ {m.realtime && (
+ <Badge
+ variant="outline"
+ className="ml-1.5 text-[10px] px-1 py-0"
+ >
+ Realtime
+ </Badge>
+ )}
+ </DropdownMenuCheckboxItem>
+ );
+ }),
+ )}
+ </div>
+ ))
+ )}
+
+ {/* TTS parameters sub-panel (hover to expand right) */}
+ {selectedTTSModel && (
+ <>
+ <DropdownMenuSeparator />
+ <DropdownMenuSub>
+ <DropdownMenuSubTrigger>
+ {t('model-parameters.ttsParameters')}
+ </DropdownMenuSubTrigger>
+ <DropdownMenuSubContent className="w-72 max-h-96 overflow-y-auto p-3">
+ <div className="mb-3">
+ <p className="text-sm font-medium">
+ {t('model-parameters.title')}
+ </p>
+ <p className="text-muted-foreground text-xs">
+ {t('model-parameters.ttsParametersDescription')}
+ </p>
+ </div>
+ {(() => {
+ if (!selectedTTSModel) return null;
+ const selType = selectedTTSModel.type;
+ const selItems = ttsGroups[selType];
+ if (!selItems) return null;
+ let selModel: TTSModelCard | undefined;
+ for (const { credential, models } of selItems) {
+ if (
+ credential.id !== selectedTTSModel.credential_id
+ )
+ continue;
+ selModel = models.find(
+ (m) => m.name === selectedTTSModel.model,
+ );
+ if (selModel) break;
+ }
+ if (!selModel) return null;
+ const mSchema = selModel.parameter_schema as
+ | ParameterSchema
+ | undefined;
+ const mProps = mSchema?.properties ?? {};
+ const mRequired = mSchema?.required ?? [];
+ const mEntries = Object.entries(mProps);
+ if (mEntries.length === 0) {
+ return (
+ <p className="text-muted-foreground text-xs">
+ {t('model-parameters.empty')}
+ </p>
+ );
+ }
+ const curParams = selectedTTSModel.parameters ?? {};
+
+ return (
+ <div
+ className="grid grid-cols-[auto_1fr] items-center gap-x-3 gap-y-3"
+ onPointerDown={(e) => e.stopPropagation()}
+ onKeyDown={(e) => e.stopPropagation()}
+ >
+ {mEntries.map(([key, prop]) => {
+ const { type: effectiveType, enumValues } =
+ resolveType(prop);
+ const label = prop.title ?? key;
+ const isReq = mRequired.includes(key);
+ const fieldProps: FieldProps = {
+ id: `tts-${selModel!.name}-${key}`,
+ label,
+ required: isReq,
+ prop,
+ value: curParams[key],
+ onChange: (v) => {
+ const next = {
+ ...curParams,
+ [key]: v,
+ };
+ if (v === '' || v === undefined) {
+ delete next[key];
+ }
+ onTTSChange({
+ ...selectedTTSModel,
+ parameters: next,
+ });
+ },
+ };
+
+ let field: React.ReactNode;
+ if (effectiveType === 'boolean') {
+ field = (
+ <BooleanField {...fieldProps} />
+ );
+ } else if (enumValues) {
+ field = <EnumField {...fieldProps} />;
+ } else if (
+ effectiveType === 'number' ||
+ effectiveType === 'integer'
+ ) {
+ field = <NumberField {...fieldProps} />;
+ } else {
+ field = <StringField {...fieldProps} />;
+ }
+
+ return (
+ <Tooltip key={key}>
+ <TooltipTrigger asChild>
+ <div className="col-span-2 grid grid-cols-subgrid items-center">
+ {field}
+ </div>
+ </TooltipTrigger>
+ {prop.description && (
+ <TooltipContent side="left">
+ {prop.description}
+ </TooltipContent>
+ )}
+ </Tooltip>
+ );
+ })}
+ </div>
+ );
+ })()}
+ </DropdownMenuSubContent>
+ </DropdownMenuSub>
+ </>
+ )}
+
+ <DropdownMenuSeparator />
+ <DropdownMenuCheckboxItem
+ checked={!selectedTTSModel}
+ onSelect={(e) => e.preventDefault()}
+ onCheckedChange={(checked) => {
+ if (checked) onTTSChange(null);
+ }}
+ >
+ {t('model-parameters.noTts')}
+ </DropdownMenuCheckboxItem>
+ </DropdownMenuSubContent>
+ </DropdownMenuSub>
</DropdownMenuContent>
</DropdownMenu>
);
diff --git a/examples/web_ui/frontend/src/context/AudioContext.tsx b/examples/web_ui/frontend/src/context/AudioContext.tsx
index 041a4cddc..6f9036a3a 100644
--- a/examples/web_ui/frontend/src/context/AudioContext.tsx
+++ b/examples/web_ui/frontend/src/context/AudioContext.tsx
@@ -28,6 +28,7 @@ export function AudioProvider({ children }: { children: ReactNode }) {
const replay: ReplayController = useMemo(
() => ({
play(el: HTMLAudioElement) {
+ manager.stopLivePlayback();
if (currentRef.current && currentRef.current !== el) {
currentRef.current.pause();
currentRef.current.currentTime = 0;
@@ -38,7 +39,7 @@ export function AudioProvider({ children }: { children: ReactNode }) {
currentRef.current = null;
},
}),
- [],
+ [manager],
);
return (
diff --git a/examples/web_ui/frontend/src/hooks/useAvailableTTSModels.ts b/examples/web_ui/frontend/src/hooks/useAvailableTTSModels.ts
new file mode 100644
index 000000000..2c5147ab1
--- /dev/null
+++ b/examples/web_ui/frontend/src/hooks/useAvailableTTSModels.ts
@@ -0,0 +1,61 @@
+import { useState, useEffect, useCallback } from 'react';
+
+import { credentialApi, ttsModelApi } from '@/api';
+import type { CredentialRecord, TTSModelCard } from '@/api';
+
+export interface CredentialWithTTSModels {
+ credential: CredentialRecord;
+ models: TTSModelCard[];
+}
+
+/**
+ * Fetches all credentials and their available TTS models, grouped by provider type.
+ * Credentials/providers that expose no TTS models are omitted.
+ */
+export function useAvailableTTSModels() {
+ const [groups, setGroups] = useState<Record<string, CredentialWithTTSModels[]>>({});
+ const [loading, setLoading] = useState(false);
+ const [error, setError] = useState<Error | null>(null);
+
+ const refetch = useCallback(async () => {
+ setLoading(true);
+ setError(null);
+ try {
+ const { credentials } = await credentialApi.list();
+ const result: Record<string, CredentialWithTTSModels[]> = {};
+
+ await Promise.all(
+ credentials.map(async (credential) => {
+ const type = credential.data.type as string | undefined;
+ if (!type) return;
+ if (!result[type]) result[type] = [];
+ try {
+ const { models } = await ttsModelApi.list(type);
+ if (models.length > 0) {
+ result[type].push({ credential, models });
+ }
+ } catch {
+ // Provider doesn't support TTS — skip silently
+ }
+ }),
+ );
+
+ // Remove provider groups with no TTS models
+ for (const key of Object.keys(result)) {
+ if (result[key].length === 0) delete result[key];
+ }
+
+ setGroups(result);
+ } catch (e) {
+ setError(e as Error);
+ } finally {
+ setLoading(false);
+ }
+ }, []);
+
+ useEffect(() => {
+ refetch();
+ }, [refetch]);
+
+ return { groups, loading, error, refetch };
+}
diff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.json
index ba9899624..3d54547d6 100644
--- a/examples/web_ui/frontend/src/i18n/locales/en.json
+++ b/examples/web_ui/frontend/src/i18n/locales/en.json
@@ -182,7 +182,8 @@
"maxContext": "Max. Context",
"maxOutput": "Max. Output",
"inputTypes": "Input",
- "outputTypes": "Output"
+ "outputTypes": "Output",
+ "availableTTSModels": "TTS Models"
},
"llm-select": {
"placeholder": "Select a model",
@@ -285,7 +286,12 @@
"empty": "No configurable parameters.",
"fallbackLabel": "Fallback model",
"fallbackLabelWithModel": "Fallback model ({{model}})",
- "parametersLabel": "Parameters"
+ "parametersLabel": "Parameters",
+ "ttsLabel": "TTS",
+ "ttsEmpty": "No TTS models available.",
+ "ttsParameters": "Parameters",
+ "ttsParametersDescription": "Adjust parameters for the selected TTS model.",
+ "noTts": "No TTS"
},
"setup": {
"title": "Connect to Service",
diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.json
index 58da0cbaf..9d230e92f 100644
--- a/examples/web_ui/frontend/src/i18n/locales/zh.json
+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json
@@ -182,7 +182,8 @@
"maxContext": "最大上下文",
"maxOutput": "最大输出",
"inputTypes": "输入模态",
- "outputTypes": "输出模态"
+ "outputTypes": "输出模态",
+ "availableTTSModels": "语音合成模型"
},
"llm-select": {
"placeholder": "选择模型",
@@ -285,7 +286,12 @@
"empty": "暂无可配置参数。",
"fallbackLabel": "兜底模型",
"fallbackLabelWithModel": "兜底模型({{model}})",
- "parametersLabel": "参数"
+ "parametersLabel": "参数",
+ "ttsLabel": "语音合成",
+ "ttsEmpty": "暂无可用的语音合成模型。",
+ "ttsParameters": "参数",
+ "ttsParametersDescription": "调整所选语音合成模型的参数。",
+ "noTts": "不使用语音合成"
},
"setup": {
"title": "连接到服务器",
diff --git a/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx b/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx
index af22962f8..5ed4efbcb 100644
--- a/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx
+++ b/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx
@@ -2,7 +2,7 @@ import type { TaskContext } from '@agentscope-ai/agentscope/state';
import { Toolbox } from 'lucide-react';
import { useCallback, useEffect, useMemo, useState } from 'react';
-import type { ChatModelConfig } from '@/api';
+import type { ChatModelConfig, TTSModelConfig } from '@/api';
import { sessionApi } from '@/api';
import { ChatContent } from '@/components/chat/ChatContent.tsx';
import { TaskPanel } from '@/components/chat/TaskPanel';
@@ -76,6 +76,7 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewport
const [selectedFallbackModel, setSelectedFallbackModel] = useState<ChatModelConfig | null>(
null,
);
+ const [selectedTTSModel, setSelectedTTSModel] = useState<TTSModelConfig | null>(null);
const [selectedPermissionMode, setSelectedPermissionMode] = useState<string>('default');
const [credentialOpen, setCredentialOpen] = useState(false);
const [credentialRefetchTrigger, setCredentialRefetchTrigger] = useState(0);
@@ -127,6 +128,7 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewport
useEffect(() => {
setSelectedModel(null);
setSelectedFallbackModel(null);
+ setSelectedTTSModel(null);
}, [sessionId]);
const selectedModelCard = useMemo(() => {
@@ -215,6 +217,7 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewport
}
setSelectedFallbackModel(view.session.config.fallback_chat_model_config ?? null);
+ setSelectedTTSModel(view.session.config.tts_model_config ?? null);
}, [view, groups, sessionId, agentId]);
// Sync selectedPermissionMode when the session changes. Same
@@ -266,6 +269,18 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewport
await refetchSessions();
};
+ /**
+ * Persist a TTS model change. `null` disables TTS.
+ *
+ * @param config - New TTS config or `null` to disable.
+ */
+ const handleTTSChange = async (config: TTSModelConfig | null) => {
+ if (!sessionId || !agentId) return;
+ setSelectedTTSModel(config);
+ await sessionApi.update(sessionId, agentId, { tts_model_config: config });
+ await refetchSessions();
+ };
+
/**
* Persist a permission-mode change.
*
@@ -296,6 +311,8 @@ export function ChatViewport({ agentId, sessionId, onTeamUpdated }: ChatViewport
onChange={handleParametersChange}
selectedFallbackModel={selectedFallbackModel}
onFallbackChange={handleFallbackChange}
+ selectedTTSModel={selectedTTSModel}
+ onTTSChange={handleTTSChange}
/>
</div>
<div id="tour-permission-mode" className="flex flex-row gap-x-2">
diff --git a/examples/web_ui/frontend/src/pages/credential/index.tsx b/examples/web_ui/frontend/src/pages/credential/index.tsx
index e951d3a45..c1bcba5bd 100644
--- a/examples/web_ui/frontend/src/pages/credential/index.tsx
+++ b/examples/web_ui/frontend/src/pages/credential/index.tsx
@@ -1,8 +1,8 @@
import { Eye, EyeOff, Plus, Pencil, Trash2 } from 'lucide-react';
import { useState, useEffect, useCallback } from 'react';
-import { credentialApi, modelApi } from '@/api';
-import type { CredentialRecord, CredentialSchema, ModelCard } from '@/api';
+import { credentialApi, modelApi, ttsModelApi } from '@/api';
+import type { CredentialRecord, CredentialSchema, ModelCard, TTSModelCard } from '@/api';
import { InputTypeBadges } from '@/components/badge/InputTypeBadges';
import { CreateCredentialDialog } from '@/components/dialog/CreateCredentialDialog';
import { DeleteDialog } from '@/components/dialog/DeleteDialog';
@@ -104,6 +104,52 @@ function ModelCardItem({ model }: { model: ModelCard }) {
);
}
+// ─── TTS Model Card ──────────────────────────────────────────────────────────
+
+function TTSModelCardItem({ model }: { model: TTSModelCard }) {
+ const { t } = useTranslation();
+
+ const statusVariant =
+ model.status === 'active'
+ ? 'default'
+ : model.status === 'deprecated'
+ ? 'secondary'
+ : 'outline';
+
+ return (
+ <Card className="shadow">
+ <CardHeader>
+ <CardTitle
+ className="text-sm font-semibold leading-tight truncate"
+ title={model.name}
+ >
+ {model.label || model.name}
+ </CardTitle>
+ {model.realtime && (
+ <CardAction>
+ <Badge variant="outline">Realtime</Badge>
+ </CardAction>
+ )}
+ </CardHeader>
+ <CardContent className="flex flex-col">
+ {model.status !== 'active' && (
+ <Badge variant={statusVariant} className="text-xs">
+ {model.status}
+ </Badge>
+ )}
+ <div className="flex justify-between items-center text-[14px]">
+ <span className="text-muted-foreground">{t('credential.inputTypes')}</span>
+ <InputTypeBadges inputTypes={model.input_types} />
+ </div>
+ <div className="flex justify-between items-center text-[14px]">
+ <span className="text-muted-foreground">{t('credential.outputTypes')}</span>
+ <InputTypeBadges inputTypes={model.output_types} />
+ </div>
+ </CardContent>
+ </Card>
+ );
+}
+
// ─── Detail panel ─────────────────────────────────────────────────────────────
interface DetailPanelProps {
@@ -116,6 +162,7 @@ interface DetailPanelProps {
function DetailPanel({ credential, schema, onEdit, onDelete }: DetailPanelProps) {
const { t } = useTranslation();
const [models, setModels] = useState<ModelCard[]>([]);
+ const [ttsModels, setTtsModels] = useState<TTSModelCard[]>([]);
const [modelsLoading, setModelsLoading] = useState(false);
const type = credential.data.type as string | undefined;
@@ -123,10 +170,20 @@ function DetailPanel({ credential, schema, onEdit, onDelete }: DetailPanelProps)
useEffect(() => {
if (!type) return;
setModelsLoading(true);
- modelApi
- .list(type)
- .then((res) => setModels(res.models))
- .catch(() => setModels([]))
+ Promise.all([
+ modelApi
+ .list(type)
+ .then((res) => res.models)
+ .catch(() => [] as ModelCard[]),
+ ttsModelApi
+ .list(type)
+ .then((res) => res.models)
+ .catch(() => [] as TTSModelCard[]),
+ ])
+ .then(([chatModels, tts]) => {
+ setModels(chatModels);
+ setTtsModels(tts);
+ })
.finally(() => setModelsLoading(false));
}, [credential.id, type]);
@@ -223,6 +280,23 @@ function DetailPanel({ credential, schema, onEdit, onDelete }: DetailPanelProps)
</div>
)}
</div>
+
+ {/* Available TTS Models */}
+ {ttsModels.length > 0 && (
+ <>
+ <Separator />
+ <div className="flex flex-col gap-y-4">
+ <h3 className="text-sm font-semibold">
+ {t('credential.availableTTSModels')}({ttsModels.length})
+ </h3>
+ <div className="grid grid-cols-2 xl:grid-cols-3 gap-3">
+ {ttsModels.map((m) => (
+ <TTSModelCardItem key={m.name} model={m} />
+ ))}
+ </div>
+ </div>
+ </>
+ )}
</div>
);
}
diff --git a/examples/web_ui/frontend/src/utils/streamingAudio.ts b/examples/web_ui/frontend/src/utils/streamingAudio.ts
index 41deb2cb0..f27b0384c 100644
--- a/examples/web_ui/frontend/src/utils/streamingAudio.ts
+++ b/examples/web_ui/frontend/src/utils/streamingAudio.ts
@@ -225,6 +225,8 @@ interface Session {
chunks: Bytes[];
totalBytes: number;
livePlayer: WavStreamPlayer | null;
+ /** True if a live player was created at start — survives dispose. */
+ hadLivePlayer: boolean;
state: StreamingAudioState;
}
@@ -267,6 +269,7 @@ export class StreamingAudioManager {
chunks: [],
totalBytes: 0,
livePlayer,
+ hadLivePlayer: livePlayer !== null,
state: { status: 'streaming', mediaType, url: null, interruptCount: 0 },
});
this.emit(blockId);
@@ -300,7 +303,7 @@ export class StreamingAudioManager {
const session = this.sessions.get(blockId);
if (!session) return;
- const hadLivePlayback = session.livePlayer !== null;
+ const hadLivePlayback = session.hadLivePlayer;
if (session.livePlayer) {
// Let queued PCM finish playing; AudioContext closes via a
// deferred timer. The livePlayer reference is kept so that
@@ -332,6 +335,22 @@ export class StreamingAudioManager {
}
}
+ /**
+ * Stop live streaming playback only (WavStreamPlayers). Does not
+ * touch replay ``<audio>`` elements or bump ``interruptCount``.
+ * Called by the replay controller so that clicking play on a past
+ * audio block silences any in-progress streaming audio without
+ * interfering with the element that's about to start playing.
+ */
+ stopLivePlayback(): void {
+ for (const session of this.sessions.values()) {
+ if (session.livePlayer) {
+ session.livePlayer.dispose();
+ session.livePlayer = null;
+ }
+ }
+ }
+
/**
* Stop all in-progress and replay audio. Called when a new reply
* starts so previous audio doesn't overlap with the new one.
diff --git a/src/agentscope/app/_app.py b/src/agentscope/app/_app.py
index 3e7403e7a..20a61fbff 100644
--- a/src/agentscope/app/_app.py
+++ b/src/agentscope/app/_app.py
@@ -9,6 +9,7 @@ from ._router import (
chat_router,
credential_router,
model_router,
+ tts_model_router,
schedule_router,
session_router,
workspace_router,
@@ -169,6 +170,7 @@ def create_app(
session_router,
workspace_router,
model_router,
+ tts_model_router,
):
app.include_router(router)
diff --git a/src/agentscope/app/_router/__init__.py b/src/agentscope/app/_router/__init__.py
index 73f2fb145..6511f3957 100644
--- a/src/agentscope/app/_router/__init__.py
+++ b/src/agentscope/app/_router/__init__.py
@@ -6,11 +6,13 @@ from ._credential import credential_router
from ._schedule import schedule_router
from ._session import session_router
from ._model import model_router
+from ._tts_model import tts_model_router
from ._workspace import workspace_router
__all__ = [
"agent_router",
"model_router",
+ "tts_model_router",
"chat_router",
"credential_router",
"schedule_router",
diff --git a/src/agentscope/app/_router/_schema/__init__.py b/src/agentscope/app/_router/_schema/__init__.py
index e9132450c..e84deeaf9 100644
--- a/src/agentscope/app/_router/_schema/__init__.py
+++ b/src/agentscope/app/_router/_schema/__init__.py
@@ -3,6 +3,7 @@
from ._chat import ChatRequest, ChatTriggerResponse
from ._model import ListModelsResponse, ListModelsRequest
+from ._tts_model import ListTTSModelsResponse, ListTTSModelsRequest
from ._schedule import (
CreateScheduleRequest,
CreateScheduleResponse,
@@ -55,6 +56,9 @@ __all__ = [
# Model
"ListModelsRequest",
"ListModelsResponse",
+ # TTS Model
+ "ListTTSModelsRequest",
+ "ListTTSModelsResponse",
# Schedule
"CreateScheduleRequest",
"CreateScheduleResponse",
diff --git a/src/agentscope/app/_router/_schema/_session.py b/src/agentscope/app/_router/_schema/_session.py
index 260af5561..3b4865c5b 100644
--- a/src/agentscope/app/_router/_schema/_session.py
+++ b/src/agentscope/app/_router/_schema/_session.py
@@ -3,7 +3,13 @@
from pydantic import BaseModel, Field
from ....permission import PermissionMode
-from ...storage import AgentRecord, ChatModelConfig, SessionRecord, TeamRecord
+from ...storage import (
+ AgentRecord,
+ ChatModelConfig,
+ TTSModelConfig,
+ SessionRecord,
+ TeamRecord,
+)
class TeamMemberView(BaseModel):
@@ -68,6 +74,10 @@ class CreateSessionRequest(BaseModel):
description="Fallback model used when the primary model fails. "
"Can be set later via PATCH.",
)
+ tts_model_config: TTSModelConfig | None = Field(
+ default=None,
+ description="TTS model configuration. Can be set later via PATCH.",
+ )
class CreateSessionResponse(BaseModel):
@@ -97,6 +107,11 @@ class UpdateSessionRequest(BaseModel):
description="New fallback model configuration. "
"Pass null to clear; omit to leave unchanged.",
)
+ tts_model_config: TTSModelConfig | None = Field(
+ default=None,
+ description="New TTS model configuration. "
+ "Pass null to clear; omit to leave unchanged.",
+ )
permission_mode: PermissionMode | None = Field(
default=None,
description="New permission mode for the session.",
diff --git a/src/agentscope/app/_router/_schema/_tts_model.py b/src/agentscope/app/_router/_schema/_tts_model.py
new file mode 100644
index 000000000..d37998a57
--- /dev/null
+++ b/src/agentscope/app/_router/_schema/_tts_model.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+"""The TTS model configuration, used as DTO layer."""
+
+from pydantic import BaseModel, Field
+
+from ....tts import TTSModelCard
+
+
+class ListTTSModelsResponse(BaseModel):
+ """List the candidate TTS models response."""
+
+ models: list[TTSModelCard] = Field(
+ description="The candidate TTS models.",
+ )
+ total: int = Field(description="The total number of candidates.")
+
+
+class ListTTSModelsRequest(BaseModel):
+ """List the candidate TTS models request."""
+
+ provider: str = Field(
+ description="The provider type, e.g. dashscope_credential.",
+ )
diff --git a/src/agentscope/app/_router/_session.py b/src/agentscope/app/_router/_session.py
index 56a3e1c0c..c7e4c4de5 100644
--- a/src/agentscope/app/_router/_session.py
+++ b/src/agentscope/app/_router/_session.py
@@ -29,6 +29,7 @@ from .._service import SessionService
from ..storage import (
AgentRecord,
ChatModelConfig,
+ TTSModelConfig,
SessionConfig,
SessionRecord,
StorageBase,
@@ -92,7 +93,7 @@ session_router = APIRouter(
async def _ensure_credential_exists(
storage: StorageBase,
user_id: str,
- config: ChatModelConfig | None,
+ config: ChatModelConfig | TTSModelConfig | None,
) -> None:
"""Validate that the credential referenced by ``config`` belongs to the
given user. No-op when ``config`` is ``None``.
@@ -100,8 +101,8 @@ async def _ensure_credential_exists(
Args:
storage (`StorageBase`): Injected storage backend.
user_id (`str`): The authenticated user ID.
- config (`ChatModelConfig | None`): Model config to validate. Pass
- ``None`` to skip the check.
+ config (`ChatModelConfig | TTSModelConfig | None`): Model config to
+ validate. Pass ``None`` to skip the check.
Raises:
`HTTPException`: 404 if the credential does not exist or does not
@@ -230,6 +231,7 @@ async def create_session(
user_id,
body.fallback_chat_model_config,
)
+ await _ensure_credential_exists(storage, user_id, body.tts_model_config)
session_record = await storage.upsert_session(
user_id=user_id,
@@ -238,6 +240,7 @@ async def create_session(
workspace_id=body.workspace_id or uuid.uuid4().hex,
chat_model_config=body.chat_model_config,
fallback_chat_model_config=body.fallback_chat_model_config,
+ tts_model_config=body.tts_model_config,
**({"name": body.name} if body.name is not None else {}),
),
)
@@ -325,6 +328,7 @@ async def update_session(
user_id,
body.fallback_chat_model_config,
)
+ await _ensure_credential_exists(storage, user_id, body.tts_model_config)
updated_state = existing.state
if body.permission_mode is not None:
diff --git a/src/agentscope/app/_router/_tts_model.py b/src/agentscope/app/_router/_tts_model.py
new file mode 100644
index 000000000..949388c5b
--- /dev/null
+++ b/src/agentscope/app/_router/_tts_model.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""The TTS model router."""
+
+from fastapi import APIRouter, Depends, HTTPException, status
+
+from ._schema import ListTTSModelsResponse, ListTTSModelsRequest
+from ...credential import CredentialFactory
+
+tts_model_router = APIRouter(
+ prefix="/tts-model",
+ tags=["tts-model"],
+ responses={404: {"description": "Not found"}},
+)
+
+
+@tts_model_router.get(
+ "/",
+ response_model=ListTTSModelsResponse,
+ summary="List all candidate TTS models under the given credential type",
+)
+async def list_tts_models(
+ body: ListTTSModelsRequest = Depends(),
+) -> ListTTSModelsResponse:
+ """Return all candidate TTS models under the given credential type.
+
+ Args:
+ body (ListTTSModelsRequest): The request body.
+
+ Returns:
+ `ListTTSModelsResponse`: The response body.
+ """
+ credential_cls = CredentialFactory.get_credential_class(body.provider)
+ if credential_cls is None:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail=f"Provider '{body.provider}' not found.",
+ )
+
+ models = credential_cls.list_tts_models()
+ return ListTTSModelsResponse(models=models, total=len(models))
diff --git a/src/agentscope/app/_service/__init__.py b/src/agentscope/app/_service/__init__.py
index 77c65bb5c..1b22f7b06 100644
--- a/src/agentscope/app/_service/__init__.py
+++ b/src/agentscope/app/_service/__init__.py
@@ -3,6 +3,7 @@
from ._chat import ChatService
from ._embedding import get_embedding_model
from ._model import get_model
+from ._tts_model import get_tts_model
from ._session import SessionService
from ._toolkit import get_toolkit
@@ -11,5 +12,6 @@ __all__ = [
"SessionService",
"get_embedding_model",
"get_model",
+ "get_tts_model",
"get_toolkit",
]
diff --git a/src/agentscope/app/_service/_chat.py b/src/agentscope/app/_service/_chat.py
index 74e812585..da7a82203 100644
--- a/src/agentscope/app/_service/_chat.py
+++ b/src/agentscope/app/_service/_chat.py
@@ -22,12 +22,14 @@ from ..middleware import (
StateChangeMiddleware,
ToolOffloadMiddleware,
)
+from ...middleware import TTSMiddleware
from .._types import (
AgentMiddlewareFactory,
AgentToolFactory,
SubAgentTemplate,
)
from ._model import get_model
+from ._tts_model import get_tts_model
from ._toolkit import get_toolkit
from ..._logging import logger
@@ -277,6 +279,18 @@ class ChatService:
),
)
+ # ----------------------------------------------------------------
+ # 3b. TTS middleware — inject when the session has a TTS config.
+ # ----------------------------------------------------------------
+ tts_cfg = session_record.config.tts_model_config
+ if tts_cfg is not None:
+ tts_model = await get_tts_model(
+ user_id,
+ tts_cfg,
+ self._storage,
+ )
+ middlewares.append(TTSMiddleware(tts_model))
+
# ----------------------------------------------------------------
# 4. Model + fallback (resolved from session's config).
# ----------------------------------------------------------------
diff --git a/src/agentscope/app/_service/_tts_model.py b/src/agentscope/app/_service/_tts_model.py
new file mode 100644
index 000000000..6f2a3dbe5
--- /dev/null
+++ b/src/agentscope/app/_service/_tts_model.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""TTS model service: builds a TTSModelBase from stored credential + config."""
+from typing import Type
+
+from fastapi import HTTPException, status
+
+from ..storage import StorageBase, TTSModelConfig
+from ...credential import CredentialFactory
+from ...tts import TTSModelBase
+
+
+async def get_tts_model(
+ user_id: str,
+ config: TTSModelConfig,
+ storage: StorageBase,
+) -> TTSModelBase:
+ """Get the TTS model instance from the configuration and storage.
+
+ Args:
+ user_id (`str`):
+ The user id.
+ config (`TTSModelConfig`):
+ The TTS model configuration.
+ storage (`StorageBase`):
+ The storage instance.
+
+ Returns:
+ `TTSModelBase`:
+ The TTS model instance.
+ """
+ credential_record = await storage.get_credential(
+ user_id,
+ config.credential_id,
+ )
+ if credential_record is None:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail=f"Credential {config.credential_id!r} not found.",
+ )
+
+ credential = CredentialFactory.from_dict(credential_record.data)
+ tts_classes = credential.get_tts_model_classes()
+ if not tts_classes:
+ raise HTTPException(
+ status_code=status.HTTP_400_BAD_REQUEST,
+ detail=f"Provider {config.type!r} does not support TTS models.",
+ )
+
+ tts_cls = _resolve_tts_class(tts_classes, config.model)
+ parameters = (
+ tts_cls.Parameters(**config.parameters) if config.parameters else None
+ )
+ return tts_cls(
+ credential=credential,
+ model=config.model,
+ parameters=parameters,
+ )
+
+
+def _resolve_tts_class(
+ classes: list[Type[TTSModelBase]],
+ model: str,
+) -> Type[TTSModelBase]:
+ """Pick the TTS class that lists the given model name."""
+ for cls in classes:
+ if any(card.name == model for card in cls.list_models()):
+ return cls
+ return classes[0]
diff --git a/src/agentscope/app/storage/__init__.py b/src/agentscope/app/storage/__init__.py
index 86da74093..4baf8f70f 100644
--- a/src/agentscope/app/storage/__init__.py
+++ b/src/agentscope/app/storage/__init__.py
@@ -14,6 +14,7 @@ from ._model import (
SessionRecord,
SessionSource,
ChatModelConfig,
+ TTSModelConfig,
EmbeddingModelConfig,
TeamData,
TeamRecord,
@@ -31,6 +32,7 @@ __all__ = [
"SessionRecord",
"SessionSource",
"ChatModelConfig",
+ "TTSModelConfig",
"EmbeddingModelConfig",
"TeamData",
"TeamRecord",
diff --git a/src/agentscope/app/storage/_model/__init__.py b/src/agentscope/app/storage/_model/__init__.py
index 74f61d06b..b68673f6d 100644
--- a/src/agentscope/app/storage/_model/__init__.py
+++ b/src/agentscope/app/storage/_model/__init__.py
@@ -8,6 +8,7 @@ from ._session import (
SessionRecord,
SessionConfig,
ChatModelConfig,
+ TTSModelConfig,
EmbeddingModelConfig,
SessionSource,
)
@@ -25,6 +26,7 @@ __all__ = [
"SessionRecord",
"SessionSource",
"ChatModelConfig",
+ "TTSModelConfig",
"EmbeddingModelConfig",
"TeamData",
"TeamRecord",
diff --git a/src/agentscope/app/storage/_model/_session.py b/src/agentscope/app/storage/_model/_session.py
index 19d97b82b..a43c0a181 100644
--- a/src/agentscope/app/storage/_model/_session.py
+++ b/src/agentscope/app/storage/_model/_session.py
@@ -32,6 +32,22 @@ class ChatModelConfig(BaseModel):
"""The model parameters."""
+class TTSModelConfig(BaseModel):
+ """The TTS model configuration class."""
+
+ type: str
+ """The provider type."""
+
+ credential_id: str
+ """The credential id."""
+
+ model: str
+ """The TTS model name."""
+
+ parameters: dict
+ """TTS parameters (voice, language, etc.)."""
+
+
class EmbeddingModelConfig(BaseModel):
"""Configuration for constructing an embedding model from a credential.
@@ -73,6 +89,9 @@ class SessionConfig(BaseModel):
"""The fallback chat model config. Used as a backup when the primary
model fails. None means no fallback configured."""
+ tts_model_config: TTSModelConfig | None = None
+ """The TTS model config. None means TTS is not enabled."""
+
class SessionRecord(_RecordBase):
"""The session record."""
diff --git a/src/agentscope/credential/_base.py b/src/agentscope/credential/_base.py
index 72f89940f..c8566acc8 100644
--- a/src/agentscope/credential/_base.py
+++ b/src/agentscope/credential/_base.py
@@ -8,6 +8,8 @@ from pydantic import BaseModel, Field
if TYPE_CHECKING:
from ..embedding import EmbeddingModelBase
from ..model import ChatModelBase, ModelCard
+ from ..tts import TTSModelBase
+ from ..tts._tts_model_card import TTSModelCard
class CredentialBase(BaseModel):
@@ -37,6 +39,33 @@ class CredentialBase(BaseModel):
f"{cls.__name__} must implement ``get_chat_model_class``.",
)
+ @classmethod
+ def get_tts_model_classes(cls) -> list[Type["TTSModelBase"]]:
+ """Return the TTS model classes supported by this credential.
+
+ Subclasses that support TTS should override this to return one or
+ more :class:`TTSModelBase` subclasses. The default returns an empty
+ list (provider does not support TTS).
+
+ Returns:
+ `list[Type[TTSModelBase]]`:
+ The TTS model classes, or an empty list.
+ """
+ return []
+
+ @classmethod
+ def list_tts_models(cls) -> list["TTSModelCard"]:
+ """List the candidate TTS models available under this credential.
+
+ Returns:
+ `list[TTSModelCard]`:
+ A list of TTS model cards, or empty if TTS is not supported.
+ """
+ cards: list["TTSModelCard"] = []
+ for tts_cls in cls.get_tts_model_classes():
+ cards.extend(tts_cls.list_models())
+ return cards
+
@classmethod
def list_models(cls) -> list["ModelCard"]:
"""List the candidate chat models that are available under this
diff --git a/src/agentscope/credential/_dashscope.py b/src/agentscope/credential/_dashscope.py
index 8e9a85666..11e95fd1c 100644
--- a/src/agentscope/credential/_dashscope.py
+++ b/src/agentscope/credential/_dashscope.py
@@ -9,6 +9,7 @@ from ._base import CredentialBase
if TYPE_CHECKING:
from ..embedding import EmbeddingModelBase
from ..model import ChatModelBase
+ from ..tts import TTSModelBase
_DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
@@ -43,6 +44,13 @@ class DashScopeCredential(CredentialBase):
return DashScopeChatModel
+ @classmethod
+ def get_tts_model_classes(cls) -> list[Type["TTSModelBase"]]:
+ """Return the DashScope TTS model classes."""
+ from ..tts import DashScopeRealtimeTTSModel, DashScopeTTSModel
+
+ return [DashScopeTTSModel, DashScopeRealtimeTTSModel]
+
@classmethod
def get_embedding_model_class(cls) -> Type["EmbeddingModelBase"]:
"""Return the DashScopeEmbeddingModel class."""
diff --git a/src/agentscope/middleware/__init__.py b/src/agentscope/middleware/__init__.py
index 3b501bd79..a15f6b5d6 100644
--- a/src/agentscope/middleware/__init__.py
+++ b/src/agentscope/middleware/__init__.py
@@ -3,8 +3,10 @@
from ._base import MiddlewareBase
from ._tracing import TracingMiddleware
+from ._tts_middleware import TTSMiddleware
__all__ = [
"MiddlewareBase",
"TracingMiddleware",
+ "TTSMiddleware",
]
diff --git a/src/agentscope/middleware/_tts_middleware.py b/src/agentscope/middleware/_tts_middleware.py
new file mode 100644
index 000000000..cff4502ee
--- /dev/null
+++ b/src/agentscope/middleware/_tts_middleware.py
@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+"""Middleware that turns reasoning text into speech and injects it as
+``DATA_BLOCK_*`` events into the agent's event stream."""
+import uuid
+from typing import TYPE_CHECKING, AsyncGenerator, Callable
+
+from ._base import MiddlewareBase
+from ..event import (
+ DataBlockDeltaEvent,
+ DataBlockEndEvent,
+ DataBlockStartEvent,
+ TextBlockDeltaEvent,
+ TextBlockEndEvent,
+)
+from ..tts import TTSModelBase, TTSResponse
+
+if TYPE_CHECKING:
+ from ..agent import Agent
+
+
+class TTSMiddleware(MiddlewareBase):
+ """Synthesize speech for every text block produced during reasoning and
+ inject the audio as ``DATA_BLOCK_*`` events into the stream.
+
+ - Non-realtime TTS (``realtime=False``): on each
+ ``TextBlockEndEvent`` the accumulated text is sent to
+ :meth:`TTSModelBase.synthesize`; the resulting audio chunks are
+ emitted as one ``DATA_BLOCK_START`` + N ``DATA_BLOCK_DELTA`` +
+ ``DATA_BLOCK_END``.
+ - Realtime TTS (``realtime=True``): each
+ ``TextBlockDeltaEvent`` is pushed into the model via
+ :meth:`TTSModelBase.push`; any audio produced is emitted immediately.
+ On ``TextBlockEndEvent`` :meth:`TTSModelBase.synthesize` is called
+ to drain remaining audio, and the data block is closed.
+
+ Each ``DataBlockDeltaEvent.data`` carries an **incremental** base64 PCM
+ chunk; the full audio is the concatenation of every delta's decoded
+ bytes (the data block is keyed by ``block_id``).
+ """
+
+ def __init__(self, tts_model: TTSModelBase) -> None:
+ """Initialize the TTS middleware.
+
+ Args:
+ tts_model (`TTSModelBase`):
+ The TTS model used to synthesize speech for assistant text
+ blocks produced during reasoning.
+ """
+ self.tts = tts_model
+
+ async def on_reply(
+ self,
+ agent: "Agent",
+ input_kwargs: dict,
+ next_handler: Callable[..., AsyncGenerator],
+ ) -> AsyncGenerator:
+ """Intercept the reply stream, synthesize speech for text blocks,
+ and inject ``DATA_BLOCK_*`` audio events into the output."""
+ text_buffer: str = ""
+ audio_block_id: str | None = None
+ audio_media_type: str | None = None
+
+ async with self.tts:
+ async for evt in next_handler(**input_kwargs):
+ yield evt
+
+ if isinstance(evt, TextBlockDeltaEvent):
+ text_buffer += evt.delta
+ if self.tts.realtime and evt.delta:
+ tts_res = await self.tts.push(evt.delta)
+ async for audio_evt in self._emit_chunk(
+ agent,
+ tts_res,
+ audio_block_id,
+ audio_media_type,
+ ):
+ if isinstance(audio_evt, DataBlockStartEvent):
+ audio_block_id = audio_evt.block_id
+ audio_media_type = audio_evt.media_type
+ yield audio_evt
+
+ elif isinstance(evt, TextBlockEndEvent):
+ text = text_buffer
+ text_buffer = ""
+
+ if self.tts.realtime:
+ res = await self.tts.synthesize()
+ async for audio_evt in self._emit_synth_result(
+ agent,
+ res,
+ audio_block_id,
+ audio_media_type,
+ ):
+ if isinstance(audio_evt, DataBlockStartEvent):
+ audio_block_id = audio_evt.block_id
+ audio_media_type = audio_evt.media_type
+ yield audio_evt
+ elif text.strip():
+ res = await self.tts.synthesize(text)
+ async for audio_evt in self._emit_synth_result(
+ agent,
+ res,
+ audio_block_id,
+ audio_media_type,
+ ):
+ if isinstance(audio_evt, DataBlockStartEvent):
+ audio_block_id = audio_evt.block_id
+ audio_media_type = audio_evt.media_type
+ yield audio_evt
+
+ if audio_block_id is not None:
+ yield DataBlockEndEvent(
+ reply_id=agent.state.reply_id,
+ block_id=audio_block_id,
+ )
+ audio_block_id = None
+ audio_media_type = None
+
+ async def _emit_synth_result(
+ self,
+ agent: "Agent",
+ res: TTSResponse | AsyncGenerator[TTSResponse, None],
+ audio_block_id: str | None,
+ audio_media_type: str | None,
+ ) -> AsyncGenerator:
+ """Normalize ``synthesize()`` returns (single response or async
+ generator) into a stream of ``DATA_BLOCK_*`` events."""
+ if isinstance(res, AsyncGenerator):
+ async for chunk in res:
+ async for ae in self._emit_chunk(
+ agent,
+ chunk,
+ audio_block_id,
+ audio_media_type,
+ ):
+ if isinstance(ae, DataBlockStartEvent):
+ audio_block_id = ae.block_id
+ audio_media_type = ae.media_type
+ yield ae
+ else:
+ async for ae in self._emit_chunk(
+ agent,
+ res,
+ audio_block_id,
+ audio_media_type,
+ ):
+ yield ae
+
+ @staticmethod
+ async def _emit_chunk(
+ agent: "Agent",
+ tts_res: TTSResponse | None,
+ audio_block_id: str | None,
+ audio_media_type: str | None,
+ ) -> AsyncGenerator:
+ """Emit one TTSResponse chunk as ``DATA_BLOCK_START`` (if needed)
+ followed by ``DATA_BLOCK_DELTA``."""
+ if tts_res is None or tts_res.content is None:
+ return
+ media_type = tts_res.content.source.media_type
+ data = tts_res.content.source.data
+ if not data:
+ return
+
+ if audio_block_id is None:
+ audio_block_id = uuid.uuid4().hex
+ audio_media_type = media_type
+ yield DataBlockStartEvent(
+ reply_id=agent.state.reply_id,
+ block_id=audio_block_id,
+ media_type=media_type,
+ )
+
+ yield DataBlockDeltaEvent(
+ reply_id=agent.state.reply_id,
+ block_id=audio_block_id,
+ data=data,
+ media_type=audio_media_type,
+ )
diff --git a/src/agentscope/tts/__init__.py b/src/agentscope/tts/__init__.py
new file mode 100644
index 000000000..42179d92d
--- /dev/null
+++ b/src/agentscope/tts/__init__.py
@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""The TTS (Text-to-Speech) module in AgentScope."""
+
+from ._tts_base import TTSModelBase
+from ._tts_model_card import TTSModelCard
+from ._tts_response import TTSResponse, TTSUsage
+from ._dashscope import DashScopeTTSModel, DashScopeRealtimeTTSModel
+
+__all__ = [
+ "TTSModelBase",
+ "TTSModelCard",
+ "TTSResponse",
+ "TTSUsage",
+ "DashScopeTTSModel",
+ "DashScopeRealtimeTTSModel",
+]
diff --git a/src/agentscope/tts/_dashscope/__init__.py b/src/agentscope/tts/_dashscope/__init__.py
new file mode 100644
index 000000000..33cb3c420
--- /dev/null
+++ b/src/agentscope/tts/_dashscope/__init__.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""The DashScope TTS module."""
+
+from ._model import DashScopeTTSModel
+from ._realtime_model import DashScopeRealtimeTTSModel
+
+__all__ = [
+ "DashScopeTTSModel",
+ "DashScopeRealtimeTTSModel",
+]
diff --git a/src/agentscope/tts/_dashscope/_model.py b/src/agentscope/tts/_dashscope/_model.py
new file mode 100644
index 000000000..8f63a60bb
--- /dev/null
+++ b/src/agentscope/tts/_dashscope/_model.py
@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+"""DashScope TTS model implementation using MultiModalConversation API."""
+import base64
+import io
+import wave
+from datetime import datetime
+from typing import (
+ Any,
+ AsyncGenerator,
+ Generator,
+ Literal,
+ TYPE_CHECKING,
+)
+
+from pydantic import BaseModel, Field
+
+from .._tts_base import TTSModelBase
+from .._tts_response import TTSResponse, TTSUsage
+from ..._utils._audio import _build_streaming_wav_header
+from ...credential import DashScopeCredential
+from ...message import DataBlock, Base64Source
+
+if TYPE_CHECKING:
+ from dashscope.api_entities.dashscope_response import (
+ MultiModalConversationResponse,
+ )
+
+
+# DashScope TTS returns raw PCM (24kHz, mono, 16-bit). We wrap it as WAV
+# on the way out so the frontend can play it: streaming deltas get a
+# streaming WAV header on the first chunk; non-streaming returns a
+# self-contained fixed-size WAV.
+_TTS_SAMPLE_RATE = 24000
+_TTS_CHANNELS = 1
+_TTS_BITS_PER_SAMPLE = 16
+_DEFAULT_MEDIA_TYPE = "audio/wav"
+_SENTINEL = object()
+
+
+def _parse_usage(usage: Any, elapsed: float) -> TTSUsage | None:
+ """Extract a TTSUsage from the DashScope usage object, or None."""
+ if usage is None:
+ return None
+ return TTSUsage(
+ input_tokens=getattr(usage, "input_tokens", 0) or 0,
+ output_tokens=getattr(usage, "output_tokens", 0) or 0,
+ time=elapsed,
+ )
+
+
+class DashScopeTTSModel(TTSModelBase):
+ """DashScope TTS model implementation using the MultiModalConversation
+ API. For more details please see the `official document
+ <https://bailian.console.aliyun.com/?tab=doc#/doc/?type=model&url=2879134>`_.
+ """
+
+ class Parameters(BaseModel):
+ """Frontend-exposed parameters for DashScope TTS models."""
+
+ voice: str = Field(
+ default="Cherry",
+ title="Voice",
+ description="The voice to use for synthesis.",
+ )
+
+ type: Literal["dashscope_tts"] = "dashscope_tts"
+ """The type of the TTS model."""
+
+ realtime: bool = False
+
+ def __init__(
+ self,
+ credential: DashScopeCredential,
+ model: str = "qwen3-tts-flash",
+ parameters: "DashScopeTTSModel.Parameters | None" = None,
+ stream: bool = True,
+ ) -> None:
+ """Initialize the DashScope TTS model.
+
+ .. note:: More details about the parameters, such as ``model``
+ and ``voice``, can be found in the `official document
+ <https://bailian.console.aliyun.com/?tab=doc#/doc/?type=model&url=2879134>`_.
+
+ Args:
+ credential (`DashScopeCredential`):
+ The DashScope credential used to authenticate the API call.
+ model (`str`, defaults to ``"qwen3-tts-flash"``):
+ The TTS model name. Supported models include
+ ``qwen3-tts-flash``, ``qwen-tts``, etc.
+ parameters (`DashScopeTTSModel.Parameters | None`, defaults to \
+ `None`):
+ The TTS parameters (voice, language, etc.). When ``None``,
+ the default parameters will be used.
+ stream (`bool`, defaults to `True`):
+ Whether to use streaming output. When `True`,
+ :meth:`synthesize` returns an async generator yielding
+ ``TTSResponse`` chunks; when `False`, it returns a single
+ aggregated ``TTSResponse``.
+ """
+ super().__init__(
+ credential=credential,
+ model=model,
+ parameters=parameters,
+ stream=stream,
+ )
+
+ async def synthesize(
+ self,
+ text: str | None = None,
+ **kwargs: Any,
+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:
+ """Call the DashScope TTS API to synthesize speech from text.
+
+ Args:
+ text (`str | None`, optional):
+ The text to be synthesized.
+ **kwargs (`Any`):
+ Additional keyword arguments to pass to the TTS API call.
+
+ Returns:
+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:
+ A single ``TTSResponse`` when ``stream=False``, or an async
+ generator yielding ``TTSResponse`` chunks when ``stream=True``.
+ """
+ if not text:
+ return TTSResponse(content=None)
+
+ import dashscope
+
+ response = dashscope.MultiModalConversation.call(
+ model=self.model,
+ api_key=self.credential.api_key.get_secret_value(),
+ text=text,
+ voice=self.parameters.voice,
+ stream=True,
+ **kwargs,
+ )
+
+ if self.stream:
+ return self._parse_into_async_generator(response)
+
+ return self._aggregate_sync(response)
+
+ @staticmethod
+ def _aggregate_sync(
+ response: Generator["MultiModalConversationResponse", None, None],
+ ) -> TTSResponse:
+ """Aggregate all streaming chunks into a single self-contained WAV."""
+ start_datetime = datetime.now()
+ audio_bytes = bytearray()
+ usage = None
+ for chunk in response:
+ if chunk.usage is not None:
+ usage = chunk.usage
+ if chunk.output is not None:
+ audio = chunk.output.audio
+ if audio and audio.data:
+ audio_bytes += base64.b64decode(audio.data)
+ elapsed = (datetime.now() - start_datetime).total_seconds()
+
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as wav:
+ wav.setnchannels(_TTS_CHANNELS)
+ wav.setsampwidth(_TTS_BITS_PER_SAMPLE // 8)
+ wav.setframerate(_TTS_SAMPLE_RATE)
+ wav.writeframes(bytes(audio_bytes))
+
+ return TTSResponse(
+ content=DataBlock(
+ source=Base64Source(
+ data=base64.b64encode(buf.getvalue()).decode("ascii"),
+ media_type=_DEFAULT_MEDIA_TYPE,
+ ),
+ ),
+ usage=_parse_usage(usage, elapsed),
+ )
+
+ @staticmethod
+ async def _parse_into_async_generator(
+ response: Generator["MultiModalConversationResponse", None, None],
+ ) -> AsyncGenerator[TTSResponse, None]:
+ """Parse the streaming TTS response into an async generator.
+
+ Each yielded ``TTSResponse`` carries an **incremental** WAV chunk:
+ the first chunk is prefixed with a streaming WAV/RIFF header so the
+ frontend can start playback immediately (without waiting for
+ end-of-stream); subsequent chunks are raw PCM bytes appended to that
+ open stream. The final response has ``is_last=True``.
+
+ Args:
+ response (`Generator[MultiModalConversationResponse, None, None]`):
+ The streaming response from the DashScope TTS API.
+
+ Yields:
+ `TTSResponse`:
+ A ``TTSResponse`` for each incremental audio chunk; the final
+ response has ``is_last=True``.
+ """
+ pending: TTSResponse | None = None
+ header_sent = False
+ usage = None
+ start_datetime = datetime.now()
+ it = iter(response)
+ while True:
+ chunk = next(it, _SENTINEL)
+ if chunk is _SENTINEL:
+ break
+ if chunk.usage is not None:
+ usage = chunk.usage
+ if chunk.output is None:
+ continue
+ audio = chunk.output.audio
+ if not audio or not audio.data:
+ continue
+ delta_bytes = base64.b64decode(audio.data)
+ if not delta_bytes:
+ continue
+ if not header_sent:
+ payload = (
+ _build_streaming_wav_header(
+ sample_rate=_TTS_SAMPLE_RATE,
+ channels=_TTS_CHANNELS,
+ bits_per_sample=_TTS_BITS_PER_SAMPLE,
+ )
+ + delta_bytes
+ )
+ header_sent = True
+ else:
+ payload = delta_bytes
+ if pending is not None:
+ yield pending
+ pending = TTSResponse(
+ content=DataBlock(
+ source=Base64Source(
+ data=base64.b64encode(payload).decode("ascii"),
+ media_type=_DEFAULT_MEDIA_TYPE,
+ ),
+ ),
+ is_last=False,
+ )
+ elapsed = (datetime.now() - start_datetime).total_seconds()
+
+ if pending is not None:
+ pending.is_last = True
+ pending.usage = _parse_usage(usage, elapsed)
+ yield pending
+ else:
+ yield TTSResponse(
+ content=None,
+ is_last=True,
+ usage=_parse_usage(usage, elapsed),
+ )
diff --git a/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash-realtime.yaml b/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash-realtime.yaml
new file mode 100644
index 000000000..a0f4f0cac
--- /dev/null
+++ b/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash-realtime.yaml
@@ -0,0 +1,19 @@
+name: qwen3-tts-flash-realtime
+label: Qwen3-TTS-Flash-Realtime
+status: active
+realtime: true
+
+input_types:
+ - text/plain
+
+output_types:
+ - audio/wav
+
+voices:
+ # Source: https://help.aliyun.com/zh/model-studio/qwen-tts-voice-list
+ - Cherry
+ - Serena
+ - Ethan
+ - Chelsie
+
+parameter_overrides: {}
diff --git a/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash.yaml b/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash.yaml
new file mode 100644
index 000000000..6750a5026
--- /dev/null
+++ b/src/agentscope/tts/_dashscope/_models/qwen3-tts-flash.yaml
@@ -0,0 +1,18 @@
+name: qwen3-tts-flash
+label: Qwen3-TTS-Flash
+status: active
+
+input_types:
+ - text/plain
+
+output_types:
+ - audio/wav
+
+voices:
+ # Source: https://help.aliyun.com/zh/model-studio/qwen-tts-voice-list
+ - Cherry
+ - Serena
+ - Ethan
+ - Chelsie
+
+parameter_overrides: {}
diff --git a/src/agentscope/tts/_dashscope/_realtime_model.py b/src/agentscope/tts/_dashscope/_realtime_model.py
new file mode 100644
index 000000000..40767c20b
--- /dev/null
+++ b/src/agentscope/tts/_dashscope/_realtime_model.py
@@ -0,0 +1,463 @@
+# -*- coding: utf-8 -*-
+"""DashScope Realtime TTS model implementation."""
+import asyncio
+import base64
+import threading
+from typing import Any, AsyncGenerator, Literal, TYPE_CHECKING
+
+from pydantic import BaseModel, Field
+
+from .._tts_base import TTSModelBase
+from .._tts_response import TTSResponse
+from ..._logging import logger
+from ..._utils._audio import _build_streaming_wav_header
+from ...credential import DashScopeCredential
+from ...message import DataBlock, Base64Source
+
+if TYPE_CHECKING:
+ from dashscope.audio.qwen_tts_realtime import (
+ QwenTtsRealtime,
+ QwenTtsRealtimeCallback,
+ )
+
+
+_MEDIA_TYPE = "audio/wav"
+_SAMPLE_RATE = 24000
+_CHANNELS = 1
+_BITS_PER_SAMPLE = 16
+
+
+def _make_callback_class() -> type["QwenTtsRealtimeCallback"]:
+ """Create the DashScope realtime TTS callback class lazily to avoid
+ importing dashscope at module level."""
+ from dashscope.audio.qwen_tts_realtime import QwenTtsRealtimeCallback
+
+ class _Callback(QwenTtsRealtimeCallback):
+ """Internal callback that accumulates PCM audio from the WebSocket.
+
+ Audio data is stored as raw bytes (decoded from base64) to allow
+ incremental delta extraction without base64 boundary issues.
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.chunk_event = threading.Event()
+ self.finish_event = threading.Event()
+ self._pcm_bytes: bytearray = bytearray()
+ self._consumed: int = 0
+
+ def on_event(self, response: dict[str, Any]) -> None:
+ """Handle incoming WebSocket events from the TTS service."""
+ try:
+ event_type = response.get("type")
+
+ if event_type == "session.created":
+ self._pcm_bytes = bytearray()
+ self._consumed = 0
+ self.finish_event.clear()
+
+ elif event_type == "response.audio.delta":
+ audio_data = response.get("delta")
+ if audio_data:
+ if isinstance(audio_data, bytes):
+ self._pcm_bytes += audio_data
+ else:
+ self._pcm_bytes += base64.b64decode(audio_data)
+ if not self.chunk_event.is_set():
+ self.chunk_event.set()
+
+ elif event_type == "session.finished":
+ self.chunk_event.set()
+ self.finish_event.set()
+
+ except Exception:
+ logger.exception("Error in TTS WebSocket callback")
+ self.finish_event.set()
+
+ def on_close(self, close_status_code: int, close_msg: str) -> None:
+ """Handle WebSocket connection closure."""
+ self.finish_event.set()
+ self.chunk_event.set()
+ if close_status_code:
+ logger.warning(
+ "TTS WebSocket closed with code %s: %s",
+ close_status_code,
+ close_msg,
+ )
+
+ def _take_delta(self, header: bool = False) -> bytes | None:
+ """Return new PCM bytes since last call, or None if empty.
+
+ Args:
+ header: If True, prepend a streaming WAV header to the
+ first returned chunk.
+ """
+ new_data = self._pcm_bytes[self._consumed :]
+ if not new_data:
+ return None
+ self._consumed = len(self._pcm_bytes)
+ if header:
+ return _build_streaming_wav_header(
+ sample_rate=_SAMPLE_RATE,
+ channels=_CHANNELS,
+ bits_per_sample=_BITS_PER_SAMPLE,
+ ) + bytes(new_data)
+ return bytes(new_data)
+
+ def get_audio_response(self, block: bool) -> TTSResponse:
+ """Return incremental audio delta (non-blocking or blocking)."""
+ if block:
+ self.finish_event.wait()
+ delta = self._take_delta(header=self._consumed == 0)
+ if delta:
+ return TTSResponse(
+ content=DataBlock(
+ source=Base64Source(
+ data=base64.b64encode(delta).decode("ascii"),
+ media_type=_MEDIA_TYPE,
+ ),
+ ),
+ )
+ return TTSResponse(content=None)
+
+ async def get_audio_chunks(self) -> AsyncGenerator[TTSResponse, None]:
+ """Yield incremental WAV audio chunks as they arrive."""
+ header_sent = False
+ while True:
+ if self.finish_event.is_set():
+ delta = self._take_delta(header=not header_sent)
+ if delta:
+ yield TTSResponse(
+ content=DataBlock(
+ source=Base64Source(
+ data=base64.b64encode(delta).decode(
+ "ascii",
+ ),
+ media_type=_MEDIA_TYPE,
+ ),
+ ),
+ is_last=True,
+ )
+ else:
+ yield TTSResponse(content=None, is_last=True)
+ self.reset()
+ break
+
+ if self.chunk_event.is_set():
+ self.chunk_event.clear()
+ else:
+ await asyncio.to_thread(self.chunk_event.wait)
+
+ if self.finish_event.is_set():
+ continue
+
+ delta = self._take_delta(header=not header_sent)
+ if delta:
+ header_sent = True
+ yield TTSResponse(
+ content=DataBlock(
+ source=Base64Source(
+ data=base64.b64encode(delta).decode("ascii"),
+ media_type=_MEDIA_TYPE,
+ ),
+ ),
+ is_last=False,
+ )
+
+ def reset(self) -> None:
+ """Reset internal state for the next utterance."""
+ self.finish_event.clear()
+ self.chunk_event.clear()
+ self._pcm_bytes = bytearray()
+ self._consumed = 0
+
+ def has_audio_data(self) -> bool:
+ """Return whether any audio data has been received."""
+ return bool(self._pcm_bytes)
+
+ return _Callback
+
+
+class DashScopeRealtimeTTSModel(TTSModelBase):
+ """DashScope Realtime TTS model using the QwenTtsRealtime WebSocket API.
+
+ This model supports streaming input: text can be pushed incrementally
+ via :meth:`push`, and :meth:`synthesize` finalizes the current utterance.
+
+ For more details see the `official document
+ <https://bailian.console.aliyun.com/?tab=doc#/doc/?type=model&url=2938790>`_.
+
+ .. note:: Only one streaming input request can be active at a time.
+ """
+
+ class Parameters(BaseModel):
+ """Frontend-exposed parameters for DashScope Realtime TTS models."""
+
+ voice: str = Field(
+ default="Cherry",
+ title="Voice",
+ description="The voice to use for synthesis.",
+ )
+
+ type: Literal["dashscope_realtime_tts"] = "dashscope_realtime_tts"
+
+ realtime: bool = True
+
+ def __init__(
+ self,
+ credential: DashScopeCredential,
+ model: str = "qwen3-tts-flash-realtime",
+ parameters: "DashScopeRealtimeTTSModel.Parameters | None" = None,
+ stream: bool = True,
+ cold_start_length: int | None = None,
+ cold_start_words: int | None = None,
+ max_retries: int = 3,
+ retry_delay: float = 5.0,
+ ) -> None:
+ """Initialize the DashScope Realtime TTS model.
+
+ Args:
+ credential (`DashScopeCredential`):
+ The DashScope credential.
+ model (`str`, defaults to ``"qwen3-tts-flash-realtime"``):
+ The realtime TTS model name.
+ parameters (`Parameters | None`, defaults to `None`):
+ The TTS parameters (voice, etc.).
+ stream (`bool`, defaults to `True`):
+ Whether :meth:`synthesize` returns a streaming async generator.
+ cold_start_length (`int | None`, defaults to `None`):
+ Minimum character count before the first text chunk is sent.
+ cold_start_words (`int | None`, defaults to `None`):
+ Minimum word count before the first text chunk is sent.
+ max_retries (`int`, defaults to `3`):
+ Max retry attempts on WebSocket failure.
+ retry_delay (`float`, defaults to `5.0`):
+ Initial retry delay in seconds (exponential backoff).
+ """
+ super().__init__(
+ credential=credential,
+ model=model,
+ parameters=parameters,
+ stream=stream,
+ )
+ self.cold_start_length = cold_start_length
+ self.cold_start_words = cold_start_words
+ self.max_retries = max_retries
+ self.retry_delay = retry_delay
+
+ self._tts_client: QwenTtsRealtime | None = None
+ self._callback: Any = None
+ self._connected = False
+ self._cold_start_buffer: str = ""
+ self._cold_start_done: bool = False
+ self._accumulated_text: str = ""
+
+ def _create_client(self) -> None:
+ """Create a fresh TTS client and callback."""
+ import dashscope
+ from dashscope.audio.qwen_tts_realtime import QwenTtsRealtime
+
+ dashscope.api_key = self.credential.api_key.get_secret_value()
+
+ callback_cls = _make_callback_class()
+ self._callback = callback_cls()
+ self._tts_client = QwenTtsRealtime(
+ model=self.model,
+ callback=self._callback,
+ )
+
+ async def connect(self) -> None:
+ """Establish the WebSocket connection."""
+ if self._connected:
+ return
+
+ self._create_client()
+ self._tts_client.connect()
+ self._tts_client.update_session(
+ voice=self.parameters.voice,
+ mode="server_commit",
+ )
+ self._connected = True
+
+ async def close(self) -> None:
+ """Close the WebSocket connection."""
+ if not self._connected:
+ return
+ self._connected = False
+ try:
+ self._tts_client.close()
+ except Exception:
+ pass
+
+ async def _reconnect(self) -> None:
+ """Reconnect by recreating the client."""
+ try:
+ self._tts_client.close()
+ except Exception:
+ pass
+ self._connected = False
+ self._cold_start_buffer = ""
+ self._cold_start_done = False
+ await self.connect()
+
+ async def push(
+ self,
+ text: str,
+ **kwargs: Any,
+ ) -> TTSResponse:
+ """Push an incremental text delta for realtime synthesis.
+
+ Args:
+ text (`str`):
+ An incremental text chunk (delta) to append.
+ **kwargs (`Any`):
+ Additional keyword arguments (unused).
+
+ Returns:
+ `TTSResponse`:
+ Audio accumulated so far, or empty if not yet available.
+ """
+ from websocket import WebSocketConnectionClosedException
+
+ if not self._connected:
+ raise RuntimeError(
+ "TTS model is not connected. Call `connect()` first.",
+ )
+
+ if not text:
+ return TTSResponse(content=None)
+
+ self._accumulated_text += text
+
+ if not self._cold_start_done:
+ self._cold_start_buffer += text
+ ready = True
+ if (
+ self.cold_start_length
+ and len(self._cold_start_buffer) < self.cold_start_length
+ ):
+ ready = False
+ if (
+ ready
+ and self.cold_start_words
+ and len(self._cold_start_buffer.split())
+ < self.cold_start_words
+ ):
+ ready = False
+ if ready:
+ try:
+ self._tts_client.append_text(self._cold_start_buffer)
+ except WebSocketConnectionClosedException:
+ return TTSResponse(content=None)
+ self._cold_start_buffer = ""
+ self._cold_start_done = True
+ else:
+ try:
+ self._tts_client.append_text(text)
+ except WebSocketConnectionClosedException:
+ return TTSResponse(content=None)
+
+ return self._callback.get_audio_response(block=False)
+
+ async def synthesize(
+ self,
+ text: str | None = None,
+ **kwargs: Any,
+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:
+ """Finalize synthesis for the current utterance.
+
+ If text was previously pushed via :meth:`push`, this flushes any
+ remaining buffered text, commits, and waits for audio. If ``text``
+ is provided, it is appended before committing.
+
+ Args:
+ text (`str | None`, defaults to `None`):
+ Optional additional text to append before finalizing.
+ If ``None``, finalizes previously pushed text.
+ **kwargs (`Any`):
+ Additional keyword arguments (unused).
+
+ Returns:
+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:
+ A single response when ``stream=False``, or an async generator
+ of incremental chunks when ``stream=True``.
+ """
+ from websocket import WebSocketConnectionClosedException
+
+ if not self._connected:
+ raise RuntimeError(
+ "TTS model is not connected. Call `connect()` first.",
+ )
+
+ if text is not None:
+ self._accumulated_text += text
+
+ unsent = self._cold_start_buffer
+ if text is not None:
+ unsent += text
+ self._cold_start_buffer = ""
+
+ full_text = self._accumulated_text
+ delay = self.retry_delay
+
+ for attempt in range(self.max_retries):
+ try:
+ if unsent:
+ self._tts_client.append_text(unsent)
+
+ self._tts_client.commit()
+ self._tts_client.finish()
+
+ await asyncio.to_thread(
+ self._callback.finish_event.wait,
+ )
+
+ if full_text and not self._callback.has_audio_data():
+ if attempt < self.max_retries - 1:
+ logger.warning(
+ "TTS: no audio received, retrying (%d/%d) in "
+ "%.1fs...",
+ attempt + 1,
+ self.max_retries,
+ delay,
+ )
+ await asyncio.sleep(delay)
+ await self._reconnect()
+ unsent = full_text
+ delay *= 2
+ continue
+ self._reset_state()
+ raise RuntimeError(
+ f"TTS synthesis failed: no audio after "
+ f"{self.max_retries} attempts",
+ )
+ break
+
+ except WebSocketConnectionClosedException:
+ if attempt < self.max_retries - 1:
+ logger.warning(
+ "TTS WebSocket closed, retrying (%d/%d) in %.1fs...",
+ attempt + 1,
+ self.max_retries,
+ delay,
+ )
+ await asyncio.sleep(delay)
+ await self._reconnect()
+ unsent = full_text
+ delay *= 2
+ else:
+ self._reset_state()
+ raise
+
+ self._reset_state()
+
+ if self.stream:
+ return self._callback.get_audio_chunks()
+
+ return self._callback.get_audio_response(block=True)
+
+ def _reset_state(self) -> None:
+ """Reset per-utterance tracking state."""
+ self._cold_start_buffer = ""
+ self._cold_start_done = False
+ self._accumulated_text = ""
diff --git a/src/agentscope/tts/_tts_base.py b/src/agentscope/tts/_tts_base.py
new file mode 100644
index 000000000..14401b9f7
--- /dev/null
+++ b/src/agentscope/tts/_tts_base.py
@@ -0,0 +1,212 @@
+# -*- coding: utf-8 -*-
+"""The TTS model base class."""
+import inspect
+from abc import abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, AsyncGenerator
+
+from pydantic import BaseModel
+
+from ._tts_response import TTSResponse
+from .._logging import logger
+from ..credential import CredentialBase
+
+if TYPE_CHECKING:
+ from ._tts_model_card import TTSModelCard
+
+
+class TTSModelBase:
+ """Base class for TTS models in AgentScope.
+
+ This base class provides a unified abstraction for both non-realtime and
+ realtime (streaming-input) TTS models, governed by the
+ ``realtime`` flag.
+
+ For non-realtime TTS models, only :meth:`synthesize` needs to be
+ implemented. For realtime TTS models, the lifecycle is managed via the
+ async context manager (``async with model: ...``) or by calling
+ :meth:`connect` / :meth:`close` manually; :meth:`push` appends text
+ chunks and returns whatever audio is currently available, while
+ :meth:`synthesize` blocks until the full speech has been synthesized.
+ """
+
+ class Parameters(BaseModel):
+ """Base parameter schema for TTS models. Subclasses should override
+ this with provider-specific parameters."""
+
+ credential: CredentialBase
+ """The credential used to authenticate against the TTS provider."""
+
+ model: str
+ """The name of the TTS model."""
+
+ parameters: BaseModel
+ """The TTS model parameters."""
+
+ stream: bool
+ """Whether to use streaming output if supported by the model."""
+
+ realtime: bool = False
+ """Whether the TTS model supports realtime (streaming-input) mode."""
+
+ def __init__(
+ self,
+ credential: CredentialBase,
+ model: str,
+ parameters: BaseModel | None = None,
+ stream: bool = True,
+ ) -> None:
+ """Initialize the TTS model base class.
+
+ Args:
+ credential (`CredentialBase`):
+ The credential used to authenticate against the TTS provider.
+ model (`str`):
+ The name of the TTS model.
+ parameters (`BaseModel | None`, defaults to `None`):
+ The TTS model parameters.
+ stream (`bool`, defaults to `True`):
+ Whether to use streaming output if supported by the model.
+ """
+ self.credential = credential
+ self.model = model
+ self.parameters = parameters or self.Parameters()
+ self.stream = stream
+
+ async def __aenter__(self) -> "TTSModelBase":
+ """Enter the async context manager and initialize resources if
+ needed."""
+ if self.realtime:
+ await self.connect()
+ return self
+
+ async def __aexit__(
+ self,
+ exc_type: Any,
+ exc_value: Any,
+ traceback: Any,
+ ) -> None:
+ """Exit the async context manager and clean up resources if needed."""
+ if self.realtime:
+ await self.close()
+
+ async def connect(self) -> None:
+ """Connect to the TTS model and initialize resources.
+
+ .. note:: Only relevant for realtime TTS models — realtime subclasses
+ must override this. The default is a no-op so that non-realtime
+ models can ignore the lifecycle hooks; :meth:`__aenter__` only
+ calls it when ``realtime`` is True.
+ """
+ return
+
+ async def close(self) -> None:
+ """Close the connection to the TTS model and clean up resources.
+
+ .. note:: Only relevant for realtime TTS models — realtime subclasses
+ must override this. See :meth:`connect` for the rationale behind
+ the no-op default.
+ """
+ return
+
+ async def push( # pylint: disable=unused-argument
+ self,
+ text: str,
+ **kwargs: Any,
+ ) -> TTSResponse:
+ """Append text to be synthesized and return the received TTS response.
+ This method is non-blocking and may return an empty response if no
+ audio is available yet.
+
+ To receive all the synthesized speech, call :meth:`synthesize` after
+ pushing all the text chunks.
+
+ .. note:: Only relevant for realtime TTS models — realtime subclasses
+ must override this. Non-realtime models should call
+ :meth:`synthesize` directly and never reach this method.
+
+ Args:
+ text (`str`):
+ The text chunk to be synthesized.
+ **kwargs (`Any`):
+ Additional keyword arguments to pass to the TTS API call.
+
+ Returns:
+ `TTSResponse`:
+ The TTSResponse containing the audio block.
+ """
+ return TTSResponse(content=None)
+
+ @classmethod
+ def list_models(
+ cls,
+ custom_yaml_dir: str | None = None,
+ ) -> list["TTSModelCard"]:
+ """List candidate TTS models by scanning YAML model cards.
+
+ Args:
+ custom_yaml_dir (`str | None`):
+ The custom YAML directory. If ``None``, uses the ``_models``
+ directory next to the concrete subclass's source file.
+
+ Returns:
+ `list[TTSModelCard]`:
+ A list of TTS model cards.
+ """
+ from ._tts_model_card import TTSModelCard
+
+ if custom_yaml_dir is None:
+ subclass_file = Path(inspect.getfile(cls))
+ yaml_dir = subclass_file.parent / "_models"
+ else:
+ yaml_dir = Path(custom_yaml_dir)
+
+ yaml_files = list(yaml_dir.glob("*.yaml"))
+
+ model_cards = []
+ for yaml_file in yaml_files:
+ try:
+ card = TTSModelCard.from_yaml(
+ yaml_path=str(yaml_file),
+ parameter_class=cls.Parameters,
+ )
+ if card.realtime != cls.realtime:
+ continue
+ model_cards.append(card)
+ except Exception as e:
+ logger.warning(
+ "Warning: Failed to load %s: %s",
+ yaml_file,
+ str(e),
+ )
+ continue
+
+ return model_cards
+
+ @abstractmethod
+ async def synthesize(
+ self,
+ text: str | None = None,
+ **kwargs: Any,
+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:
+ """Synthesize speech from the appended text. Different from
+ :meth:`push`, this method blocks until the full speech has been
+ synthesized.
+
+ Args:
+ text (`str | None`, defaults to `None`):
+ The text to be synthesized. If `None`, this method will
+ wait for all previously pushed text to be synthesized and
+ return the last synthesized TTSResponse.
+ **kwargs (`Any`):
+ Additional keyword arguments to pass to the TTS API call.
+
+ Returns:
+ `TTSResponse | AsyncGenerator[TTSResponse, None]`:
+ A single TTSResponse containing the full audio when
+ ``stream=False``. When ``stream=True``, an async generator
+ yielding TTSResponse chunks where each chunk carries an
+ **incremental** audio delta (not a cumulative buffer); the
+ full audio is the concatenation of every chunk's decoded
+ bytes. The final yielded chunk has ``is_last=True``.
+ """
diff --git a/src/agentscope/tts/_tts_model_card.py b/src/agentscope/tts/_tts_model_card.py
new file mode 100644
index 000000000..55ad4f2a8
--- /dev/null
+++ b/src/agentscope/tts/_tts_model_card.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+"""The TTS model card class."""
+import copy
+from datetime import datetime
+from typing import Literal, Self, Type
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class TTSModelCard(BaseModel):
+ """The model card class for TTS models."""
+
+ type: Literal["tts_model"] = "tts_model"
+ """The model card type discriminator."""
+
+ name: str = Field(description="The name of the TTS model")
+ """The model name."""
+
+ label: str = Field(description="The model label.")
+ """The model label used for frontend rendering."""
+
+ status: Literal["active", "deprecated", "sunset"] = Field(
+ default="active",
+ title="Status",
+ description="The model status",
+ )
+ """The model status."""
+
+ deprecated_at: datetime | None = Field(
+ default=None,
+ description="The model deprecation date and time.",
+ title="Deprecation date",
+ )
+ """The model deprecated at."""
+
+ input_types: list[str] = Field(
+ description="The supported model input types.",
+ title="Input types",
+ default=["text/plain"],
+ )
+ """The model supported input types."""
+
+ output_types: list[str] = Field(
+ description="The supported model output types.",
+ title="Output types",
+ default=["audio/wav"],
+ )
+ """The model supported output types."""
+
+ realtime: bool = Field(
+ default=False,
+ description="Whether the model supports streaming input (realtime).",
+ title="Realtime",
+ )
+ """Whether the model supports streaming input."""
+
+ parameter_schema: dict
+ """The parameters schema."""
+
+ parameters_overrides: dict[str, dict]
+ """The parameter overrides."""
+
+ @classmethod
+ def from_yaml(
+ cls,
+ yaml_path: str,
+ parameter_class: Type[BaseModel],
+ ) -> Self:
+ """Read a TTS model card from a YAML file.
+
+ Args:
+ yaml_path (`str`):
+ Path to the YAML file
+ parameter_class (`Type[BaseModel]`):
+ The parameter class (e.g., DashScopeTTSModel.Parameters)
+
+ Returns:
+ `TTSModelCard`:
+ TTSModelCard instance with merged parameter schema
+ """
+ with open(yaml_path, "r", encoding="utf-8") as file:
+ config = yaml.safe_load(file)
+
+ base_schema = parameter_class.model_json_schema()
+ properties = copy.deepcopy(base_schema.get("properties", {}))
+
+ # Auto-inject: populate voice enum from YAML voices list
+ voices = config.get("voices", [])
+ if voices and "voice" in properties:
+ properties["voice"] = {
+ **properties["voice"],
+ "default": voices[0],
+ "enum": voices,
+ }
+
+ # Apply parameter_overrides
+ overrides = config.get("parameter_overrides", {})
+ for param_name, override in overrides.items():
+ if override is None:
+ properties.pop(param_name, None)
+ continue
+
+ if isinstance(override, dict):
+ if override.get("hidden"):
+ properties.pop(param_name, None)
+ continue
+
+ if param_name in properties:
+ properties[param_name] = {
+ **properties[param_name],
+ **override,
+ }
+
+ required = [
+ r for r in base_schema.get("required", []) if r in properties
+ ]
+ final_schema = {
+ "type": "object",
+ "properties": properties,
+ "required": required,
+ }
+
+ return cls(
+ name=config["name"],
+ label=config["label"],
+ status=config.get("status", "active"),
+ deprecated_at=config.get("deprecated_at"),
+ input_types=config.get("input_types", ["text/plain"]),
+ output_types=config.get("output_types", ["audio/wav"]),
+ realtime=config.get("realtime", False),
+ parameter_schema=final_schema,
+ parameters_overrides=config.get("parameter_overrides", {}),
+ )
diff --git a/src/agentscope/tts/_tts_response.py b/src/agentscope/tts/_tts_response.py
new file mode 100644
index 000000000..0fe660582
--- /dev/null
+++ b/src/agentscope/tts/_tts_response.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+"""The TTS response module."""
+from dataclasses import dataclass, field
+from typing import Literal
+
+from .._utils._common import _get_timestamp
+from .._utils._mixin import DictMixin
+from ..message import DataBlock
+from ..types import JSONSerializableObject
+
+
+@dataclass
+class TTSUsage(DictMixin):
+ """The usage of a TTS model API invocation."""
+
+ input_tokens: int
+ """The number of input tokens."""
+
+ output_tokens: int
+ """The number of output tokens."""
+
+ time: float
+ """The time used in seconds."""
+
+ type: Literal["tts"] = field(default_factory=lambda: "tts")
+ """The type of the usage, must be `tts`."""
+
+
+@dataclass
+class TTSResponse(DictMixin):
+ """The response of TTS models."""
+
+ content: DataBlock | None
+ """The audio chunk produced by the TTS model. The audio format is
+ indicated by ``content.source.media_type`` (e.g.
+ ``"audio/pcm;rate=24000"``, ``"audio/mpeg"``)."""
+
+ id: str = field(default_factory=lambda: _get_timestamp(True))
+ """The unique identifier of the response."""
+
+ created_at: str = field(default_factory=_get_timestamp)
+ """When the response was created."""
+
+ type: Literal["tts"] = field(default_factory=lambda: "tts")
+ """The type of the response, which is always 'tts'."""
+
+ usage: TTSUsage | None = field(default_factory=lambda: None)
+ """The usage information of the TTS response, if available."""
+
+ metadata: dict[str, JSONSerializableObject] | None = field(
+ default_factory=lambda: None,
+ )
+ """The metadata of the TTS response."""
+
+ is_last: bool = field(default_factory=lambda: True)
+ """Whether this is the last response in a stream of TTS responses."""
diff --git a/tests/tts_dashscope_test.py b/tests/tts_dashscope_test.py
new file mode 100644
index 000000000..adf27521f
--- /dev/null
+++ b/tests/tts_dashscope_test.py
@@ -0,0 +1,683 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=protected-access
+"""Unit tests for the TTS module.
+
+Covers:
+ * ``TTSModelBase`` default no-op behaviour for ``connect`` / ``close`` /
+ ``push`` (so non-realtime subclasses needn't override them).
+ * ``DashScopeTTSModel`` non-streaming aggregation.
+ * ``DashScopeTTSModel`` streaming: incremental deltas and ``is_last``
+ placement at the final chunk only.
+ * ``DashScopeRealtimeTTSModel`` connect / close / push / synthesize
+ lifecycle over a mocked WebSocket.
+"""
+import base64
+import io
+import wave
+from typing import Any, AsyncGenerator
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import MagicMock, Mock, patch
+
+from agentscope.credential import DashScopeCredential
+from agentscope.tts import (
+ DashScopeTTSModel,
+ DashScopeRealtimeTTSModel,
+ TTSModelBase,
+ TTSResponse,
+)
+
+
+_MEDIA_TYPE = "audio/wav"
+# DashScope TTS emits 24kHz / mono / 16-bit PCM; the WAV wrapping in the
+# model layer uses the same parameters.
+_TTS_SAMPLE_RATE = 24000
+_TTS_CHANNELS = 1
+_TTS_SAMPLE_WIDTH = 2 # bytes (= 16 bit)
+_WAV_HEADER_LEN = 44
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_api_chunk(
+ data_bytes: bytes | None,
+ usage: Any = None,
+) -> MagicMock:
+ """Build a chunk shaped like what dashscope.MultiModalConversation
+ yields. ``data_bytes=None`` represents a chunk with no output."""
+ chunk = MagicMock()
+ chunk.usage = usage
+ if data_bytes is None:
+ chunk.output = None
+ return chunk
+ chunk.output = MagicMock()
+ chunk.output.audio = MagicMock()
+ chunk.output.audio.data = base64.b64encode(data_bytes).decode("ascii")
+ return chunk
+
+
+def _make_usage(
+ input_tokens: int = 0,
+ output_tokens: int = 0,
+ characters: int = 0,
+) -> MagicMock:
+ """Build a usage object shaped like what the DashScope API returns."""
+ usage = MagicMock()
+ usage.input_tokens = input_tokens
+ usage.output_tokens = output_tokens
+ usage.characters = characters
+ return usage
+
+
+def _make_api_generator(chunks: list[bytes | None]) -> Any:
+ """Build a sync generator like ``MultiModalConversation.call`` returns.
+ The last chunk carries a usage object."""
+
+ def _gen() -> Any:
+ for i, data in enumerate(chunks):
+ is_last = i == len(chunks) - 1
+ usage = _make_usage(characters=10) if is_last else None
+ yield _make_api_chunk(data, usage=usage)
+
+ return _gen()
+
+
+# ---------------------------------------------------------------------------
+# TTSModelBase — default no-op surface for non-realtime subclasses
+# ---------------------------------------------------------------------------
+
+
+class _DummyTTS(TTSModelBase):
+ """Minimal subclass that implements only ``synthesize`` — exercises the
+ base class's no-op ``connect`` / ``close`` / ``push`` defaults."""
+
+ async def synthesize(
+ self,
+ text: str | None = None,
+ **kwargs: Any,
+ ) -> TTSResponse | AsyncGenerator[TTSResponse, None]:
+ del text, kwargs
+ return TTSResponse(content=None)
+
+
+class _RealtimeDummyTTS(_DummyTTS):
+ """Realtime-flavoured dummy to assert ``__aenter__`` drives the lifecycle
+ hooks when ``realtime`` is True."""
+
+ realtime = True
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self.connect_calls = 0
+ self.close_calls = 0
+
+ async def connect(self) -> None:
+ self.connect_calls += 1
+
+ async def close(self) -> None:
+ self.close_calls += 1
+
+
+def _make_dummy(cls: type = _DummyTTS) -> TTSModelBase:
+ return cls(
+ credential=DashScopeCredential(api_key="test"),
+ model="x",
+ stream=False,
+ )
+
+
+class TestTTSModelBaseDefaults(IsolatedAsyncioTestCase):
+ """The base class supplies safe no-op defaults so non-realtime subclasses
+ don't need to implement realtime-only hooks."""
+
+ async def test_default_connect_close_noop(self) -> None:
+ """Default connect/close return without raising."""
+ model = _make_dummy()
+ await model.connect()
+ await model.close()
+
+ async def test_default_push_returns_empty(self) -> None:
+ """Default push returns an empty TTSResponse rather than raising,
+ so a misuse on a non-realtime model degrades gracefully."""
+ model = _make_dummy()
+ resp = await model.push("ignored")
+ self.assertIsNone(resp.content)
+
+ async def test_aenter_skips_hooks_for_non_realtime(self) -> None:
+ """``async with`` on a non-realtime model must not invoke connect/
+ close (gated by ``realtime``)."""
+ model = _make_dummy(_DummyTTS)
+ async with model as m:
+ self.assertIs(m, model)
+
+ async def test_aenter_invokes_hooks_for_realtime(self) -> None:
+ """For ``realtime=True`` subclasses, connect/close fire on
+ enter/exit exactly once."""
+ model = _make_dummy(_RealtimeDummyTTS)
+ async with model:
+ self.assertEqual(model.connect_calls, 1)
+ self.assertEqual(model.close_calls, 0)
+ self.assertEqual(model.close_calls, 1)
+
+
+# ---------------------------------------------------------------------------
+# DashScopeTTSModel — non-streaming and streaming
+# ---------------------------------------------------------------------------
+
+
+def _parse_wav_payload(wav_bytes: bytes) -> bytes:
+ """Decode a full WAV file and return its raw PCM frames."""
+ with wave.open(io.BytesIO(wav_bytes), "rb") as wav:
+ return wav.readframes(wav.getnframes())
+
+
+class TestDashScopeTTSModel(IsolatedAsyncioTestCase):
+ """The unittests for DashScope TTS model (non-realtime)."""
+
+ def setUp(self) -> None:
+ """Set up the test case."""
+ self.patcher = patch("dashscope.MultiModalConversation")
+ self.mock_mmc = self.patcher.start()
+
+ def tearDown(self) -> None:
+ """Tear down the test case."""
+ self.patcher.stop()
+
+ def _make_model(self, stream: bool = False) -> DashScopeTTSModel:
+ """Create a DashScopeTTSModel with test credentials."""
+ return DashScopeTTSModel(
+ credential=DashScopeCredential(api_key="test"),
+ model="qwen3-tts-flash",
+ parameters=DashScopeTTSModel.Parameters(voice="Cherry"),
+ stream=stream,
+ )
+
+ # -- non-streaming --
+
+ async def test_aggregates_chunks(self) -> None:
+ """All API chunks are aggregated into one self-contained WAV."""
+ self.mock_mmc.call.return_value = _make_api_generator(
+ [b"AAAA", b"BBBB", b"CCCC"],
+ )
+ model = self._make_model(stream=False)
+
+ result = await model.synthesize("Hello world")
+
+ self.assertIsInstance(result, TTSResponse)
+ self.assertEqual(result.content.source.media_type, _MEDIA_TYPE)
+ wav_bytes = base64.b64decode(result.content.source.data)
+ with wave.open(io.BytesIO(wav_bytes), "rb") as wav:
+ self.assertEqual(
+ {
+ "framerate": wav.getframerate(),
+ "channels": wav.getnchannels(),
+ "sampwidth": wav.getsampwidth(),
+ "frames": wav.readframes(wav.getnframes()),
+ },
+ {
+ "framerate": _TTS_SAMPLE_RATE,
+ "channels": _TTS_CHANNELS,
+ "sampwidth": _TTS_SAMPLE_WIDTH,
+ "frames": b"AAAABBBBCCCC",
+ },
+ )
+ self.assertTrue(result.is_last)
+
+ async def test_none_short_circuits(self) -> None:
+ """``synthesize(None)`` returns an empty response without touching
+ the API."""
+ model = self._make_model(stream=False)
+
+ result = await model.synthesize(None)
+
+ self.assertIsNone(result.content)
+ self.mock_mmc.call.assert_not_called()
+
+ async def test_empty_string_short_circuits(self) -> None:
+ """``synthesize("")`` returns an empty response without touching
+ the API."""
+ model = self._make_model(stream=False)
+
+ result = await model.synthesize("")
+
+ self.assertIsNone(result.content)
+ self.mock_mmc.call.assert_not_called()
+
+ async def test_skips_empty_chunks(self) -> None:
+ """Chunks without ``output`` are ignored during aggregation."""
+ self.mock_mmc.call.return_value = _make_api_generator(
+ [None, b"AAAA", None, b"BBBB"],
+ )
+ model = self._make_model(stream=False)
+
+ result = await model.synthesize("Hello world")
+
+ wav_bytes = base64.b64decode(result.content.source.data)
+ self.assertEqual(_parse_wav_payload(wav_bytes), b"AAAABBBB")
+
+ # -- streaming --
+
+ async def test_incremental_deltas(self) -> None:
+ """Each API chunk yields one TTSResponse with incremental PCM."""
+ self.mock_mmc.call.return_value = _make_api_generator(
+ [b"AAAA", b"BBBB", b"CCCC"],
+ )
+ model = self._make_model(stream=True)
+
+ gen = await model.synthesize("Hello world")
+ chunks = [c async for c in gen]
+
+ payloads = [base64.b64decode(c.content.source.data) for c in chunks]
+
+ self.assertTrue(payloads[0].startswith(b"RIFF"))
+ self.assertEqual(payloads[0][8:12], b"WAVE")
+ self.assertEqual(payloads[0][_WAV_HEADER_LEN:], b"AAAA")
+ self.assertEqual(payloads[1], b"BBBB")
+ self.assertEqual(payloads[2], b"CCCC")
+
+ self.assertEqual(
+ [c.is_last for c in chunks],
+ [False, False, True],
+ )
+ self.assertEqual(
+ [c.content.source.media_type for c in chunks],
+ [_MEDIA_TYPE, _MEDIA_TYPE, _MEDIA_TYPE],
+ )
+
+ async def test_single_chunk_marked_last(self) -> None:
+ """A lone audio chunk is flagged ``is_last=True`` with a streaming
+ WAV header."""
+ self.mock_mmc.call.return_value = _make_api_generator([b"ONLYCHUNK"])
+ model = self._make_model(stream=True)
+
+ gen = await model.synthesize("Hello world")
+ chunks = [c async for c in gen]
+
+ self.assertEqual(len(chunks), 1)
+ self.assertTrue(chunks[0].is_last)
+ payload = base64.b64decode(chunks[0].content.source.data)
+ self.assertTrue(payload.startswith(b"RIFF"))
+ self.assertEqual(payload[_WAV_HEADER_LEN:], b"ONLYCHUNK")
+
+ async def test_empty_stream_yields_terminal(self) -> None:
+ """When the API yields no audio, the generator emits a terminal
+ sentinel so consumers can detect EOS."""
+ self.mock_mmc.call.return_value = _make_api_generator([None, None])
+ model = self._make_model(stream=True)
+
+ gen = await model.synthesize("Hello world")
+ chunks = [c async for c in gen]
+
+ self.assertEqual(len(chunks), 1)
+ self.assertIsNone(chunks[0].content)
+ self.assertTrue(chunks[0].is_last)
+
+
+# ---------------------------------------------------------------------------
+# DashScopeRealtimeTTSModel — realtime push / synthesize lifecycle
+# ---------------------------------------------------------------------------
+
+
+class TestDashScopeRealtimeTTSModel( # pylint: disable=too-many-public-methods
+ IsolatedAsyncioTestCase,
+):
+ """The unittests for DashScope Realtime TTS model."""
+
+ def setUp(self) -> None:
+ self.mock_modules = self._create_mock_dashscope_modules()
+ self.mock_client = self._create_mock_tts_client()
+ mock_tts_class = Mock(return_value=self.mock_client)
+ self.mock_modules[
+ "dashscope.audio.qwen_tts_realtime"
+ ].QwenTtsRealtime = mock_tts_class
+
+ @staticmethod
+ def _create_mock_dashscope_modules() -> dict:
+ mock_qwen_tts_realtime = MagicMock()
+ mock_qwen_tts_realtime.QwenTtsRealtime = Mock
+ mock_qwen_tts_realtime.QwenTtsRealtimeCallback = Mock
+
+ mock_audio = MagicMock()
+ mock_audio.qwen_tts_realtime = mock_qwen_tts_realtime
+
+ mock_dashscope = MagicMock()
+ mock_dashscope.api_key = None
+ mock_dashscope.audio = mock_audio
+
+ return {
+ "dashscope": mock_dashscope,
+ "dashscope.audio": mock_audio,
+ "dashscope.audio.qwen_tts_realtime": mock_qwen_tts_realtime,
+ }
+
+ @staticmethod
+ def _create_mock_tts_client() -> Mock:
+ client = Mock()
+ client.connect = Mock()
+ client.close = Mock()
+ client.finish = Mock()
+ client.commit = Mock()
+ client.update_session = Mock()
+ client.append_text = Mock()
+ return client
+
+ def _make_model(self, **kwargs: Any) -> DashScopeRealtimeTTSModel:
+ defaults: dict[str, Any] = {
+ "credential": DashScopeCredential(api_key="test"),
+ "model": "qwen3-tts-flash-realtime",
+ "stream": True,
+ "max_retries": 1,
+ "retry_delay": 0.0,
+ }
+ defaults.update(kwargs)
+ return DashScopeRealtimeTTSModel(**defaults)
+
+ def _mock_synthesize_callback(self, model: Any) -> None:
+ """Set up callback mocks so ``synthesize()`` doesn't block."""
+ model._callback.finish_event = Mock()
+ model._callback.finish_event.wait = Mock()
+ model._callback.has_audio_data = Mock(return_value=True)
+ model._callback.get_audio_response = Mock(
+ return_value=TTSResponse(content=None),
+ )
+
+ # -- connect / close --
+
+ async def test_connect_creates_client(self) -> None:
+ """connect() creates a WebSocket client and calls update_session."""
+ with patch.dict("sys.modules", self.mock_modules):
+ model = self._make_model()
+ await model.connect()
+
+ self.assertTrue(model._connected)
+ self.mock_client.connect.assert_called_once()
+ self.mock_client.update_session.assert_called_once()
+
+ async def test_close_disconnects(self) -> None:
+ """close() sets _connected to False and closes the client."""
+ with patch.dict("sys.modules", self.mock_modules):
+ model = self._make_model()
+ await model.connect()
+ await model.close()
+
+ self.assertFalse(model._connected)
+ self.mock_client.close.assert_called_once()
+
+ async def test_close_idempotent(self) -> None:
+ """Calling close() when already disconnected is a no-op."""
+ model = self._make_model()
+ model._connected = False
+ await model.close()
+ self.assertFalse(model._connected)
+
+ async def test_async_context_manager(self) -> None:
+ """``async with`` triggers connect on enter and close on exit."""
+ with patch.dict("sys.modules", self.mock_modules):
+ model = self._make_model()
+ async with model:
+ self.assertTrue(model._connected)
+ self.assertFalse(model._connected)
+
+ # -- push --
+
+ async def test_push_appends_text(self) -> None:
+ """A single push forwards the delta to append_text."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ await model.push("Hello")
+
+ self.mock_client.append_text.assert_called_once_with("Hello")
+ self.assertEqual(model._accumulated_text, "Hello")
+ self.assertTrue(model._cold_start_done)
+
+ async def test_push_incremental_deltas(self) -> None:
+ """Consecutive deltas are each forwarded verbatim."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ await model.push("Hello")
+ await model.push(" world")
+
+ self.assertEqual(
+ self.mock_client.append_text.call_count,
+ 2,
+ )
+ self.mock_client.append_text.assert_any_call("Hello")
+ self.mock_client.append_text.assert_any_call(" world")
+
+ async def test_push_empty_text_no_call(self) -> None:
+ """Empty string does not call append_text."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ res = await model.push("")
+
+ self.mock_client.append_text.assert_not_called()
+ self.assertIsNone(res.content)
+
+ async def test_push_not_connected_raises(self) -> None:
+ """push() raises RuntimeError when not connected."""
+ model = self._make_model()
+ model._connected = False
+
+ with self.assertRaises(RuntimeError):
+ await model.push("Hello")
+
+ async def test_push_returns_audio_when_available(self) -> None:
+ """push() returns audio data from the callback when available."""
+ mock_audio_data = base64.b64encode(b"PCMDATA").decode("ascii")
+
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ model._callback.get_audio_response = Mock(
+ return_value=TTSResponse(
+ content=MagicMock(
+ source=MagicMock(
+ data=mock_audio_data,
+ media_type=_MEDIA_TYPE,
+ ),
+ ),
+ ),
+ )
+ res = await model.push("Hello")
+
+ self.assertIsNotNone(res.content)
+ self.assertEqual(res.content.source.data, mock_audio_data)
+
+ async def test_push_returns_empty_when_no_audio(self) -> None:
+ """push() returns empty response when no audio is ready."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ model._callback.get_audio_response = Mock(
+ return_value=TTSResponse(content=None),
+ )
+ res = await model.push("Hello")
+ self.assertIsNone(res.content)
+
+ async def test_push_cold_start_buffers_across_deltas(self) -> None:
+ """Multiple small deltas are buffered until cold_start_length is
+ met, then flushed as a single ``append_text`` call."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model(cold_start_length=10) as model:
+ await model.push("Hi")
+ await model.push(" there")
+ self.mock_client.append_text.assert_not_called()
+ self.assertFalse(model._cold_start_done)
+
+ await model.push(" friend!")
+ self.mock_client.append_text.assert_called_once_with(
+ "Hi there friend!",
+ )
+ self.assertTrue(model._cold_start_done)
+
+ async def test_push_cold_start_single_delta_meets_threshold(self) -> None:
+ """A single delta exceeding cold_start_length sends immediately."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model(cold_start_length=5) as model:
+ await model.push("Hello world")
+
+ self.mock_client.append_text.assert_called_once_with(
+ "Hello world",
+ )
+
+ async def test_push_after_cold_start_forwards_directly(self) -> None:
+ """Once cold start is done, subsequent deltas bypass the buffer."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model(cold_start_length=3) as model:
+ await model.push("Hello")
+ self.mock_client.append_text.assert_called_with("Hello")
+
+ await model.push(" world")
+ self.mock_client.append_text.assert_called_with(" world")
+ self.assertEqual(
+ self.mock_client.append_text.call_count,
+ 2,
+ )
+
+ async def test_push_cold_start_words_buffers(self) -> None:
+ """Deltas below cold_start_words are buffered."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model(cold_start_words=3) as model:
+ await model.push("Hello")
+ await model.push(" world")
+
+ self.mock_client.append_text.assert_not_called()
+ self.assertFalse(model._cold_start_done)
+
+ # -- synthesize --
+
+ async def test_synthesize_commits_and_finishes(self) -> None:
+ """synthesize(text=...) appends text, commits, and finishes."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ self._mock_synthesize_callback(model)
+
+ await model.synthesize(text="Hello")
+
+ self.mock_client.append_text.assert_called_once_with("Hello")
+ self.mock_client.commit.assert_called_once()
+ self.mock_client.finish.assert_called_once()
+
+ async def test_synthesize_appends_extra_text(self) -> None:
+ """synthesize(text=...) after push appends the extra text."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ model._callback.get_audio_response = Mock(
+ return_value=TTSResponse(content=None),
+ )
+ await model.push("Hello")
+ self.mock_client.append_text.reset_mock()
+
+ self._mock_synthesize_callback(model)
+ await model.synthesize(text=" world")
+
+ self.mock_client.append_text.assert_called_once_with(
+ " world",
+ )
+ self.mock_client.commit.assert_called_once()
+
+ async def test_synthesize_no_text_drain(self) -> None:
+ """synthesize(None) after push commits without extra append."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ model._callback.get_audio_response = Mock(
+ return_value=TTSResponse(content=None),
+ )
+ await model.push("Hello")
+ self.mock_client.append_text.reset_mock()
+
+ self._mock_synthesize_callback(model)
+ await model.synthesize()
+
+ self.mock_client.append_text.assert_not_called()
+ self.mock_client.commit.assert_called_once()
+
+ async def test_synthesize_resets_state(self) -> None:
+ """State is reset after synthesize completes."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model() as model:
+ self._mock_synthesize_callback(model)
+ await model.synthesize(text="Hello")
+
+ self.assertFalse(model._cold_start_done)
+ self.assertEqual(model._accumulated_text, "")
+ self.assertEqual(model._cold_start_buffer, "")
+
+ async def test_synthesize_stream_returns_generator(self) -> None:
+ """stream=True returns an async generator with is_last."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model(stream=True) as model:
+ self._mock_synthesize_callback(model)
+
+ async def mock_chunks() -> AsyncGenerator[TTSResponse, None]:
+ yield TTSResponse(content=None, is_last=True)
+
+ model._callback.get_audio_chunks = mock_chunks
+
+ gen = await model.synthesize(text="Hello")
+ chunks = [c async for c in gen]
+
+ self.assertTrue(len(chunks) >= 1)
+ self.assertTrue(chunks[-1].is_last)
+
+ async def test_synthesize_non_stream_returns_single(self) -> None:
+ """stream=False returns a single TTSResponse."""
+ mock_audio_data = base64.b64encode(b"PCMDATA").decode("ascii")
+
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model(stream=False) as model:
+ self._mock_synthesize_callback(model)
+ model._callback.get_audio_response = Mock(
+ return_value=TTSResponse(
+ content=MagicMock(
+ source=MagicMock(
+ data=mock_audio_data,
+ media_type=_MEDIA_TYPE,
+ ),
+ ),
+ ),
+ )
+
+ res = await model.synthesize(text="Hello")
+
+ self.assertIsInstance(res, TTSResponse)
+ self.assertIsNotNone(res.content)
+
+ async def test_synthesize_flushes_cold_start_buffer(self) -> None:
+ """If cold start was never met during push, synthesize flushes the
+ buffered text before committing."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model(cold_start_length=100) as model:
+ await model.push("Hi")
+ await model.push(" there")
+ self.mock_client.append_text.assert_not_called()
+
+ self._mock_synthesize_callback(model)
+ await model.synthesize()
+
+ self.mock_client.append_text.assert_called_once_with(
+ "Hi there",
+ )
+ self.mock_client.commit.assert_called_once()
+
+ async def test_synthesize_not_connected_raises(self) -> None:
+ """synthesize() raises RuntimeError when not connected."""
+ model = self._make_model()
+ model._connected = False
+
+ with self.assertRaises(RuntimeError):
+ await model.synthesize(text="Hello")
+
+ async def test_synthesize_no_audio_raises_after_retries(self) -> None:
+ """RuntimeError after max_retries with no audio received."""
+ with patch.dict("sys.modules", self.mock_modules):
+ async with self._make_model(
+ max_retries=1,
+ retry_delay=0.0,
+ ) as model:
+ model._callback.finish_event = Mock()
+ model._callback.finish_event.wait = Mock()
+ model._callback.has_audio_data = Mock(return_value=False)
+
+ with self.assertRaises(RuntimeError):
+ await model.synthesize(text="Hello")
diff --git a/tests/tts_middleware_test.py b/tests/tts_middleware_test.py
new file mode 100644
index 000000000..3f296c631
--- /dev/null
+++ b/tests/tts_middleware_test.py
@@ -0,0 +1,388 @@
+# -*- coding: utf-8 -*-
+"""Unit tests for TTSMiddleware."""
+import base64
+from typing import Any, AsyncGenerator
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, MagicMock
+
+from agentscope.event import (
+ DataBlockDeltaEvent,
+ DataBlockEndEvent,
+ DataBlockStartEvent,
+ TextBlockDeltaEvent,
+ TextBlockEndEvent,
+)
+from agentscope.message import Base64Source, DataBlock
+from agentscope.middleware import TTSMiddleware
+from agentscope.tts import TTSModelBase, TTSResponse
+
+_EXCLUDE = {"id", "created_at", "metadata"}
+
+
+def _dump(evt: Any) -> dict:
+ """Dump an event to a dict excluding auto-generated fields."""
+ return evt.model_dump(exclude=_EXCLUDE)
+
+
+def _make_tts_response(
+ data: str,
+ media_type: str = "audio/wav",
+) -> TTSResponse:
+ return TTSResponse(
+ content=DataBlock(
+ source=Base64Source(data=data, media_type=media_type),
+ ),
+ )
+
+
+def _make_agent_stub(
+ reply_id: str = "reply-1",
+ name: str = "agent",
+) -> MagicMock:
+ agent = MagicMock()
+ agent.name = name
+ agent.state.reply_id = reply_id
+ return agent
+
+
+class TestTTSMiddlewareNonRealtime(IsolatedAsyncioTestCase):
+ """Tests for non-realtime (non-streaming-input) TTS path."""
+
+ async def test_synthesize_on_text_block_end(self) -> None:
+ """After TextBlockEnd, synthesize() is called with accumulated text
+ and DATA_BLOCK_START + DATA_BLOCK_DELTA + DATA_BLOCK_END are emitted.
+ """
+ audio_b64 = base64.b64encode(b"\x00\x01\x02").decode()
+
+ tts = MagicMock(spec=TTSModelBase)
+ tts.realtime = False
+ tts.__aenter__ = AsyncMock(return_value=tts)
+ tts.__aexit__ = AsyncMock(return_value=None)
+ tts.synthesize = AsyncMock(
+ return_value=_make_tts_response(audio_b64),
+ )
+
+ middleware = TTSMiddleware(tts_model=tts)
+ agent = _make_agent_stub()
+
+ upstream_events = [
+ TextBlockDeltaEvent(
+ reply_id="reply-1",
+ block_id="blk-1",
+ delta="Hello ",
+ ),
+ TextBlockDeltaEvent(
+ reply_id="reply-1",
+ block_id="blk-1",
+ delta="world",
+ ),
+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),
+ ]
+
+ async def next_handler(**_kwargs: Any) -> AsyncGenerator:
+ for evt in upstream_events:
+ yield evt
+
+ emitted = []
+ async for evt in middleware.on_reply(agent, {}, next_handler):
+ emitted.append(evt)
+
+ # Upstream events are passed through
+ self.assertEqual(
+ [_dump(e) for e in emitted[:3]],
+ [_dump(e) for e in upstream_events],
+ )
+
+ # After TextBlockEnd: START + DELTA + END
+ block_id = emitted[3].block_id
+ self.assertEqual(
+ [_dump(e) for e in emitted[3:]],
+ [
+ {
+ "type": "DATA_BLOCK_START",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_DELTA",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "data": audio_b64,
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_END",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ },
+ ],
+ )
+
+ # synthesize was called with the full text
+ tts.synthesize.assert_called_once_with("Hello world")
+
+ async def test_empty_text_skips_synthesize(self) -> None:
+ """When accumulated text is whitespace-only, synthesize is not called
+ and no DATA_BLOCK events are emitted."""
+ tts = MagicMock(spec=TTSModelBase)
+ tts.realtime = False
+ tts.__aenter__ = AsyncMock(return_value=tts)
+ tts.__aexit__ = AsyncMock(return_value=None)
+ tts.synthesize = AsyncMock()
+
+ middleware = TTSMiddleware(tts_model=tts)
+ agent = _make_agent_stub()
+
+ upstream_events = [
+ TextBlockDeltaEvent(
+ reply_id="reply-1",
+ block_id="blk-1",
+ delta=" ",
+ ),
+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),
+ ]
+
+ async def next_handler(**_kwargs: Any) -> AsyncGenerator:
+ for evt in upstream_events:
+ yield evt
+
+ emitted = []
+ async for evt in middleware.on_reply(agent, {}, next_handler):
+ emitted.append(evt)
+
+ # Only the upstream events are passed through
+ self.assertEqual(
+ [_dump(e) for e in emitted],
+ [_dump(e) for e in upstream_events],
+ )
+ tts.synthesize.assert_not_called()
+
+ async def test_streaming_output_multiple_chunks(self) -> None:
+ """When synthesize() returns an async generator, each chunk produces
+ a DATA_BLOCK_DELTA under the same block_id."""
+ chunk1 = _make_tts_response("AAAA")
+ chunk2 = _make_tts_response("BBBB")
+
+ async def synth_gen(*_args: Any, **_kwargs: Any) -> AsyncGenerator:
+ yield chunk1
+ yield chunk2
+
+ tts = MagicMock(spec=TTSModelBase)
+ tts.realtime = False
+ tts.__aenter__ = AsyncMock(return_value=tts)
+ tts.__aexit__ = AsyncMock(return_value=None)
+ tts.synthesize = AsyncMock(return_value=synth_gen())
+
+ middleware = TTSMiddleware(tts_model=tts)
+ agent = _make_agent_stub()
+
+ upstream_events = [
+ TextBlockDeltaEvent(
+ reply_id="reply-1",
+ block_id="blk-1",
+ delta="hi",
+ ),
+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),
+ ]
+
+ async def next_handler(**_kwargs: Any) -> AsyncGenerator:
+ for evt in upstream_events:
+ yield evt
+
+ emitted = []
+ async for evt in middleware.on_reply(agent, {}, next_handler):
+ emitted.append(evt)
+
+ # Upstream pass-through + START + 2x DELTA + END
+ data_events = emitted[2:]
+ block_id = data_events[0].block_id
+ self.assertEqual(
+ [_dump(e) for e in data_events],
+ [
+ {
+ "type": "DATA_BLOCK_START",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_DELTA",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "data": "AAAA",
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_DELTA",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "data": "BBBB",
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_END",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ },
+ ],
+ )
+
+
+class TestTTSMiddlewareRealtime(IsolatedAsyncioTestCase):
+ """Tests for realtime (streaming-input) TTS path."""
+
+ async def test_push_on_delta_and_drain_on_end(self) -> None:
+ """In realtime mode, push() is called on each TextBlockDelta and
+ synthesize() drains on TextBlockEnd."""
+ push_audio = _make_tts_response("PUSH1")
+ drain_audio = _make_tts_response("DRAIN")
+
+ tts = MagicMock(spec=TTSModelBase)
+ tts.realtime = True
+ tts.__aenter__ = AsyncMock(return_value=tts)
+ tts.__aexit__ = AsyncMock(return_value=None)
+ tts.push = AsyncMock(return_value=push_audio)
+ tts.synthesize = AsyncMock(return_value=drain_audio)
+
+ middleware = TTSMiddleware(tts_model=tts)
+ agent = _make_agent_stub()
+
+ upstream_events = [
+ TextBlockDeltaEvent(
+ reply_id="reply-1",
+ block_id="blk-1",
+ delta="Hello",
+ ),
+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),
+ ]
+
+ async def next_handler(**_kwargs: Any) -> AsyncGenerator:
+ for evt in upstream_events:
+ yield evt
+
+ emitted = []
+ async for evt in middleware.on_reply(agent, {}, next_handler):
+ emitted.append(evt)
+
+ # push() called with the delta text
+ tts.push.assert_called_once_with("Hello")
+
+ # synthesize() called to drain
+ tts.synthesize.assert_called_once()
+
+ data_events = [
+ e
+ for e in emitted
+ if isinstance(
+ e,
+ (DataBlockStartEvent, DataBlockDeltaEvent, DataBlockEndEvent),
+ )
+ ]
+ # START + DELTA(push) + DELTA(drain) + END
+ block_id = data_events[0].block_id
+ self.assertEqual(
+ [_dump(e) for e in data_events],
+ [
+ {
+ "type": "DATA_BLOCK_START",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_DELTA",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "data": "PUSH1",
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_DELTA",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "data": "DRAIN",
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_END",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ },
+ ],
+ )
+
+ async def test_push_returns_none_no_audio_emitted(self) -> None:
+ """When push() returns empty content, no DATA_BLOCK events are emitted
+ until synthesize() drains."""
+ empty_response = TTSResponse(content=None)
+ drain_audio = _make_tts_response("FINAL")
+
+ tts = MagicMock(spec=TTSModelBase)
+ tts.realtime = True
+ tts.__aenter__ = AsyncMock(return_value=tts)
+ tts.__aexit__ = AsyncMock(return_value=None)
+ tts.push = AsyncMock(return_value=empty_response)
+ tts.synthesize = AsyncMock(return_value=drain_audio)
+
+ middleware = TTSMiddleware(tts_model=tts)
+ agent = _make_agent_stub()
+
+ upstream_events = [
+ TextBlockDeltaEvent(
+ reply_id="reply-1",
+ block_id="blk-1",
+ delta="Hi",
+ ),
+ TextBlockDeltaEvent(
+ reply_id="reply-1",
+ block_id="blk-1",
+ delta=" there",
+ ),
+ TextBlockEndEvent(reply_id="reply-1", block_id="blk-1"),
+ ]
+
+ async def next_handler(**_kwargs: Any) -> AsyncGenerator:
+ for evt in upstream_events:
+ yield evt
+
+ emitted = []
+ async for evt in middleware.on_reply(agent, {}, next_handler):
+ emitted.append(evt)
+
+ # push called twice but produced no audio
+ self.assertEqual(tts.push.call_count, 2)
+
+ data_events = [
+ e
+ for e in emitted
+ if isinstance(
+ e,
+ (DataBlockStartEvent, DataBlockDeltaEvent, DataBlockEndEvent),
+ )
+ ]
+ # Only drain produces: START + DELTA + END
+ block_id = data_events[0].block_id
+ self.assertEqual(
+ [_dump(e) for e in data_events],
+ [
+ {
+ "type": "DATA_BLOCK_START",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_DELTA",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ "data": "FINAL",
+ "media_type": "audio/wav",
+ },
+ {
+ "type": "DATA_BLOCK_END",
+ "reply_id": "reply-1",
+ "block_id": block_id,
+ },
+ ],
+ )