Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(omni): support real-time audio caption streaming via DashScope and OpenAI
qbc58c24fa7c85a
diff --git a/examples/web_ui/frontend/package.json b/examples/web_ui/frontend/package.jsonindex 90c71dc94..659fddf82 100644--- a/examples/web_ui/frontend/package.json+++ b/examples/web_ui/frontend/package.json@@ -11,7 +11,7 @@"preview": "vite preview"},"dependencies": {- "@agentscope-ai/agentscope": "^0.0.8",+ "@agentscope-ai/agentscope": "^0.0.9","@fontsource-variable/geist": "^5.2.8","@tailwindcss/vite": "^4.3.0","class-variance-authority": "^0.7.1",diff --git a/examples/web_ui/frontend/src/components/chat/MessageBubble.tsx b/examples/web_ui/frontend/src/components/chat/MessageBubble.tsxindex a81666082..b66267b5c 100644--- a/examples/web_ui/frontend/src/components/chat/MessageBubble.tsx+++ b/examples/web_ui/frontend/src/components/chat/MessageBubble.tsx@@ -12,12 +12,13 @@ import {CalendarClock,CheckCircle,ChevronDownIcon,+ CirclePlay,Copy,Loader2,MessageSquareQuote,Wrench,} from 'lucide-react';-import { useEffect, useState } from 'react';+import { useEffect, useRef, useState } from 'react';import ReactMarkdown from 'react-markdown';import remarkGfm from 'remark-gfm';@@ -32,6 +33,7 @@ import {CollapsibleTrigger,} from '@/components/ui/collapsible.tsx';import { Item, ItemContent } from '@/components/ui/item.tsx';+import { useAudioBlock, useReplayController } from '@/context/AudioContext';import { useTranslation } from '@/i18n/useI18n';import { formatNumber, formatTime } from '@/utils/common';@@ -144,6 +146,157 @@ function groupToolCalls(content: ContentBlock[]): ExtendedContentBlock[] {return result;}+const AUDIO_WAVE_LINES: Array<{ x: number; y1: number; y2: number }> = [+ { x: 2, y1: 10, y2: 13 },+ { x: 6, y1: 6, y2: 17 },+ { x: 10, y1: 3, y2: 21 },+ { x: 14, y1: 8, y2: 15 },+ { x: 18, y1: 5, y2: 18 },+ { x: 22, y1: 10, y2: 13 },+];++function AudioWave({ isPlaying = true, className }: { isPlaying?: boolean; className?: string }) {+ return (+ <>+ {isPlaying && (+ <style>{`+ @keyframes audioWave {+ 0%, 100% { transform: scaleY(1); }+ 50% { transform: scaleY(0.3); }+ }+ `}</style>+ )}+ <svg+ xmlns="http://www.w3.org/2000/svg"+ width="24"+ height="24"+ viewBox="0 0 24 24"+ fill="none"+ stroke="currentColor"+ strokeWidth={2}+ strokeLinecap="round"+ strokeLinejoin="round"+ className={className}+ >+ {AUDIO_WAVE_LINES.map(({ x, y1, y2 }, i) => (+ <line+ key={x}+ x1={x}+ x2={x}+ y1={y1}+ y2={y2}+ style={{+ transformOrigin: `${x}px 12px`,+ animation: isPlaying+ ? `audioWave 0.8s ease-in-out ${i * 0.12}s infinite`+ : 'none',+ }}+ />+ ))}+ </svg>+ </>+ );+}++/**+ * Inline audio control rendered *inside* the time/usage Badge so the play+ * icon visually merges into the same chip rather than floating as its own+ * pill.+ */+function AudioInlineControl({ block }: { block: DataBlock }) {+ const { t } = useTranslation();+ const audioState = useAudioBlock(block.id);+ const replayController = useReplayController();+ const audioRef = useRef<HTMLAudioElement | null>(null);+ const [isPlaying, setIsPlaying] = useState(false);++ const isStreaming = audioState?.status === 'streaming';++ // Don't build the giant base64 data URL while bytes are still streaming —+ // it would re-allocate on every DATA_BLOCK_DELTA. Live playback during+ // that window is handled by the manager's WavStreamPlayer; we only need+ // `src` for replay after the stream ends (or for historical messages).+ let src: string | null = null;+ if (!isStreaming) {+ if (audioState?.url) {+ src = audioState.url;+ } else if (block.source.type === 'url') {+ src = block.source.url;+ } else if (block.source.type === 'base64' && block.source.data) {+ src = `data:${block.source.media_type};base64,${block.source.data}`;+ }+ }++ // Reset the hidden <audio> when the source URL changes (e.g. streaming+ // just transitioned to a Blob URL). Without an explicit load() some+ // browsers keep the previous (or empty) source bound to the element.+ useEffect(() => {+ const el = audioRef.current;+ if (!el || !src) return;+ setIsPlaying(false);+ el.load();+ }, [src]);++ // Pause when a newer reply interrupts this block's playback.+ const interruptCount = audioState?.interruptCount ?? 0;+ useEffect(() => {+ if (interruptCount === 0) return;+ const el = audioRef.current;+ if (el && !el.paused) {+ el.pause();+ }+ }, [interruptCount]);++ if (isStreaming) {+ return <AudioWave isPlaying className="ml-1" />;+ }++ if (!src) return null;++ const toggle = async () => {+ const el = audioRef.current;+ if (!el) return;+ if (el.paused) {+ replayController?.play(el);+ try {+ await el.play();+ } catch (err) {+ console.error('Audio playback failed', err);+ }+ } else {+ el.pause();+ replayController?.stop();+ }+ };++ return (+ <>+ <button+ type="button"+ onClick={toggle}+ aria-label={+ isPlaying ? t('messageBubble.pauseAudio') : t('messageBubble.playAudio')+ }+ className="ml-1 inline-flex cursor-pointer items-center transition-opacity hover:opacity-70"+ >+ {isPlaying ? (+ <AudioWave isPlaying className="size-3" />+ ) : (+ <CirclePlay className="size-3" />+ )}+ </button>+ <audio+ ref={audioRef}+ src={src}+ preload="auto"+ onPlay={() => setIsPlaying(true)}+ onPause={() => setIsPlaying(false)}+ onEnded={() => setIsPlaying(false)}+ />+ </>+ );+}+/*** Render a single content block. Tool call groups are dispatched to* `renderToolGroup`; the per-group truncation at the first `asking` call@@ -237,6 +390,9 @@ function renderBlock(case 'data': {const dataType = block.source.media_type.split('/')[0];+ // Audio data blocks render in the footer (see AudioFooterControl),+ // not inline alongside text.+ if (dataType === 'audio') return null;let data: string;if (block.source.type === 'url') {data = block.source.url;@@ -246,8 +402,6 @@ function renderBlock(switch (dataType) {case 'image':return <img key={index} src={data} alt="Uploaded image" />;- case 'audio':- return <audio key={index} controls src={data} />;case 'video':return <video key={index} controls src={data} />;}@@ -358,7 +512,15 @@ export function MessageBubble({ message, onUserConfirm }: MessageBubbleProps) {}, [isRunning]);const blocks = groupToolCalls(message.content);- const showBody = blocks.length > 0;+ const audioBlocks = message.content.filter(+ (b): b is DataBlock => b.type === 'data' && b.source.media_type.split('/')[0] === 'audio',+ );+ // Audio data blocks are rendered in the footer, so they shouldn't keep an+ // otherwise-empty body bubble alive.+ const hasBodyContent = blocks.some(+ (b) => !(b.type === 'data' && b.source.media_type.split('/')[0] === 'audio'),+ );+ const showBody = hasBodyContent;const showFooter = !isUser;const startMs = new Date(message.created_at).getTime();@@ -417,6 +579,9 @@ export function MessageBubble({ message, onUserConfirm }: MessageBubbleProps) {</span></>)}+ {audioBlocks.map((block) => (+ <AudioInlineControl key={block.id} block={block} />+ ))}</Badge></div>)}diff --git a/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx b/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsxindex a72f55629..22f3783d3 100644--- a/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx+++ b/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx@@ -1,4 +1,4 @@-import { SlidersHorizontal } from 'lucide-react';+import { ChevronDown, SlidersHorizontal } from 'lucide-react';import { useCallback, useEffect, useState } from 'react';import type { ChatModelConfig, ModelCard } from '@/api';@@ -8,6 +8,8 @@ import {DropdownMenuCheckboxItem,DropdownMenuContent,DropdownMenuLabel,+ DropdownMenuRadioGroup,+ DropdownMenuRadioItem,DropdownMenuSeparator,DropdownMenuSub,DropdownMenuSubContent,@@ -17,6 +19,7 @@ import {import { Input } from '@/components/ui/input';import { Label } from '@/components/ui/label';import { Switch } from '@/components/ui/switch';+import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';import { useAvailableModels } from '@/hooks/useAvailableModels';import { useTranslation } from '@/i18n/useI18n';@@ -29,7 +32,8 @@ interface ParameterProperty {maximum?: number;exclusiveMinimum?: number;exclusiveMaximum?: number;- anyOf?: Array<{ type: string }>;+ enum?: unknown[];+ anyOf?: ParameterProperty[];}interface ParameterSchema {@@ -40,6 +44,155 @@ interface ParameterSchema {required?: string[];}+interface ResolvedType {+ type: string;+ enumValues: unknown[] | null;+}++/** Resolve a property's effective scalar type and enum values, looking+ * through ``anyOf`` and ignoring ``null`` variants. */+function resolveType(prop: ParameterProperty): ResolvedType {+ if (prop.type) {+ return { type: prop.type, enumValues: prop.enum ?? null };+ }+ for (const variant of prop.anyOf ?? []) {+ if (variant.type && variant.type !== 'null') {+ return {+ type: variant.type,+ enumValues: variant.enum ?? prop.enum ?? null,+ };+ }+ }+ return { type: 'string', enumValues: null };+}++// ---------------------------------------------------------------------------+// Field components+// ---------------------------------------------------------------------------++interface FieldProps {+ id: string;+ label: string;+ required: boolean;+ prop: ParameterProperty;+ value: unknown;+ onChange: (next: unknown) => void;+}++function BooleanField({ id, label, prop, value, onChange }: FieldProps) {+ return (+ <>+ <Label htmlFor={id} className="whitespace-nowrap">+ {label}+ </Label>+ <Switch+ id={id}+ checked={value !== undefined ? !!value : !!prop.default}+ onCheckedChange={(checked) => onChange(!!checked)}+ />+ </>+ );+}++function EnumField({ id, label, required, prop, value, onChange }: FieldProps) {+ const enumValues = resolveType(prop).enumValues ?? [];+ // TODO: experiment with using prop.description as placeholder text+ const displayValue = value !== undefined && value !== null ? String(value) : '';++ return (+ <>+ <Label htmlFor={id} className="whitespace-nowrap">+ {label}+ {required && <span className="text-destructive ml-0.5">*</span>}+ </Label>+ <DropdownMenu>+ <DropdownMenuTrigger asChild>+ <Button id={id} variant="outline" className="w-full justify-between gap-1">+ <span className="truncate">{displayValue}</span>+ <ChevronDown className="size-3.5 opacity-50 shrink-0" />+ </Button>+ </DropdownMenuTrigger>+ <DropdownMenuContent+ align="start"+ className="max-h-60 overflow-y-auto"+ onPointerDown={(e) => e.stopPropagation()}+ >+ <DropdownMenuRadioGroup value={displayValue} onValueChange={(v) => onChange(v)}>+ {enumValues.map((opt) => (+ <DropdownMenuRadioItem key={String(opt)} value={String(opt)}>+ {String(opt)}+ </DropdownMenuRadioItem>+ ))}+ </DropdownMenuRadioGroup>+ </DropdownMenuContent>+ </DropdownMenu>+ </>+ );+}++function NumberField({ id, label, required, prop, value, onChange }: FieldProps) {+ const { type: effectiveType } = resolveType(prop);++ return (+ <>+ <Label htmlFor={id} className="whitespace-nowrap">+ {label}+ {required && <span className="text-destructive ml-0.5">*</span>}+ </Label>+ <Input+ id={id}+ type="number"+ value={value !== undefined ? String(value) : ''}+ placeholder={prop.default != null ? String(prop.default) : undefined}+ min={prop.minimum}+ max={prop.maximum}+ step={effectiveType === 'number' ? 'any' : undefined}+ onChange={(e) => {+ const raw = e.target.value;+ onChange(raw === '' ? undefined : Number(raw));+ }}+ onBlur={(e) => {+ if (e.target.value === '') return;+ let num = Number(e.target.value);+ if (prop.minimum !== undefined && num < prop.minimum) num = prop.minimum;+ if (prop.maximum !== undefined && num > prop.maximum) num = prop.maximum;+ if (prop.exclusiveMinimum !== undefined && num <= prop.exclusiveMinimum)+ num =+ prop.exclusiveMinimum ++ (effectiveType === 'integer' ? 1 : Number.EPSILON);+ if (prop.exclusiveMaximum !== undefined && num >= prop.exclusiveMaximum)+ num =+ prop.exclusiveMaximum -+ (effectiveType === 'integer' ? 1 : Number.EPSILON);+ if (num !== Number(e.target.value)) onChange(num);+ }}+ />+ </>+ );+}++function StringField({ id, label, required, prop, value, onChange }: FieldProps) {+ return (+ <>+ <Label htmlFor={id} className="whitespace-nowrap">+ {label}+ {required && <span className="text-destructive ml-0.5">*</span>}+ </Label>+ <Input+ id={id}+ type="text"+ value={value !== undefined ? String(value) : ''}+ placeholder={prop.default != null ? String(prop.default) : undefined}+ onChange={(e) => onChange(e.target.value)}+ />+ </>+ );+}++// ---------------------------------------------------------------------------+// Main component+// ---------------------------------------------------------------------------+interface Props {/** Currently selected primary model — used to read the parameter schema. */selectedModel: ChatModelConfig | null;@@ -115,10 +268,6 @@ export function ModelParametersPopover({</DropdownMenuTrigger><DropdownMenuContent align="start" className="min-w-40">{/* ----- Fallback model selection ----- */}- {/* The sub-trigger reflects the current fallback selection so the- user can see the active model without drilling in. The label- is truncated with an ellipsis to keep the trigger on one line- when the model name is long. */}<DropdownMenuSub><DropdownMenuSubTrigger><span className="truncate">@@ -147,10 +296,6 @@ export function ModelParametersPopover({selectedFallbackModel?.credential_id ===credential.id &&selectedFallbackModel?.model === m.name;- // Checkbox-style indicator: a check appears next- // to the active fallback. Toggling off the active- // item clears the selection (mirrors the explicit- // "No fallback" entry below).return (<DropdownMenuCheckboxItemkey={`${credential.id}-${m.name}`}@@ -176,9 +321,6 @@ export function ModelParametersPopover({)))}<DropdownMenuSeparator />- {/* "No fallback" is checked exactly when no fallback is set,- making the list behave like a single-select group. Clicking- it while already checked is a no-op. */}<DropdownMenuCheckboxItemchecked={!selectedFallbackModel}onCheckedChange={(checked) => {@@ -207,134 +349,51 @@ export function ModelParametersPopover({{t('model-parameters.empty')}</p>) : (- <div className="grid grid-cols-[auto_1fr] items-center gap-x-3 gap-y-3">+ <div+ className="grid grid-cols-[auto_1fr] items-center gap-x-3 gap-y-3"+ onPointerDown={(e) => e.stopPropagation()}+ onKeyDown={(e) => e.stopPropagation()}+ >{entries.map(([key, prop]) => {- const effectiveType =- prop.type ??- prop.anyOf?.find((t) => t.type !== 'null')?.type ??- 'string';- const isBoolean = effectiveType === 'boolean';- const isNumber =- effectiveType === 'number' || effectiveType === 'integer';+ const { type: effectiveType, enumValues } = resolveType(prop);const label = prop.title ?? key;const isRequired = required.includes(key);+ const fieldProps: FieldProps = {+ id: `param-${key}`,+ label,+ required: isRequired,+ prop,+ value: values[key],+ onChange: (v) => handleChange(key, v),+ };- if (isBoolean) {- return (- <div- key={key}- className="contents"- // Keep clicks/keys inside the form from- // closing the dropdown menu.- onPointerDown={(e) => e.stopPropagation()}- onKeyDown={(e) => e.stopPropagation()}- >- <Label- htmlFor={`param-${key}`}- className="whitespace-nowrap"- >- {label}- </Label>- <Switch- id={`param-${key}`}- checked={- values[key] !== undefined- ? !!values[key]- : !!prop.default- }- onCheckedChange={(checked) =>- handleChange(key, !!checked)- }- />- </div>- );+ let field: React.ReactNode;+ if (effectiveType === 'boolean') {+ field = <BooleanField {...fieldProps} />;+ } else if (enumValues) {+ field = <EnumField {...fieldProps} />;+ } else if (+ effectiveType === 'number' ||+ effectiveType === 'integer'+ ) {+ field = <NumberField {...fieldProps} />;+ } else {+ field = <StringField {...fieldProps} />;}return (- <div- key={key}- className="contents"- onPointerDown={(e) => e.stopPropagation()}- onKeyDown={(e) => e.stopPropagation()}- >- <Label- htmlFor={`param-${key}`}- className="whitespace-nowrap"- >- {label}- {isRequired && (- <span className="text-destructive ml-0.5">- *- </span>- )}- </Label>- <Input- id={`param-${key}`}- type={isNumber ? 'number' : 'text'}- value={- values[key] !== undefined- ? String(values[key])- : ''- }- placeholder={- prop.default !== undefined- ? String(prop.default)- : undefined- }- min={prop.minimum}- max={prop.maximum}- step={- isNumber && effectiveType === 'number'- ? 'any'- : undefined- }- onChange={(e) => {- const raw = e.target.value;- if (isNumber) {- handleChange(- key,- raw === '' ? undefined : Number(raw),- );- } else {- handleChange(key, raw);- }- }}- onBlur={(e) => {- if (!isNumber || e.target.value === '') return;- let num = Number(e.target.value);- if (- prop.minimum !== undefined &&- num < prop.minimum- )- num = prop.minimum;- if (- prop.maximum !== undefined &&- num > prop.maximum- )- num = prop.maximum;- if (- prop.exclusiveMinimum !== undefined &&- num <= prop.exclusiveMinimum- )- num =- prop.exclusiveMinimum +- (effectiveType === 'integer'- ? 1- : Number.EPSILON);- if (- prop.exclusiveMaximum !== undefined &&- num >= prop.exclusiveMaximum- )- num =- prop.exclusiveMaximum -- (effectiveType === 'integer'- ? 1- : Number.EPSILON);- if (num !== Number(e.target.value))- handleChange(key, num);- }}- />- </div>+ <Tooltip key={key}>+ <TooltipTrigger asChild>+ <div className="col-span-2 grid grid-cols-subgrid items-center">+ {field}+ </div>+ </TooltipTrigger>+ {prop.description && (+ <TooltipContent side="left">+ {prop.description}+ </TooltipContent>+ )}+ </Tooltip>);})}</div>diff --git a/examples/web_ui/frontend/src/context/AudioContext.tsx b/examples/web_ui/frontend/src/context/AudioContext.tsxnew file mode 100644index 000000000..041a4cddc--- /dev/null+++ b/examples/web_ui/frontend/src/context/AudioContext.tsx@@ -0,0 +1,85 @@+import { createContext, useContext, useEffect, useMemo, useRef, useSyncExternalStore } from 'react';+import type { ReactNode } from 'react';++import { StreamingAudioManager } from '@/utils/streamingAudio';+import type { StreamingAudioState } from '@/utils/streamingAudio';++const AudioContext = createContext<StreamingAudioManager | null>(null);++interface ReplayController {+ play: (el: HTMLAudioElement) => void;+ stop: () => void;+}++const ReplayContext = createContext<ReplayController | null>(null);++/**+ * Provides a {@link StreamingAudioManager} to the component tree. The manager+ * collects DATA_BLOCK_* events for audio blocks and exposes per-block state+ * that ``MessageBubble`` consumes via {@link useAudioBlock}.+ *+ * Mount this once around any subtree that renders assistant messages.+ */+export function AudioProvider({ children }: { children: ReactNode }) {+ const manager = useMemo(() => new StreamingAudioManager(), []);+ useEffect(() => () => manager.disposeAll(), [manager]);++ const currentRef = useRef<HTMLAudioElement | null>(null);+ const replay: ReplayController = useMemo(+ () => ({+ play(el: HTMLAudioElement) {+ if (currentRef.current && currentRef.current !== el) {+ currentRef.current.pause();+ currentRef.current.currentTime = 0;+ }+ currentRef.current = el;+ },+ stop() {+ currentRef.current = null;+ },+ }),+ [],+ );++ return (+ <AudioContext.Provider value={manager}>+ <ReplayContext.Provider value={replay}>{children}</ReplayContext.Provider>+ </AudioContext.Provider>+ );+}++/**+ * Access the streaming audio manager. Returns ``null`` outside of an+ * ``AudioProvider`` — callers must handle that, since audio handling is+ * an optional feature.+ */+export function useAudioManager(): StreamingAudioManager | null {+ return useContext(AudioContext);+}++/**+ * Access the replay controller that ensures only one audio element plays+ * at a time across all message bubbles.+ */+export function useReplayController(): ReplayController | null {+ return useContext(ReplayContext);+}++/**+ * Subscribe to the streaming state for a single audio DataBlock.+ * Re-renders when the block transitions from ``streaming`` to ``ready``.+ *+ * Returns ``null`` if the block isn't being tracked (e.g. a historical+ * message loaded from the server, where the bytes are already complete).+ */+export function useAudioBlock(blockId: string | undefined): StreamingAudioState | null {+ const manager = useAudioManager();+ return useSyncExternalStore(+ (fn) => {+ if (!manager || !blockId) return () => undefined;+ return manager.subscribe(blockId, fn);+ },+ () => (manager && blockId ? manager.getState(blockId) : null),+ () => null,+ );+}diff --git a/examples/web_ui/frontend/src/hooks/useMessages.ts b/examples/web_ui/frontend/src/hooks/useMessages.tsindex 3b9c9b130..31727c20a 100644--- a/examples/web_ui/frontend/src/hooks/useMessages.ts+++ b/examples/web_ui/frontend/src/hooks/useMessages.ts@@ -2,6 +2,9 @@ import { EventType } from '@agentscope-ai/agentscope/event';import type {AgentEvent,CustomEvent,+ DataBlockStartEvent,+ DataBlockDeltaEvent,+ DataBlockEndEvent,ReplyStartEvent,UserConfirmResultEvent,} from '@agentscope-ai/agentscope/event';@@ -12,6 +15,7 @@ import { useState, useCallback, useRef, useEffect } from 'react';import { sessionApi } from '@/api';import { chatApi } from '@/api';+import { useAudioManager } from '@/context/AudioContext';/*** Manages messages for a single ``(agentId, sessionId)`` pair.@@ -70,19 +74,12 @@ export function useMessages(const abortRef = useRef<AbortController | null>(null);const rafRef = useRef<number | null>(null);- // Mirror `options` into a ref so `processEvent` (a single-creation- // useCallback to keep the SSE stable) always reads the freshest- // callbacks. Without this, the closure freezes the first render's- // `options` — typically captured when `agentId` is still `null` —- // so `team_updated` / `state_updated` events fire stale no-op- // handlers, which in turn call `useSessions(null).refetch`- // (`if (!agentId) setSessions([])`) and blank out the session list.+ const audioManager = useAudioManager();+const optionsRef = useRef(options);useEffect(() => {optionsRef.current = options;}, [options]);-- /** Batch UI updates via requestAnimationFrame. */const scheduleUpdate = useCallback(() => {if (rafRef.current !== null) return;rafRef.current = requestAnimationFrame(() => {@@ -106,6 +103,7 @@ export function useMessages(return;}if (event.type === EventType.REPLY_START) {+ audioManager?.stopAllPlayback();const e = event as ReplyStartEvent;const msg = AssistantMsg({ id: e.reply_id, name: e.name, content: [] });msgsRef.current = [...msgsRef.current, msg];@@ -120,9 +118,33 @@ export function useMessages(} else if (currentReplyRef.current) {appendEvent(currentReplyRef.current, event);}++ // Route streaming audio DataBlocks to the audio manager. They still+ // flow through `appendEvent` above (which builds up `source.data`+ // in the Msg), but MessageBubble reads playback state from the+ // manager so it can show progress and autoplay on completion.+ if (audioManager) {+ if (event.type === EventType.DATA_BLOCK_START) {+ const e = event as DataBlockStartEvent;+ if (e.media_type.startsWith('audio/')) {+ audioManager.start(e.block_id, e.media_type);+ }+ } else if (event.type === EventType.DATA_BLOCK_DELTA) {+ const e = event as DataBlockDeltaEvent;+ if (e.media_type.startsWith('audio/')) {+ audioManager.append(e.block_id, e.data);+ }+ } else if (event.type === EventType.DATA_BLOCK_END) {+ const e = event as DataBlockEndEvent;+ // `end` is a no-op when the block isn't being tracked, so+ // we can call it unconditionally.+ audioManager.end(e.block_id);+ }+ }+scheduleUpdate();},- [scheduleUpdate],+ [scheduleUpdate, audioManager],);// ── Lifecycle: fetch history + open SSE stream ──────────────────@@ -132,6 +154,7 @@ export function useMessages(setMsgs([]);setError(null);setStreaming(false);+ audioManager?.disposeAll();if (!agentId || !sessionId) return;@@ -176,7 +199,7 @@ export function useMessages(controller.abort();abortRef.current = null;};- }, [agentId, sessionId, scheduleUpdate, processEvent]);+ }, [agentId, sessionId, scheduleUpdate, processEvent, audioManager]);/*** Send a user message. Appends the message to the local listdiff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.jsonindex ce3ba649d..ba9899624 100644--- a/examples/web_ui/frontend/src/i18n/locales/en.json+++ b/examples/web_ui/frontend/src/i18n/locales/en.json@@ -76,6 +76,9 @@},"messageBubble": {"thinking": "Thinking",+ "audioGenerating": "Generating audio…",+ "playAudio": "Play audio",+ "pauseAudio": "Pause audio","running": "Running...","hintSource": {"tool_output": "Tool Output",diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.jsonindex fee9dd65a..0fd0c2497 100644--- a/examples/web_ui/frontend/src/i18n/locales/zh.json+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json@@ -76,6 +76,9 @@},"messageBubble": {"thinking": "思考中",+ "audioGenerating": "正在生成音频…",+ "playAudio": "播放音频",+ "pauseAudio": "暂停音频","running": "运行中...","hintSource": {"tool_output": "工具输出",diff --git a/examples/web_ui/frontend/src/pages/chat/index.tsx b/examples/web_ui/frontend/src/pages/chat/index.tsxindex cfa6cbe5b..e6fd177b0 100644--- a/examples/web_ui/frontend/src/pages/chat/index.tsx+++ b/examples/web_ui/frontend/src/pages/chat/index.tsx@@ -55,6 +55,7 @@ import {SidebarMenuButton,SidebarMenuItem,} from '@/components/ui/sidebar';+import { AudioProvider } from '@/context/AudioContext';import { useAgents } from '@/hooks/useAgents';import { useSessions } from '@/hooks/useSessions';import { useTranslation } from '@/i18n/useI18n.ts';@@ -434,4 +435,8 @@ const ChatPageInner = () => {);};-export const ChatPage = () => <ChatPageInner />;+export const ChatPage = () => (+ <AudioProvider>+ <ChatPageInner />+ </AudioProvider>+);diff --git a/examples/web_ui/frontend/src/utils/streamingAudio.ts b/examples/web_ui/frontend/src/utils/streamingAudio.tsnew file mode 100644index 000000000..41deb2cb0--- /dev/null+++ b/examples/web_ui/frontend/src/utils/streamingAudio.ts@@ -0,0 +1,368 @@+/**+ * Streaming audio playback support for assistant DataBlocks.+ *+ * Omni-style models (e.g. qwen3.5-omni-plus) deliver speech as a stream of+ * DATA_BLOCK_DELTA events carrying base64 audio bytes for one logical+ * DataBlock. This module routes those chunks to two playback paths:+ *+ * 1. **Live chunked playback** — only for audio/wav. We parse the RIFF+ * header from the first chunk and schedule subsequent PCM bytes onto a+ * Web Audio timeline as they arrive, so the assistant starts speaking+ * the moment the first chunk arrives — without waiting for+ * DATA_BLOCK_END, which the agent only emits after the *entire* model+ * stream (including text) finishes.+ *+ * 2. **Buffered final playback** — for non-wav formats (mp3/opus/etc.) we+ * accumulate bytes and, on DATA_BLOCK_END, build a Blob URL that backs+ * autoplay + an ``<audio controls>`` element for replay/scrubbing.+ *+ * For wav, the controls element built on DATA_BLOCK_END is replay-only —+ * we intentionally skip the second autoplay there to avoid colliding with+ * the live player. ``WavStreamPlayer.finalize`` defers AudioContext+ * teardown until queued PCM has finished playing, so live audio is never+ * cut short by the end event arriving mid-playback.+ */++/** Status of a streaming audio block. */+export type StreamingAudioStatus = 'streaming' | 'ready' | 'error';++/** Per-block state surfaced to React. */+export interface StreamingAudioState {+ status: StreamingAudioStatus;+ mediaType: string;+ /** Object URL for ``<audio>`` once playback-ready. */+ url: string | null;+ /** Cause for ``status === 'error'``. */+ error?: string;+ /** Incremented each time a newer reply interrupts this block. */+ interruptCount: number;+}++type Listener = () => void;++type Bytes = Uint8Array<ArrayBufferLike>;++function base64ToBytes(b64: string): Bytes {+ const binary = atob(b64);+ const bytes = new Uint8Array(binary.length);+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);+ return bytes;+}++function concatBytes(a: Bytes, b: Bytes): Bytes {+ const out = new Uint8Array(a.length + b.length);+ out.set(a, 0);+ out.set(b, a.length);+ return out;+}++interface WavHeader {+ sampleRate: number;+ channels: number;+ bitsPerSample: number;+ /** Byte offset where PCM samples begin (right after the ``data`` chunk header). */+ dataOffset: number;+}++/**+ * Parse a minimal RIFF/WAVE header. Returns ``null`` if the buffer doesn't+ * yet contain enough bytes to locate the ``fmt `` and ``data`` chunks.+ */+function parseWavHeader(bytes: Bytes): WavHeader | null {+ if (bytes.length < 44) return null;+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);+ // "RIFF"...."WAVE"+ if (view.getUint32(0, false) !== 0x52494646 || view.getUint32(8, false) !== 0x57415645) {+ return null;+ }++ let offset = 12;+ let sampleRate = 0;+ let channels = 0;+ let bitsPerSample = 0;+ let dataOffset = -1;++ while (offset + 8 <= bytes.length) {+ const chunkId = view.getUint32(offset, false);+ const chunkSize = view.getUint32(offset + 4, true);+ const chunkBody = offset + 8;++ if (chunkId === 0x666d7420 /* "fmt " */) {+ if (chunkBody + 16 > bytes.length) return null;+ channels = view.getUint16(chunkBody + 2, true);+ sampleRate = view.getUint32(chunkBody + 4, true);+ bitsPerSample = view.getUint16(chunkBody + 14, true);+ } else if (chunkId === 0x64617461 /* "data" */) {+ dataOffset = chunkBody;+ break;+ }+ offset = chunkBody + chunkSize;+ }++ if (!sampleRate || !channels || !bitsPerSample || dataOffset < 0) return null;+ return { sampleRate, channels, bitsPerSample, dataOffset };+}++/**+ * Convert raw little-endian PCM into a Float32 AudioBuffer.+ * Supports 16-bit and 8-bit; falls back to silence for other depths.+ */+function pcmToAudioBuffer(ctx: AudioContext, pcm: Bytes, header: WavHeader): AudioBuffer | null {+ const { channels, bitsPerSample, sampleRate } = header;+ const bytesPerSample = bitsPerSample / 8;+ const frameSize = bytesPerSample * channels;+ const frameCount = Math.floor(pcm.length / frameSize);+ if (frameCount === 0) return null;++ const buffer = ctx.createBuffer(channels, frameCount, sampleRate);+ const view = new DataView(pcm.buffer, pcm.byteOffset, pcm.byteLength);++ for (let ch = 0; ch < channels; ch++) {+ const channelData = buffer.getChannelData(ch);+ for (let i = 0; i < frameCount; i++) {+ const sampleOffset = i * frameSize + ch * bytesPerSample;+ if (bitsPerSample === 16) {+ channelData[i] = view.getInt16(sampleOffset, true) / 0x8000;+ } else if (bitsPerSample === 8) {+ channelData[i] = (view.getUint8(sampleOffset) - 128) / 128;+ } else {+ channelData[i] = 0;+ }+ }+ }+ return buffer;+}++/** Live PCM playback for an in-progress WAV stream. */+class WavStreamPlayer {+ private ctx: AudioContext | null = null;+ private header: WavHeader | null = null;+ private pending: Bytes = new Uint8Array(0);+ private nextStartTime = 0;+ private failed = false;+ private deferredCloseTimer: ReturnType<typeof setTimeout> | null = null;++ append(chunk: Bytes): void {+ if (this.failed) return;+ this.pending = concatBytes(this.pending, chunk);++ if (!this.header) {+ const header = parseWavHeader(this.pending);+ if (!header) return;+ this.header = header;+ try {+ this.ctx = new AudioContext({ sampleRate: header.sampleRate });+ if (this.ctx.state === 'suspended') {+ void this.ctx.resume();+ }+ this.nextStartTime = this.ctx.currentTime;+ } catch {+ this.failed = true;+ return;+ }+ this.pending = this.pending.slice(header.dataOffset);+ }++ if (!this.ctx || !this.header) return;++ const frameSize = (this.header.bitsPerSample / 8) * this.header.channels;+ const playableBytes = Math.floor(this.pending.length / frameSize) * frameSize;+ if (playableBytes === 0) return;++ const toPlay = this.pending.slice(0, playableBytes);+ this.pending = this.pending.slice(playableBytes);++ const audioBuffer = pcmToAudioBuffer(this.ctx, toPlay, this.header);+ if (!audioBuffer) return;++ const source = this.ctx.createBufferSource();+ source.buffer = audioBuffer;+ source.connect(this.ctx.destination);+ const startAt = Math.max(this.nextStartTime, this.ctx.currentTime);+ source.start(startAt);+ this.nextStartTime = startAt + audioBuffer.duration;+ }++ /**+ * Graceful end-of-stream: stop accepting more bytes, then close the+ * AudioContext only after the last queued buffer has finished playing.+ * Without this, ``end()`` calling ``dispose()`` would cut live playback+ * short by closing the context while sources are still scheduled.+ *+ * The AudioContext reference is kept on ``this.ctx`` so that+ * ``dispose()`` can still close it immediately if a newer reply+ * interrupts playback before the timer fires.+ */+ finalize(): void {+ if (!this.ctx || this.failed) {+ this.dispose();+ return;+ }+ const remainingMs = Math.max(0, this.nextStartTime - this.ctx.currentTime) * 1000;+ this.deferredCloseTimer = setTimeout(() => {+ this.dispose();+ }, remainingMs + 200);+ this.header = null;+ this.pending = new Uint8Array(0);+ }++ dispose(): void {+ if (this.deferredCloseTimer) {+ clearTimeout(this.deferredCloseTimer);+ this.deferredCloseTimer = null;+ }+ if (this.ctx) {+ void this.ctx.close().catch(() => undefined);+ this.ctx = null;+ }+ this.pending = new Uint8Array(0);+ this.header = null;+ }+}++interface Session {+ mediaType: string;+ chunks: Bytes[];+ totalBytes: number;+ livePlayer: WavStreamPlayer | null;+ state: StreamingAudioState;+}++/**+ * Manages the lifecycle of streaming audio DataBlocks for one chat session.+ * React components subscribe via {@link subscribe} to re-render when a block's+ * state changes.+ */+export class StreamingAudioManager {+ private sessions = new Map<string, Session>();+ private listeners = new Map<string, Set<Listener>>();++ private emit(blockId: string): void {+ this.listeners.get(blockId)?.forEach((fn) => fn());+ }++ subscribe(blockId: string, fn: Listener): () => void {+ let set = this.listeners.get(blockId);+ if (!set) {+ set = new Set();+ this.listeners.set(blockId, set);+ }+ set.add(fn);+ return () => {+ set?.delete(fn);+ if (set && set.size === 0) this.listeners.delete(blockId);+ };+ }++ getState(blockId: string): StreamingAudioState | null {+ return this.sessions.get(blockId)?.state ?? null;+ }++ /** Called on DATA_BLOCK_START for an audio block. */+ start(blockId: string, mediaType: string): void {+ if (this.sessions.has(blockId)) return;+ const livePlayer = mediaType === 'audio/wav' ? new WavStreamPlayer() : null;+ this.sessions.set(blockId, {+ mediaType,+ chunks: [],+ totalBytes: 0,+ livePlayer,+ state: { status: 'streaming', mediaType, url: null, interruptCount: 0 },+ });+ this.emit(blockId);+ }++ /** Called on DATA_BLOCK_DELTA. ``data`` is the base64 chunk payload. */+ append(blockId: string, data: string): void {+ const session = this.sessions.get(blockId);+ if (!session || !data) return;+ let bytes: Uint8Array;+ try {+ bytes = base64ToBytes(data);+ } catch {+ return;+ }+ session.chunks.push(bytes);+ session.totalBytes += bytes.length;+ session.livePlayer?.append(bytes);+ // No emit — partial bytes don't change the rendered UI yet.+ }++ /**+ * Called on DATA_BLOCK_END. Builds the replay URL and (for non-wav)+ * autoplays. For wav we skip autoplay because ``WavStreamPlayer`` has+ * already been playing the bytes as they arrived; a second ``Audio``+ * playback here would either double up or — given ``end`` fires only+ * after the whole model stream (text included) finishes — make the+ * user perceive audio as starting only after text streaming completes.+ */+ end(blockId: string): void {+ const session = this.sessions.get(blockId);+ if (!session) return;++ const hadLivePlayback = session.livePlayer !== null;+ if (session.livePlayer) {+ // Let queued PCM finish playing; AudioContext closes via a+ // deferred timer. The livePlayer reference is kept so that+ // stopAllPlayback() can dispose() it immediately if a newer+ // reply starts before the timer fires.+ session.livePlayer.finalize();+ }++ const blob = new Blob(session.chunks as BlobPart[], { type: session.mediaType });+ const url = URL.createObjectURL(blob);++ // Free the per-chunk buffers; the Blob owns the bytes from here on.+ session.chunks = [];++ session.state = {+ status: 'ready',+ mediaType: session.mediaType,+ url,+ interruptCount: session.state.interruptCount,+ };+ this.emit(blockId);++ if (!hadLivePlayback) {+ // Non-wav: no live playback path, so kick off autoplay now. Some+ // browsers gate this behind a user gesture; if blocked, the user+ // can still hit play on the <audio controls> element.+ const el = new Audio(url);+ void el.play().catch(() => undefined);+ }+ }++ /**+ * Stop all in-progress and replay audio. Called when a new reply+ * starts so previous audio doesn't overlap with the new one.+ * Blob URLs are preserved so the user can still manually replay.+ */+ stopAllPlayback(): void {+ for (const [blockId, session] of this.sessions) {+ if (session.livePlayer) {+ session.livePlayer.dispose();+ session.livePlayer = null;+ }+ session.state = {+ ...session.state,+ interruptCount: session.state.interruptCount + 1,+ };+ this.emit(blockId);+ }+ }++ /** Release a single block's resources (Object URL etc.). */+ dispose(blockId: string): void {+ const session = this.sessions.get(blockId);+ if (!session) return;+ session.livePlayer?.dispose();+ if (session.state.url) URL.revokeObjectURL(session.state.url);+ this.sessions.delete(blockId);+ this.listeners.delete(blockId);+ }++ /** Release every block — call on unmount or session switch. */+ disposeAll(): void {+ for (const id of [...this.sessions.keys()]) this.dispose(id);+ }+}diff --git a/examples/web_ui/pnpm-lock.yaml b/examples/web_ui/pnpm-lock.yamlindex aa04b72a7..f895ec6e6 100644--- a/examples/web_ui/pnpm-lock.yaml+++ b/examples/web_ui/pnpm-lock.yaml@@ -58,8 +58,8 @@ importers:frontend:dependencies:'@agentscope-ai/agentscope':- specifier: ^0.0.8- version: 0.0.8+ specifier: ^0.0.9+ version: 0.0.9'@fontsource-variable/geist':specifier: ^5.2.8version: 5.2.8@@ -123,9 +123,6 @@ importers:react-router-dom:specifier: ^7.15.1- rehype-raw:- specifier: ^7.0.0- version: 7.0.0remark-gfm:specifier: ^4.0.1version: 4.0.1@@ -202,8 +199,8 @@ importers:packages:- '@agentscope-ai/[email protected]':- resolution: {integrity: sha512-Z0hcvCekvsbOxhEkgrT5+epMImfiUrt5cdeY8aGt73MWrSVxg0opErk6KrPVd1/WK+zPs5yIvhuaZ/sslJRFxQ==}+ '@agentscope-ai/[email protected]':+ resolution: {integrity: sha512-FuELfct2hovIINATNPWEpYktyqe1SYoVb9TCJvQoXqQcj5upik0VJM+8+OBA2y79xWe+rTBO1lr0bhuSGcHrPA==}'@babel/[email protected]':resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==}@@ -2491,10 +2488,6 @@ packages:resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}engines: {node: '>=0.12'}- resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==}- engines: {node: '>=0.12'}-resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==}engines: {node: '>=6'}@@ -2885,27 +2878,12 @@ packages:resolution: {integrity: sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==}engines: {node: '>= 0.4'}- resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==}-- resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==}-- resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==}-resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==}- resolution: {integrity: sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==}-resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==}- resolution: {integrity: sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==}-resolution: {integrity: sha512-1TJ6Fih/b8h5TIcv+1+Hw0PDQWJTKDKzFZzcKOiW1wJza3XoAQlkCuXLbymPYB8+ZQyw8mHvdw560e8zVFIWyA==}@@ -2925,9 +2903,6 @@ packages:resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==}- resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}-resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==}engines: {node: '>= 0.8'}@@ -3703,9 +3678,6 @@ packages:resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==}engines: {node: '>=18'}- resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==}-resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==}engines: {node: '>= 0.8'}@@ -3936,9 +3908,6 @@ packages:resolution: {integrity: sha512-YTUo+Flmw4ZXiWfQKGcwwc11KnoRAYgzAE2E7mXKCjSviTKShtxBsN6YUUBB2gtaBzKzeKunxhUwNHQuRryhWA==}engines: {node: '>= 4'}- resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==}-resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==}@@ -4473,9 +4442,6 @@ packages:react: ^16.8 || ^17.0 || ^18.0 || ^19.0.0 || ^19.0.0-rcreact-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0.0 || ^19.0.0-rc- resolution: {integrity: sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==}-resolution: {integrity: sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==}@@ -4534,9 +4500,6 @@ packages:resolution: {integrity: sha512-Dhxzh5HZuiHQhbvTW9AMetFfBHDMYpo23Uo9btPXgdYP+3T5S+p+jgNy7spra+veYhBP2dCSgxR/i2Y02h5/6w==}engines: {node: '>=0.10.0'}- resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==}-resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}engines: {node: '>= 8'}@@ -4635,7 +4598,7 @@ packages:snapshots:- '@agentscope-ai/[email protected]':+ '@agentscope-ai/[email protected]':dependencies:'@cfworker/json-schema': 4.1.1'@modelcontextprotocol/sdk': 1.29.0(@cfworker/[email protected])([email protected])@@ -6866,8 +6829,6 @@ snapshots:- [email protected]: {}-@@ -7326,37 +7287,6 @@ snapshots:dependencies:function-bind: 1.1.2- dependencies:- '@types/hast': 3.0.4- '@types/unist': 3.0.3- devlop: 1.1.0- hastscript: 9.0.1- property-information: 7.1.0- vfile: 6.0.3- vfile-location: 5.0.3- web-namespaces: 2.0.1-- dependencies:- '@types/hast': 3.0.4-- dependencies:- '@types/hast': 3.0.4- '@types/unist': 3.0.3- '@ungap/structured-clone': 1.3.1- hast-util-from-parse5: 8.0.3- hast-util-to-parse5: 8.0.1- html-void-elements: 3.0.0- mdast-util-to-hast: 13.2.1- parse5: 7.3.0- unist-util-position: 5.0.0- unist-util-visit: 5.1.0- vfile: 6.0.3- web-namespaces: 2.0.1- zwitch: 2.0.4-dependencies:'@types/estree': 1.0.9@@ -7377,28 +7307,10 @@ snapshots:transitivePeerDependencies:- supports-color- dependencies:- '@types/hast': 3.0.4- comma-separated-tokens: 2.0.3- devlop: 1.1.0- property-information: 7.1.0- space-separated-tokens: 2.0.2- web-namespaces: 2.0.1- zwitch: 2.0.4-dependencies:'@types/hast': 3.0.4- dependencies:- '@types/hast': 3.0.4- comma-separated-tokens: 2.0.3- hast-util-parse-selector: 4.0.0- property-information: 7.1.0- space-separated-tokens: 2.0.2-dependencies:'@types/set-cookie-parser': 2.4.10@@ -7418,8 +7330,6 @@ snapshots:- [email protected]: {}-dependencies:depd: 2.0.0@@ -8319,10 +8229,6 @@ snapshots:- dependencies:- entities: 6.0.1-@@ -8583,12 +8489,6 @@ snapshots:tiny-invariant: 1.3.3tslib: 2.8.1- dependencies:- '@types/hast': 3.0.4- hast-util-raw: 9.1.0- vfile: 6.0.3-dependencies:'@types/mdast': 4.0.4@@ -9234,11 +9134,6 @@ snapshots:- '@types/react'- '@types/react-dom'- dependencies:- '@types/unist': 3.0.3- vfile: 6.0.3-dependencies:'@types/unist': 3.0.3@@ -9275,8 +9170,6 @@ snapshots:- [email protected]: {}-diff --git a/scripts/model_examples/_utils.py b/scripts/model_examples/_utils.pyindex 534d0a6d5..ad79692c6 100644--- a/scripts/model_examples/_utils.py+++ b/scripts/model_examples/_utils.py@@ -1,8 +1,14 @@# -*- coding: utf-8 -*-"""Shared utility helpers for model-examples scripts."""+import base64 as _b64from typing import AsyncGenerator-from agentscope.message import TextBlock, ThinkingBlock, DataBlock+from agentscope.message import (+ TextBlock,+ ThinkingBlock,+ DataBlock,+ Base64Source,+)from agentscope.model import ChatResponse@@ -16,10 +22,18 @@ async def stream_and_collect(from it without re-printing the entire content. Text from the final chunkis printed only when no text was streamed in any delta chunk (e.g. somemodels batch the answer in the last chunk).++ Streaming ``DataBlock`` chunks (e.g. omni audio output) are reported as+ per-chunk size summaries while the stream is being consumed, so callers+ can see the audio arriving incrementally rather than only in the final+ cumulative chunk."""final: ChatResponse | None = Nonein_thinking = Falsetext_was_streamed = False+ # Track per-block audio progress: block_id -> (media_type, chunk_count,+ # total_bytes).+ audio_progress: dict[str, list] = {}async for chunk in gen:if chunk.is_last:final = chunk@@ -37,6 +51,27 @@ async def stream_and_collect(in_thinking = Falseprint(block.text, end="", flush=True)text_was_streamed = True+ elif isinstance(block, DataBlock) and isinstance(+ block.source,+ Base64Source,+ ):+ # Streaming binary delta (e.g. omni audio output).+ if in_thinking:+ print()+ print("--- Answer ---")+ in_thinking = False+ delta_bytes = len(_b64.b64decode(block.source.data))+ state = audio_progress.setdefault(+ block.id,+ [block.source.media_type, 0, 0],+ )+ state[1] += 1+ state[2] += delta_bytes+ print(+ f"\n[Audio chunk #{state[1]} ({state[0]}): "+ f"+{delta_bytes}B, total={state[2]}B]",+ flush=True,+ )# If text was not streamed in any delta, print it from the final chunk now.if not text_was_streamed and final is not None:final_text = "".join(@@ -52,16 +87,17 @@ async def stream_and_collect(print(final_text)if in_thinking:print()- # Report audio output if present in the final chunk+ # Report cumulative audio output from the final chunk.if final is not None:- import base64 as _b64-for block in final.content:- if isinstance(block, DataBlock):+ if isinstance(block, DataBlock) and isinstance(+ block.source,+ Base64Source,+ ):media = block.source.media_typebyte_size = len(_b64.b64decode(block.source.data))print(- f"[Audio output: {media}, " f"{byte_size} bytes]",+ f"[Audio output (cumulative): {media}, {byte_size} bytes]",)print()assert final is not Nonediff --git a/src/agentscope/_utils/_audio.py b/src/agentscope/_utils/_audio.pynew file mode 100644index 000000000..b93b5cbf6--- /dev/null+++ b/src/agentscope/_utils/_audio.py@@ -0,0 +1,42 @@+# -*- coding: utf-8 -*-+"""Audio utilities shared across model providers."""+import struct+++def _build_streaming_wav_header(+ sample_rate: int = 24000,+ channels: int = 1,+ bits_per_sample: int = 16,+) -> bytes:+ """Build a 44-byte WAV/RIFF header for streaming PCM.++ The RIFF and ``data`` chunk sizes are set to ``0xFFFFFFFF`` since the+ total length isn't known yet. Decoders that only need sample-rate,+ channel count and bit depth (e.g. the web ``WavStreamPlayer``) treat+ everything after the ``data`` chunk header as PCM, so this is+ sufficient for live decoding of an open-ended stream.++ Both DashScope omni and OpenAI streaming deliver raw PCM upstream;+ prefixing the first chunk with this header lets the frontend start+ playback immediately without buffering the whole response.+ """+ byte_rate = sample_rate * channels * bits_per_sample // 8+ block_align = channels * bits_per_sample // 8+ return (+ b"RIFF"+ + struct.pack("<I", 0xFFFFFFFF)+ + b"WAVE"+ + b"fmt "+ + struct.pack("<I", 16)+ + struct.pack(+ "<HHIIHH",+ 1,+ channels,+ sample_rate,+ byte_rate,+ block_align,+ bits_per_sample,+ )+ + b"data"+ + struct.pack("<I", 0xFFFFFFFF)+ )diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.pyindex 9bbbd2b0f..8258fce80 100644--- a/src/agentscope/agent/_agent.py+++ b/src/agentscope/agent/_agent.py@@ -790,7 +790,12 @@ class Agent:**kwargs,)- block_ids: dict = {"text": None, "thinking": None, "tools": []}+ block_ids: dict = {+ "text": None,+ "thinking": None,+ "tools": [],+ "data": [],+ }completed_response: ChatResponse | None = None# Check if res is an async generator (streaming response)@@ -832,6 +837,11 @@ class Agent:reply_id=self.state.reply_id,tool_call_id=tool_call_id,)+ for data_block_id in block_ids["data"]:+ yield DataBlockEndEvent(+ reply_id=self.state.reply_id,+ block_id=data_block_id,+ )# Send the model call ended event with usage if availableyield ModelCallEndEvent(@@ -2197,12 +2207,29 @@ class Agent:else None)+ # Assistant-produced audio (e.g. qwen-omni speaking aloud) is delivered+ # to the user via streaming events; the raw bytes don't belong in+ # conversation memory. Filtering here keeps every downstream walker+ # (formatter, count_tokens, persistence) honest without each having+ # to remember.+ persisted_blocks = [+ b+ for b in blocks+ if not (+ isinstance(b, DataBlock)+ and isinstance(b.source, (Base64Source, URLSource))+ and b.source.media_type.startswith("audio/")+ )+ ]+ if not persisted_blocks and msg_usage is None:+ return+if len(self.state.context) == 0:self.state.context.append(AssistantMsg(id=self.state.reply_id,name=self.name,- content=list(blocks),+ content=persisted_blocks,usage=msg_usage,),)@@ -2211,7 +2238,7 @@ class Agent:if last_msg.role == "assistant" and last_msg.name == self.name:if isinstance(last_msg.content, str):last_msg.content = [TextBlock(text=last_msg.content)]- last_msg.content.extend(blocks)+ last_msg.content.extend(persisted_blocks)if msg_usage is not None:if last_msg.usage is None:last_msg.usage = msg_usage@@ -2223,7 +2250,7 @@ class Agent:AssistantMsg(id=self.state.reply_id,name=self.name,- content=list(blocks),+ content=persisted_blocks,usage=msg_usage,),)@@ -2384,6 +2411,7 @@ class Agent:# Classify the content blocks into different typestext_blocks, thinking_blocks, tool_call_blocks = [], [], []+ data_blocks: list = []for block in chunk.content:if isinstance(block, TextBlock):text_blocks.append(block)@@ -2391,8 +2419,17 @@ class Agent:thinking_blocks.append(block)elif isinstance(block, ToolCallBlock):tool_call_blocks.append(block)-- # Handle the text blocks+ elif isinstance(block, DataBlock):+ data_blocks.append(block)++ # Handle the text blocks. We only auto-close the open text block+ # when the current chunk has neither text NOR data — a chunk that+ # carries only data (e.g. an omni-style audio PCM delta arriving+ # between two text deltas) must keep the text stream alive so the+ # frontend doesn't fragment one logical text stream into many+ # separate bubbles. A chunk with tool calls (and no text/data)+ # still closes text, which preserves text → tool → text render+ # order via distinct text blocks.if text_blocks:# If the current chunk has text blocks but no text block id,# start with a start event@@ -2409,14 +2446,15 @@ class Agent:delta="".join([_.text for _ in text_blocks]),)- elif block_ids.get("text"):+ elif block_ids.get("text") and not data_blocks:yield TextBlockEndEvent(reply_id=self.state.reply_id,block_id=block_ids["text"],)block_ids["text"] = None- # Handle the thinking blocks+ # Same reasoning as the text block above — keep the thinking+ # stream open across data-only chunks.if thinking_blocks:# Generate a new thinking block id and start eventif not block_ids.get("thinking"):@@ -2432,7 +2470,7 @@ class Agent:delta="".join([_.thinking for _ in thinking_blocks]),)- elif block_ids.get("thinking"):+ elif block_ids.get("thinking") and not data_blocks:yield ThinkingBlockEndEvent(reply_id=self.state.reply_id,block_id=block_ids["thinking"],@@ -2467,6 +2505,29 @@ class Agent:)block_ids["tools"].remove(finished_id)+ # Handle the data blocks (streaming binary content, e.g. omni audio).+ # Each DataBlock chunk from the model carries a delta payload with a+ # stable block id; we open a stream the first time we see an id and+ # emit delta events for subsequent chunks with the same id.+ for data_block in data_blocks:+ if not isinstance(data_block.source, Base64Source):+ # Only Base64Source carries inline delta bytes; URLSource is+ # one-shot and not part of the streaming protocol.+ continue+ if data_block.id not in block_ids["data"]:+ block_ids["data"].append(data_block.id)+ yield DataBlockStartEvent(+ reply_id=self.state.reply_id,+ block_id=data_block.id,+ media_type=data_block.source.media_type,+ )+ yield DataBlockDeltaEvent(+ reply_id=self.state.reply_id,+ block_id=data_block.id,+ data=data_block.source.data,+ media_type=data_block.source.media_type,+ )+async def _convert_tool_chunk_to_event(self,tool_call_id: str,diff --git a/src/agentscope/app/_router/_session.py b/src/agentscope/app/_router/_session.pyindex e8846f737..56a3e1c0c 100644--- a/src/agentscope/app/_router/_session.py+++ b/src/agentscope/app/_router/_session.py@@ -350,7 +350,9 @@ async def update_session(return await storage.upsert_session(user_id=user_id,agent_id=agent_id,- config=existing.config.model_copy(update=config_updates),+ config=SessionConfig.model_validate(+ {**existing.config.model_dump(mode="json"), **config_updates},+ ),state=updated_state,session_id=session_id,)diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.pyindex bfe431623..dcd0a1be4 100644--- a/src/agentscope/formatter/_dashscope_formatter.py+++ b/src/agentscope/formatter/_dashscope_formatter.py@@ -63,7 +63,6 @@ class _DashScopeFormatterBase(FormatterBase, ABC):def _format_dashscope_data_block(self,block: DataBlock,- role: str = "user",) -> dict[str, Any] | None:"""Format a DataBlock into the OpenAI-compatible format forDashScope API.@@ -76,10 +75,6 @@ class _DashScopeFormatterBase(FormatterBase, ABC):Args:block (`DataBlock`):The DataBlock to format.- role (`str`, defaults to ``"user"``):- The role of the message containing this block. Audio blocks- in assistant messages are skipped to avoid errors in- subsequent model calls.Returns:`dict[str, Any] | None`:@@ -107,8 +102,6 @@ class _DashScopeFormatterBase(FormatterBase, ABC):return self._format_video_source(block.source)if main_type == "audio":- if role == "assistant":- return Nonereturn self._format_audio_source(block.source)logger.warning(@@ -263,7 +256,6 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):elif isinstance(block, DataBlock):formatted_block = self._format_dashscope_data_block(block,- role=msg.role,)if formatted_block:content_blocks.append(formatted_block)@@ -305,7 +297,6 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):formatted_sub = (self._format_dashscope_data_block(sub,- role="user",))if formatted_sub:@@ -372,7 +363,6 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):elif isinstance(item, DataBlock):fmt_item = self._format_dashscope_data_block(item,- role="user",)if fmt_item is not None:promo_content.append(fmt_item)@@ -519,7 +509,6 @@ class DashScopeMultiAgentFormatter(_DashScopeFormatterBase):elif isinstance(block, DataBlock):formatted_block = self._format_dashscope_data_block(block,- role=msg.role,)if formatted_block is not None:media_blocks.append(formatted_block)diff --git a/src/agentscope/formatter/_moonshot_formatter.py b/src/agentscope/formatter/_moonshot_formatter.pyindex 075c012c4..7d05b51b8 100644--- a/src/agentscope/formatter/_moonshot_formatter.py+++ b/src/agentscope/formatter/_moonshot_formatter.py@@ -119,10 +119,7 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):content_blocks.append({"type": "text", "text": block.text})elif isinstance(block, DataBlock):- formatted = self._format_openai_data_block(- block,- role=msg.role,- )+ formatted = self._format_openai_data_block(block)if formatted is not None:content_blocks.append(formatted)@@ -165,7 +162,6 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):elif isinstance(sub, DataBlock):formatted_sub = self._format_openai_data_block(sub,- role="user",)if formatted_sub is not None:hint_parts.append(formatted_sub)@@ -230,7 +226,6 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):elif isinstance(item, DataBlock):fmt_item = self._format_openai_data_block(item,- role="user",)if fmt_item is not None:promo_content.append(fmt_item)@@ -390,10 +385,7 @@ class MoonshotMultiAgentFormatter(_OpenAIFormatterBase):if isinstance(block, TextBlock):accumulated_text.append(f"{msg.name}: {block.text}")elif isinstance(block, DataBlock):- formatted = self._format_openai_data_block(- block,- role=msg.role,- )+ formatted = self._format_openai_data_block(block)if formatted is not None:media_blocks.append(formatted)diff --git a/src/agentscope/formatter/_openai_formatter.py b/src/agentscope/formatter/_openai_formatter.pyindex 0e842b9ab..245fc62dc 100644--- a/src/agentscope/formatter/_openai_formatter.py+++ b/src/agentscope/formatter/_openai_formatter.py@@ -31,7 +31,6 @@ class _OpenAIFormatterBase(FormatterBase, ABC):def _format_openai_data_block(self,block: DataBlock,- role: str = "user",) -> dict[str, Any] | None:"""Format a DataBlock into the required format for OpenAI API.@@ -42,10 +41,6 @@ class _OpenAIFormatterBase(FormatterBase, ABC):Args:block (`DataBlock`):The DataBlock to format.- role (`str`, defaults to ``"user"``):- The role of the message that contains this block. Audio blocks- in assistant messages are skipped to avoid errors in subsequent- model calls.Returns:`dict[str, Any] | None`:@@ -70,10 +65,6 @@ class _OpenAIFormatterBase(FormatterBase, ABC):return self._format_image_source(block.source)if main_type == "audio":- # Filter out audio content when the multimodal model outputs both- # text and audio, to prevent errors in subsequent model calls- if role == "assistant":- return Nonereturn self._format_audio_source(block.source)logger.warning(@@ -240,7 +231,6 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):elif isinstance(block, DataBlock):formatted = self._format_openai_data_block(block,- role=msg.role,)if formatted is not None:content_blocks.append(formatted)@@ -277,7 +267,6 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):elif isinstance(sub, DataBlock):formatted_sub = self._format_openai_data_block(sub,- role="user",)if formatted_sub is not None:hint_parts.append(formatted_sub)@@ -335,7 +324,6 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):elif isinstance(item, DataBlock):fmt_item = self._format_openai_data_block(item,- role="user",)if fmt_item is not None:promo_content.append(fmt_item)@@ -471,7 +459,6 @@ class OpenAIMultiAgentFormatter(_OpenAIFormatterBase):elif isinstance(block, DataBlock):formatted = self._format_openai_data_block(block,- role=msg.role,)if formatted is not None:media_blocks.append(formatted)diff --git a/src/agentscope/formatter/_openai_response_formatter.py b/src/agentscope/formatter/_openai_response_formatter.pyindex 283f1188c..d44d64f9c 100644--- a/src/agentscope/formatter/_openai_response_formatter.py+++ b/src/agentscope/formatter/_openai_response_formatter.py@@ -38,7 +38,6 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):def _format_response_data_block(self,block: DataBlock,- role: str = "user",) -> dict[str, Any] | None:"""Format a DataBlock into the Response API format.@@ -53,8 +52,6 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):Args:block (`DataBlock`):The DataBlock to format.- role (`str`, defaults to ``"user"``):- The role of the message that contains this block.Returns:`dict[str, Any] | None`:@@ -75,7 +72,7 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):)return None- base_result = self._format_openai_data_block(block, role)+ base_result = self._format_openai_data_block(block)if base_result is None:return None@@ -134,10 +131,7 @@ class OpenAIResponseFormatter(_OpenAIResponseFormatterBase):)elif isinstance(block, DataBlock):- formatted = self._format_response_data_block(- block,- role=msg.role,- )+ formatted = self._format_response_data_block(block)if formatted is not None:content_parts.append(formatted)@@ -188,7 +182,6 @@ class OpenAIResponseFormatter(_OpenAIResponseFormatterBase):formatted_sub = (self._format_response_data_block(sub,- role="user",))if formatted_sub is not None:@@ -302,7 +295,6 @@ class OpenAIResponseFormatter(_OpenAIResponseFormatterBase):elif isinstance(item, DataBlock):fmt_item = self._format_response_data_block(item,- role="user",)if fmt_item is not None:promo_content.append(fmt_item)@@ -452,10 +444,7 @@ class OpenAIResponseMultiAgentFormatter(_OpenAIResponseFormatterBase):if isinstance(block, TextBlock):accumulated_text.append(f"{msg.name}: {block.text}")elif isinstance(block, DataBlock):- formatted = self._format_response_data_block(- block,- role=msg.role,- )+ formatted = self._format_response_data_block(block)if formatted is not None:media_blocks.append(formatted)diff --git a/src/agentscope/message/_base.py b/src/agentscope/message/_base.pyindex 6ef60edc6..6dd2d08b1 100644--- a/src/agentscope/message/_base.py+++ b/src/agentscope/message/_base.py@@ -1,5 +1,6 @@# -*- coding: utf-8 -*-"""The message class in agentscope."""+import base64import uuidfrom datetime import datetimefrom typing import Literal, List, overload, Sequence, Self, TYPE_CHECKING, Any@@ -283,7 +284,18 @@ class Msg(BaseModel):event.block_id,)elif event.data:- block.source.data += event.data+ # Each delta is an independently base64-encoded chunk+ # (with its own padding); naive string concat would+ # corrupt the byte stream. Decode, concat bytes, re-encode.+ existing = (+ base64.b64decode(block.source.data)+ if block.source.data+ else b""+ )+ incoming = base64.b64decode(event.data)+ block.source.data = base64.b64encode(+ existing + incoming,+ ).decode("ascii")case EventType.DATA_BLOCK_END:passdiff --git a/src/agentscope/model/_anthropic/_model.py b/src/agentscope/model/_anthropic/_model.pyindex dc36659a4..426a85ac8 100644--- a/src/agentscope/model/_anthropic/_model.py+++ b/src/agentscope/model/_anthropic/_model.py@@ -49,7 +49,7 @@ class AnthropicChatModel(ChatModelBase):thinking_budget: int | None = Field(default=None,- title="Thinking budget",+ title="Thinking Budget",description="The thinking budget for the LLM output.",gt=0,)diff --git a/src/agentscope/model/_dashscope/_model.py b/src/agentscope/model/_dashscope/_model.pyindex e97c5a390..5de1941ad 100644--- a/src/agentscope/model/_dashscope/_model.py+++ b/src/agentscope/model/_dashscope/_model.py@@ -1,6 +1,10 @@# -*- coding: utf-8 -*-"""The DashScope chat model class (OpenAI-compatible implementation)."""+import base64+import io+import uuidimport warnings+import wavefrom collections import OrderedDictfrom datetime import datetimefrom typing import Any, AsyncGenerator, List, Literal, Type, TYPE_CHECKING@@ -10,6 +14,7 @@ from pydantic import BaseModel, Fieldfrom .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODESfrom .._model_response import ChatResponse, StructuredResponsefrom .._model_usage import ChatUsage+from ..._utils._audio import _build_streaming_wav_headerfrom ...credential import DashScopeCredentialfrom ...formatter import FormatterBase, DashScopeChatFormatterfrom ...message import (@@ -57,7 +62,7 @@ class DashScopeChatModel(ChatModelBase):thinking_budget: int | None = Field(default=None,- title="Thinking budget",+ title="Thinking Budget",description="The thinking budget for the LLM output.",gt=0,)@@ -92,6 +97,21 @@ class DashScopeChatModel(ChatModelBase):description="If enable parallel tool calls for the LLM output.",)+ voice: str | None = Field(+ default=None,+ title="Voice",+ description=(+ "Voice for audio output on omni-style models (e.g. "+ "``qwen3.5-omni-plus``). Setting this implicitly asks the "+ "model to speak its response — ``modalities`` is filled in "+ "automatically. Supported voices vary by model — see the "+ "model card's ``voice.suggestions``. Any value the API "+ "accepts works — the suggestions are convenience-only. "+ "Leave unset for text-only "+ "responses."+ ),+ )+type: Literal["dashscope_chat"] = "dashscope_chat""""The type of the chat model."""@@ -208,6 +228,18 @@ class DashScopeChatModel(ChatModelBase):if self.parameters.top_p is not None:request_kwargs["top_p"] = self.parameters.top_p+ if self.parameters.voice is not None:+ # Requesting audio output implies ``modalities`` must include+ # ``"audio"``; set it automatically so callers don't have to.+ # ``format`` is forced to ``pcm16``: omni streaming delivers raw+ # PCM upstream regardless of the requested format, and we wrap+ # it as WAV in ``_parse_stream_response`` before yielding.+ request_kwargs["audio"] = {+ "voice": self.parameters.voice,+ "format": "pcm16",+ }+ request_kwargs["modalities"] = ["text", "audio"]+request_kwargs.update(kwargs)fmt_tools, fmt_tool_choice = self._format_tools(tools, tool_choice)@@ -237,17 +269,7 @@ class DashScopeChatModel(ChatModelBase):response = await client.chat.completions.create(**request_kwargs)if self.stream:- audio_cfg = request_kwargs.get("audio")- audio_fmt = (- audio_cfg.get("format", "wav")- if isinstance(audio_cfg, dict)- else "wav"- )- return self._parse_stream_response(- start_datetime,- response,- audio_fmt,- )+ return self._parse_stream_response(start_datetime, response)return self._parse_completion_response(start_datetime, response)@@ -255,7 +277,6 @@ class DashScopeChatModel(ChatModelBase):self,start_datetime: datetime,response: AsyncStream,- audio_format: str = "wav",) -> AsyncGenerator[ChatResponse, None]:"""Parse the DashScope streaming response (OpenAI-compatible format).@@ -264,9 +285,6 @@ class DashScopeChatModel(ChatModelBase):The start datetime of the response generation.response (`AsyncStream`):The OpenAI-compatible async stream object.- audio_format (`str`, defaults to ``"wav"``):- The audio format requested (used to set the media type on- the output ``DataBlock``).Yields:`ChatResponse`:@@ -278,7 +296,15 @@ class DashScopeChatModel(ChatModelBase):acc_text = TextBlock(text="")acc_thinking = ThinkingBlock(thinking="")acc_tool_calls: OrderedDict = OrderedDict()- acc_audio_data: str = ""+ # Raw PCM bytes accumulated across chunks. Storing the decoded form+ # (rather than concatenated base64 strings) avoids the risk of+ # corrupting the byte stream when an intermediate chunk happens to+ # carry base64 padding (``=``).+ acc_audio_data: bytearray = bytearray()+ audio_block_id: str | None = None+ # ``True`` once the first audio chunk has been prefixed with a+ # streaming WAV header and yielded.+ audio_header_sent: bool = Falseasync with response as stream:async for chunk in stream:@@ -309,15 +335,37 @@ class DashScopeChatModel(ChatModelBase):)delta_text = getattr(delta, "content", None) or ""- # Collect audio output from Omni models (delta.audio.data)+ # Collect audio output from Omni models (delta.audio.data).+ # Upstream sends raw PCM (24kHz, 16-bit mono); we prefix the+ # first chunk with a streaming WAV header so the frontend+ # can start playback immediately rather than waiting for+ # end-of-stream.delta_audio = getattr(delta, "audio", None)+ delta_audio_block: DataBlock | None = Noneif delta_audio is not None:if isinstance(delta_audio, dict):audio_chunk = delta_audio.get("data", "")else:audio_chunk = getattr(delta_audio, "data", "") or ""if audio_chunk:- acc_audio_data += audio_chunk+ if audio_block_id is None:+ audio_block_id = uuid.uuid4().hex+ pcm_bytes = base64.b64decode(audio_chunk)+ acc_audio_data += pcm_bytes+ if not audio_header_sent:+ payload = _build_streaming_wav_header() + pcm_bytes+ audio_header_sent = True+ else:+ payload = pcm_bytes+ delta_audio_block = DataBlock(+ id=audio_block_id,+ source=Base64Source(+ data=base64.b64encode(payload).decode(+ "ascii",+ ),+ media_type="audio/wav",+ ),+ )acc_thinking.thinking += delta_thinkingacc_text.text += delta_text@@ -352,7 +400,7 @@ class DashScopeChatModel(ChatModelBase):)delta_contents: List[- TextBlock | ToolCallBlock | ThinkingBlock+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock] = []if delta_thinking:delta_contents.append(@@ -366,6 +414,8 @@ class DashScopeChatModel(ChatModelBase):TextBlock(id=acc_text.id, text=delta_text),)delta_contents.extend(delta_tool_call_blocks)+ if delta_audio_block is not None:+ delta_contents.append(delta_audio_block)if delta_contents:_kwargs: dict[str, Any] = {@@ -393,11 +443,25 @@ class DashScopeChatModel(ChatModelBase):),)if acc_audio_data:+ # PCM bytes were already streamed incrementally above (first+ # chunk prefixed with a WAV header). Here we also assemble a+ # standalone fixed-size WAV and attach it to the ``is_last``+ # chunk so callers that consume the model directly (i.e.+ # without going through ``Agent``, which filters audio blocks+ # out of context) get a self-contained audio block for+ # downstream serialization / display.+ buf = io.BytesIO()+ with wave.open(buf, "wb") as wav:+ wav.setnchannels(1)+ wav.setsampwidth(2)+ wav.setframerate(24000)+ wav.writeframes(bytes(acc_audio_data))final_contents.append(DataBlock(+ id=audio_block_id,source=Base64Source(- data=acc_audio_data,- media_type=f"audio/{audio_format}",+ data=base64.b64encode(buf.getvalue()).decode("ascii"),+ media_type="audio/wav",),),)diff --git a/src/agentscope/model/_dashscope/_models/qwen-long.yaml b/src/agentscope/model/_dashscope/_models/qwen-long.yamlindex a60794b4b..56b7f3474 100644--- a/src/agentscope/model/_dashscope/_models/qwen-long.yaml+++ b/src/agentscope/model/_dashscope/_models/qwen-long.yaml@@ -15,3 +15,7 @@ parameter_overrides:max_tokens: {"maximum": 8192}thinking_enable:hidden: true+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_dashscope/_models/qwen-max-3.7.yaml b/src/agentscope/model/_dashscope/_models/qwen-max-3.7.yamlindex 817429e55..e626af494 100644--- a/src/agentscope/model/_dashscope/_models/qwen-max-3.7.yaml+++ b/src/agentscope/model/_dashscope/_models/qwen-max-3.7.yaml@@ -14,3 +14,7 @@ output_size: 65536parameter_overrides:max_tokens: {"maximum": 65536}+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_dashscope/_models/qwen-plus.yaml b/src/agentscope/model/_dashscope/_models/qwen-plus.yamlindex f7f77d55e..ba5e41ecd 100644--- a/src/agentscope/model/_dashscope/_models/qwen-plus.yaml+++ b/src/agentscope/model/_dashscope/_models/qwen-plus.yaml@@ -14,3 +14,7 @@ output_size: 16384parameter_overrides:max_tokens: {"maximum": 16384}+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_dashscope/_models/qwen3.5-omni-plus.yaml b/src/agentscope/model/_dashscope/_models/qwen3.5-omni-plus.yamlindex 973e282dd..dcf3c8dfd 100644--- a/src/agentscope/model/_dashscope/_models/qwen3.5-omni-plus.yaml+++ b/src/agentscope/model/_dashscope/_models/qwen3.5-omni-plus.yaml@@ -24,5 +24,15 @@ context_size: 262144output_size: 65536parameter_overrides:- max_tokens: {"maximum": 65536}-+ max_tokens:+ maximum: 65536+ # Source: https://help.aliyun.com/zh/model-studio/omni-voice-list+ voice:+ default: Tina+ enum:+ - Tina+ - Cherry+ - Ethan+ - Serena+ - Chelsie+ - Cindydiff --git a/src/agentscope/model/_dashscope/_models/qwen3.5-plus.yaml b/src/agentscope/model/_dashscope/_models/qwen3.5-plus.yamlnew file mode 100644index 000000000..4682425d0--- /dev/null+++ b/src/agentscope/model/_dashscope/_models/qwen3.5-plus.yaml@@ -0,0 +1,28 @@+name: qwen3.5-plus+label: Qwen3.5-Plus+status: active++input_types:+ - text/plain+ - application/x-thinking+ - image/bmp+ - image/jpeg+ - image/png+ - image/tiff+ - image/webp+ - image/heic+ - video/mp4++output_types:+ - text/plain+ - application/x-thinking++context_size: 1000000+output_size: 65536++parameter_overrides:+ max_tokens: {"maximum": 65536}+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_dashscope/_models/qwen3.6-max-preview.yaml b/src/agentscope/model/_dashscope/_models/qwen3.6-max-preview.yamlindex e4978f56f..501f5a374 100644--- a/src/agentscope/model/_dashscope/_models/qwen3.6-max-preview.yaml+++ b/src/agentscope/model/_dashscope/_models/qwen3.6-max-preview.yaml@@ -14,3 +14,7 @@ output_size: 64000parameter_overrides:max_tokens: {"maximum": 64000}+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_dashscope/_models/qwen3.6-plus.yaml b/src/agentscope/model/_dashscope/_models/qwen3.6-plus.yamlindex 3ae5e677d..3dbc91988 100644--- a/src/agentscope/model/_dashscope/_models/qwen3.6-plus.yaml+++ b/src/agentscope/model/_dashscope/_models/qwen3.6-plus.yaml@@ -22,3 +22,7 @@ output_size: 65536parameter_overrides:max_tokens: {"maximum": 65536}+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_model_card.py b/src/agentscope/model/_model_card.pyindex 711ae6654..c5795ecfc 100644--- a/src/agentscope/model/_model_card.py+++ b/src/agentscope/model/_model_card.py@@ -103,6 +103,15 @@ class ModelCard(BaseModel):properties.pop("thinking_enable", None)properties.pop("thinking_budget", None)+ # Auto-filter: only omni-style models that declare an ``audio/*``+ # output type expose the ``voice`` parameter to the frontend+ # popover. Defense in depth — a model yaml can also drop ``voice``+ # explicitly via ``parameter_overrides: { voice: { hidden: true } }``.+ if not any(+ isinstance(t, str) and t.startswith("audio/") for t in output_types+ ):+ properties.pop("voice", None)+# Auto-inject: set max_tokens maximum from output_sizeif "max_tokens" in properties and "output_size" in config:properties["max_tokens"]["maximum"] = config["output_size"]diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.pyindex d1e534ead..6056f2db4 100644--- a/src/agentscope/model/_openai_chat/_model.py+++ b/src/agentscope/model/_openai_chat/_model.py@@ -1,5 +1,9 @@# -*- coding: utf-8 -*-"""The OpenAI Chat Completions model implementation."""+import base64+import io+import uuid+import wavefrom collections import OrderedDictfrom datetime import datetimefrom typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type@@ -9,6 +13,7 @@ from pydantic import BaseModel, Fieldfrom .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODESfrom .._model_response import ChatResponsefrom .._model_usage import ChatUsage+from ..._utils._audio import _build_streaming_wav_headerfrom ...credential import OpenAICredentialfrom ...formatter import FormatterBase, OpenAIChatFormatterfrom ...message import (@@ -87,6 +92,21 @@ class OpenAIChatModel(ChatModelBase):description="Whether to enable parallel tool calls.",)+ voice: str | None = Field(+ default=None,+ title="Voice",+ description=(+ "Voice for audio output on omni-style models (e.g. "+ "``gpt-audio-mini``). Setting this implicitly asks the "+ "model to speak its response — ``modalities`` is filled in "+ "automatically. Supported voices vary by model — see the "+ "model card's ``voice.suggestions``. Any value the API "+ "accepts works — the suggestions are convenience-only. "+ "Leave unset for text-only "+ "responses."+ ),+ )+type: Literal["openai_chat"] = "openai_chat""""The type of the chat model."""@@ -214,6 +234,18 @@ class OpenAIChatModel(ChatModelBase):):kwargs["reasoning_effort"] = self.parameters.reasoning_effort+ if self.parameters.voice is not None:+ # Requesting audio output implies ``modalities`` must include+ # ``"audio"``; set it automatically so callers don't have to.+ # ``format`` is forced to ``pcm16``: OpenAI streaming only+ # supports ``pcm16`` (other formats raise 400), and we re-wrap+ # as WAV downstream so the frontend receives a playable block.+ kwargs["audio"] = {+ "voice": self.parameters.voice,+ "format": "pcm16",+ }+ kwargs["modalities"] = ["text", "audio"]+kwargs.update(generate_kwargs)fmt_tools, fmt_tool_choice = self._format_tools(tools, tool_choice)@@ -240,10 +272,11 @@ class OpenAIChatModel(ChatModelBase):)if self.stream:+ # Streaming wire format is always ``pcm16`` (forced above) and we+ # re-wrap it as WAV before yielding, so downstream sees ``wav``.return self._parse_stream_response(start_datetime,response,- audio_fmt,)return self._parse_completion_response(@@ -256,18 +289,20 @@ class OpenAIChatModel(ChatModelBase):self,start_datetime: datetime,response: AsyncStream,- audio_format: str = "wav",) -> AsyncGenerator[ChatResponse, None]:"""Parse the OpenAI Chat streaming response.+ Upstream sends raw PCM16 (24kHz, 16-bit mono — OpenAI's only+ streaming-supported audio format). We prefix the first audio+ chunk with a streaming WAV header so the frontend can start+ playback immediately, and assemble a fixed-size WAV on the+ final chunk for non-streaming consumers.+Args:start_datetime (`datetime`):The start datetime of the response generation.response (`AsyncStream`):The OpenAI async stream object.- audio_format (`str`, defaults to ``"wav"``):- The audio format requested (used to set the media type on- the output ``DataBlock``).Yields:`ChatResponse`:@@ -280,8 +315,15 @@ class OpenAIChatModel(ChatModelBase):acc_text = TextBlock(text="")acc_thinking = ThinkingBlock(thinking="")acc_tool_calls: OrderedDict = OrderedDict()- acc_audio_data: str = ""- acc_audio_transcript: str = ""+ # Raw audio bytes accumulated across chunks. Storing the decoded+ # form (rather than concatenated base64 strings) avoids corrupting+ # the byte stream when an intermediate chunk happens to carry+ # base64 padding (``=``).+ acc_audio_data: bytearray = bytearray()+ audio_block_id: str | None = None+ # ``True`` once the first audio chunk has been prefixed with a+ # streaming WAV header and yielded.+ audio_header_sent: bool = Falseasync with response as stream:async for chunk in stream:@@ -320,6 +362,8 @@ class OpenAIChatModel(ChatModelBase):# Collect audio output (delta.audio.data /# delta.audio.transcript)+ delta_audio_block: DataBlock | None = None+ transcript_chunk: str = ""delta_audio = getattr(delta, "audio", None)if delta_audio is not None:if isinstance(delta_audio, dict):@@ -331,9 +375,30 @@ class OpenAIChatModel(ChatModelBase):getattr(delta_audio, "transcript", "") or "")if audio_chunk:- acc_audio_data += audio_chunk- if transcript_chunk:- acc_audio_transcript += transcript_chunk+ if audio_block_id is None:+ audio_block_id = uuid.uuid4().hex+ pcm_bytes = base64.b64decode(audio_chunk)+ acc_audio_data += pcm_bytes+ if not audio_header_sent:+ payload = _build_streaming_wav_header() + pcm_bytes+ audio_header_sent = True+ else:+ payload = pcm_bytes+ delta_audio_block = DataBlock(+ id=audio_block_id,+ source=Base64Source(+ data=base64.b64encode(payload).decode(+ "ascii",+ ),+ media_type="audio/wav",+ ),+ )+ # Omni models deliver text via ``delta.audio.transcript``+ # (not ``delta.content``); fold it into ``delta_text`` so+ # the agent's streaming pipeline emits ``TextBlockDelta``+ # events alongside the audio chunks.+ if transcript_chunk:+ delta_text += transcript_chunkacc_thinking.thinking += delta_thinkingacc_text.text += delta_text@@ -360,7 +425,7 @@ class OpenAIChatModel(ChatModelBase):)delta_contents: List[- TextBlock | ToolCallBlock | ThinkingBlock+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock] = []if delta_thinking:delta_contents.append(@@ -374,6 +439,8 @@ class OpenAIChatModel(ChatModelBase):TextBlock(id=acc_text.id, text=delta_text),)delta_contents.extend(delta_tool_call_blocks)+ if delta_audio_block is not None:+ delta_contents.append(delta_audio_block)if delta_contents:_kwargs: dict[str, Any] = {@@ -392,18 +459,30 @@ class OpenAIChatModel(ChatModelBase):final_contents.append(acc_thinking)if acc_text.text:final_contents.append(acc_text)- elif acc_audio_transcript:- final_contents.append(TextBlock(text=acc_audio_transcript))for tc in acc_tool_calls.values():final_contents.append(ToolCallBlock(id=tc["id"], name=tc["name"], input=tc["input"]),)if acc_audio_data:+ # PCM bytes were already streamed incrementally above (first+ # chunk prefixed with a WAV header). Here we also assemble a+ # standalone fixed-size WAV and attach it to the ``is_last``+ # chunk so callers that consume the model directly (i.e.+ # without going through ``Agent``, which filters audio blocks+ # out of context) get a self-contained audio block for+ # downstream serialization / display.+ buf = io.BytesIO()+ with wave.open(buf, "wb") as wav:+ wav.setnchannels(1)+ wav.setsampwidth(2)+ wav.setframerate(24000)+ wav.writeframes(bytes(acc_audio_data))final_contents.append(DataBlock(+ id=audio_block_id,source=Base64Source(- data=acc_audio_data,- media_type=f"audio/{audio_format}",+ data=base64.b64encode(buf.getvalue()).decode("ascii"),+ media_type="audio/wav",),),)diff --git a/src/agentscope/model/_openai_chat/_models/gpt-4.1-mini.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4.1-mini.yamlindex a6bdad35e..cc66dcaa5 100644--- a/src/agentscope/model/_openai_chat/_models/gpt-4.1-mini.yaml+++ b/src/agentscope/model/_openai_chat/_models/gpt-4.1-mini.yaml@@ -22,3 +22,7 @@ parameter_overrides:hidden: truereasoning_effort:hidden: true+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_openai_chat/_models/gpt-4.1-nano.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4.1-nano.yamlindex 5a067a036..bf26d9d9b 100644--- a/src/agentscope/model/_openai_chat/_models/gpt-4.1-nano.yaml+++ b/src/agentscope/model/_openai_chat/_models/gpt-4.1-nano.yaml@@ -22,3 +22,7 @@ parameter_overrides:hidden: truereasoning_effort:hidden: true+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_openai_chat/_models/gpt-4.1.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4.1.yamlindex 40ebbe051..f26b14301 100644--- a/src/agentscope/model/_openai_chat/_models/gpt-4.1.yaml+++ b/src/agentscope/model/_openai_chat/_models/gpt-4.1.yaml@@ -20,3 +20,7 @@ parameter_overrides:maximum: 32768thinking_enable:hidden: true+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_openai_chat/_models/gpt-4o-mini.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4o-mini.yamlindex da2ec3901..b41ac171c 100644--- a/src/agentscope/model/_openai_chat/_models/gpt-4o-mini.yaml+++ b/src/agentscope/model/_openai_chat/_models/gpt-4o-mini.yaml@@ -22,3 +22,7 @@ parameter_overrides:hidden: truereasoning_effort:hidden: true+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_openai_chat/_models/gpt-4o.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4o.yamlindex e19c9be73..5b19e6883 100644--- a/src/agentscope/model/_openai_chat/_models/gpt-4o.yaml+++ b/src/agentscope/model/_openai_chat/_models/gpt-4o.yaml@@ -22,3 +22,7 @@ parameter_overrides:hidden: truereasoning_effort:hidden: true+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_openai_chat/_models/gpt-5.4.yaml b/src/agentscope/model/_openai_chat/_models/gpt-5.4.yamlindex ef1908708..dec7e31bd 100644--- a/src/agentscope/model/_openai_chat/_models/gpt-5.4.yaml+++ b/src/agentscope/model/_openai_chat/_models/gpt-5.4.yaml@@ -20,3 +20,7 @@ parameter_overrides:maximum: 128000thinking_enable:hidden: true+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_openai_chat/_models/gpt-5.5.yaml b/src/agentscope/model/_openai_chat/_models/gpt-5.5.yamlindex 746c4f516..1e009e393 100644--- a/src/agentscope/model/_openai_chat/_models/gpt-5.5.yaml+++ b/src/agentscope/model/_openai_chat/_models/gpt-5.5.yaml@@ -24,3 +24,7 @@ parameter_overrides:hidden: truereasoning_effort:hidden: true+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_openai_chat/_models/gpt-audio-mini.yaml b/src/agentscope/model/_openai_chat/_models/gpt-audio-mini.yamlnew file mode 100644index 000000000..5877d6832--- /dev/null+++ b/src/agentscope/model/_openai_chat/_models/gpt-audio-mini.yaml@@ -0,0 +1,31 @@+name: gpt-audio-mini+label: GPT Audio Mini+status: active++input_types:+ - text/plain+ - audio/wav+ - audio/mp3++output_types:+ - text/plain+ - audio/wav++context_size: 128000+output_size: 16384++parameter_overrides:+ max_tokens:+ maximum: 16384+ thinking_enable:+ hidden: true+ # Source: https://platform.openai.com/docs/guides/text-to-speech+ voice:+ default: alloy+ enum:+ - alloy+ - echo+ - nova+ - shimmer+ - sage+ - versediff --git a/src/agentscope/model/_openai_chat/_models/o3.yaml b/src/agentscope/model/_openai_chat/_models/o3.yamlindex 7aedacce8..13e7762aa 100644--- a/src/agentscope/model/_openai_chat/_models/o3.yaml+++ b/src/agentscope/model/_openai_chat/_models/o3.yaml@@ -19,3 +19,7 @@ output_size: 100000parameter_overrides:max_tokens:maximum: 100000+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/src/agentscope/model/_openai_chat/_models/o4-mini.yaml b/src/agentscope/model/_openai_chat/_models/o4-mini.yamlindex 931013282..ab2e91192 100644--- a/src/agentscope/model/_openai_chat/_models/o4-mini.yaml+++ b/src/agentscope/model/_openai_chat/_models/o4-mini.yaml@@ -19,3 +19,7 @@ output_size: 100000parameter_overrides:max_tokens:maximum: 100000+ # Voice only applies to omni-style models; hide the field for this+ # non-omni model so the frontend popover doesn't render it.+ voice:+ hidden: truediff --git a/tests/event_to_message_test.py b/tests/event_to_message_test.pyindex e78786bee..3f261af53 100644--- a/tests/event_to_message_test.py+++ b/tests/event_to_message_test.py@@ -263,31 +263,35 @@ class EventToMessageTest(IsolatedAsyncioTestCase):],)+ # Each delta carries an independently base64-encoded chunk (with its+ # own padding); ``append_event`` decodes -> concats bytes -> re-encodes+ # so the message reflects the concatenated underlying bytes, not a+ # string-concat of the chunks. Here: b"abc" + b"def" -> "YWJjZGVm".ev_data_delta1 = DataBlockDeltaEvent(reply_id=_REPLY_ID,block_id=_B_DATA,- data="abc",+ data="YWJj", # base64(b"abc")media_type="image/png",)gt_data_delta1 = _base([_tb(_B_TEXT, "Hello World"),_thb(_B_THINK, "Let me think"),- _db_b64(_B_DATA, "abc", "image/png"),+ _db_b64(_B_DATA, "YWJj", "image/png"),],)ev_data_delta2 = DataBlockDeltaEvent(reply_id=_REPLY_ID,block_id=_B_DATA,- data="def",+ data="ZGVm", # base64(b"def")media_type="image/png",)gt_data_delta2 = _base([_tb(_B_TEXT, "Hello World"),_thb(_B_THINK, "Let me think"),- _db_b64(_B_DATA, "abcdef", "image/png"),+ _db_b64(_B_DATA, "YWJjZGVm", "image/png"),],)@@ -296,7 +300,7 @@ class EventToMessageTest(IsolatedAsyncioTestCase):[ # unchanged_tb(_B_TEXT, "Hello World"),_thb(_B_THINK, "Let me think"),- _db_b64(_B_DATA, "abcdef", "image/png"),+ _db_b64(_B_DATA, "YWJjZGVm", "image/png"),],)@@ -312,7 +316,7 @@ class EventToMessageTest(IsolatedAsyncioTestCase):_s4_prefix = [_tb(_B_TEXT, "Hello World"),_thb(_B_THINK, "Let me think"),- _db_b64(_B_DATA, "abcdef", "image/png"),+ _db_b64(_B_DATA, "YWJjZGVm", "image/png"),]gt_tc_allow_start = _base(_s4_prefix + [_tcb(_TC_ALLOW, "search", "", "pending")],diff --git a/tests/model_dashscope_test.py b/tests/model_dashscope_test.pyindex 709e1bdbb..f4ebc6830 100644--- a/tests/model_dashscope_test.py+++ b/tests/model_dashscope_test.py@@ -4,6 +4,9 @@Tests cover both non-streaming and streaming modes."""+import base64+import io+import wavefrom typing import Anyimport unittestfrom unittest import IsolatedAsyncioTestCase@@ -11,7 +14,12 @@ from unittest.mock import AsyncMock, MagicMock, patchfrom utils import AnyString-from agentscope.message import TextBlock, ToolCallBlock, ThinkingBlock+from agentscope.message import (+ TextBlock,+ ToolCallBlock,+ ThinkingBlock,+ DataBlock,+)from agentscope.model import DashScopeChatModelfrom agentscope.credential import DashScopeCredentialfrom agentscope.tool import ToolChoice@@ -78,6 +86,7 @@ def _make_stream_chunk(delta_text: str | None = None,delta_reasoning: str | None = None,tool_calls: list | None = None,+ delta_audio: dict | None = None,response_id: str = "req-1",usage: dict | None = None,has_choices: bool = True,@@ -99,7 +108,7 @@ def _make_stream_chunk(delta.content = delta_textdelta.reasoning_content = delta_reasoningdelta.tool_calls = tool_calls- delta.audio = None+ delta.audio = delta_audiochoice = MagicMock()choice.delta = deltachunk.choices = [choice]@@ -411,6 +420,78 @@ class TestDashScopeStream(IsolatedAsyncioTestCase):self.assertEqual(responses[-1].usage.input_tokens, 50)self.assertEqual(responses[-1].usage.output_tokens, 10)+ @patch("openai.AsyncClient")+ async def test_stream_audio_response(+ self,+ mock_client_cls: MagicMock,+ ) -> None:+ """Stream PCM deltas produce per-chunk DataBlocks (first chunk+ prefixed with a streaming WAV header) sharing a stable id, plus a+ final fixed-size WAV block readable by the ``wave`` module."""+ pcm1 = bytes([1, 2, 3, 4])+ pcm2 = bytes([5, 6, 7, 8])+ pcm3 = bytes([9, 10, 11, 12])+ pcm_full = pcm1 + pcm2 + pcm3++ chunks = [+ _make_stream_chunk(+ delta_audio={"data": base64.b64encode(pcm1).decode()},+ ),+ _make_stream_chunk(+ delta_audio={"data": base64.b64encode(pcm2).decode()},+ ),+ _make_stream_chunk(+ delta_audio={"data": base64.b64encode(pcm3).decode()},+ ),+ _make_stream_chunk(+ has_choices=False,+ usage={"prompt_tokens": 5, "completion_tokens": 3},+ ),+ ]+ mock_create = AsyncMock(return_value=_MockAsyncStream(chunks))+ mock_client_cls.return_value.chat.completions.create = mock_create++ gen = await self.model([])+ responses = [r async for r in gen]+ self.assertEqual(len(responses), 4)++ # All four chunks (3 deltas + 1 final) must share the same audio+ # block id so downstream consumers stitch them as one stream.+ all_audio_ids = {+ block.id+ for r in responses+ for block in r.content+ if isinstance(block, DataBlock)+ }+ self.assertEqual(len(all_audio_ids), 1)++ # First delta: WAV header (44 bytes, "RIFF"..."WAVE") + pcm1.+ first_payload = base64.b64decode(responses[0].content[0].source.data)+ self.assertEqual(len(first_payload), 44 + len(pcm1))+ self.assertEqual(first_payload[:4], b"RIFF")+ self.assertEqual(first_payload[8:12], b"WAVE")+ self.assertEqual(first_payload[44:], pcm1)++ # Subsequent deltas: raw PCM only, no header.+ for resp, pcm in zip(responses[1:3], [pcm2, pcm3]):+ payload = base64.b64decode(resp.content[0].source.data)+ self.assertEqual(payload, pcm)++ # Final ``is_last`` block: a fixed-size WAV the ``wave`` module+ # can parse end-to-end at 24kHz / mono / 16-bit.+ final = responses[-1]+ self.assertTrue(final.is_last)+ final_audio = next(+ b for b in final.content if isinstance(b, DataBlock)+ )+ wav_bytes = base64.b64decode(final_audio.source.data)+ with wave.open(io.BytesIO(wav_bytes), "rb") as wav:+ self.assertEqual(wav.getnchannels(), 1)+ self.assertEqual(wav.getsampwidth(), 2)+ self.assertEqual(wav.getframerate(), 24000)+ frames = wav.readframes(wav.getnframes())+ self.assertEqual(frames, pcm_full)+# ---------------------------------------------------------------------------# _format_tools testsdiff --git a/tests/model_openai_chat_test.py b/tests/model_openai_chat_test.pyindex 31e36e376..8921a7e33 100644--- a/tests/model_openai_chat_test.py+++ b/tests/model_openai_chat_test.py@@ -7,6 +7,9 @@ Tests cover both non-streaming and streaming modes, verifying that:- Stream mode yields n delta ChatResponses (is_last=False) followed by1 final ChatResponse (is_last=True) with the full accumulated content."""+import base64+import io+import wavefrom typing import Anyimport unittestfrom unittest import IsolatedAsyncioTestCase@@ -477,17 +480,35 @@ class TestOpenAIChatStream(IsolatedAsyncioTestCase):self,mock_client_cls: MagicMock,) -> None:- """Stream audio deltas accumulate into a final DataBlock with the- concatenated base64 payload and a transcript TextBlock."""+ """Stream PCM deltas produce per-chunk DataBlocks (first chunk+ prefixed with a streaming WAV header) sharing a stable id, plus a+ final fixed-size WAV block readable by the ``wave`` module.+ Transcript chunks ride alongside as TextBlock deltas so the agent+ can stream caption text live; the final block carries the full+ accumulated transcript."""+ pcm1 = bytes([1, 2, 3, 4])+ pcm2 = bytes([5, 6, 7, 8])+ pcm3 = bytes([9, 10, 11, 12])+ pcm_full = pcm1 + pcm2 + pcm3+chunks = [_make_stream_chunk(- delta_audio={"data": "AAAA", "transcript": "Hello"},+ delta_audio={+ "data": base64.b64encode(pcm1).decode(),+ "transcript": "Hello",+ },),_make_stream_chunk(- delta_audio={"data": "BBBB", "transcript": " world"},+ delta_audio={+ "data": base64.b64encode(pcm2).decode(),+ "transcript": " world",+ },),_make_stream_chunk(- delta_audio={"data": "CCCC", "transcript": "!"},+ delta_audio={+ "data": base64.b64encode(pcm3).decode(),+ "transcript": "!",+ },),_make_stream_chunk(has_choices=False,@@ -499,27 +520,67 @@ class TestOpenAIChatStream(IsolatedAsyncioTestCase):gen = await self.model([])responses = [r async for r in gen]-- # Only the final response carries the assembled audio content; the- # per-chunk deltas have no text/thinking/tool_call so no delta- # ChatResponse is yielded mid-stream.- self.assertEqual(len(responses), 1)- final = responses[0]+ self.assertEqual(len(responses), 4)++ # All four chunks (3 deltas + 1 final) must share the same audio+ # block id so downstream consumers stitch them as one stream.+ all_audio_ids = {+ block.id+ for r in responses+ for block in r.content+ if isinstance(block, DataBlock)+ }+ self.assertEqual(len(all_audio_ids), 1)++ # First delta: WAV header (44 bytes, "RIFF"..."WAVE") + pcm1.+ first_audio = next(+ b for b in responses[0].content if isinstance(b, DataBlock)+ )+ first_payload = base64.b64decode(first_audio.source.data)+ self.assertEqual(len(first_payload), 44 + len(pcm1))+ self.assertEqual(first_payload[:4], b"RIFF")+ self.assertEqual(first_payload[8:12], b"WAVE")+ self.assertEqual(first_payload[44:], pcm1)+ self.assertEqual(first_audio.source.media_type, "audio/wav")++ # Subsequent deltas: raw PCM only, no header.+ for resp, pcm in zip(responses[1:3], [pcm2, pcm3]):+ audio_block = next(+ b for b in resp.content if isinstance(b, DataBlock)+ )+ self.assertEqual(base64.b64decode(audio_block.source.data), pcm)+ self.assertEqual(audio_block.source.media_type, "audio/wav")++ # Transcript rides alongside: each delta carries a TextBlock with+ # only that chunk's text (so the agent emits TextBlockDeltaEvents+ # in real time).+ for resp, expected_text in zip(+ responses[:3],+ ["Hello", " world", "!"],+ ):+ text_block = next(+ b for b in resp.content if isinstance(b, TextBlock)+ )+ self.assertEqual(text_block.text, expected_text)++ # Final ``is_last`` block: a fixed-size WAV the ``wave`` module+ # can parse end-to-end at 24kHz / mono / 16-bit.+ final = responses[-1]self.assertTrue(final.is_last)- self.assertEqual(- final.content,- [- TextBlock.model_construct(id=A, text="Hello world!"),- DataBlock.model_construct(- id=A,- source=Base64Source.model_construct(- type="base64",- media_type="audio/wav",- data="AAAABBBBCCCC",- ),- ),- ],+ final_audio = next(+ b for b in final.content if isinstance(b, DataBlock))+ wav_bytes = base64.b64decode(final_audio.source.data)+ with wave.open(io.BytesIO(wav_bytes), "rb") as wav:+ self.assertEqual(wav.getnchannels(), 1)+ self.assertEqual(wav.getsampwidth(), 2)+ self.assertEqual(wav.getframerate(), 24000)+ frames = wav.readframes(wav.getnframes())+ self.assertEqual(frames, pcm_full)++ # Transcript is accumulated and emitted as a TextBlock alongside.+ final_text = next(b for b in final.content if isinstance(b, TextBlock))+ self.assertEqual(final_text.text, "Hello world!")class TestOpenAIChatModelParameters(unittest.TestCase):