Skip to content

Build and run agents you can see, understand and trust.

feat(omni): support real-time audio caption streaming via DashScope and OpenAI

qbc58c24fa7c85a
diff --git a/examples/web_ui/frontend/package.json b/examples/web_ui/frontend/package.json
index 90c71dc94..659fddf82 100644
--- a/examples/web_ui/frontend/package.json
+++ b/examples/web_ui/frontend/package.json
@@ -11,7 +11,7 @@
"preview": "vite preview"
},
"dependencies": {
- "@agentscope-ai/agentscope": "^0.0.8",
+ "@agentscope-ai/agentscope": "^0.0.9",
"@fontsource-variable/geist": "^5.2.8",
"@tailwindcss/vite": "^4.3.0",
"class-variance-authority": "^0.7.1",
diff --git a/examples/web_ui/frontend/src/components/chat/MessageBubble.tsx b/examples/web_ui/frontend/src/components/chat/MessageBubble.tsx
index a81666082..b66267b5c 100644
--- a/examples/web_ui/frontend/src/components/chat/MessageBubble.tsx
+++ b/examples/web_ui/frontend/src/components/chat/MessageBubble.tsx
@@ -12,12 +12,13 @@ import {
CalendarClock,
CheckCircle,
ChevronDownIcon,
+ CirclePlay,
Copy,
Loader2,
MessageSquareQuote,
Wrench,
} from 'lucide-react';
-import { useEffect, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
@@ -32,6 +33,7 @@ import {
CollapsibleTrigger,
} from '@/components/ui/collapsible.tsx';
import { Item, ItemContent } from '@/components/ui/item.tsx';
+import { useAudioBlock, useReplayController } from '@/context/AudioContext';
import { useTranslation } from '@/i18n/useI18n';
import { formatNumber, formatTime } from '@/utils/common';
@@ -144,6 +146,157 @@ function groupToolCalls(content: ContentBlock[]): ExtendedContentBlock[] {
return result;
}
+const AUDIO_WAVE_LINES: Array<{ x: number; y1: number; y2: number }> = [
+ { x: 2, y1: 10, y2: 13 },
+ { x: 6, y1: 6, y2: 17 },
+ { x: 10, y1: 3, y2: 21 },
+ { x: 14, y1: 8, y2: 15 },
+ { x: 18, y1: 5, y2: 18 },
+ { x: 22, y1: 10, y2: 13 },
+];
+
+function AudioWave({ isPlaying = true, className }: { isPlaying?: boolean; className?: string }) {
+ return (
+ <>
+ {isPlaying && (
+ <style>{`
+ @keyframes audioWave {
+ 0%, 100% { transform: scaleY(1); }
+ 50% { transform: scaleY(0.3); }
+ }
+ `}</style>
+ )}
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ width="24"
+ height="24"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ strokeWidth={2}
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ className={className}
+ >
+ {AUDIO_WAVE_LINES.map(({ x, y1, y2 }, i) => (
+ <line
+ key={x}
+ x1={x}
+ x2={x}
+ y1={y1}
+ y2={y2}
+ style={{
+ transformOrigin: `${x}px 12px`,
+ animation: isPlaying
+ ? `audioWave 0.8s ease-in-out ${i * 0.12}s infinite`
+ : 'none',
+ }}
+ />
+ ))}
+ </svg>
+ </>
+ );
+}
+
+/**
+ * Inline audio control rendered *inside* the time/usage Badge so the play
+ * icon visually merges into the same chip rather than floating as its own
+ * pill.
+ */
+function AudioInlineControl({ block }: { block: DataBlock }) {
+ const { t } = useTranslation();
+ const audioState = useAudioBlock(block.id);
+ const replayController = useReplayController();
+ const audioRef = useRef<HTMLAudioElement | null>(null);
+ const [isPlaying, setIsPlaying] = useState(false);
+
+ const isStreaming = audioState?.status === 'streaming';
+
+ // Don't build the giant base64 data URL while bytes are still streaming —
+ // it would re-allocate on every DATA_BLOCK_DELTA. Live playback during
+ // that window is handled by the manager's WavStreamPlayer; we only need
+ // `src` for replay after the stream ends (or for historical messages).
+ let src: string | null = null;
+ if (!isStreaming) {
+ if (audioState?.url) {
+ src = audioState.url;
+ } else if (block.source.type === 'url') {
+ src = block.source.url;
+ } else if (block.source.type === 'base64' && block.source.data) {
+ src = `data:${block.source.media_type};base64,${block.source.data}`;
+ }
+ }
+
+ // Reset the hidden <audio> when the source URL changes (e.g. streaming
+ // just transitioned to a Blob URL). Without an explicit load() some
+ // browsers keep the previous (or empty) source bound to the element.
+ useEffect(() => {
+ const el = audioRef.current;
+ if (!el || !src) return;
+ setIsPlaying(false);
+ el.load();
+ }, [src]);
+
+ // Pause when a newer reply interrupts this block's playback.
+ const interruptCount = audioState?.interruptCount ?? 0;
+ useEffect(() => {
+ if (interruptCount === 0) return;
+ const el = audioRef.current;
+ if (el && !el.paused) {
+ el.pause();
+ }
+ }, [interruptCount]);
+
+ if (isStreaming) {
+ return <AudioWave isPlaying className="ml-1" />;
+ }
+
+ if (!src) return null;
+
+ const toggle = async () => {
+ const el = audioRef.current;
+ if (!el) return;
+ if (el.paused) {
+ replayController?.play(el);
+ try {
+ await el.play();
+ } catch (err) {
+ console.error('Audio playback failed', err);
+ }
+ } else {
+ el.pause();
+ replayController?.stop();
+ }
+ };
+
+ return (
+ <>
+ <button
+ type="button"
+ onClick={toggle}
+ aria-label={
+ isPlaying ? t('messageBubble.pauseAudio') : t('messageBubble.playAudio')
+ }
+ className="ml-1 inline-flex cursor-pointer items-center transition-opacity hover:opacity-70"
+ >
+ {isPlaying ? (
+ <AudioWave isPlaying className="size-3" />
+ ) : (
+ <CirclePlay className="size-3" />
+ )}
+ </button>
+ <audio
+ ref={audioRef}
+ src={src}
+ preload="auto"
+ onPlay={() => setIsPlaying(true)}
+ onPause={() => setIsPlaying(false)}
+ onEnded={() => setIsPlaying(false)}
+ />
+ </>
+ );
+}
+
/**
* Render a single content block. Tool call groups are dispatched to
* `renderToolGroup`; the per-group truncation at the first `asking` call
@@ -237,6 +390,9 @@ function renderBlock(
case 'data': {
const dataType = block.source.media_type.split('/')[0];
+ // Audio data blocks render in the footer (see AudioFooterControl),
+ // not inline alongside text.
+ if (dataType === 'audio') return null;
let data: string;
if (block.source.type === 'url') {
data = block.source.url;
@@ -246,8 +402,6 @@ function renderBlock(
switch (dataType) {
case 'image':
return <img key={index} src={data} alt="Uploaded image" />;
- case 'audio':
- return <audio key={index} controls src={data} />;
case 'video':
return <video key={index} controls src={data} />;
}
@@ -358,7 +512,15 @@ export function MessageBubble({ message, onUserConfirm }: MessageBubbleProps) {
}, [isRunning]);
const blocks = groupToolCalls(message.content);
- const showBody = blocks.length > 0;
+ const audioBlocks = message.content.filter(
+ (b): b is DataBlock => b.type === 'data' && b.source.media_type.split('/')[0] === 'audio',
+ );
+ // Audio data blocks are rendered in the footer, so they shouldn't keep an
+ // otherwise-empty body bubble alive.
+ const hasBodyContent = blocks.some(
+ (b) => !(b.type === 'data' && b.source.media_type.split('/')[0] === 'audio'),
+ );
+ const showBody = hasBodyContent;
const showFooter = !isUser;
const startMs = new Date(message.created_at).getTime();
@@ -417,6 +579,9 @@ export function MessageBubble({ message, onUserConfirm }: MessageBubbleProps) {
</span>
</>
)}
+ {audioBlocks.map((block) => (
+ <AudioInlineControl key={block.id} block={block} />
+ ))}
</Badge>
</div>
)}
diff --git a/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx b/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx
index a72f55629..22f3783d3 100644
--- a/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx
+++ b/examples/web_ui/frontend/src/components/popover/ModelParametersPopover.tsx
@@ -1,4 +1,4 @@
-import { SlidersHorizontal } from 'lucide-react';
+import { ChevronDown, SlidersHorizontal } from 'lucide-react';
import { useCallback, useEffect, useState } from 'react';
import type { ChatModelConfig, ModelCard } from '@/api';
@@ -8,6 +8,8 @@ import {
DropdownMenuCheckboxItem,
DropdownMenuContent,
DropdownMenuLabel,
+ DropdownMenuRadioGroup,
+ DropdownMenuRadioItem,
DropdownMenuSeparator,
DropdownMenuSub,
DropdownMenuSubContent,
@@ -17,6 +19,7 @@ import {
import { Input } from '@/components/ui/input';
import { Label } from '@/components/ui/label';
import { Switch } from '@/components/ui/switch';
+import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
import { useAvailableModels } from '@/hooks/useAvailableModels';
import { useTranslation } from '@/i18n/useI18n';
@@ -29,7 +32,8 @@ interface ParameterProperty {
maximum?: number;
exclusiveMinimum?: number;
exclusiveMaximum?: number;
- anyOf?: Array<{ type: string }>;
+ enum?: unknown[];
+ anyOf?: ParameterProperty[];
}
interface ParameterSchema {
@@ -40,6 +44,155 @@ interface ParameterSchema {
required?: string[];
}
+interface ResolvedType {
+ type: string;
+ enumValues: unknown[] | null;
+}
+
+/** Resolve a property's effective scalar type and enum values, looking
+ * through ``anyOf`` and ignoring ``null`` variants. */
+function resolveType(prop: ParameterProperty): ResolvedType {
+ if (prop.type) {
+ return { type: prop.type, enumValues: prop.enum ?? null };
+ }
+ for (const variant of prop.anyOf ?? []) {
+ if (variant.type && variant.type !== 'null') {
+ return {
+ type: variant.type,
+ enumValues: variant.enum ?? prop.enum ?? null,
+ };
+ }
+ }
+ return { type: 'string', enumValues: null };
+}
+
+// ---------------------------------------------------------------------------
+// Field components
+// ---------------------------------------------------------------------------
+
+interface FieldProps {
+ id: string;
+ label: string;
+ required: boolean;
+ prop: ParameterProperty;
+ value: unknown;
+ onChange: (next: unknown) => void;
+}
+
+function BooleanField({ id, label, prop, value, onChange }: FieldProps) {
+ return (
+ <>
+ <Label htmlFor={id} className="whitespace-nowrap">
+ {label}
+ </Label>
+ <Switch
+ id={id}
+ checked={value !== undefined ? !!value : !!prop.default}
+ onCheckedChange={(checked) => onChange(!!checked)}
+ />
+ </>
+ );
+}
+
+function EnumField({ id, label, required, prop, value, onChange }: FieldProps) {
+ const enumValues = resolveType(prop).enumValues ?? [];
+ // TODO: experiment with using prop.description as placeholder text
+ const displayValue = value !== undefined && value !== null ? String(value) : '';
+
+ return (
+ <>
+ <Label htmlFor={id} className="whitespace-nowrap">
+ {label}
+ {required && <span className="text-destructive ml-0.5">*</span>}
+ </Label>
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild>
+ <Button id={id} variant="outline" className="w-full justify-between gap-1">
+ <span className="truncate">{displayValue}</span>
+ <ChevronDown className="size-3.5 opacity-50 shrink-0" />
+ </Button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent
+ align="start"
+ className="max-h-60 overflow-y-auto"
+ onPointerDown={(e) => e.stopPropagation()}
+ >
+ <DropdownMenuRadioGroup value={displayValue} onValueChange={(v) => onChange(v)}>
+ {enumValues.map((opt) => (
+ <DropdownMenuRadioItem key={String(opt)} value={String(opt)}>
+ {String(opt)}
+ </DropdownMenuRadioItem>
+ ))}
+ </DropdownMenuRadioGroup>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ </>
+ );
+}
+
+function NumberField({ id, label, required, prop, value, onChange }: FieldProps) {
+ const { type: effectiveType } = resolveType(prop);
+
+ return (
+ <>
+ <Label htmlFor={id} className="whitespace-nowrap">
+ {label}
+ {required && <span className="text-destructive ml-0.5">*</span>}
+ </Label>
+ <Input
+ id={id}
+ type="number"
+ value={value !== undefined ? String(value) : ''}
+ placeholder={prop.default != null ? String(prop.default) : undefined}
+ min={prop.minimum}
+ max={prop.maximum}
+ step={effectiveType === 'number' ? 'any' : undefined}
+ onChange={(e) => {
+ const raw = e.target.value;
+ onChange(raw === '' ? undefined : Number(raw));
+ }}
+ onBlur={(e) => {
+ if (e.target.value === '') return;
+ let num = Number(e.target.value);
+ if (prop.minimum !== undefined && num < prop.minimum) num = prop.minimum;
+ if (prop.maximum !== undefined && num > prop.maximum) num = prop.maximum;
+ if (prop.exclusiveMinimum !== undefined && num <= prop.exclusiveMinimum)
+ num =
+ prop.exclusiveMinimum +
+ (effectiveType === 'integer' ? 1 : Number.EPSILON);
+ if (prop.exclusiveMaximum !== undefined && num >= prop.exclusiveMaximum)
+ num =
+ prop.exclusiveMaximum -
+ (effectiveType === 'integer' ? 1 : Number.EPSILON);
+ if (num !== Number(e.target.value)) onChange(num);
+ }}
+ />
+ </>
+ );
+}
+
+function StringField({ id, label, required, prop, value, onChange }: FieldProps) {
+ return (
+ <>
+ <Label htmlFor={id} className="whitespace-nowrap">
+ {label}
+ {required && <span className="text-destructive ml-0.5">*</span>}
+ </Label>
+ <Input
+ id={id}
+ type="text"
+ value={value !== undefined ? String(value) : ''}
+ placeholder={prop.default != null ? String(prop.default) : undefined}
+ onChange={(e) => onChange(e.target.value)}
+ />
+ </>
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Main component
+// ---------------------------------------------------------------------------
+
interface Props {
/** Currently selected primary model — used to read the parameter schema. */
selectedModel: ChatModelConfig | null;
@@ -115,10 +268,6 @@ export function ModelParametersPopover({
</DropdownMenuTrigger>
<DropdownMenuContent align="start" className="min-w-40">
{/* ----- Fallback model selection ----- */}
- {/* The sub-trigger reflects the current fallback selection so the
- user can see the active model without drilling in. The label
- is truncated with an ellipsis to keep the trigger on one line
- when the model name is long. */}
<DropdownMenuSub>
<DropdownMenuSubTrigger>
<span className="truncate">
@@ -147,10 +296,6 @@ export function ModelParametersPopover({
selectedFallbackModel?.credential_id ===
credential.id &&
selectedFallbackModel?.model === m.name;
- // Checkbox-style indicator: a check appears next
- // to the active fallback. Toggling off the active
- // item clears the selection (mirrors the explicit
- // "No fallback" entry below).
return (
<DropdownMenuCheckboxItem
key={`${credential.id}-${m.name}`}
@@ -176,9 +321,6 @@ export function ModelParametersPopover({
))
)}
<DropdownMenuSeparator />
- {/* "No fallback" is checked exactly when no fallback is set,
- making the list behave like a single-select group. Clicking
- it while already checked is a no-op. */}
<DropdownMenuCheckboxItem
checked={!selectedFallbackModel}
onCheckedChange={(checked) => {
@@ -207,134 +349,51 @@ export function ModelParametersPopover({
{t('model-parameters.empty')}
</p>
) : (
- <div className="grid grid-cols-[auto_1fr] items-center gap-x-3 gap-y-3">
+ <div
+ className="grid grid-cols-[auto_1fr] items-center gap-x-3 gap-y-3"
+ onPointerDown={(e) => e.stopPropagation()}
+ onKeyDown={(e) => e.stopPropagation()}
+ >
{entries.map(([key, prop]) => {
- const effectiveType =
- prop.type ??
- prop.anyOf?.find((t) => t.type !== 'null')?.type ??
- 'string';
- const isBoolean = effectiveType === 'boolean';
- const isNumber =
- effectiveType === 'number' || effectiveType === 'integer';
+ const { type: effectiveType, enumValues } = resolveType(prop);
const label = prop.title ?? key;
const isRequired = required.includes(key);
+ const fieldProps: FieldProps = {
+ id: `param-${key}`,
+ label,
+ required: isRequired,
+ prop,
+ value: values[key],
+ onChange: (v) => handleChange(key, v),
+ };
- if (isBoolean) {
- return (
- <div
- key={key}
- className="contents"
- // Keep clicks/keys inside the form from
- // closing the dropdown menu.
- onPointerDown={(e) => e.stopPropagation()}
- onKeyDown={(e) => e.stopPropagation()}
- >
- <Label
- htmlFor={`param-${key}`}
- className="whitespace-nowrap"
- >
- {label}
- </Label>
- <Switch
- id={`param-${key}`}
- checked={
- values[key] !== undefined
- ? !!values[key]
- : !!prop.default
- }
- onCheckedChange={(checked) =>
- handleChange(key, !!checked)
- }
- />
- </div>
- );
+ let field: React.ReactNode;
+ if (effectiveType === 'boolean') {
+ field = <BooleanField {...fieldProps} />;
+ } else if (enumValues) {
+ field = <EnumField {...fieldProps} />;
+ } else if (
+ effectiveType === 'number' ||
+ effectiveType === 'integer'
+ ) {
+ field = <NumberField {...fieldProps} />;
+ } else {
+ field = <StringField {...fieldProps} />;
}
return (
- <div
- key={key}
- className="contents"
- onPointerDown={(e) => e.stopPropagation()}
- onKeyDown={(e) => e.stopPropagation()}
- >
- <Label
- htmlFor={`param-${key}`}
- className="whitespace-nowrap"
- >
- {label}
- {isRequired && (
- <span className="text-destructive ml-0.5">
- *
- </span>
- )}
- </Label>
- <Input
- id={`param-${key}`}
- type={isNumber ? 'number' : 'text'}
- value={
- values[key] !== undefined
- ? String(values[key])
- : ''
- }
- placeholder={
- prop.default !== undefined
- ? String(prop.default)
- : undefined
- }
- min={prop.minimum}
- max={prop.maximum}
- step={
- isNumber && effectiveType === 'number'
- ? 'any'
- : undefined
- }
- onChange={(e) => {
- const raw = e.target.value;
- if (isNumber) {
- handleChange(
- key,
- raw === '' ? undefined : Number(raw),
- );
- } else {
- handleChange(key, raw);
- }
- }}
- onBlur={(e) => {
- if (!isNumber || e.target.value === '') return;
- let num = Number(e.target.value);
- if (
- prop.minimum !== undefined &&
- num < prop.minimum
- )
- num = prop.minimum;
- if (
- prop.maximum !== undefined &&
- num > prop.maximum
- )
- num = prop.maximum;
- if (
- prop.exclusiveMinimum !== undefined &&
- num <= prop.exclusiveMinimum
- )
- num =
- prop.exclusiveMinimum +
- (effectiveType === 'integer'
- ? 1
- : Number.EPSILON);
- if (
- prop.exclusiveMaximum !== undefined &&
- num >= prop.exclusiveMaximum
- )
- num =
- prop.exclusiveMaximum -
- (effectiveType === 'integer'
- ? 1
- : Number.EPSILON);
- if (num !== Number(e.target.value))
- handleChange(key, num);
- }}
- />
- </div>
+ <Tooltip key={key}>
+ <TooltipTrigger asChild>
+ <div className="col-span-2 grid grid-cols-subgrid items-center">
+ {field}
+ </div>
+ </TooltipTrigger>
+ {prop.description && (
+ <TooltipContent side="left">
+ {prop.description}
+ </TooltipContent>
+ )}
+ </Tooltip>
);
})}
</div>
diff --git a/examples/web_ui/frontend/src/context/AudioContext.tsx b/examples/web_ui/frontend/src/context/AudioContext.tsx
new file mode 100644
index 000000000..041a4cddc
--- /dev/null
+++ b/examples/web_ui/frontend/src/context/AudioContext.tsx
@@ -0,0 +1,85 @@
+import { createContext, useContext, useEffect, useMemo, useRef, useSyncExternalStore } from 'react';
+import type { ReactNode } from 'react';
+
+import { StreamingAudioManager } from '@/utils/streamingAudio';
+import type { StreamingAudioState } from '@/utils/streamingAudio';
+
+const AudioContext = createContext<StreamingAudioManager | null>(null);
+
+interface ReplayController {
+ play: (el: HTMLAudioElement) => void;
+ stop: () => void;
+}
+
+const ReplayContext = createContext<ReplayController | null>(null);
+
+/**
+ * Provides a {@link StreamingAudioManager} to the component tree. The manager
+ * collects DATA_BLOCK_* events for audio blocks and exposes per-block state
+ * that ``MessageBubble`` consumes via {@link useAudioBlock}.
+ *
+ * Mount this once around any subtree that renders assistant messages.
+ */
+export function AudioProvider({ children }: { children: ReactNode }) {
+ const manager = useMemo(() => new StreamingAudioManager(), []);
+ useEffect(() => () => manager.disposeAll(), [manager]);
+
+ const currentRef = useRef<HTMLAudioElement | null>(null);
+ const replay: ReplayController = useMemo(
+ () => ({
+ play(el: HTMLAudioElement) {
+ if (currentRef.current && currentRef.current !== el) {
+ currentRef.current.pause();
+ currentRef.current.currentTime = 0;
+ }
+ currentRef.current = el;
+ },
+ stop() {
+ currentRef.current = null;
+ },
+ }),
+ [],
+ );
+
+ return (
+ <AudioContext.Provider value={manager}>
+ <ReplayContext.Provider value={replay}>{children}</ReplayContext.Provider>
+ </AudioContext.Provider>
+ );
+}
+
+/**
+ * Access the streaming audio manager. Returns ``null`` outside of an
+ * ``AudioProvider`` — callers must handle that, since audio handling is
+ * an optional feature.
+ */
+export function useAudioManager(): StreamingAudioManager | null {
+ return useContext(AudioContext);
+}
+
+/**
+ * Access the replay controller that ensures only one audio element plays
+ * at a time across all message bubbles.
+ */
+export function useReplayController(): ReplayController | null {
+ return useContext(ReplayContext);
+}
+
+/**
+ * Subscribe to the streaming state for a single audio DataBlock.
+ * Re-renders when the block transitions from ``streaming`` to ``ready``.
+ *
+ * Returns ``null`` if the block isn't being tracked (e.g. a historical
+ * message loaded from the server, where the bytes are already complete).
+ */
+export function useAudioBlock(blockId: string | undefined): StreamingAudioState | null {
+ const manager = useAudioManager();
+ return useSyncExternalStore(
+ (fn) => {
+ if (!manager || !blockId) return () => undefined;
+ return manager.subscribe(blockId, fn);
+ },
+ () => (manager && blockId ? manager.getState(blockId) : null),
+ () => null,
+ );
+}
diff --git a/examples/web_ui/frontend/src/hooks/useMessages.ts b/examples/web_ui/frontend/src/hooks/useMessages.ts
index 3b9c9b130..31727c20a 100644
--- a/examples/web_ui/frontend/src/hooks/useMessages.ts
+++ b/examples/web_ui/frontend/src/hooks/useMessages.ts
@@ -2,6 +2,9 @@ import { EventType } from '@agentscope-ai/agentscope/event';
import type {
AgentEvent,
CustomEvent,
+ DataBlockStartEvent,
+ DataBlockDeltaEvent,
+ DataBlockEndEvent,
ReplyStartEvent,
UserConfirmResultEvent,
} from '@agentscope-ai/agentscope/event';
@@ -12,6 +15,7 @@ import { useState, useCallback, useRef, useEffect } from 'react';
import { sessionApi } from '@/api';
import { chatApi } from '@/api';
+import { useAudioManager } from '@/context/AudioContext';
/**
* Manages messages for a single ``(agentId, sessionId)`` pair.
@@ -70,19 +74,12 @@ export function useMessages(
const abortRef = useRef<AbortController | null>(null);
const rafRef = useRef<number | null>(null);
- // Mirror `options` into a ref so `processEvent` (a single-creation
- // useCallback to keep the SSE stable) always reads the freshest
- // callbacks. Without this, the closure freezes the first render's
- // `options` — typically captured when `agentId` is still `null` —
- // so `team_updated` / `state_updated` events fire stale no-op
- // handlers, which in turn call `useSessions(null).refetch`
- // (`if (!agentId) setSessions([])`) and blank out the session list.
+ const audioManager = useAudioManager();
+
const optionsRef = useRef(options);
useEffect(() => {
optionsRef.current = options;
}, [options]);
-
- /** Batch UI updates via requestAnimationFrame. */
const scheduleUpdate = useCallback(() => {
if (rafRef.current !== null) return;
rafRef.current = requestAnimationFrame(() => {
@@ -106,6 +103,7 @@ export function useMessages(
return;
}
if (event.type === EventType.REPLY_START) {
+ audioManager?.stopAllPlayback();
const e = event as ReplyStartEvent;
const msg = AssistantMsg({ id: e.reply_id, name: e.name, content: [] });
msgsRef.current = [...msgsRef.current, msg];
@@ -120,9 +118,33 @@ export function useMessages(
} else if (currentReplyRef.current) {
appendEvent(currentReplyRef.current, event);
}
+
+ // Route streaming audio DataBlocks to the audio manager. They still
+ // flow through `appendEvent` above (which builds up `source.data`
+ // in the Msg), but MessageBubble reads playback state from the
+ // manager so it can show progress and autoplay on completion.
+ if (audioManager) {
+ if (event.type === EventType.DATA_BLOCK_START) {
+ const e = event as DataBlockStartEvent;
+ if (e.media_type.startsWith('audio/')) {
+ audioManager.start(e.block_id, e.media_type);
+ }
+ } else if (event.type === EventType.DATA_BLOCK_DELTA) {
+ const e = event as DataBlockDeltaEvent;
+ if (e.media_type.startsWith('audio/')) {
+ audioManager.append(e.block_id, e.data);
+ }
+ } else if (event.type === EventType.DATA_BLOCK_END) {
+ const e = event as DataBlockEndEvent;
+ // `end` is a no-op when the block isn't being tracked, so
+ // we can call it unconditionally.
+ audioManager.end(e.block_id);
+ }
+ }
+
scheduleUpdate();
},
- [scheduleUpdate],
+ [scheduleUpdate, audioManager],
);
// ── Lifecycle: fetch history + open SSE stream ──────────────────
@@ -132,6 +154,7 @@ export function useMessages(
setMsgs([]);
setError(null);
setStreaming(false);
+ audioManager?.disposeAll();
if (!agentId || !sessionId) return;
@@ -176,7 +199,7 @@ export function useMessages(
controller.abort();
abortRef.current = null;
};
- }, [agentId, sessionId, scheduleUpdate, processEvent]);
+ }, [agentId, sessionId, scheduleUpdate, processEvent, audioManager]);
/**
* Send a user message. Appends the message to the local list
diff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.json
index ce3ba649d..ba9899624 100644
--- a/examples/web_ui/frontend/src/i18n/locales/en.json
+++ b/examples/web_ui/frontend/src/i18n/locales/en.json
@@ -76,6 +76,9 @@
},
"messageBubble": {
"thinking": "Thinking",
+ "audioGenerating": "Generating audio…",
+ "playAudio": "Play audio",
+ "pauseAudio": "Pause audio",
"running": "Running...",
"hintSource": {
"tool_output": "Tool Output",
diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.json
index fee9dd65a..0fd0c2497 100644
--- a/examples/web_ui/frontend/src/i18n/locales/zh.json
+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json
@@ -76,6 +76,9 @@
},
"messageBubble": {
"thinking": "思考中",
+ "audioGenerating": "正在生成音频…",
+ "playAudio": "播放音频",
+ "pauseAudio": "暂停音频",
"running": "运行中...",
"hintSource": {
"tool_output": "工具输出",
diff --git a/examples/web_ui/frontend/src/pages/chat/index.tsx b/examples/web_ui/frontend/src/pages/chat/index.tsx
index cfa6cbe5b..e6fd177b0 100644
--- a/examples/web_ui/frontend/src/pages/chat/index.tsx
+++ b/examples/web_ui/frontend/src/pages/chat/index.tsx
@@ -55,6 +55,7 @@ import {
SidebarMenuButton,
SidebarMenuItem,
} from '@/components/ui/sidebar';
+import { AudioProvider } from '@/context/AudioContext';
import { useAgents } from '@/hooks/useAgents';
import { useSessions } from '@/hooks/useSessions';
import { useTranslation } from '@/i18n/useI18n.ts';
@@ -434,4 +435,8 @@ const ChatPageInner = () => {
);
};
-export const ChatPage = () => <ChatPageInner />;
+export const ChatPage = () => (
+ <AudioProvider>
+ <ChatPageInner />
+ </AudioProvider>
+);
diff --git a/examples/web_ui/frontend/src/utils/streamingAudio.ts b/examples/web_ui/frontend/src/utils/streamingAudio.ts
new file mode 100644
index 000000000..41deb2cb0
--- /dev/null
+++ b/examples/web_ui/frontend/src/utils/streamingAudio.ts
@@ -0,0 +1,368 @@
+/**
+ * Streaming audio playback support for assistant DataBlocks.
+ *
+ * Omni-style models (e.g. qwen3.5-omni-plus) deliver speech as a stream of
+ * DATA_BLOCK_DELTA events carrying base64 audio bytes for one logical
+ * DataBlock. This module routes those chunks to two playback paths:
+ *
+ * 1. **Live chunked playback** — only for audio/wav. We parse the RIFF
+ * header from the first chunk and schedule subsequent PCM bytes onto a
+ * Web Audio timeline as they arrive, so the assistant starts speaking
+ * the moment the first chunk arrives — without waiting for
+ * DATA_BLOCK_END, which the agent only emits after the *entire* model
+ * stream (including text) finishes.
+ *
+ * 2. **Buffered final playback** — for non-wav formats (mp3/opus/etc.) we
+ * accumulate bytes and, on DATA_BLOCK_END, build a Blob URL that backs
+ * autoplay + an ``<audio controls>`` element for replay/scrubbing.
+ *
+ * For wav, the controls element built on DATA_BLOCK_END is replay-only —
+ * we intentionally skip the second autoplay there to avoid colliding with
+ * the live player. ``WavStreamPlayer.finalize`` defers AudioContext
+ * teardown until queued PCM has finished playing, so live audio is never
+ * cut short by the end event arriving mid-playback.
+ */
+
+/** Status of a streaming audio block. */
+export type StreamingAudioStatus = 'streaming' | 'ready' | 'error';
+
+/** Per-block state surfaced to React. */
+export interface StreamingAudioState {
+ status: StreamingAudioStatus;
+ mediaType: string;
+ /** Object URL for ``<audio>`` once playback-ready. */
+ url: string | null;
+ /** Cause for ``status === 'error'``. */
+ error?: string;
+ /** Incremented each time a newer reply interrupts this block. */
+ interruptCount: number;
+}
+
+type Listener = () => void;
+
+type Bytes = Uint8Array<ArrayBufferLike>;
+
+function base64ToBytes(b64: string): Bytes {
+ const binary = atob(b64);
+ const bytes = new Uint8Array(binary.length);
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+ return bytes;
+}
+
+function concatBytes(a: Bytes, b: Bytes): Bytes {
+ const out = new Uint8Array(a.length + b.length);
+ out.set(a, 0);
+ out.set(b, a.length);
+ return out;
+}
+
+interface WavHeader {
+ sampleRate: number;
+ channels: number;
+ bitsPerSample: number;
+ /** Byte offset where PCM samples begin (right after the ``data`` chunk header). */
+ dataOffset: number;
+}
+
+/**
+ * Parse a minimal RIFF/WAVE header. Returns ``null`` if the buffer doesn't
+ * yet contain enough bytes to locate the ``fmt `` and ``data`` chunks.
+ */
+function parseWavHeader(bytes: Bytes): WavHeader | null {
+ if (bytes.length < 44) return null;
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+ // "RIFF"...."WAVE"
+ if (view.getUint32(0, false) !== 0x52494646 || view.getUint32(8, false) !== 0x57415645) {
+ return null;
+ }
+
+ let offset = 12;
+ let sampleRate = 0;
+ let channels = 0;
+ let bitsPerSample = 0;
+ let dataOffset = -1;
+
+ while (offset + 8 <= bytes.length) {
+ const chunkId = view.getUint32(offset, false);
+ const chunkSize = view.getUint32(offset + 4, true);
+ const chunkBody = offset + 8;
+
+ if (chunkId === 0x666d7420 /* "fmt " */) {
+ if (chunkBody + 16 > bytes.length) return null;
+ channels = view.getUint16(chunkBody + 2, true);
+ sampleRate = view.getUint32(chunkBody + 4, true);
+ bitsPerSample = view.getUint16(chunkBody + 14, true);
+ } else if (chunkId === 0x64617461 /* "data" */) {
+ dataOffset = chunkBody;
+ break;
+ }
+ offset = chunkBody + chunkSize;
+ }
+
+ if (!sampleRate || !channels || !bitsPerSample || dataOffset < 0) return null;
+ return { sampleRate, channels, bitsPerSample, dataOffset };
+}
+
+/**
+ * Convert raw little-endian PCM into a Float32 AudioBuffer.
+ * Supports 16-bit and 8-bit; falls back to silence for other depths.
+ */
+function pcmToAudioBuffer(ctx: AudioContext, pcm: Bytes, header: WavHeader): AudioBuffer | null {
+ const { channels, bitsPerSample, sampleRate } = header;
+ const bytesPerSample = bitsPerSample / 8;
+ const frameSize = bytesPerSample * channels;
+ const frameCount = Math.floor(pcm.length / frameSize);
+ if (frameCount === 0) return null;
+
+ const buffer = ctx.createBuffer(channels, frameCount, sampleRate);
+ const view = new DataView(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+
+ for (let ch = 0; ch < channels; ch++) {
+ const channelData = buffer.getChannelData(ch);
+ for (let i = 0; i < frameCount; i++) {
+ const sampleOffset = i * frameSize + ch * bytesPerSample;
+ if (bitsPerSample === 16) {
+ channelData[i] = view.getInt16(sampleOffset, true) / 0x8000;
+ } else if (bitsPerSample === 8) {
+ channelData[i] = (view.getUint8(sampleOffset) - 128) / 128;
+ } else {
+ channelData[i] = 0;
+ }
+ }
+ }
+ return buffer;
+}
+
+/** Live PCM playback for an in-progress WAV stream. */
+class WavStreamPlayer {
+ private ctx: AudioContext | null = null;
+ private header: WavHeader | null = null;
+ private pending: Bytes = new Uint8Array(0);
+ private nextStartTime = 0;
+ private failed = false;
+ private deferredCloseTimer: ReturnType<typeof setTimeout> | null = null;
+
+ append(chunk: Bytes): void {
+ if (this.failed) return;
+ this.pending = concatBytes(this.pending, chunk);
+
+ if (!this.header) {
+ const header = parseWavHeader(this.pending);
+ if (!header) return;
+ this.header = header;
+ try {
+ this.ctx = new AudioContext({ sampleRate: header.sampleRate });
+ if (this.ctx.state === 'suspended') {
+ void this.ctx.resume();
+ }
+ this.nextStartTime = this.ctx.currentTime;
+ } catch {
+ this.failed = true;
+ return;
+ }
+ this.pending = this.pending.slice(header.dataOffset);
+ }
+
+ if (!this.ctx || !this.header) return;
+
+ const frameSize = (this.header.bitsPerSample / 8) * this.header.channels;
+ const playableBytes = Math.floor(this.pending.length / frameSize) * frameSize;
+ if (playableBytes === 0) return;
+
+ const toPlay = this.pending.slice(0, playableBytes);
+ this.pending = this.pending.slice(playableBytes);
+
+ const audioBuffer = pcmToAudioBuffer(this.ctx, toPlay, this.header);
+ if (!audioBuffer) return;
+
+ const source = this.ctx.createBufferSource();
+ source.buffer = audioBuffer;
+ source.connect(this.ctx.destination);
+ const startAt = Math.max(this.nextStartTime, this.ctx.currentTime);
+ source.start(startAt);
+ this.nextStartTime = startAt + audioBuffer.duration;
+ }
+
+ /**
+ * Graceful end-of-stream: stop accepting more bytes, then close the
+ * AudioContext only after the last queued buffer has finished playing.
+ * Without this, ``end()`` calling ``dispose()`` would cut live playback
+ * short by closing the context while sources are still scheduled.
+ *
+ * The AudioContext reference is kept on ``this.ctx`` so that
+ * ``dispose()`` can still close it immediately if a newer reply
+ * interrupts playback before the timer fires.
+ */
+ finalize(): void {
+ if (!this.ctx || this.failed) {
+ this.dispose();
+ return;
+ }
+ const remainingMs = Math.max(0, this.nextStartTime - this.ctx.currentTime) * 1000;
+ this.deferredCloseTimer = setTimeout(() => {
+ this.dispose();
+ }, remainingMs + 200);
+ this.header = null;
+ this.pending = new Uint8Array(0);
+ }
+
+ dispose(): void {
+ if (this.deferredCloseTimer) {
+ clearTimeout(this.deferredCloseTimer);
+ this.deferredCloseTimer = null;
+ }
+ if (this.ctx) {
+ void this.ctx.close().catch(() => undefined);
+ this.ctx = null;
+ }
+ this.pending = new Uint8Array(0);
+ this.header = null;
+ }
+}
+
+interface Session {
+ mediaType: string;
+ chunks: Bytes[];
+ totalBytes: number;
+ livePlayer: WavStreamPlayer | null;
+ state: StreamingAudioState;
+}
+
+/**
+ * Manages the lifecycle of streaming audio DataBlocks for one chat session.
+ * React components subscribe via {@link subscribe} to re-render when a block's
+ * state changes.
+ */
+export class StreamingAudioManager {
+ private sessions = new Map<string, Session>();
+ private listeners = new Map<string, Set<Listener>>();
+
+ private emit(blockId: string): void {
+ this.listeners.get(blockId)?.forEach((fn) => fn());
+ }
+
+ subscribe(blockId: string, fn: Listener): () => void {
+ let set = this.listeners.get(blockId);
+ if (!set) {
+ set = new Set();
+ this.listeners.set(blockId, set);
+ }
+ set.add(fn);
+ return () => {
+ set?.delete(fn);
+ if (set && set.size === 0) this.listeners.delete(blockId);
+ };
+ }
+
+ getState(blockId: string): StreamingAudioState | null {
+ return this.sessions.get(blockId)?.state ?? null;
+ }
+
+ /** Called on DATA_BLOCK_START for an audio block. */
+ start(blockId: string, mediaType: string): void {
+ if (this.sessions.has(blockId)) return;
+ const livePlayer = mediaType === 'audio/wav' ? new WavStreamPlayer() : null;
+ this.sessions.set(blockId, {
+ mediaType,
+ chunks: [],
+ totalBytes: 0,
+ livePlayer,
+ state: { status: 'streaming', mediaType, url: null, interruptCount: 0 },
+ });
+ this.emit(blockId);
+ }
+
+ /** Called on DATA_BLOCK_DELTA. ``data`` is the base64 chunk payload. */
+ append(blockId: string, data: string): void {
+ const session = this.sessions.get(blockId);
+ if (!session || !data) return;
+ let bytes: Uint8Array;
+ try {
+ bytes = base64ToBytes(data);
+ } catch {
+ return;
+ }
+ session.chunks.push(bytes);
+ session.totalBytes += bytes.length;
+ session.livePlayer?.append(bytes);
+ // No emit — partial bytes don't change the rendered UI yet.
+ }
+
+ /**
+ * Called on DATA_BLOCK_END. Builds the replay URL and (for non-wav)
+ * autoplays. For wav we skip autoplay because ``WavStreamPlayer`` has
+ * already been playing the bytes as they arrived; a second ``Audio``
+ * playback here would either double up or — given ``end`` fires only
+ * after the whole model stream (text included) finishes — make the
+ * user perceive audio as starting only after text streaming completes.
+ */
+ end(blockId: string): void {
+ const session = this.sessions.get(blockId);
+ if (!session) return;
+
+ const hadLivePlayback = session.livePlayer !== null;
+ if (session.livePlayer) {
+ // Let queued PCM finish playing; AudioContext closes via a
+ // deferred timer. The livePlayer reference is kept so that
+ // stopAllPlayback() can dispose() it immediately if a newer
+ // reply starts before the timer fires.
+ session.livePlayer.finalize();
+ }
+
+ const blob = new Blob(session.chunks as BlobPart[], { type: session.mediaType });
+ const url = URL.createObjectURL(blob);
+
+ // Free the per-chunk buffers; the Blob owns the bytes from here on.
+ session.chunks = [];
+
+ session.state = {
+ status: 'ready',
+ mediaType: session.mediaType,
+ url,
+ interruptCount: session.state.interruptCount,
+ };
+ this.emit(blockId);
+
+ if (!hadLivePlayback) {
+ // Non-wav: no live playback path, so kick off autoplay now. Some
+ // browsers gate this behind a user gesture; if blocked, the user
+ // can still hit play on the <audio controls> element.
+ const el = new Audio(url);
+ void el.play().catch(() => undefined);
+ }
+ }
+
+ /**
+ * Stop all in-progress and replay audio. Called when a new reply
+ * starts so previous audio doesn't overlap with the new one.
+ * Blob URLs are preserved so the user can still manually replay.
+ */
+ stopAllPlayback(): void {
+ for (const [blockId, session] of this.sessions) {
+ if (session.livePlayer) {
+ session.livePlayer.dispose();
+ session.livePlayer = null;
+ }
+ session.state = {
+ ...session.state,
+ interruptCount: session.state.interruptCount + 1,
+ };
+ this.emit(blockId);
+ }
+ }
+
+ /** Release a single block's resources (Object URL etc.). */
+ dispose(blockId: string): void {
+ const session = this.sessions.get(blockId);
+ if (!session) return;
+ session.livePlayer?.dispose();
+ if (session.state.url) URL.revokeObjectURL(session.state.url);
+ this.sessions.delete(blockId);
+ this.listeners.delete(blockId);
+ }
+
+ /** Release every block — call on unmount or session switch. */
+ disposeAll(): void {
+ for (const id of [...this.sessions.keys()]) this.dispose(id);
+ }
+}
diff --git a/examples/web_ui/pnpm-lock.yaml b/examples/web_ui/pnpm-lock.yaml
index aa04b72a7..f895ec6e6 100644
--- a/examples/web_ui/pnpm-lock.yaml
+++ b/examples/web_ui/pnpm-lock.yaml
@@ -58,8 +58,8 @@ importers:
frontend:
dependencies:
'@agentscope-ai/agentscope':
- specifier: ^0.0.8
- version: 0.0.8
+ specifier: ^0.0.9
+ version: 0.0.9
'@fontsource-variable/geist':
specifier: ^5.2.8
version: 5.2.8
@@ -123,9 +123,6 @@ importers:
react-router-dom:
specifier: ^7.15.1
- rehype-raw:
- specifier: ^7.0.0
- version: 7.0.0
remark-gfm:
specifier: ^4.0.1
version: 4.0.1
@@ -202,8 +199,8 @@ importers:
packages:
- '@agentscope-ai/[email protected]':
- resolution: {integrity: sha512-Z0hcvCekvsbOxhEkgrT5+epMImfiUrt5cdeY8aGt73MWrSVxg0opErk6KrPVd1/WK+zPs5yIvhuaZ/sslJRFxQ==}
+ '@agentscope-ai/[email protected]':
+ resolution: {integrity: sha512-FuELfct2hovIINATNPWEpYktyqe1SYoVb9TCJvQoXqQcj5upik0VJM+8+OBA2y79xWe+rTBO1lr0bhuSGcHrPA==}
resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==}
@@ -2491,10 +2488,6 @@ packages:
resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
engines: {node: '>=0.12'}
- resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==}
- engines: {node: '>=0.12'}
-
resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==}
engines: {node: '>=6'}
@@ -2885,27 +2878,12 @@ packages:
resolution: {integrity: sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==}
engines: {node: '>= 0.4'}
- resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==}
-
- resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==}
-
- resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==}
-
resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==}
- resolution: {integrity: sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==}
-
resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==}
- resolution: {integrity: sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==}
-
resolution: {integrity: sha512-1TJ6Fih/b8h5TIcv+1+Hw0PDQWJTKDKzFZzcKOiW1wJza3XoAQlkCuXLbymPYB8+ZQyw8mHvdw560e8zVFIWyA==}
@@ -2925,9 +2903,6 @@ packages:
resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==}
- resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}
-
resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==}
engines: {node: '>= 0.8'}
@@ -3703,9 +3678,6 @@ packages:
resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==}
engines: {node: '>=18'}
- resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==}
-
resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==}
engines: {node: '>= 0.8'}
@@ -3936,9 +3908,6 @@ packages:
resolution: {integrity: sha512-YTUo+Flmw4ZXiWfQKGcwwc11KnoRAYgzAE2E7mXKCjSviTKShtxBsN6YUUBB2gtaBzKzeKunxhUwNHQuRryhWA==}
engines: {node: '>= 4'}
- resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==}
-
resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==}
@@ -4473,9 +4442,6 @@ packages:
react: ^16.8 || ^17.0 || ^18.0 || ^19.0.0 || ^19.0.0-rc
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0.0 || ^19.0.0-rc
- resolution: {integrity: sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==}
-
resolution: {integrity: sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==}
@@ -4534,9 +4500,6 @@ packages:
resolution: {integrity: sha512-Dhxzh5HZuiHQhbvTW9AMetFfBHDMYpo23Uo9btPXgdYP+3T5S+p+jgNy7spra+veYhBP2dCSgxR/i2Y02h5/6w==}
engines: {node: '>=0.10.0'}
- resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==}
-
resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
engines: {node: '>= 8'}
@@ -4635,7 +4598,7 @@ packages:
snapshots:
- '@agentscope-ai/[email protected]':
+ '@agentscope-ai/[email protected]':
dependencies:
'@cfworker/json-schema': 4.1.1
'@modelcontextprotocol/sdk': 1.29.0(@cfworker/[email protected])([email protected])
@@ -6866,8 +6829,6 @@ snapshots:
-
@@ -7326,37 +7287,6 @@ snapshots:
dependencies:
function-bind: 1.1.2
- dependencies:
- '@types/hast': 3.0.4
- '@types/unist': 3.0.3
- devlop: 1.1.0
- hastscript: 9.0.1
- property-information: 7.1.0
- vfile: 6.0.3
- vfile-location: 5.0.3
- web-namespaces: 2.0.1
-
- dependencies:
- '@types/hast': 3.0.4
-
- dependencies:
- '@types/hast': 3.0.4
- '@types/unist': 3.0.3
- '@ungap/structured-clone': 1.3.1
- hast-util-from-parse5: 8.0.3
- hast-util-to-parse5: 8.0.1
- html-void-elements: 3.0.0
- mdast-util-to-hast: 13.2.1
- parse5: 7.3.0
- unist-util-position: 5.0.0
- unist-util-visit: 5.1.0
- vfile: 6.0.3
- web-namespaces: 2.0.1
- zwitch: 2.0.4
-
dependencies:
'@types/estree': 1.0.9
@@ -7377,28 +7307,10 @@ snapshots:
transitivePeerDependencies:
- supports-color
- dependencies:
- '@types/hast': 3.0.4
- comma-separated-tokens: 2.0.3
- devlop: 1.1.0
- property-information: 7.1.0
- space-separated-tokens: 2.0.2
- web-namespaces: 2.0.1
- zwitch: 2.0.4
-
dependencies:
'@types/hast': 3.0.4
- dependencies:
- '@types/hast': 3.0.4
- comma-separated-tokens: 2.0.3
- hast-util-parse-selector: 4.0.0
- property-information: 7.1.0
- space-separated-tokens: 2.0.2
-
dependencies:
'@types/set-cookie-parser': 2.4.10
@@ -7418,8 +7330,6 @@ snapshots:
-
dependencies:
depd: 2.0.0
@@ -8319,10 +8229,6 @@ snapshots:
- dependencies:
- entities: 6.0.1
-
@@ -8583,12 +8489,6 @@ snapshots:
tiny-invariant: 1.3.3
tslib: 2.8.1
- dependencies:
- '@types/hast': 3.0.4
- hast-util-raw: 9.1.0
- vfile: 6.0.3
-
dependencies:
'@types/mdast': 4.0.4
@@ -9234,11 +9134,6 @@ snapshots:
- '@types/react'
- '@types/react-dom'
- dependencies:
- '@types/unist': 3.0.3
- vfile: 6.0.3
-
dependencies:
'@types/unist': 3.0.3
@@ -9275,8 +9170,6 @@ snapshots:
-
diff --git a/scripts/model_examples/_utils.py b/scripts/model_examples/_utils.py
index 534d0a6d5..ad79692c6 100644
--- a/scripts/model_examples/_utils.py
+++ b/scripts/model_examples/_utils.py
@@ -1,8 +1,14 @@
# -*- coding: utf-8 -*-
"""Shared utility helpers for model-examples scripts."""
+import base64 as _b64
from typing import AsyncGenerator
-from agentscope.message import TextBlock, ThinkingBlock, DataBlock
+from agentscope.message import (
+ TextBlock,
+ ThinkingBlock,
+ DataBlock,
+ Base64Source,
+)
from agentscope.model import ChatResponse
@@ -16,10 +22,18 @@ async def stream_and_collect(
from it without re-printing the entire content. Text from the final chunk
is printed only when no text was streamed in any delta chunk (e.g. some
models batch the answer in the last chunk).
+
+ Streaming ``DataBlock`` chunks (e.g. omni audio output) are reported as
+ per-chunk size summaries while the stream is being consumed, so callers
+ can see the audio arriving incrementally rather than only in the final
+ cumulative chunk.
"""
final: ChatResponse | None = None
in_thinking = False
text_was_streamed = False
+ # Track per-block audio progress: block_id -> (media_type, chunk_count,
+ # total_bytes).
+ audio_progress: dict[str, list] = {}
async for chunk in gen:
if chunk.is_last:
final = chunk
@@ -37,6 +51,27 @@ async def stream_and_collect(
in_thinking = False
print(block.text, end="", flush=True)
text_was_streamed = True
+ elif isinstance(block, DataBlock) and isinstance(
+ block.source,
+ Base64Source,
+ ):
+ # Streaming binary delta (e.g. omni audio output).
+ if in_thinking:
+ print()
+ print("--- Answer ---")
+ in_thinking = False
+ delta_bytes = len(_b64.b64decode(block.source.data))
+ state = audio_progress.setdefault(
+ block.id,
+ [block.source.media_type, 0, 0],
+ )
+ state[1] += 1
+ state[2] += delta_bytes
+ print(
+ f"\n[Audio chunk #{state[1]} ({state[0]}): "
+ f"+{delta_bytes}B, total={state[2]}B]",
+ flush=True,
+ )
# If text was not streamed in any delta, print it from the final chunk now.
if not text_was_streamed and final is not None:
final_text = "".join(
@@ -52,16 +87,17 @@ async def stream_and_collect(
print(final_text)
if in_thinking:
print()
- # Report audio output if present in the final chunk
+ # Report cumulative audio output from the final chunk.
if final is not None:
- import base64 as _b64
-
for block in final.content:
- if isinstance(block, DataBlock):
+ if isinstance(block, DataBlock) and isinstance(
+ block.source,
+ Base64Source,
+ ):
media = block.source.media_type
byte_size = len(_b64.b64decode(block.source.data))
print(
- f"[Audio output: {media}, " f"{byte_size} bytes]",
+ f"[Audio output (cumulative): {media}, {byte_size} bytes]",
)
print()
assert final is not None
diff --git a/src/agentscope/_utils/_audio.py b/src/agentscope/_utils/_audio.py
new file mode 100644
index 000000000..b93b5cbf6
--- /dev/null
+++ b/src/agentscope/_utils/_audio.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""Audio utilities shared across model providers."""
+import struct
+
+
+def _build_streaming_wav_header(
+ sample_rate: int = 24000,
+ channels: int = 1,
+ bits_per_sample: int = 16,
+) -> bytes:
+ """Build a 44-byte WAV/RIFF header for streaming PCM.
+
+ The RIFF and ``data`` chunk sizes are set to ``0xFFFFFFFF`` since the
+ total length isn't known yet. Decoders that only need sample-rate,
+ channel count and bit depth (e.g. the web ``WavStreamPlayer``) treat
+ everything after the ``data`` chunk header as PCM, so this is
+ sufficient for live decoding of an open-ended stream.
+
+ Both DashScope omni and OpenAI streaming deliver raw PCM upstream;
+ prefixing the first chunk with this header lets the frontend start
+ playback immediately without buffering the whole response.
+ """
+ byte_rate = sample_rate * channels * bits_per_sample // 8
+ block_align = channels * bits_per_sample // 8
+ return (
+ b"RIFF"
+ + struct.pack("<I", 0xFFFFFFFF)
+ + b"WAVE"
+ + b"fmt "
+ + struct.pack("<I", 16)
+ + struct.pack(
+ "<HHIIHH",
+ 1,
+ channels,
+ sample_rate,
+ byte_rate,
+ block_align,
+ bits_per_sample,
+ )
+ + b"data"
+ + struct.pack("<I", 0xFFFFFFFF)
+ )
diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.py
index 9bbbd2b0f..8258fce80 100644
--- a/src/agentscope/agent/_agent.py
+++ b/src/agentscope/agent/_agent.py
@@ -790,7 +790,12 @@ class Agent:
**kwargs,
)
- block_ids: dict = {"text": None, "thinking": None, "tools": []}
+ block_ids: dict = {
+ "text": None,
+ "thinking": None,
+ "tools": [],
+ "data": [],
+ }
completed_response: ChatResponse | None = None
# Check if res is an async generator (streaming response)
@@ -832,6 +837,11 @@ class Agent:
reply_id=self.state.reply_id,
tool_call_id=tool_call_id,
)
+ for data_block_id in block_ids["data"]:
+ yield DataBlockEndEvent(
+ reply_id=self.state.reply_id,
+ block_id=data_block_id,
+ )
# Send the model call ended event with usage if available
yield ModelCallEndEvent(
@@ -2197,12 +2207,29 @@ class Agent:
else None
)
+ # Assistant-produced audio (e.g. qwen-omni speaking aloud) is delivered
+ # to the user via streaming events; the raw bytes don't belong in
+ # conversation memory. Filtering here keeps every downstream walker
+ # (formatter, count_tokens, persistence) honest without each having
+ # to remember.
+ persisted_blocks = [
+ b
+ for b in blocks
+ if not (
+ isinstance(b, DataBlock)
+ and isinstance(b.source, (Base64Source, URLSource))
+ and b.source.media_type.startswith("audio/")
+ )
+ ]
+ if not persisted_blocks and msg_usage is None:
+ return
+
if len(self.state.context) == 0:
self.state.context.append(
AssistantMsg(
id=self.state.reply_id,
name=self.name,
- content=list(blocks),
+ content=persisted_blocks,
usage=msg_usage,
),
)
@@ -2211,7 +2238,7 @@ class Agent:
if last_msg.role == "assistant" and last_msg.name == self.name:
if isinstance(last_msg.content, str):
last_msg.content = [TextBlock(text=last_msg.content)]
- last_msg.content.extend(blocks)
+ last_msg.content.extend(persisted_blocks)
if msg_usage is not None:
if last_msg.usage is None:
last_msg.usage = msg_usage
@@ -2223,7 +2250,7 @@ class Agent:
AssistantMsg(
id=self.state.reply_id,
name=self.name,
- content=list(blocks),
+ content=persisted_blocks,
usage=msg_usage,
),
)
@@ -2384,6 +2411,7 @@ class Agent:
# Classify the content blocks into different types
text_blocks, thinking_blocks, tool_call_blocks = [], [], []
+ data_blocks: list = []
for block in chunk.content:
if isinstance(block, TextBlock):
text_blocks.append(block)
@@ -2391,8 +2419,17 @@ class Agent:
thinking_blocks.append(block)
elif isinstance(block, ToolCallBlock):
tool_call_blocks.append(block)
-
- # Handle the text blocks
+ elif isinstance(block, DataBlock):
+ data_blocks.append(block)
+
+ # Handle the text blocks. We only auto-close the open text block
+ # when the current chunk has neither text NOR data — a chunk that
+ # carries only data (e.g. an omni-style audio PCM delta arriving
+ # between two text deltas) must keep the text stream alive so the
+ # frontend doesn't fragment one logical text stream into many
+ # separate bubbles. A chunk with tool calls (and no text/data)
+ # still closes text, which preserves text → tool → text render
+ # order via distinct text blocks.
if text_blocks:
# If the current chunk has text blocks but no text block id,
# start with a start event
@@ -2409,14 +2446,15 @@ class Agent:
delta="".join([_.text for _ in text_blocks]),
)
- elif block_ids.get("text"):
+ elif block_ids.get("text") and not data_blocks:
yield TextBlockEndEvent(
reply_id=self.state.reply_id,
block_id=block_ids["text"],
)
block_ids["text"] = None
- # Handle the thinking blocks
+ # Same reasoning as the text block above — keep the thinking
+ # stream open across data-only chunks.
if thinking_blocks:
# Generate a new thinking block id and start event
if not block_ids.get("thinking"):
@@ -2432,7 +2470,7 @@ class Agent:
delta="".join([_.thinking for _ in thinking_blocks]),
)
- elif block_ids.get("thinking"):
+ elif block_ids.get("thinking") and not data_blocks:
yield ThinkingBlockEndEvent(
reply_id=self.state.reply_id,
block_id=block_ids["thinking"],
@@ -2467,6 +2505,29 @@ class Agent:
)
block_ids["tools"].remove(finished_id)
+ # Handle the data blocks (streaming binary content, e.g. omni audio).
+ # Each DataBlock chunk from the model carries a delta payload with a
+ # stable block id; we open a stream the first time we see an id and
+ # emit delta events for subsequent chunks with the same id.
+ for data_block in data_blocks:
+ if not isinstance(data_block.source, Base64Source):
+ # Only Base64Source carries inline delta bytes; URLSource is
+ # one-shot and not part of the streaming protocol.
+ continue
+ if data_block.id not in block_ids["data"]:
+ block_ids["data"].append(data_block.id)
+ yield DataBlockStartEvent(
+ reply_id=self.state.reply_id,
+ block_id=data_block.id,
+ media_type=data_block.source.media_type,
+ )
+ yield DataBlockDeltaEvent(
+ reply_id=self.state.reply_id,
+ block_id=data_block.id,
+ data=data_block.source.data,
+ media_type=data_block.source.media_type,
+ )
+
async def _convert_tool_chunk_to_event(
self,
tool_call_id: str,
diff --git a/src/agentscope/app/_router/_session.py b/src/agentscope/app/_router/_session.py
index e8846f737..56a3e1c0c 100644
--- a/src/agentscope/app/_router/_session.py
+++ b/src/agentscope/app/_router/_session.py
@@ -350,7 +350,9 @@ async def update_session(
return await storage.upsert_session(
user_id=user_id,
agent_id=agent_id,
- config=existing.config.model_copy(update=config_updates),
+ config=SessionConfig.model_validate(
+ {**existing.config.model_dump(mode="json"), **config_updates},
+ ),
state=updated_state,
session_id=session_id,
)
diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.py
index bfe431623..dcd0a1be4 100644
--- a/src/agentscope/formatter/_dashscope_formatter.py
+++ b/src/agentscope/formatter/_dashscope_formatter.py
@@ -63,7 +63,6 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
def _format_dashscope_data_block(
self,
block: DataBlock,
- role: str = "user",
) -> dict[str, Any] | None:
"""Format a DataBlock into the OpenAI-compatible format for
DashScope API.
@@ -76,10 +75,6 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
Args:
block (`DataBlock`):
The DataBlock to format.
- role (`str`, defaults to ``"user"``):
- The role of the message containing this block. Audio blocks
- in assistant messages are skipped to avoid errors in
- subsequent model calls.
Returns:
`dict[str, Any] | None`:
@@ -107,8 +102,6 @@ class _DashScopeFormatterBase(FormatterBase, ABC):
return self._format_video_source(block.source)
if main_type == "audio":
- if role == "assistant":
- return None
return self._format_audio_source(block.source)
logger.warning(
@@ -263,7 +256,6 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):
elif isinstance(block, DataBlock):
formatted_block = self._format_dashscope_data_block(
block,
- role=msg.role,
)
if formatted_block:
content_blocks.append(formatted_block)
@@ -305,7 +297,6 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):
formatted_sub = (
self._format_dashscope_data_block(
sub,
- role="user",
)
)
if formatted_sub:
@@ -372,7 +363,6 @@ class DashScopeChatFormatter(_DashScopeFormatterBase):
elif isinstance(item, DataBlock):
fmt_item = self._format_dashscope_data_block(
item,
- role="user",
)
if fmt_item is not None:
promo_content.append(fmt_item)
@@ -519,7 +509,6 @@ class DashScopeMultiAgentFormatter(_DashScopeFormatterBase):
elif isinstance(block, DataBlock):
formatted_block = self._format_dashscope_data_block(
block,
- role=msg.role,
)
if formatted_block is not None:
media_blocks.append(formatted_block)
diff --git a/src/agentscope/formatter/_moonshot_formatter.py b/src/agentscope/formatter/_moonshot_formatter.py
index 075c012c4..7d05b51b8 100644
--- a/src/agentscope/formatter/_moonshot_formatter.py
+++ b/src/agentscope/formatter/_moonshot_formatter.py
@@ -119,10 +119,7 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):
content_blocks.append({"type": "text", "text": block.text})
elif isinstance(block, DataBlock):
- formatted = self._format_openai_data_block(
- block,
- role=msg.role,
- )
+ formatted = self._format_openai_data_block(block)
if formatted is not None:
content_blocks.append(formatted)
@@ -165,7 +162,6 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):
elif isinstance(sub, DataBlock):
formatted_sub = self._format_openai_data_block(
sub,
- role="user",
)
if formatted_sub is not None:
hint_parts.append(formatted_sub)
@@ -230,7 +226,6 @@ class MoonshotChatFormatter(_OpenAIFormatterBase):
elif isinstance(item, DataBlock):
fmt_item = self._format_openai_data_block(
item,
- role="user",
)
if fmt_item is not None:
promo_content.append(fmt_item)
@@ -390,10 +385,7 @@ class MoonshotMultiAgentFormatter(_OpenAIFormatterBase):
if isinstance(block, TextBlock):
accumulated_text.append(f"{msg.name}: {block.text}")
elif isinstance(block, DataBlock):
- formatted = self._format_openai_data_block(
- block,
- role=msg.role,
- )
+ formatted = self._format_openai_data_block(block)
if formatted is not None:
media_blocks.append(formatted)
diff --git a/src/agentscope/formatter/_openai_formatter.py b/src/agentscope/formatter/_openai_formatter.py
index 0e842b9ab..245fc62dc 100644
--- a/src/agentscope/formatter/_openai_formatter.py
+++ b/src/agentscope/formatter/_openai_formatter.py
@@ -31,7 +31,6 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
def _format_openai_data_block(
self,
block: DataBlock,
- role: str = "user",
) -> dict[str, Any] | None:
"""Format a DataBlock into the required format for OpenAI API.
@@ -42,10 +41,6 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
Args:
block (`DataBlock`):
The DataBlock to format.
- role (`str`, defaults to ``"user"``):
- The role of the message that contains this block. Audio blocks
- in assistant messages are skipped to avoid errors in subsequent
- model calls.
Returns:
`dict[str, Any] | None`:
@@ -70,10 +65,6 @@ class _OpenAIFormatterBase(FormatterBase, ABC):
return self._format_image_source(block.source)
if main_type == "audio":
- # Filter out audio content when the multimodal model outputs both
- # text and audio, to prevent errors in subsequent model calls
- if role == "assistant":
- return None
return self._format_audio_source(block.source)
logger.warning(
@@ -240,7 +231,6 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):
elif isinstance(block, DataBlock):
formatted = self._format_openai_data_block(
block,
- role=msg.role,
)
if formatted is not None:
content_blocks.append(formatted)
@@ -277,7 +267,6 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):
elif isinstance(sub, DataBlock):
formatted_sub = self._format_openai_data_block(
sub,
- role="user",
)
if formatted_sub is not None:
hint_parts.append(formatted_sub)
@@ -335,7 +324,6 @@ class OpenAIChatFormatter(_OpenAIFormatterBase):
elif isinstance(item, DataBlock):
fmt_item = self._format_openai_data_block(
item,
- role="user",
)
if fmt_item is not None:
promo_content.append(fmt_item)
@@ -471,7 +459,6 @@ class OpenAIMultiAgentFormatter(_OpenAIFormatterBase):
elif isinstance(block, DataBlock):
formatted = self._format_openai_data_block(
block,
- role=msg.role,
)
if formatted is not None:
media_blocks.append(formatted)
diff --git a/src/agentscope/formatter/_openai_response_formatter.py b/src/agentscope/formatter/_openai_response_formatter.py
index 283f1188c..d44d64f9c 100644
--- a/src/agentscope/formatter/_openai_response_formatter.py
+++ b/src/agentscope/formatter/_openai_response_formatter.py
@@ -38,7 +38,6 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):
def _format_response_data_block(
self,
block: DataBlock,
- role: str = "user",
) -> dict[str, Any] | None:
"""Format a DataBlock into the Response API format.
@@ -53,8 +52,6 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):
Args:
block (`DataBlock`):
The DataBlock to format.
- role (`str`, defaults to ``"user"``):
- The role of the message that contains this block.
Returns:
`dict[str, Any] | None`:
@@ -75,7 +72,7 @@ class _OpenAIResponseFormatterBase(_OpenAIFormatterBase, ABC):
)
return None
- base_result = self._format_openai_data_block(block, role)
+ base_result = self._format_openai_data_block(block)
if base_result is None:
return None
@@ -134,10 +131,7 @@ class OpenAIResponseFormatter(_OpenAIResponseFormatterBase):
)
elif isinstance(block, DataBlock):
- formatted = self._format_response_data_block(
- block,
- role=msg.role,
- )
+ formatted = self._format_response_data_block(block)
if formatted is not None:
content_parts.append(formatted)
@@ -188,7 +182,6 @@ class OpenAIResponseFormatter(_OpenAIResponseFormatterBase):
formatted_sub = (
self._format_response_data_block(
sub,
- role="user",
)
)
if formatted_sub is not None:
@@ -302,7 +295,6 @@ class OpenAIResponseFormatter(_OpenAIResponseFormatterBase):
elif isinstance(item, DataBlock):
fmt_item = self._format_response_data_block(
item,
- role="user",
)
if fmt_item is not None:
promo_content.append(fmt_item)
@@ -452,10 +444,7 @@ class OpenAIResponseMultiAgentFormatter(_OpenAIResponseFormatterBase):
if isinstance(block, TextBlock):
accumulated_text.append(f"{msg.name}: {block.text}")
elif isinstance(block, DataBlock):
- formatted = self._format_response_data_block(
- block,
- role=msg.role,
- )
+ formatted = self._format_response_data_block(block)
if formatted is not None:
media_blocks.append(formatted)
diff --git a/src/agentscope/message/_base.py b/src/agentscope/message/_base.py
index 6ef60edc6..6dd2d08b1 100644
--- a/src/agentscope/message/_base.py
+++ b/src/agentscope/message/_base.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
"""The message class in agentscope."""
+import base64
import uuid
from datetime import datetime
from typing import Literal, List, overload, Sequence, Self, TYPE_CHECKING, Any
@@ -283,7 +284,18 @@ class Msg(BaseModel):
event.block_id,
)
elif event.data:
- block.source.data += event.data
+ # Each delta is an independently base64-encoded chunk
+ # (with its own padding); naive string concat would
+ # corrupt the byte stream. Decode, concat bytes, re-encode.
+ existing = (
+ base64.b64decode(block.source.data)
+ if block.source.data
+ else b""
+ )
+ incoming = base64.b64decode(event.data)
+ block.source.data = base64.b64encode(
+ existing + incoming,
+ ).decode("ascii")
case EventType.DATA_BLOCK_END:
pass
diff --git a/src/agentscope/model/_anthropic/_model.py b/src/agentscope/model/_anthropic/_model.py
index dc36659a4..426a85ac8 100644
--- a/src/agentscope/model/_anthropic/_model.py
+++ b/src/agentscope/model/_anthropic/_model.py
@@ -49,7 +49,7 @@ class AnthropicChatModel(ChatModelBase):
thinking_budget: int | None = Field(
default=None,
- title="Thinking budget",
+ title="Thinking Budget",
description="The thinking budget for the LLM output.",
gt=0,
)
diff --git a/src/agentscope/model/_dashscope/_model.py b/src/agentscope/model/_dashscope/_model.py
index e97c5a390..5de1941ad 100644
--- a/src/agentscope/model/_dashscope/_model.py
+++ b/src/agentscope/model/_dashscope/_model.py
@@ -1,6 +1,10 @@
# -*- coding: utf-8 -*-
"""The DashScope chat model class (OpenAI-compatible implementation)."""
+import base64
+import io
+import uuid
import warnings
+import wave
from collections import OrderedDict
from datetime import datetime
from typing import Any, AsyncGenerator, List, Literal, Type, TYPE_CHECKING
@@ -10,6 +14,7 @@ from pydantic import BaseModel, Field
from .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODES
from .._model_response import ChatResponse, StructuredResponse
from .._model_usage import ChatUsage
+from ..._utils._audio import _build_streaming_wav_header
from ...credential import DashScopeCredential
from ...formatter import FormatterBase, DashScopeChatFormatter
from ...message import (
@@ -57,7 +62,7 @@ class DashScopeChatModel(ChatModelBase):
thinking_budget: int | None = Field(
default=None,
- title="Thinking budget",
+ title="Thinking Budget",
description="The thinking budget for the LLM output.",
gt=0,
)
@@ -92,6 +97,21 @@ class DashScopeChatModel(ChatModelBase):
description="If enable parallel tool calls for the LLM output.",
)
+ voice: str | None = Field(
+ default=None,
+ title="Voice",
+ description=(
+ "Voice for audio output on omni-style models (e.g. "
+ "``qwen3.5-omni-plus``). Setting this implicitly asks the "
+ "model to speak its response — ``modalities`` is filled in "
+ "automatically. Supported voices vary by model — see the "
+ "model card's ``voice.suggestions``. Any value the API "
+ "accepts works — the suggestions are convenience-only. "
+ "Leave unset for text-only "
+ "responses."
+ ),
+ )
+
type: Literal["dashscope_chat"] = "dashscope_chat"
"""The type of the chat model."""
@@ -208,6 +228,18 @@ class DashScopeChatModel(ChatModelBase):
if self.parameters.top_p is not None:
request_kwargs["top_p"] = self.parameters.top_p
+ if self.parameters.voice is not None:
+ # Requesting audio output implies ``modalities`` must include
+ # ``"audio"``; set it automatically so callers don't have to.
+ # ``format`` is forced to ``pcm16``: omni streaming delivers raw
+ # PCM upstream regardless of the requested format, and we wrap
+ # it as WAV in ``_parse_stream_response`` before yielding.
+ request_kwargs["audio"] = {
+ "voice": self.parameters.voice,
+ "format": "pcm16",
+ }
+ request_kwargs["modalities"] = ["text", "audio"]
+
request_kwargs.update(kwargs)
fmt_tools, fmt_tool_choice = self._format_tools(tools, tool_choice)
@@ -237,17 +269,7 @@ class DashScopeChatModel(ChatModelBase):
response = await client.chat.completions.create(**request_kwargs)
if self.stream:
- audio_cfg = request_kwargs.get("audio")
- audio_fmt = (
- audio_cfg.get("format", "wav")
- if isinstance(audio_cfg, dict)
- else "wav"
- )
- return self._parse_stream_response(
- start_datetime,
- response,
- audio_fmt,
- )
+ return self._parse_stream_response(start_datetime, response)
return self._parse_completion_response(start_datetime, response)
@@ -255,7 +277,6 @@ class DashScopeChatModel(ChatModelBase):
self,
start_datetime: datetime,
response: AsyncStream,
- audio_format: str = "wav",
) -> AsyncGenerator[ChatResponse, None]:
"""Parse the DashScope streaming response (OpenAI-compatible format).
@@ -264,9 +285,6 @@ class DashScopeChatModel(ChatModelBase):
The start datetime of the response generation.
response (`AsyncStream`):
The OpenAI-compatible async stream object.
- audio_format (`str`, defaults to ``"wav"``):
- The audio format requested (used to set the media type on
- the output ``DataBlock``).
Yields:
`ChatResponse`:
@@ -278,7 +296,15 @@ class DashScopeChatModel(ChatModelBase):
acc_text = TextBlock(text="")
acc_thinking = ThinkingBlock(thinking="")
acc_tool_calls: OrderedDict = OrderedDict()
- acc_audio_data: str = ""
+ # Raw PCM bytes accumulated across chunks. Storing the decoded form
+ # (rather than concatenated base64 strings) avoids the risk of
+ # corrupting the byte stream when an intermediate chunk happens to
+ # carry base64 padding (``=``).
+ acc_audio_data: bytearray = bytearray()
+ audio_block_id: str | None = None
+ # ``True`` once the first audio chunk has been prefixed with a
+ # streaming WAV header and yielded.
+ audio_header_sent: bool = False
async with response as stream:
async for chunk in stream:
@@ -309,15 +335,37 @@ class DashScopeChatModel(ChatModelBase):
)
delta_text = getattr(delta, "content", None) or ""
- # Collect audio output from Omni models (delta.audio.data)
+ # Collect audio output from Omni models (delta.audio.data).
+ # Upstream sends raw PCM (24kHz, 16-bit mono); we prefix the
+ # first chunk with a streaming WAV header so the frontend
+ # can start playback immediately rather than waiting for
+ # end-of-stream.
delta_audio = getattr(delta, "audio", None)
+ delta_audio_block: DataBlock | None = None
if delta_audio is not None:
if isinstance(delta_audio, dict):
audio_chunk = delta_audio.get("data", "")
else:
audio_chunk = getattr(delta_audio, "data", "") or ""
if audio_chunk:
- acc_audio_data += audio_chunk
+ if audio_block_id is None:
+ audio_block_id = uuid.uuid4().hex
+ pcm_bytes = base64.b64decode(audio_chunk)
+ acc_audio_data += pcm_bytes
+ if not audio_header_sent:
+ payload = _build_streaming_wav_header() + pcm_bytes
+ audio_header_sent = True
+ else:
+ payload = pcm_bytes
+ delta_audio_block = DataBlock(
+ id=audio_block_id,
+ source=Base64Source(
+ data=base64.b64encode(payload).decode(
+ "ascii",
+ ),
+ media_type="audio/wav",
+ ),
+ )
acc_thinking.thinking += delta_thinking
acc_text.text += delta_text
@@ -352,7 +400,7 @@ class DashScopeChatModel(ChatModelBase):
)
delta_contents: List[
- TextBlock | ToolCallBlock | ThinkingBlock
+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock
] = []
if delta_thinking:
delta_contents.append(
@@ -366,6 +414,8 @@ class DashScopeChatModel(ChatModelBase):
TextBlock(id=acc_text.id, text=delta_text),
)
delta_contents.extend(delta_tool_call_blocks)
+ if delta_audio_block is not None:
+ delta_contents.append(delta_audio_block)
if delta_contents:
_kwargs: dict[str, Any] = {
@@ -393,11 +443,25 @@ class DashScopeChatModel(ChatModelBase):
),
)
if acc_audio_data:
+ # PCM bytes were already streamed incrementally above (first
+ # chunk prefixed with a WAV header). Here we also assemble a
+ # standalone fixed-size WAV and attach it to the ``is_last``
+ # chunk so callers that consume the model directly (i.e.
+ # without going through ``Agent``, which filters audio blocks
+ # out of context) get a self-contained audio block for
+ # downstream serialization / display.
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as wav:
+ wav.setnchannels(1)
+ wav.setsampwidth(2)
+ wav.setframerate(24000)
+ wav.writeframes(bytes(acc_audio_data))
final_contents.append(
DataBlock(
+ id=audio_block_id,
source=Base64Source(
- data=acc_audio_data,
- media_type=f"audio/{audio_format}",
+ data=base64.b64encode(buf.getvalue()).decode("ascii"),
+ media_type="audio/wav",
),
),
)
diff --git a/src/agentscope/model/_dashscope/_models/qwen-long.yaml b/src/agentscope/model/_dashscope/_models/qwen-long.yaml
index a60794b4b..56b7f3474 100644
--- a/src/agentscope/model/_dashscope/_models/qwen-long.yaml
+++ b/src/agentscope/model/_dashscope/_models/qwen-long.yaml
@@ -15,3 +15,7 @@ parameter_overrides:
max_tokens: {"maximum": 8192}
thinking_enable:
hidden: true
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_dashscope/_models/qwen-max-3.7.yaml b/src/agentscope/model/_dashscope/_models/qwen-max-3.7.yaml
index 817429e55..e626af494 100644
--- a/src/agentscope/model/_dashscope/_models/qwen-max-3.7.yaml
+++ b/src/agentscope/model/_dashscope/_models/qwen-max-3.7.yaml
@@ -14,3 +14,7 @@ output_size: 65536
parameter_overrides:
max_tokens: {"maximum": 65536}
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_dashscope/_models/qwen-plus.yaml b/src/agentscope/model/_dashscope/_models/qwen-plus.yaml
index f7f77d55e..ba5e41ecd 100644
--- a/src/agentscope/model/_dashscope/_models/qwen-plus.yaml
+++ b/src/agentscope/model/_dashscope/_models/qwen-plus.yaml
@@ -14,3 +14,7 @@ output_size: 16384
parameter_overrides:
max_tokens: {"maximum": 16384}
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_dashscope/_models/qwen3.5-omni-plus.yaml b/src/agentscope/model/_dashscope/_models/qwen3.5-omni-plus.yaml
index 973e282dd..dcf3c8dfd 100644
--- a/src/agentscope/model/_dashscope/_models/qwen3.5-omni-plus.yaml
+++ b/src/agentscope/model/_dashscope/_models/qwen3.5-omni-plus.yaml
@@ -24,5 +24,15 @@ context_size: 262144
output_size: 65536
parameter_overrides:
- max_tokens: {"maximum": 65536}
-
+ max_tokens:
+ maximum: 65536
+ # Source: https://help.aliyun.com/zh/model-studio/omni-voice-list
+ voice:
+ default: Tina
+ enum:
+ - Tina
+ - Cherry
+ - Ethan
+ - Serena
+ - Chelsie
+ - Cindy
diff --git a/src/agentscope/model/_dashscope/_models/qwen3.5-plus.yaml b/src/agentscope/model/_dashscope/_models/qwen3.5-plus.yaml
new file mode 100644
index 000000000..4682425d0
--- /dev/null
+++ b/src/agentscope/model/_dashscope/_models/qwen3.5-plus.yaml
@@ -0,0 +1,28 @@
+name: qwen3.5-plus
+label: Qwen3.5-Plus
+status: active
+
+input_types:
+ - text/plain
+ - application/x-thinking
+ - image/bmp
+ - image/jpeg
+ - image/png
+ - image/tiff
+ - image/webp
+ - image/heic
+ - video/mp4
+
+output_types:
+ - text/plain
+ - application/x-thinking
+
+context_size: 1000000
+output_size: 65536
+
+parameter_overrides:
+ max_tokens: {"maximum": 65536}
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_dashscope/_models/qwen3.6-max-preview.yaml b/src/agentscope/model/_dashscope/_models/qwen3.6-max-preview.yaml
index e4978f56f..501f5a374 100644
--- a/src/agentscope/model/_dashscope/_models/qwen3.6-max-preview.yaml
+++ b/src/agentscope/model/_dashscope/_models/qwen3.6-max-preview.yaml
@@ -14,3 +14,7 @@ output_size: 64000
parameter_overrides:
max_tokens: {"maximum": 64000}
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_dashscope/_models/qwen3.6-plus.yaml b/src/agentscope/model/_dashscope/_models/qwen3.6-plus.yaml
index 3ae5e677d..3dbc91988 100644
--- a/src/agentscope/model/_dashscope/_models/qwen3.6-plus.yaml
+++ b/src/agentscope/model/_dashscope/_models/qwen3.6-plus.yaml
@@ -22,3 +22,7 @@ output_size: 65536
parameter_overrides:
max_tokens: {"maximum": 65536}
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_model_card.py b/src/agentscope/model/_model_card.py
index 711ae6654..c5795ecfc 100644
--- a/src/agentscope/model/_model_card.py
+++ b/src/agentscope/model/_model_card.py
@@ -103,6 +103,15 @@ class ModelCard(BaseModel):
properties.pop("thinking_enable", None)
properties.pop("thinking_budget", None)
+ # Auto-filter: only omni-style models that declare an ``audio/*``
+ # output type expose the ``voice`` parameter to the frontend
+ # popover. Defense in depth — a model yaml can also drop ``voice``
+ # explicitly via ``parameter_overrides: { voice: { hidden: true } }``.
+ if not any(
+ isinstance(t, str) and t.startswith("audio/") for t in output_types
+ ):
+ properties.pop("voice", None)
+
# Auto-inject: set max_tokens maximum from output_size
if "max_tokens" in properties and "output_size" in config:
properties["max_tokens"]["maximum"] = config["output_size"]
diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.py
index d1e534ead..6056f2db4 100644
--- a/src/agentscope/model/_openai_chat/_model.py
+++ b/src/agentscope/model/_openai_chat/_model.py
@@ -1,5 +1,9 @@
# -*- coding: utf-8 -*-
"""The OpenAI Chat Completions model implementation."""
+import base64
+import io
+import uuid
+import wave
from collections import OrderedDict
from datetime import datetime
from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
@@ -9,6 +13,7 @@ from pydantic import BaseModel, Field
from .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODES
from .._model_response import ChatResponse
from .._model_usage import ChatUsage
+from ..._utils._audio import _build_streaming_wav_header
from ...credential import OpenAICredential
from ...formatter import FormatterBase, OpenAIChatFormatter
from ...message import (
@@ -87,6 +92,21 @@ class OpenAIChatModel(ChatModelBase):
description="Whether to enable parallel tool calls.",
)
+ voice: str | None = Field(
+ default=None,
+ title="Voice",
+ description=(
+ "Voice for audio output on omni-style models (e.g. "
+ "``gpt-audio-mini``). Setting this implicitly asks the "
+ "model to speak its response — ``modalities`` is filled in "
+ "automatically. Supported voices vary by model — see the "
+ "model card's ``voice.suggestions``. Any value the API "
+ "accepts works — the suggestions are convenience-only. "
+ "Leave unset for text-only "
+ "responses."
+ ),
+ )
+
type: Literal["openai_chat"] = "openai_chat"
"""The type of the chat model."""
@@ -214,6 +234,18 @@ class OpenAIChatModel(ChatModelBase):
):
kwargs["reasoning_effort"] = self.parameters.reasoning_effort
+ if self.parameters.voice is not None:
+ # Requesting audio output implies ``modalities`` must include
+ # ``"audio"``; set it automatically so callers don't have to.
+ # ``format`` is forced to ``pcm16``: OpenAI streaming only
+ # supports ``pcm16`` (other formats raise 400), and we re-wrap
+ # as WAV downstream so the frontend receives a playable block.
+ kwargs["audio"] = {
+ "voice": self.parameters.voice,
+ "format": "pcm16",
+ }
+ kwargs["modalities"] = ["text", "audio"]
+
kwargs.update(generate_kwargs)
fmt_tools, fmt_tool_choice = self._format_tools(tools, tool_choice)
@@ -240,10 +272,11 @@ class OpenAIChatModel(ChatModelBase):
)
if self.stream:
+ # Streaming wire format is always ``pcm16`` (forced above) and we
+ # re-wrap it as WAV before yielding, so downstream sees ``wav``.
return self._parse_stream_response(
start_datetime,
response,
- audio_fmt,
)
return self._parse_completion_response(
@@ -256,18 +289,20 @@ class OpenAIChatModel(ChatModelBase):
self,
start_datetime: datetime,
response: AsyncStream,
- audio_format: str = "wav",
) -> AsyncGenerator[ChatResponse, None]:
"""Parse the OpenAI Chat streaming response.
+ Upstream sends raw PCM16 (24kHz, 16-bit mono — OpenAI's only
+ streaming-supported audio format). We prefix the first audio
+ chunk with a streaming WAV header so the frontend can start
+ playback immediately, and assemble a fixed-size WAV on the
+ final chunk for non-streaming consumers.
+
Args:
start_datetime (`datetime`):
The start datetime of the response generation.
response (`AsyncStream`):
The OpenAI async stream object.
- audio_format (`str`, defaults to ``"wav"``):
- The audio format requested (used to set the media type on
- the output ``DataBlock``).
Yields:
`ChatResponse`:
@@ -280,8 +315,15 @@ class OpenAIChatModel(ChatModelBase):
acc_text = TextBlock(text="")
acc_thinking = ThinkingBlock(thinking="")
acc_tool_calls: OrderedDict = OrderedDict()
- acc_audio_data: str = ""
- acc_audio_transcript: str = ""
+ # Raw audio bytes accumulated across chunks. Storing the decoded
+ # form (rather than concatenated base64 strings) avoids corrupting
+ # the byte stream when an intermediate chunk happens to carry
+ # base64 padding (``=``).
+ acc_audio_data: bytearray = bytearray()
+ audio_block_id: str | None = None
+ # ``True`` once the first audio chunk has been prefixed with a
+ # streaming WAV header and yielded.
+ audio_header_sent: bool = False
async with response as stream:
async for chunk in stream:
@@ -320,6 +362,8 @@ class OpenAIChatModel(ChatModelBase):
# Collect audio output (delta.audio.data /
# delta.audio.transcript)
+ delta_audio_block: DataBlock | None = None
+ transcript_chunk: str = ""
delta_audio = getattr(delta, "audio", None)
if delta_audio is not None:
if isinstance(delta_audio, dict):
@@ -331,9 +375,30 @@ class OpenAIChatModel(ChatModelBase):
getattr(delta_audio, "transcript", "") or ""
)
if audio_chunk:
- acc_audio_data += audio_chunk
- if transcript_chunk:
- acc_audio_transcript += transcript_chunk
+ if audio_block_id is None:
+ audio_block_id = uuid.uuid4().hex
+ pcm_bytes = base64.b64decode(audio_chunk)
+ acc_audio_data += pcm_bytes
+ if not audio_header_sent:
+ payload = _build_streaming_wav_header() + pcm_bytes
+ audio_header_sent = True
+ else:
+ payload = pcm_bytes
+ delta_audio_block = DataBlock(
+ id=audio_block_id,
+ source=Base64Source(
+ data=base64.b64encode(payload).decode(
+ "ascii",
+ ),
+ media_type="audio/wav",
+ ),
+ )
+ # Omni models deliver text via ``delta.audio.transcript``
+ # (not ``delta.content``); fold it into ``delta_text`` so
+ # the agent's streaming pipeline emits ``TextBlockDelta``
+ # events alongside the audio chunks.
+ if transcript_chunk:
+ delta_text += transcript_chunk
acc_thinking.thinking += delta_thinking
acc_text.text += delta_text
@@ -360,7 +425,7 @@ class OpenAIChatModel(ChatModelBase):
)
delta_contents: List[
- TextBlock | ToolCallBlock | ThinkingBlock
+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock
] = []
if delta_thinking:
delta_contents.append(
@@ -374,6 +439,8 @@ class OpenAIChatModel(ChatModelBase):
TextBlock(id=acc_text.id, text=delta_text),
)
delta_contents.extend(delta_tool_call_blocks)
+ if delta_audio_block is not None:
+ delta_contents.append(delta_audio_block)
if delta_contents:
_kwargs: dict[str, Any] = {
@@ -392,18 +459,30 @@ class OpenAIChatModel(ChatModelBase):
final_contents.append(acc_thinking)
if acc_text.text:
final_contents.append(acc_text)
- elif acc_audio_transcript:
- final_contents.append(TextBlock(text=acc_audio_transcript))
for tc in acc_tool_calls.values():
final_contents.append(
ToolCallBlock(id=tc["id"], name=tc["name"], input=tc["input"]),
)
if acc_audio_data:
+ # PCM bytes were already streamed incrementally above (first
+ # chunk prefixed with a WAV header). Here we also assemble a
+ # standalone fixed-size WAV and attach it to the ``is_last``
+ # chunk so callers that consume the model directly (i.e.
+ # without going through ``Agent``, which filters audio blocks
+ # out of context) get a self-contained audio block for
+ # downstream serialization / display.
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as wav:
+ wav.setnchannels(1)
+ wav.setsampwidth(2)
+ wav.setframerate(24000)
+ wav.writeframes(bytes(acc_audio_data))
final_contents.append(
DataBlock(
+ id=audio_block_id,
source=Base64Source(
- data=acc_audio_data,
- media_type=f"audio/{audio_format}",
+ data=base64.b64encode(buf.getvalue()).decode("ascii"),
+ media_type="audio/wav",
),
),
)
diff --git a/src/agentscope/model/_openai_chat/_models/gpt-4.1-mini.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4.1-mini.yaml
index a6bdad35e..cc66dcaa5 100644
--- a/src/agentscope/model/_openai_chat/_models/gpt-4.1-mini.yaml
+++ b/src/agentscope/model/_openai_chat/_models/gpt-4.1-mini.yaml
@@ -22,3 +22,7 @@ parameter_overrides:
hidden: true
reasoning_effort:
hidden: true
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_openai_chat/_models/gpt-4.1-nano.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4.1-nano.yaml
index 5a067a036..bf26d9d9b 100644
--- a/src/agentscope/model/_openai_chat/_models/gpt-4.1-nano.yaml
+++ b/src/agentscope/model/_openai_chat/_models/gpt-4.1-nano.yaml
@@ -22,3 +22,7 @@ parameter_overrides:
hidden: true
reasoning_effort:
hidden: true
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_openai_chat/_models/gpt-4.1.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4.1.yaml
index 40ebbe051..f26b14301 100644
--- a/src/agentscope/model/_openai_chat/_models/gpt-4.1.yaml
+++ b/src/agentscope/model/_openai_chat/_models/gpt-4.1.yaml
@@ -20,3 +20,7 @@ parameter_overrides:
maximum: 32768
thinking_enable:
hidden: true
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_openai_chat/_models/gpt-4o-mini.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4o-mini.yaml
index da2ec3901..b41ac171c 100644
--- a/src/agentscope/model/_openai_chat/_models/gpt-4o-mini.yaml
+++ b/src/agentscope/model/_openai_chat/_models/gpt-4o-mini.yaml
@@ -22,3 +22,7 @@ parameter_overrides:
hidden: true
reasoning_effort:
hidden: true
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_openai_chat/_models/gpt-4o.yaml b/src/agentscope/model/_openai_chat/_models/gpt-4o.yaml
index e19c9be73..5b19e6883 100644
--- a/src/agentscope/model/_openai_chat/_models/gpt-4o.yaml
+++ b/src/agentscope/model/_openai_chat/_models/gpt-4o.yaml
@@ -22,3 +22,7 @@ parameter_overrides:
hidden: true
reasoning_effort:
hidden: true
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_openai_chat/_models/gpt-5.4.yaml b/src/agentscope/model/_openai_chat/_models/gpt-5.4.yaml
index ef1908708..dec7e31bd 100644
--- a/src/agentscope/model/_openai_chat/_models/gpt-5.4.yaml
+++ b/src/agentscope/model/_openai_chat/_models/gpt-5.4.yaml
@@ -20,3 +20,7 @@ parameter_overrides:
maximum: 128000
thinking_enable:
hidden: true
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_openai_chat/_models/gpt-5.5.yaml b/src/agentscope/model/_openai_chat/_models/gpt-5.5.yaml
index 746c4f516..1e009e393 100644
--- a/src/agentscope/model/_openai_chat/_models/gpt-5.5.yaml
+++ b/src/agentscope/model/_openai_chat/_models/gpt-5.5.yaml
@@ -24,3 +24,7 @@ parameter_overrides:
hidden: true
reasoning_effort:
hidden: true
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_openai_chat/_models/gpt-audio-mini.yaml b/src/agentscope/model/_openai_chat/_models/gpt-audio-mini.yaml
new file mode 100644
index 000000000..5877d6832
--- /dev/null
+++ b/src/agentscope/model/_openai_chat/_models/gpt-audio-mini.yaml
@@ -0,0 +1,31 @@
+name: gpt-audio-mini
+label: GPT Audio Mini
+status: active
+
+input_types:
+ - text/plain
+ - audio/wav
+ - audio/mp3
+
+output_types:
+ - text/plain
+ - audio/wav
+
+context_size: 128000
+output_size: 16384
+
+parameter_overrides:
+ max_tokens:
+ maximum: 16384
+ thinking_enable:
+ hidden: true
+ # Source: https://platform.openai.com/docs/guides/text-to-speech
+ voice:
+ default: alloy
+ enum:
+ - alloy
+ - echo
+ - nova
+ - shimmer
+ - sage
+ - verse
diff --git a/src/agentscope/model/_openai_chat/_models/o3.yaml b/src/agentscope/model/_openai_chat/_models/o3.yaml
index 7aedacce8..13e7762aa 100644
--- a/src/agentscope/model/_openai_chat/_models/o3.yaml
+++ b/src/agentscope/model/_openai_chat/_models/o3.yaml
@@ -19,3 +19,7 @@ output_size: 100000
parameter_overrides:
max_tokens:
maximum: 100000
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/src/agentscope/model/_openai_chat/_models/o4-mini.yaml b/src/agentscope/model/_openai_chat/_models/o4-mini.yaml
index 931013282..ab2e91192 100644
--- a/src/agentscope/model/_openai_chat/_models/o4-mini.yaml
+++ b/src/agentscope/model/_openai_chat/_models/o4-mini.yaml
@@ -19,3 +19,7 @@ output_size: 100000
parameter_overrides:
max_tokens:
maximum: 100000
+ # Voice only applies to omni-style models; hide the field for this
+ # non-omni model so the frontend popover doesn't render it.
+ voice:
+ hidden: true
diff --git a/tests/event_to_message_test.py b/tests/event_to_message_test.py
index e78786bee..3f261af53 100644
--- a/tests/event_to_message_test.py
+++ b/tests/event_to_message_test.py
@@ -263,31 +263,35 @@ class EventToMessageTest(IsolatedAsyncioTestCase):
],
)
+ # Each delta carries an independently base64-encoded chunk (with its
+ # own padding); ``append_event`` decodes -> concats bytes -> re-encodes
+ # so the message reflects the concatenated underlying bytes, not a
+ # string-concat of the chunks. Here: b"abc" + b"def" -> "YWJjZGVm".
ev_data_delta1 = DataBlockDeltaEvent(
reply_id=_REPLY_ID,
block_id=_B_DATA,
- data="abc",
+ data="YWJj", # base64(b"abc")
media_type="image/png",
)
gt_data_delta1 = _base(
[
_tb(_B_TEXT, "Hello World"),
_thb(_B_THINK, "Let me think"),
- _db_b64(_B_DATA, "abc", "image/png"),
+ _db_b64(_B_DATA, "YWJj", "image/png"),
],
)
ev_data_delta2 = DataBlockDeltaEvent(
reply_id=_REPLY_ID,
block_id=_B_DATA,
- data="def",
+ data="ZGVm", # base64(b"def")
media_type="image/png",
)
gt_data_delta2 = _base(
[
_tb(_B_TEXT, "Hello World"),
_thb(_B_THINK, "Let me think"),
- _db_b64(_B_DATA, "abcdef", "image/png"),
+ _db_b64(_B_DATA, "YWJjZGVm", "image/png"),
],
)
@@ -296,7 +300,7 @@ class EventToMessageTest(IsolatedAsyncioTestCase):
[ # unchanged
_tb(_B_TEXT, "Hello World"),
_thb(_B_THINK, "Let me think"),
- _db_b64(_B_DATA, "abcdef", "image/png"),
+ _db_b64(_B_DATA, "YWJjZGVm", "image/png"),
],
)
@@ -312,7 +316,7 @@ class EventToMessageTest(IsolatedAsyncioTestCase):
_s4_prefix = [
_tb(_B_TEXT, "Hello World"),
_thb(_B_THINK, "Let me think"),
- _db_b64(_B_DATA, "abcdef", "image/png"),
+ _db_b64(_B_DATA, "YWJjZGVm", "image/png"),
]
gt_tc_allow_start = _base(
_s4_prefix + [_tcb(_TC_ALLOW, "search", "", "pending")],
diff --git a/tests/model_dashscope_test.py b/tests/model_dashscope_test.py
index 709e1bdbb..f4ebc6830 100644
--- a/tests/model_dashscope_test.py
+++ b/tests/model_dashscope_test.py
@@ -4,6 +4,9 @@
Tests cover both non-streaming and streaming modes.
"""
+import base64
+import io
+import wave
from typing import Any
import unittest
from unittest import IsolatedAsyncioTestCase
@@ -11,7 +14,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
from utils import AnyString
-from agentscope.message import TextBlock, ToolCallBlock, ThinkingBlock
+from agentscope.message import (
+ TextBlock,
+ ToolCallBlock,
+ ThinkingBlock,
+ DataBlock,
+)
from agentscope.model import DashScopeChatModel
from agentscope.credential import DashScopeCredential
from agentscope.tool import ToolChoice
@@ -78,6 +86,7 @@ def _make_stream_chunk(
delta_text: str | None = None,
delta_reasoning: str | None = None,
tool_calls: list | None = None,
+ delta_audio: dict | None = None,
response_id: str = "req-1",
usage: dict | None = None,
has_choices: bool = True,
@@ -99,7 +108,7 @@ def _make_stream_chunk(
delta.content = delta_text
delta.reasoning_content = delta_reasoning
delta.tool_calls = tool_calls
- delta.audio = None
+ delta.audio = delta_audio
choice = MagicMock()
choice.delta = delta
chunk.choices = [choice]
@@ -411,6 +420,78 @@ class TestDashScopeStream(IsolatedAsyncioTestCase):
self.assertEqual(responses[-1].usage.input_tokens, 50)
self.assertEqual(responses[-1].usage.output_tokens, 10)
+ @patch("openai.AsyncClient")
+ async def test_stream_audio_response(
+ self,
+ mock_client_cls: MagicMock,
+ ) -> None:
+ """Stream PCM deltas produce per-chunk DataBlocks (first chunk
+ prefixed with a streaming WAV header) sharing a stable id, plus a
+ final fixed-size WAV block readable by the ``wave`` module."""
+ pcm1 = bytes([1, 2, 3, 4])
+ pcm2 = bytes([5, 6, 7, 8])
+ pcm3 = bytes([9, 10, 11, 12])
+ pcm_full = pcm1 + pcm2 + pcm3
+
+ chunks = [
+ _make_stream_chunk(
+ delta_audio={"data": base64.b64encode(pcm1).decode()},
+ ),
+ _make_stream_chunk(
+ delta_audio={"data": base64.b64encode(pcm2).decode()},
+ ),
+ _make_stream_chunk(
+ delta_audio={"data": base64.b64encode(pcm3).decode()},
+ ),
+ _make_stream_chunk(
+ has_choices=False,
+ usage={"prompt_tokens": 5, "completion_tokens": 3},
+ ),
+ ]
+ mock_create = AsyncMock(return_value=_MockAsyncStream(chunks))
+ mock_client_cls.return_value.chat.completions.create = mock_create
+
+ gen = await self.model([])
+ responses = [r async for r in gen]
+ self.assertEqual(len(responses), 4)
+
+ # All four chunks (3 deltas + 1 final) must share the same audio
+ # block id so downstream consumers stitch them as one stream.
+ all_audio_ids = {
+ block.id
+ for r in responses
+ for block in r.content
+ if isinstance(block, DataBlock)
+ }
+ self.assertEqual(len(all_audio_ids), 1)
+
+ # First delta: WAV header (44 bytes, "RIFF"..."WAVE") + pcm1.
+ first_payload = base64.b64decode(responses[0].content[0].source.data)
+ self.assertEqual(len(first_payload), 44 + len(pcm1))
+ self.assertEqual(first_payload[:4], b"RIFF")
+ self.assertEqual(first_payload[8:12], b"WAVE")
+ self.assertEqual(first_payload[44:], pcm1)
+
+ # Subsequent deltas: raw PCM only, no header.
+ for resp, pcm in zip(responses[1:3], [pcm2, pcm3]):
+ payload = base64.b64decode(resp.content[0].source.data)
+ self.assertEqual(payload, pcm)
+
+ # Final ``is_last`` block: a fixed-size WAV the ``wave`` module
+ # can parse end-to-end at 24kHz / mono / 16-bit.
+ final = responses[-1]
+ self.assertTrue(final.is_last)
+ final_audio = next(
+ b for b in final.content if isinstance(b, DataBlock)
+ )
+ wav_bytes = base64.b64decode(final_audio.source.data)
+ with wave.open(io.BytesIO(wav_bytes), "rb") as wav:
+ self.assertEqual(wav.getnchannels(), 1)
+ self.assertEqual(wav.getsampwidth(), 2)
+ self.assertEqual(wav.getframerate(), 24000)
+ frames = wav.readframes(wav.getnframes())
+ self.assertEqual(frames, pcm_full)
+
# ---------------------------------------------------------------------------
# _format_tools tests
diff --git a/tests/model_openai_chat_test.py b/tests/model_openai_chat_test.py
index 31e36e376..8921a7e33 100644
--- a/tests/model_openai_chat_test.py
+++ b/tests/model_openai_chat_test.py
@@ -7,6 +7,9 @@ Tests cover both non-streaming and streaming modes, verifying that:
- Stream mode yields n delta ChatResponses (is_last=False) followed by
1 final ChatResponse (is_last=True) with the full accumulated content.
"""
+import base64
+import io
+import wave
from typing import Any
import unittest
from unittest import IsolatedAsyncioTestCase
@@ -477,17 +480,35 @@ class TestOpenAIChatStream(IsolatedAsyncioTestCase):
self,
mock_client_cls: MagicMock,
) -> None:
- """Stream audio deltas accumulate into a final DataBlock with the
- concatenated base64 payload and a transcript TextBlock."""
+ """Stream PCM deltas produce per-chunk DataBlocks (first chunk
+ prefixed with a streaming WAV header) sharing a stable id, plus a
+ final fixed-size WAV block readable by the ``wave`` module.
+ Transcript chunks ride alongside as TextBlock deltas so the agent
+ can stream caption text live; the final block carries the full
+ accumulated transcript."""
+ pcm1 = bytes([1, 2, 3, 4])
+ pcm2 = bytes([5, 6, 7, 8])
+ pcm3 = bytes([9, 10, 11, 12])
+ pcm_full = pcm1 + pcm2 + pcm3
+
chunks = [
_make_stream_chunk(
- delta_audio={"data": "AAAA", "transcript": "Hello"},
+ delta_audio={
+ "data": base64.b64encode(pcm1).decode(),
+ "transcript": "Hello",
+ },
),
_make_stream_chunk(
- delta_audio={"data": "BBBB", "transcript": " world"},
+ delta_audio={
+ "data": base64.b64encode(pcm2).decode(),
+ "transcript": " world",
+ },
),
_make_stream_chunk(
- delta_audio={"data": "CCCC", "transcript": "!"},
+ delta_audio={
+ "data": base64.b64encode(pcm3).decode(),
+ "transcript": "!",
+ },
),
_make_stream_chunk(
has_choices=False,
@@ -499,27 +520,67 @@ class TestOpenAIChatStream(IsolatedAsyncioTestCase):
gen = await self.model([])
responses = [r async for r in gen]
-
- # Only the final response carries the assembled audio content; the
- # per-chunk deltas have no text/thinking/tool_call so no delta
- # ChatResponse is yielded mid-stream.
- self.assertEqual(len(responses), 1)
- final = responses[0]
+ self.assertEqual(len(responses), 4)
+
+ # All four chunks (3 deltas + 1 final) must share the same audio
+ # block id so downstream consumers stitch them as one stream.
+ all_audio_ids = {
+ block.id
+ for r in responses
+ for block in r.content
+ if isinstance(block, DataBlock)
+ }
+ self.assertEqual(len(all_audio_ids), 1)
+
+ # First delta: WAV header (44 bytes, "RIFF"..."WAVE") + pcm1.
+ first_audio = next(
+ b for b in responses[0].content if isinstance(b, DataBlock)
+ )
+ first_payload = base64.b64decode(first_audio.source.data)
+ self.assertEqual(len(first_payload), 44 + len(pcm1))
+ self.assertEqual(first_payload[:4], b"RIFF")
+ self.assertEqual(first_payload[8:12], b"WAVE")
+ self.assertEqual(first_payload[44:], pcm1)
+ self.assertEqual(first_audio.source.media_type, "audio/wav")
+
+ # Subsequent deltas: raw PCM only, no header.
+ for resp, pcm in zip(responses[1:3], [pcm2, pcm3]):
+ audio_block = next(
+ b for b in resp.content if isinstance(b, DataBlock)
+ )
+ self.assertEqual(base64.b64decode(audio_block.source.data), pcm)
+ self.assertEqual(audio_block.source.media_type, "audio/wav")
+
+ # Transcript rides alongside: each delta carries a TextBlock with
+ # only that chunk's text (so the agent emits TextBlockDeltaEvents
+ # in real time).
+ for resp, expected_text in zip(
+ responses[:3],
+ ["Hello", " world", "!"],
+ ):
+ text_block = next(
+ b for b in resp.content if isinstance(b, TextBlock)
+ )
+ self.assertEqual(text_block.text, expected_text)
+
+ # Final ``is_last`` block: a fixed-size WAV the ``wave`` module
+ # can parse end-to-end at 24kHz / mono / 16-bit.
+ final = responses[-1]
self.assertTrue(final.is_last)
- self.assertEqual(
- final.content,
- [
- TextBlock.model_construct(id=A, text="Hello world!"),
- DataBlock.model_construct(
- id=A,
- source=Base64Source.model_construct(
- type="base64",
- media_type="audio/wav",
- data="AAAABBBBCCCC",
- ),
- ),
- ],
+ final_audio = next(
+ b for b in final.content if isinstance(b, DataBlock)
)
+ wav_bytes = base64.b64decode(final_audio.source.data)
+ with wave.open(io.BytesIO(wav_bytes), "rb") as wav:
+ self.assertEqual(wav.getnchannels(), 1)
+ self.assertEqual(wav.getsampwidth(), 2)
+ self.assertEqual(wav.getframerate(), 24000)
+ frames = wav.readframes(wav.getnframes())
+ self.assertEqual(frames, pcm_full)
+
+ # Transcript is accumulated and emitted as a TextBlock alongside.
+ final_text = next(b for b in final.content if isinstance(b, TextBlock))
+ self.assertEqual(final_text.text, "Hello world!")
class TestOpenAIChatModelParameters(unittest.TestCase):