Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(realtime): add GPT Live Voice Integration with Chat Agent Delegation (#3272)
qbc692066f370b1
diff --git a/examples/realtime/README.md b/examples/realtime/README.mdindex eac9be9f8..2a973f4f8 100644--- a/examples/realtime/README.md+++ b/examples/realtime/README.md@@ -40,6 +40,22 @@ python examples/realtime/local_mic.pySpeak, hear the reply, and speak over it to interrupt. `Ctrl-Q` quits.+### GPT-Live prototype++The GPT-Live integration uses a server-side WebSocket and client delegation.+The local example builds a chat agent from the selected backend model; the Web+UI automatically reuses the session's current Chat model and tools:++```bash+export OPENAI_API_KEY=sk-...+export OPENAI_LIVE_BACKEND_MODEL=your-responses-model+python examples/realtime/openai_live_local.py+```++This integration supports continuous full-duplex audio, independent user and+assistant captions, and application-managed task execution through the normal+AgentScope chat agent.+The default model is `qwen-audio-3.0-realtime-plus`. Any card listed for thecredential works; the class is looked up from the card, the same way theservice layer does it:diff --git a/examples/realtime/openai_live_local.py b/examples/realtime/openai_live_local.pynew file mode 100644index 000000000..2fce7ad1a--- /dev/null+++ b/examples/realtime/openai_live_local.py@@ -0,0 +1,77 @@+# -*- coding: utf-8 -*-+"""Talk to GPT-Live through the local microphone.++Set ``OPENAI_API_KEY`` and ``OPENAI_LIVE_BACKEND_MODEL`` before running.+Use headphones because the local transport does not cancel acoustic echo.+"""+import asyncio+import os++from agentscope.agent import Agent, RealtimeAgent+from agentscope.credential import OpenAICredential+from agentscope.model import OpenAIChatModel+from agentscope.realtime import LocalAudioTransport, OpenAILiveModel+from agentscope.state import AgentState+from agentscope.tui import launch_realtime_ui+++async def main() -> None:+ """Run one local GPT-Live voice session."""+ api_key = os.environ.get("OPENAI_API_KEY")+ backend_model = os.environ.get("OPENAI_LIVE_BACKEND_MODEL")+ if not api_key:+ raise SystemExit("Set OPENAI_API_KEY first.")+ if not backend_model:+ raise SystemExit("Set OPENAI_LIVE_BACKEND_MODEL first.")++ credential = OpenAICredential(api_key=api_key)+ live_model = OpenAILiveModel(+ "gpt-live-1",+ credential,+ )+ backend = OpenAIChatModel(credential=credential, model=backend_model)++ def _create_backend_agent(state: AgentState) -> Agent:+ """Build the task agent used for one client delegation."""+ return Agent(+ name="Friday",+ system_prompt=(+ "Answer accurately and keep task results concise for speech."+ ),+ model=backend,+ state=state,+ )++ agent = RealtimeAgent(+ name="Friday",+ system_prompt=(+ "Speak briefly. Listen while the user is talking. "+ "Delegate questions that need deeper reasoning."+ ),+ model=live_model,+ delegation_agent_factory=_create_backend_agent,+ )+ transport = LocalAudioTransport(+ input_sample_rate=live_model.input_sample_rate,+ output_sample_rate=live_model.output_sample_rate,+ input_device=_device("REALTIME_INPUT_DEVICE"),+ output_device=_device("REALTIME_OUTPUT_DEVICE"),+ )++ async with agent, transport:+ await launch_realtime_ui(agent, transport)+++def _device(env: str) -> int | str | None:+ """Return a sounddevice index or name from an environment variable."""+ value = os.environ.get(env)+ if value is None:+ return None+ return int(value) if value.isdigit() else value+++if __name__ == "__main__":+ try:+ asyncio.run(main())+ except KeyboardInterrupt:+ print("\nbye")diff --git a/examples/web_ui/frontend/src/api/types.ts b/examples/web_ui/frontend/src/api/types.tsindex 4774350e8..48453b255 100644--- a/examples/web_ui/frontend/src/api/types.ts+++ b/examples/web_ui/frontend/src/api/types.ts@@ -888,6 +888,8 @@ export interface RealtimeModelCard {input_sample_rate: number;output_sample_rate: number;supports_tools: boolean;+ interaction_mode: 'turn_based' | 'full_duplex';+ delegation_backend: 'none' | 'session_chat';max_context_tokens: number | null;max_audio_turns: number | null;max_audio_duration_s: number | null;diff --git a/examples/web_ui/frontend/src/components/chat/ChatContent.tsx b/examples/web_ui/frontend/src/components/chat/ChatContent.tsxindex 9cd39663b..775cfde84 100644--- a/examples/web_ui/frontend/src/components/chat/ChatContent.tsx+++ b/examples/web_ui/frontend/src/components/chat/ChatContent.tsx@@ -1,7 +1,9 @@-import { ReplyFinishedReason } from '@agentscope-ai/agentscope/event';+import {+ type ExternalExecutionResultEvent,+ ReplyFinishedReason,+} from '@agentscope-ai/agentscope/event';import {type ContentBlock,- getContentBlocks,type Msg,type TextBlock,type ToolCallBlock,@@ -13,6 +15,7 @@ import type { GitStatus } from '@/api';import type { ChatQueueItem } from '@/api/chat';import { ASMessageBubble, SteeringMessageBubble } from '@/components/chat/ASMessageBubble.tsx';import { ConfirmCard } from '@/components/chat/ConfirmCard.tsx';+import { ExternalExecutionCard } from '@/components/chat/ExternalExecutionCard.tsx';import { FlipCard } from '@/components/chat/FlipCard.tsx';import { QueuedMessages } from '@/components/chat/QueuedMessages.tsx';import { TextInput } from '@/components/chat/TextInput.tsx';@@ -34,6 +37,7 @@ import type { ReplyPhase } from '@/hooks/useMessages';import { useTranslation } from '@/i18n/useI18n';import type { RealtimeASRConfig } from '@/lib/browserRealtimeASRTransport';import type { RealtimeConnectionState } from '@/lib/browserWebRTCTransport';+import { getPendingHitlCalls } from '@/lib/pendingToolCalls';import { cn } from '@/lib/utils';/** How long a load may run before it is worth showing a spinner. */@@ -90,6 +94,7 @@ interface ChatContentProps {replyId: string,rules?: ToolCallBlock['suggested_rules'],) => Promise<void>;+ onExternalExecutionResult: (event: ExternalExecutionResultEvent) => Promise<void>;autoComplete?: (input: string) => string | null;className?: string;/** Called when the user clicks the stop button. */@@ -151,6 +156,7 @@ const ChatContentComponent: React.FC<ChatContentProps> = ({disabled,onSend,onUserConfirm,+ onExternalExecutionResult,autoComplete,className,onInterrupt,@@ -217,14 +223,9 @@ const ChatContentComponent: React.FC<ChatContentProps> = ({return () => clearTimeout(timer);}, [loading]);- const toConfirmedToolCalls = useMemo(() => {- if (msgs.length === 0) return [];-- const lastMsg = msgs[msgs.length - 1];- return getContentBlocks(lastMsg, 'tool_call')- .filter((tc) => tc.state === 'asking')- .map((tc) => ({ replyId: lastMsg.id, toolCall: tc }));- }, [msgs]);+ const pendingHitlCalls = useMemo(() => getPendingHitlCalls(msgs), [msgs]);+ const toConfirmedToolCalls = pendingHitlCalls.filter((call) => call.kind === 'confirmation');+ const externalToolCalls = pendingHitlCalls.filter((call) => call.kind === 'external_execution');// On an empty session the prompt and the input centre together, so every box// down to the message list shrinks to its content instead of filling.@@ -342,7 +343,11 @@ const ChatContentComponent: React.FC<ChatContentProps> = ({{loading ? null : (<div className="relative min-w-full max-w-full w-full"><FlipCard- visible={toConfirmedToolCalls.length > 0 || footerSlot !== null}+ visible={+ toConfirmedToolCalls.length > 0 ||+ externalToolCalls.length > 0 ||+ footerSlot !== null+ }className="absolute bottom-full left-0 right-0 mb-2 z-50">{toConfirmedToolCalls.length > 0 ? (@@ -358,6 +363,13 @@ const ChatContentComponent: React.FC<ChatContentProps> = ({)}/>+ ) : externalToolCalls.length > 0 ? (+ <ExternalExecutionCard+ key={`${externalToolCalls[0].replyId}:${externalToolCalls[0].toolCall.id}`}+ replyId={externalToolCalls[0].replyId}+ toolCall={externalToolCalls[0].toolCall}+ onResult={onExternalExecutionResult}+ />) : (footerSlot)}diff --git a/examples/web_ui/frontend/src/components/chat/ExternalExecutionCard.tsx b/examples/web_ui/frontend/src/components/chat/ExternalExecutionCard.tsxnew file mode 100644index 000000000..a1dc33d4d--- /dev/null+++ b/examples/web_ui/frontend/src/components/chat/ExternalExecutionCard.tsx@@ -0,0 +1,283 @@+import { EventType, type ExternalExecutionResultEvent } from '@agentscope-ai/agentscope/event';+import type { ToolCallBlock } from '@agentscope-ai/agentscope/message';+import { Check, CircleHelp, PlugZap, Send } from 'lucide-react';+import { useMemo, useRef, useState } from 'react';++import { Button } from '@/components/ui/button';+import { Input } from '@/components/ui/input';+import { Spinner } from '@/components/ui/spinner';+import { useTranslation } from '@/i18n/useI18n';+import { cn } from '@/lib/utils';++interface AskUserOption {+ label: string;+ description: string;+ preview?: string | null;+}++interface AskUserQuestion {+ question: string;+ header: string;+ context?: string | null;+ options: AskUserOption[];+ multi_select?: boolean;+}++interface AskUserParams {+ questions: AskUserQuestion[];+}++interface AnswerDraft {+ selected: string[];+ other: string;+ usesOther: boolean;+}++function parseAskUserParams(input: string): AskUserParams | null {+ try {+ const value = JSON.parse(input) as Partial<AskUserParams>;+ if (!Array.isArray(value.questions) || value.questions.length === 0) return null;+ if (+ !value.questions.every(+ (question) =>+ typeof question?.question === 'string' &&+ typeof question.header === 'string' &&+ Array.isArray(question.options) &&+ question.options.length > 0 &&+ question.options.every(+ (option) =>+ typeof option?.label === 'string' &&+ typeof option.description === 'string',+ ),+ )+ ) {+ return null;+ }+ return value as AskUserParams;+ } catch {+ return null;+ }+}++function AskUserCard({+ replyId,+ toolCall,+ onResult,+}: {+ replyId: string;+ toolCall: ToolCallBlock;+ onResult: (event: ExternalExecutionResultEvent) => Promise<void>;+}) {+ const { t } = useTranslation();+ const params = useMemo(() => parseAskUserParams(toolCall.input), [toolCall.input]);+ const [answers, setAnswers] = useState<AnswerDraft[]>(() =>+ (params?.questions ?? []).map(() => ({ selected: [], other: '', usesOther: false })),+ );+ const [submitting, setSubmitting] = useState(false);+ const submittingRef = useRef(false);++ if (!params) {+ return (+ <div className="rounded-[28px] bg-muted p-5 ring ring-border">+ <div className="flex items-center gap-2 text-sm font-medium">+ <CircleHelp className="size-4" />+ {t('chat.askUser.invalid')}+ </div>+ </div>+ );+ }++ const updateAnswer = (index: number, update: (answer: AnswerDraft) => AnswerDraft) => {+ setAnswers((current) =>+ current.map((answer, answerIndex) => (answerIndex === index ? update(answer) : answer)),+ );+ };++ const complete = answers.every(+ (answer) => answer.selected.length > 0 || answer.other.trim().length > 0,+ );++ const submit = async () => {+ if (!complete || submittingRef.current) return;+ submittingRef.current = true;+ setSubmitting(true);+ const submittedAnswers = params.questions.map((question, index) => ({+ question: question.question,+ selected: answers[index].usesOther ? [] : answers[index].selected,+ other: answers[index].usesOther ? answers[index].other.trim() : null,+ }));+ const output = submittedAnswers+ .map((answer) => `${answer.question}\n${answer.other ?? answer.selected.join(', ')}`)+ .join('\n\n');+ const createdAt = new Date().toISOString();+ try {+ await onResult({+ type: EventType.EXTERNAL_EXECUTION_RESULT,+ id: crypto.randomUUID(),+ created_at: createdAt,+ reply_id: replyId,+ execution_results: [+ {+ type: 'tool_result',+ id: toolCall.id,+ name: toolCall.name,+ output,+ state: 'success',+ metadata: { answers: submittedAnswers },+ created_at: createdAt,+ finished_at: createdAt,+ },+ ],+ });+ } catch {+ submittingRef.current = false;+ setSubmitting(false);+ }+ };++ return (+ <div className="max-h-[min(70vh,560px)] w-full space-y-5 overflow-y-auto rounded-[28px] bg-muted px-6 py-5 text-sm ring ring-border">+ <div className="flex items-center gap-2 text-secondary-foreground">+ <CircleHelp className="size-4 shrink-0" />+ <strong>{t('chat.askUser.title')}</strong>+ </div>+ {params.questions.map((question, questionIndex) => {+ const answer = answers[questionIndex];+ const preview = question.options.find(+ (option) => answer.selected.includes(option.label) && option.preview,+ )?.preview;+ return (+ <section key={question.question} className="space-y-3">+ <div className="space-y-1">+ <span className="text-xs font-medium uppercase tracking-wide text-muted-foreground">+ {question.header}+ </span>+ <p className="font-medium text-foreground">{question.question}</p>+ {question.context ? (+ <p className="whitespace-pre-wrap text-muted-foreground">+ {question.context}+ </p>+ ) : null}+ </div>+ <div className="grid gap-2 sm:grid-cols-2">+ {question.options.map((option) => {+ const selected = answer.selected.includes(option.label);+ return (+ <Button+ key={option.label}+ type="button"+ variant={selected ? 'secondary' : 'outline'}+ className="h-auto min-h-14 justify-start gap-3 whitespace-normal px-3 py-2 text-left"+ aria-pressed={selected}+ disabled={submitting}+ onClick={() =>+ updateAnswer(questionIndex, (current) => ({+ selected: question.multi_select+ ? selected+ ? current.selected.filter(+ (label) => label !== option.label,+ )+ : [...current.selected, option.label]+ : [option.label],+ other: '',+ usesOther: false,+ }))+ }+ >+ <span+ className={cn(+ 'flex size-5 shrink-0 items-center justify-center rounded-full border',+ selected &&+ 'border-primary bg-primary text-primary-foreground',+ )}+ >+ {selected ? <Check className="size-3" /> : null}+ </span>+ <span className="min-w-0">+ <span className="block font-medium">+ {option.label}+ </span>+ <span className="block text-xs text-muted-foreground">+ {option.description}+ </span>+ </span>+ </Button>+ );+ })}+ </div>+ <div className="space-y-2">+ <Button+ type="button"+ variant={answer.usesOther ? 'secondary' : 'ghost'}+ size="sm"+ disabled={submitting}+ onClick={() =>+ updateAnswer(questionIndex, (current) => ({+ selected: [],+ other: current.usesOther ? '' : current.other,+ usesOther: !current.usesOther,+ }))+ }+ >+ {t('chat.askUser.other')}+ </Button>+ {answer.usesOther ? (+ <Input+ autoFocus+ value={answer.other}+ placeholder={t('chat.askUser.otherPlaceholder')}+ disabled={submitting}+ onChange={(event) =>+ updateAnswer(questionIndex, (current) => ({+ ...current,+ other: event.target.value,+ }))+ }+ />+ ) : null}+ </div>+ {preview ? (+ <pre className="max-h-44 overflow-auto whitespace-pre-wrap rounded-lg border bg-background p-3 font-mono text-xs">+ {preview}+ </pre>+ ) : null}+ </section>+ );+ })}+ <div className="flex justify-end border-t pt-4">+ <Button+ type="button"+ disabled={!complete || submitting}+ onClick={() => void submit()}+ >+ {submitting ? <Spinner className="size-4" /> : <Send className="size-4" />}+ {t('chat.askUser.submit')}+ </Button>+ </div>+ </div>+ );+}++export function ExternalExecutionCard({+ replyId,+ toolCall,+ onResult,+}: {+ replyId: string;+ toolCall: ToolCallBlock;+ onResult: (event: ExternalExecutionResultEvent) => Promise<void>;+}) {+ const { t } = useTranslation();+ if (toolCall.name === 'AskUser') {+ return <AskUserCard replyId={replyId} toolCall={toolCall} onResult={onResult} />;+ }+ return (+ <div className="rounded-[28px] bg-muted px-6 py-5 text-sm ring ring-border">+ <div className="flex items-center gap-2 font-medium text-secondary-foreground">+ <PlugZap className="size-4 shrink-0" />+ {t('chat.externalExecution.title', { name: toolCall.name })}+ </div>+ <p className="mt-2 text-muted-foreground">{t('chat.externalExecution.description')}</p>+ </div>+ );+}diff --git a/examples/web_ui/frontend/src/hooks/useMessages.ts b/examples/web_ui/frontend/src/hooks/useMessages.tsindex 9a9269d22..068a59f13 100644--- a/examples/web_ui/frontend/src/hooks/useMessages.ts+++ b/examples/web_ui/frontend/src/hooks/useMessages.ts@@ -5,7 +5,9 @@ import type {DataBlockStartEvent,DataBlockDeltaEvent,DataBlockEndEvent,+ ExternalExecutionResultEvent,ReplyStartEvent,+ ToolResultEndEvent,UserConfirmResultEvent,} from '@agentscope-ai/agentscope/event';import { appendEvent, AssistantMsg, UserMsg } from '@agentscope-ai/agentscope/message';@@ -24,6 +26,7 @@ import {useChatQueue,} from '@/hooks/useChatQueue';import { useTranslation } from '@/i18n/useI18n';+import { getPendingHitlCalls } from '@/lib/pendingToolCalls';/*** One pending subagent HITL request, projected from a team *member*@@ -43,23 +46,6 @@ export type SubagentHitlEntry = {created_at: string;};-/**- * Return true if ``msg`` is an assistant reply currently parked on a- * pending tool_call (awaiting user confirmation or an external- * execution result). Used both to detect the "in-flight reply on page- * load" case and as the SDK-gap workaround that hides the confirm- * card once the paired tool_result lands.- */-const hasPendingToolCall = (msg: Msg | undefined): boolean => {- if (!msg || msg.role !== 'assistant') return false;- for (const block of msg.content) {- if (block.type !== 'tool_call') continue;- const state = (block as ToolCallBlock).state;- if (state === 'asking' || state === 'submitted') return true;- }- return false;-};-const hitlKey = (e: { worker_session_id: string; reply_id: string }) =>`${e.worker_session_id}:${e.reply_id}`;@@ -144,16 +130,36 @@ export function useMessages(isRealtimeAudioActive?: () => boolean;/** Send a tool confirmation over the active realtime transport. */sendRealtimeUserConfirm?: (event: UserConfirmResultEvent) => Promise<void>;+ /** Return an external tool result over the realtime transport. */+ sendRealtimeExternalExecutionResult?: (+ event: ExternalExecutionResultEvent,+ ) => Promise<void>;},) {const { t } = useTranslation();// A new identity for each visit also rejects late responses after A -> B -> A.const session = useMemo(() => ({ agentId, sessionId }), [agentId, sessionId]);+ const externalSubmissionsRef = useRef(+ new Map<+ string,+ {+ replyId: string;+ toolCallIds: Set<string>;+ resolve: () => void;+ reject: (error: Error) => void;+ }+ >(),+ );const activeSessionRef = useRef<typeof session | null>(null);useLayoutEffect(() => {activeSessionRef.current = session;+ const externalSubmissions = externalSubmissionsRef.current;return () => {activeSessionRef.current = null;+ for (const pending of externalSubmissions.values()) {+ pending.reject(new Error('The session closed before the result was acknowledged.'));+ }+ externalSubmissions.clear();};}, [session]);const [msgs, setMsgs] = useState<Msg[]>([]);@@ -225,7 +231,23 @@ export function useMessages(// reply content — route them to callbacks and skip appendEvent.if (event.type === EventType.CUSTOM) {const custom = event as CustomEvent;- if (custom.name === 'tts_error') {+ if (custom.name === 'external_execution_result_rejected') {+ const failure = custom.value as {+ request_id: string;+ reply_id: string;+ tool_call_ids: string[];+ };+ const pending = externalSubmissionsRef.current.get(failure.request_id);+ if (+ pending?.replyId === failure.reply_id &&+ failure.tool_call_ids.some((id) => pending.toolCallIds.has(id))+ ) {+ externalSubmissionsRef.current.delete(failure.request_id);+ const error = new Error(t('chat.externalExecution.rejected'));+ pending.reject(error);+ toast.error(error.message);+ }+ } else if (custom.name === 'tts_error') {const failure = custom.value as { stage?: string; reply_id?: string };const sessionKey = JSON.stringify([agentId, sessionId]);if (failure.stage === 'configuration') {@@ -298,6 +320,23 @@ export function useMessages(}return;}+ if (event.type === EventType.TOOL_RESULT_END) {+ const result = event as ToolResultEndEvent;+ for (const [requestId, pending] of externalSubmissionsRef.current) {+ if (pending.replyId !== result.reply_id) continue;+ pending.toolCallIds.delete(result.tool_call_id);+ if (pending.toolCallIds.size === 0) {+ externalSubmissionsRef.current.delete(requestId);+ pending.resolve();+ }+ }+ } else if (event.type === EventType.REPLY_END) {+ for (const [requestId, pending] of externalSubmissionsRef.current) {+ if (pending.replyId !== event.reply_id) continue;+ externalSubmissionsRef.current.delete(requestId);+ pending.reject(new Error(t('chat.externalExecution.rejected')));+ }+ }if (event.type === EventType.REPLY_START) {const e = event as ReplyStartEvent;// A continuation (the run resuming after a confirmation or an@@ -441,25 +480,27 @@ export function useMessages(if (cancelled) return;eventCursor = event_cursor;msgsRef.current = messages;- // If a reply is in flight (running on a worker) OR the- // tail msg is parked on a pending tool_call (awaiting+ // If a reply is in flight (running on a worker) OR any+ // persisted msg is parked on a pending tool_call (awaiting// user confirmation / external execution), initialise the// phase to ``streaming`` so the interrupt button is// available immediately — otherwise a fresh page load// while parked leaves the UI stuck on ``idle`` with no// way to abort.const tail = messages[messages.length - 1];- const pendingToolCall = hasPendingToolCall(tail);- if (is_running || pendingToolCall) {+ const pendingHitl = getPendingHitlCalls(messages)[0];+ if (is_running || pendingHitl) {setPhase('streaming');- if (is_running && !pendingToolCall && tail?.role === 'assistant') {+ if (is_running && !pendingHitl && tail?.role === 'assistant') {setActiveReplyId(tail.id);}- if (pendingToolCall) {+ if (pendingHitl) {// Prime the ref so continuation events (which// arrive without a fresh REPLY_START) apply to// the right msg.- currentReplyRef.current = tail ?? null;+ currentReplyRef.current =+ messages.find((message) => message.id === pendingHitl.replyId) ??+ null;}}// Published synchronously, not through `scheduleUpdate`:@@ -615,6 +656,47 @@ export function useMessages([agentId, sessionId],);+ const onExternalExecutionResult = useCallback(+ async (event: ExternalExecutionResultEvent) => {+ if (!agentId || !sessionId || activeSessionRef.current !== session) return;+ try {+ if (+ optionsRef.current?.isRealtimeAudioActive?.() &&+ optionsRef.current.sendRealtimeExternalExecutionResult+ ) {+ // Sending a DataChannel frame does not acknowledge its result.+ // Register first so even an immediate SSE response is matched.+ const receipt = new Promise<void>((resolve, reject) => {+ externalSubmissionsRef.current.set(event.id, {+ replyId: event.reply_id,+ toolCallIds: new Set(+ event.execution_results.map((result) => result.id),+ ),+ resolve,+ reject,+ });+ });+ await Promise.all([+ receipt,+ optionsRef.current.sendRealtimeExternalExecutionResult(event),+ ]);+ } else {+ await chatApi.trigger({+ agent_id: agentId,+ session_id: sessionId,+ input: event,+ });+ }+ } catch (e) {+ if (activeSessionRef.current === session) setError(e as Error);+ throw e;+ } finally {+ externalSubmissionsRef.current.delete(event.id);+ }+ },+ [agentId, sessionId, session],+ );+/** Abort the current SSE connection. */const abort = useCallback(() => {abortRef.current?.abort();@@ -744,6 +826,7 @@ export function useMessages(steerQueued,send,onUserConfirm,+ onExternalExecutionResult,onSubagentConfirm,subagentHitl: ownsConversation ? subagentHitl : [],abort,diff --git a/examples/web_ui/frontend/src/hooks/useRealtimeVoice.ts b/examples/web_ui/frontend/src/hooks/useRealtimeVoice.tsindex da08717c2..5b79a87ed 100644--- a/examples/web_ui/frontend/src/hooks/useRealtimeVoice.ts+++ b/examples/web_ui/frontend/src/hooks/useRealtimeVoice.ts@@ -1,4 +1,7 @@-import type { UserConfirmResultEvent } from '@agentscope-ai/agentscope/event';+import type {+ ExternalExecutionResultEvent,+ UserConfirmResultEvent,+} from '@agentscope-ai/agentscope/event';import { useCallback, useEffect, useRef, useState } from 'react';import { toast } from 'sonner';@@ -65,5 +68,11 @@ export function useRealtimeVoice(agentId: string | null, sessionId: string | nultransport.userConfirm(event);}, []);- return { state, toggle, userConfirm };+ const externalExecutionResult = useCallback(async (event: ExternalExecutionResultEvent) => {+ const transport = transportRef.current;+ if (!transport) throw new Error('Realtime voice mode is not active.');+ transport.externalExecutionResult(event);+ }, []);++ return { state, toggle, userConfirm, externalExecutionResult };}diff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.jsonindex 0253ef626..f5b2b9ea5 100644--- a/examples/web_ui/frontend/src/i18n/locales/en.json+++ b/examples/web_ui/frontend/src/i18n/locales/en.json@@ -175,6 +175,18 @@"inputPlaceholder": "Type a message...","confirmToolCall": "Allow this action?","subagentConfirmTitle": "{{name}} requests confirmation",+ "askUser": {+ "title": "Your input is needed",+ "other": "Other",+ "otherPlaceholder": "Enter your answer",+ "submit": "Submit answers",+ "invalid": "This question could not be displayed."+ },+ "externalExecution": {+ "title": "{{name}} is waiting for an external executor",+ "rejected": "The result was not accepted. Please check your answer and submit again.",+ "description": "This tool must be completed by an integration provided by the host application."+ },"newSession": "New Session","greeting": "Where should we start?","maxItersExceeded": {@@ -485,6 +497,9 @@"noModels": "No realtime voice models are available.","disable": "Disable voice model","selectFirst": "Realtime voice is off. Select a model from Voice model (the waveform icon) at the top.",+ "backendFollows": "Task backend follows Chat model · {{model}}",+ "backendMissing": "Select a Chat model before starting GPT Live",+ "pendingChatInteraction": "Complete the pending Chat confirmation or external execution before starting GPT Live.","start": "Start voice conversation","stop": "End voice conversation"},diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.jsonindex ccc4eab42..1021743e1 100644--- a/examples/web_ui/frontend/src/i18n/locales/zh.json+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json@@ -175,6 +175,18 @@"inputPlaceholder": "输入消息...","confirmToolCall": "允许此操作?","subagentConfirmTitle": "{{name}} 请求确认",+ "askUser": {+ "title": "需要你的选择",+ "other": "其他",+ "otherPlaceholder": "输入你的回答",+ "submit": "提交回答",+ "invalid": "无法显示这个问题。"+ },+ "externalExecution": {+ "title": "{{name}} 正在等待外部执行器",+ "rejected": "提交的结果未被接受,请检查回答后重新提交。",+ "description": "该工具需要由宿主应用提供的集成完成。"+ },"newSession": "新会话","greeting": "我们从哪儿开始?","maxItersExceeded": {@@ -485,6 +497,9 @@"noModels": "暂无可用的实时语音模型。","disable": "停用语音模型","selectFirst": "实时语音未开启。请在顶部的「语音模型」(声波图标)中选择模型。",+ "backendFollows": "任务后端跟随 Chat 模型 · {{model}}",+ "backendMissing": "请先选择 Chat 模型,再启动 GPT Live",+ "pendingChatInteraction": "请先完成 Chat 中待处理的工具确认或外部执行,再启动 GPT Live。","start": "开始语音对话","stop": "结束语音对话"},diff --git a/examples/web_ui/frontend/src/lib/browserWebRTCTransport.ts b/examples/web_ui/frontend/src/lib/browserWebRTCTransport.tsindex eb66ffc0b..d7f470873 100644--- a/examples/web_ui/frontend/src/lib/browserWebRTCTransport.ts+++ b/examples/web_ui/frontend/src/lib/browserWebRTCTransport.ts@@ -1,4 +1,7 @@-import type { UserConfirmResultEvent } from '@agentscope-ai/agentscope/event';+import type {+ ExternalExecutionResultEvent,+ UserConfirmResultEvent,+} from '@agentscope-ai/agentscope/event';import { realtimeApi } from '@/api';@@ -180,6 +183,19 @@ export class BrowserWebRTCTransport {);}+ externalExecutionResult(event: ExternalExecutionResultEvent): void {+ if (this.controlChannel?.readyState !== 'open') {+ throw new Error('The realtime control channel is not open.');+ }+ this.controlChannel.send(+ JSON.stringify({+ type: 'control',+ control: 'external_execution_result',+ data: event,+ }),+ );+ }+private attachRemoteTrack(track: MediaStreamTrack): void {if (track.kind !== 'audio') return;this.clearPlayoutFence();diff --git a/examples/web_ui/frontend/src/lib/pendingToolCalls.ts b/examples/web_ui/frontend/src/lib/pendingToolCalls.tsnew file mode 100644index 000000000..05f7cccb8--- /dev/null+++ b/examples/web_ui/frontend/src/lib/pendingToolCalls.ts@@ -0,0 +1,30 @@+import { getContentBlocks, type Msg, type ToolCallBlock } from '@agentscope-ai/agentscope/message';++export type PendingHitlCall = {+ kind: 'confirmation' | 'external_execution';+ replyId: string;+ toolCall: ToolCallBlock;+};++/** Return unresolved HITL calls, newest message first. */+export function getPendingHitlCalls(messages: readonly Msg[]): PendingHitlCall[] {+ const pending: PendingHitlCall[] = [];+ for (let index = messages.length - 1; index >= 0; index -= 1) {+ const message = messages[index];+ if (message.role !== 'assistant') continue;+ const resultIds = new Set(+ getContentBlocks(message, 'tool_result')+ .filter((result) => result.state !== 'running')+ .map((result) => result.id),+ );+ for (const toolCall of getContentBlocks(message, 'tool_call')) {+ if (resultIds.has(toolCall.id)) continue;+ if (toolCall.state === 'asking') {+ pending.push({ kind: 'confirmation', replyId: message.id, toolCall });+ } else if (toolCall.state === 'submitted') {+ pending.push({ kind: 'external_execution', replyId: message.id, toolCall });+ }+ }+ }+ return pending;+}diff --git a/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx b/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsxindex 85decd4ef..2f5cd5318 100644--- a/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx+++ b/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx@@ -3,6 +3,7 @@ import type { TaskContext } from '@agentscope-ai/agentscope/state';import {Activity,BookText,+ BrainCircuit,ChevronDown,Database,ListTodo,@@ -13,6 +14,7 @@ import {X,} from 'lucide-react';import { useCallback, useEffect, useMemo, useRef, useState } from 'react';+import { toast } from 'sonner';import type {ASRModelConfig,@@ -70,6 +72,7 @@ import { useSessions } from '@/hooks/useSessions';import { useWorkspace } from '@/hooks/useWorkspace.ts';import { useWorkspaceStatus } from '@/hooks/useWorkspaceStatus';import { useTranslation } from '@/i18n/useI18n';+import { getPendingHitlCalls } from '@/lib/pendingToolCalls';const REALTIME_EXCLUDED_ENUM_VALUES: Record<string, unknown[]> = {turn_detection: ['none'],@@ -319,6 +322,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatViewmoveQueued,reorderQueued,onUserConfirm,+ onExternalExecutionResult,onSubagentConfirm,subagentHitl,interrupt,@@ -328,6 +332,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatViewonSessionUpdated: handleSessionUpdated,isRealtimeAudioActive: () => voiceBusy,sendRealtimeUserConfirm: realtimeVoice.userConfirm,+ sendRealtimeExternalExecutionResult: realtimeVoice.externalExecutionResult,});const {mcps,@@ -665,7 +670,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatViewif (card) return card;}return null;- }, [groups, selectedModel?.type, selectedModel?.model, selectedModel?.model_class]);+ }, [groups, selectedModel]);const selectedRealtimeModelCard = useMemo(() => {if (!selectedRealtimeModel) return null;@@ -682,6 +687,8 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView}return null;}, [realtimeGroups, selectedRealtimeModel]);+ const realtimeUsesSessionChat =+ selectedRealtimeModelCard?.delegation_backend === 'session_chat';const selectedASRModelCard = useMemo(() => {if (!selectedASRModel) return null;@@ -985,7 +992,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatViewvalue={selectedModel}onChange={handleLlmChange}onAddCredential={() => setCredentialOpen(true)}- disabled={configPending}+ disabled={configPending || voiceBusy}/><ModelParametersPopoverselectedModel={selectedModel}@@ -997,7 +1004,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatViewonTTSChange={handleTTSChange}selectedASRModel={selectedASRModel}onASRChange={handleASRChange}- disabled={configPending}+ disabled={configPending || voiceBusy}/><PermissionModeSelectid="tour-permission-mode"@@ -1060,6 +1067,23 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView</DropdownMenu></div></div>+ {realtimeUsesSessionChat && (+ <div+ className={+ 'flex min-h-7 items-center justify-end gap-1.5 px-2 ' ++ 'text-xs text-muted-foreground'+ }+ >+ <BrainCircuit className="size-3.5 shrink-0" />+ <span className="max-w-full truncate">+ {selectedModel+ ? t('realtime.backendFollows', {+ model: selectedModel.model,+ })+ : t('realtime.backendMissing')}+ </span>+ </div>+ )}<div className="flex flex-1 justify-center min-h-0 overflow-hidden relative [--chat-content-w:48rem]"><ChatContentclassName={'max-w-[var(--chat-content-w)] w-full'}@@ -1079,6 +1103,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatViewdisabled={selectedModel === null || voiceBusy}onSend={send}onUserConfirm={onUserConfirm}+ onExternalExecutionResult={onExternalExecutionResult}onInterrupt={interrupt}onUpdateQueued={updateQueued}onDeleteQueued={deleteQueued}@@ -1086,8 +1111,23 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatViewonMoveQueued={moveQueued}onReorderQueued={reorderQueued}voiceState={realtimeVoice.state}- onVoiceToggle={() => void realtimeVoice.toggle()}- voiceDisabled={!selectedRealtimeModel || !agentId || !sessionId}+ onVoiceToggle={() => {+ if (+ !voiceBusy &&+ realtimeUsesSessionChat &&+ getPendingHitlCalls(msgs).length > 0+ ) {+ toast.error(t('realtime.pendingChatInteraction'));+ return;+ }+ void realtimeVoice.toggle();+ }}+ voiceDisabled={+ !selectedRealtimeModel ||+ !agentId ||+ !sessionId ||+ (realtimeUsesSessionChat && !selectedModel)+ }asrInputTypes={selectedASRModelCard?.input_types ?? []}onTranscribe={handleTranscribe}asrRealtime={!!selectedASRModelCard?.realtime}diff --git a/pyproject.toml b/pyproject.tomlindex e82d54589..043ff6eb9 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -113,7 +113,7 @@ tools = []# ------------ Realtime voice ------------realtime = [- "websockets>=13",+ "websockets>=14",]realtime-local = ["sounddevice"]# ------------ Terminal UI ------------diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.pyindex 012358e48..71a38bc1d 100644--- a/src/agentscope/agent/_agent.py+++ b/src/agentscope/agent/_agent.py@@ -741,10 +741,12 @@ class Agent:and self._get_reply_usage() is None):self.state.append_context(self.name, [])- self.state.context[-1].usage = current_reply_usage+ reply = self.state.get_reply_message(self.name)+ assert reply is not None+ reply.usage = current_reply_usage# The compression call is not covered by the model call events,- # so record its cost on the context tail to keep it in the token+ # so record its cost in the context to keep it in the token# accountingif res is not None and res.usage is not None:if not self.state.context:@@ -752,7 +754,11 @@ class Agent:# empty message, which is skipped by the formattersself.state.append_context(self.name, [])- self.state.context[-1].append_usage(+ usage_message = (+ self.state.get_reply_message(self.name)+ or self.state.context[-1]+ )+ usage_message.append_usage(Usage(input_tokens=res.usage.input_tokens,output_tokens=res.usage.output_tokens,@@ -970,11 +976,8 @@ class Agent:]:"""Close the unfinished tool calls on interruption, so the nextinput will be handled normally."""- if not self.state.context:- return-- last_msg = self.state.context[-1]- if last_msg.role != "assistant" or last_msg.name != self.name:+ last_msg = self.state.get_reply_message(self.name)+ if last_msg is None:return# Searching for tool calls that requires user confirmation or external@@ -1954,7 +1957,8 @@ class Agent:The events generated during the handling of the incoming event."""# Return directly if no event- if event is None or len(self.state.context) == 0:+ last_msg = self.state.get_reply_message(self.name)+ if event is None or last_msg is None:returnif isinstance(event, UserConfirmResultEvent):@@ -1964,7 +1968,6 @@ class Agent:}# Update the state with the confirmed tool calls- last_msg = self.state.context[-1]for tool_call in last_msg.get_content_blocks("tool_call"):if len(confirmed_tool_calls) == 0:break@@ -3489,10 +3492,8 @@ class Agent:state (`ToolCallState`):The new state of the tool call."""- if len(self.state.context) == 0:- return- last_msg = self.state.context[-1]- if last_msg.role != "assistant" or last_msg.name != self.name:+ last_msg = self.state.get_reply_message(self.name)+ if last_msg is None:returnfor block in last_msg.get_content_blocks():if isinstance(block, ToolCallBlock) and block.id == tool_call_id:@@ -3549,16 +3550,13 @@ class Agent:self.state.append_context(self.name, persisted_blocks)if msg_usage is not None:- self.state.context[-1].append_usage(msg_usage)+ reply = self.state.get_reply_message(self.name)+ assert reply is not None+ reply.append_usage(msg_usage)def _get_last_msg(self) -> Msg | None:- """Get the last message in the context that belongs to this agent."""- if len(self.state.context) == 0:- return None- last_msg = self.state.context[-1]- if last_msg.role == "assistant" and last_msg.name == self.name:- return last_msg- return None+ """Get the current reply without assuming it is the context tail."""+ return self.state.get_reply_message(self.name)def _get_reply_usage(self) -> Usage | None:"""Get a copy of the accumulated usage for the current reply."""diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.pyindex 25653d128..ee335dc40 100644--- a/src/agentscope/agent/_realtime/_agent.py+++ b/src/agentscope/agent/_realtime/_agent.py@@ -4,12 +4,19 @@ import asyncioimport base64import timeimport uuid+from collections import deque+from collections.abc import Callable, Iterator+from copy import deepcopyfrom dataclasses import dataclass, field-from typing import Any, AsyncIterator+from typing import Any, AsyncIterator, TYPE_CHECKINGfrom ...realtime import _events as mefrom ._aggregator import TurnAggregator-from ...realtime._base import ModelDisconnectedError, RealtimeModelBase+from ...realtime._base import (+ InteractionMode,+ ModelDisconnectedError,+ RealtimeModelBase,+)from ._metrics import TurnMetricsfrom ...realtime._transport._base import (AudioFrame,@@ -23,6 +30,7 @@ from ..._utils._common import _json_loads_with_repairfrom ...event import (AgentEvent,ConfirmResult,+ CustomEvent,DataBlockDeltaEvent,DataBlockEndEvent,DataBlockStartEvent,@@ -30,6 +38,7 @@ from ...event import (ModelCallStartEvent,ReplyEndEvent,ReplyStartEvent,+ RequireExternalExecutionEvent,RequireUserConfirmEvent,TextBlockDeltaEvent,TextBlockEndEvent,@@ -38,8 +47,10 @@ from ...event import (ToolCallEndEvent,ToolCallStartEvent,ToolResultEndEvent,+ ToolResultDataDeltaEvent,ToolResultStartEvent,ToolResultTextDeltaEvent,+ ExternalExecutionResultEvent,UserConfirmResultEvent,UserInterruptEvent,)@@ -59,10 +70,16 @@ from ...state import AgentStatefrom ...tool import ToolChunk, ToolResponse, Toolkitfrom ...types import ReplyFinishedReason+if TYPE_CHECKING:+ from .._agent import Agent+# Audio buffered while the model is being reconnected: 10 s at 100 ms chunks._BACKLOG_FRAMES = 100_HISTORY_MAX_MESSAGES = 100_HISTORY_MAX_TEXT_CHARS = 32_000+_LIVE_SEGMENT_MAX_TEXT_CHARS = 4_000+_LIVE_CHECKPOINT_INTERVAL_SECS = 2.0+_DELEGATION_INPUT_TIMEOUT_SECS = 300.0@dataclass@@ -79,6 +96,7 @@ class _Reply:audio_block_id: str = field(default_factory=lambda: uuid.uuid4().hex)text_started: bool = Falseaudio_started: bool = False+ text_chars: int = 0@dataclass@@ -99,6 +117,23 @@ class _QueuedEvent:checkpoint: AgentState | None = None+@dataclass+class _ClientDelegation:+ """One application-managed task requested by a Live session."""++ delegation_id: str+ prompt: Msg+ generation: int+ agent: "Agent | None" = None+ prompt_id: str = ""+ baseline_ids: set[str] = field(default_factory=set)+ synced_ids: set[str] = field(default_factory=set)+ buffered_events: list[AgentEvent] = field(default_factory=list)+ visible: bool = False+ timeout_task: asyncio.Task[None] | None = None+ expiring: bool = False++class RealtimeAgent:"""A voice agent: a realtime model on one side, a transport on theother, and the turn-taking state machine in between.@@ -139,6 +174,9 @@ class RealtimeAgent:state: AgentState | None = None,vad: VADBase | None = None,aggregator: TurnAggregator | None = None,+ delegation_agent_factory: (+ Callable[[AgentState], "Agent"] | None+ ) = None,) -> None:"""Initialize the realtime agent.@@ -173,6 +211,15 @@ class RealtimeAgent:acknowledgements. Subclass it to change what counts as aturn. A default with no backchannel list is used ifomitted.+ delegation_agent_factory (`Callable[[AgentState], Agent] | None`,+ optional):+ Builds a chat agent for each client-delegated Live task from+ the supplied isolated session state. Its model, toolkit, and+ permission state normally come from the current persisted+ chat session. The returned agent must have the same name+ as this realtime agent so tool routing and checkpoint+ recovery use the same assistant identity. A mismatched+ name fails the delegation before backend execution."""self.name = nameself.system_prompt = system_prompt@@ -181,6 +228,14 @@ class RealtimeAgent:self.state = state or AgentState()self.vad = vadself.aggregator = aggregator or TurnAggregator()+ self._delegation_agent_factory = delegation_agent_factory+ if (+ self.model.interaction_mode is InteractionMode.FULL_DUPLEX+ and self.vad is not None+ ):+ raise ValueError(+ "A full-duplex model cannot use client-side turn VAD.",+ )self._engine = PermissionEngine(self.state.permission_context)self._transport: TransportBase | None = None@@ -200,6 +255,19 @@ class RealtimeAgent:self._confirmations: dict[str, asyncio.Future[ConfirmResult]] = {}self._tasks: set[asyncio.Task] = set()self._barge_lock = asyncio.Lock()+ self._live_session_id = ""+ self._live_replies: dict[str, _Reply] = {}+ self._live_message_ids: set[str] = set()+ self._live_transcript: deque[tuple[str, str, int, int]] = deque()+ self._live_transcript_chars = 0+ self._last_live_checkpoint_at: float | None = None+ self._last_live_checkpoint_end_ms: int | None = None+ self._delegations: dict[str, _ClientDelegation] = {}+ self._delegation_queue: deque[str] = deque()+ self._active_delegation_id: str | None = None+ self._delegated_tool_calls: dict[str, str] = {}+ self._delegation_lock = asyncio.Lock()+ self._closing = False# Model session: connected flag, the downlink pump that outlives# any transport, and reconnect bookkeeping.@@ -229,11 +297,16 @@ class RealtimeAgent:closed it; a no-op while connected."""if self._connected:return-if self._connection_generation:async with self._barge_lock:self._reset_user_turns()self._finish_reply(ReplyFinishedReason.ERROR)+ self._finish_live_replies(ReplyFinishedReason.ERROR)+ if self.model.interaction_mode is InteractionMode.FULL_DUPLEX:+ # A new Live session cannot accept old delegation IDs.+ # Close and persist its tasks before opening the new socket.+ await self.close()+ self._closing = Falseself._connection_generation += 1instructions = self.system_prompt@@ -328,20 +401,45 @@ class RealtimeAgent:async def close(self) -> None:"""Cancel everything in flight and close the model session."""+ self._closing = True+ if self._downlink is not None:+ self._downlink.cancel()+ await asyncio.gather(self._downlink, return_exceptions=True)+ self._downlink = Nonefor future in self._confirmations.values():future.cancel()self._confirmations.clear()+ timeout_tasks = [+ delegation.timeout_task+ for delegation in self._delegations.values()+ if delegation.timeout_task is not None+ ]+ for task in timeout_tasks:+ task.cancel()for task in list(self._tasks):task.cancel()if self._tasks:await asyncio.gather(*self._tasks, return_exceptions=True)- if self._downlink is not None:- self._downlink.cancel()- await asyncio.gather(self._downlink, return_exceptions=True)- self._downlink = None+ if timeout_tasks:+ await asyncio.gather(*timeout_tasks, return_exceptions=True)+ for delegation in list(self._delegations.values()):+ if delegation.agent is not None:+ if delegation.agent.state.has_awaiting_tool_calls(self.name):+ await self._drive_client_delegation(+ delegation,+ UserInterruptEvent(+ reply_id=delegation.agent.state.reply_id,+ ),+ )+ self._sync_delegation_state(delegation)+ self._delegations.clear()+ self._delegation_queue.clear()+ self._delegated_tool_calls.clear()+ self._active_delegation_id = Noneself._connected = Falseself._connected_event.clear()await self.model.close()+ self._finish_live_replies(ReplyFinishedReason.COMPLETED)def _on_downlink_done(self, task: asyncio.Task) -> None:"""Surface a crashed downlink instead of a silently dead session."""@@ -460,19 +558,13 @@ class RealtimeAgent:# the model stops talking to nobody, then hand the caller the# events that produced (the interrupted ReplyEnd) before the# stream ends rather than leaking them into the next run.- await self._barge_in()+ if self.model.interaction_mode is InteractionMode.FULL_DUPLEX:+ await self._interrupt_live_speech()+ else:+ await self._barge_in()self._retire_unbound_user_turns()- while not self._out.empty():- queued = self._out.get_nowait()- self._yielded_checkpoint = (- (queued.event.id, queued.checkpoint)- if queued.checkpoint is not None- else None- )- try:- yield queued.event- finally:- self._yielded_checkpoint = None+ for event in self.drain_events():+ yield event# A transport failure must not look like a clean disconnect.if not uplink.cancelled() and uplink.exception() is not None:raise uplink.exception() # type: ignore[misc]@@ -489,13 +581,38 @@ class RealtimeAgent:self._retire_unbound_user_turns()self._transport = None+ def drain_events(self) -> Iterator[AgentEvent]:+ """Yield pending events, including cleanup emitted by ``close``.++ Each yielded event retains its checkpoint until iteration resumes.+ Session owners should drain after closing to publish final tool and+ transcript states before persisting the final session snapshot.+ """+ while not self._out.empty():+ queued = self._out.get_nowait()+ self._yielded_checkpoint = (+ (queued.event.id, queued.checkpoint)+ if queued.checkpoint is not None+ else None+ )+ try:+ yield queued.event+ finally:+ self._yielded_checkpoint = None+# ------------------------------------------------------------------# Discrete input# ------------------------------------------------------------------async def send(self,- inputs: (str | Msg | UserConfirmResultEvent | UserInterruptEvent),+ inputs: (+ str+ | Msg+ | UserConfirmResultEvent+ | ExternalExecutionResultEvent+ | UserInterruptEvent+ ),) -> None:"""Feed the agent anything that is not audio.@@ -514,11 +631,32 @@ class RealtimeAgent:await self._barge_in()case UserConfirmResultEvent():+ tool_call_ids = [+ result.tool_call.id for result in inputs.confirm_results+ ]+ delegation = self._delegation_for_tool_calls(tool_call_ids)+ if delegation is not None:+ self._schedule_client_delegation(delegation, inputs)+ returnfor result in inputs.confirm_results:future = self._confirmations.get(result.tool_call.id)if future and not future.done():future.set_result(result)+ case ExternalExecutionResultEvent():+ tool_call_ids = [+ result.id for result in inputs.execution_results+ ]+ delegation = self._delegation_for_tool_calls(tool_call_ids)+ if delegation is not None:+ self._schedule_client_delegation(delegation, inputs)+ return+ # pylint: disable-next=logging-fstring-interpolation+ logger.warning(+ f"RealtimeAgent: ignored external execution results "+ f"without an active delegation: {tool_call_ids}",+ )+case str() | Msg():if not self.model.supports_text_input:raise NotImplementedError(@@ -553,6 +691,11 @@ class RealtimeAgent:"""Latency breakdown of the most recent turn."""return self._metrics+ @property+ def allows_open_user_reply_checkpoint(self) -> bool:+ """Whether an incomplete user transcript is safe to persist."""+ return self.model.interaction_mode is InteractionMode.FULL_DUPLEX+# ------------------------------------------------------------------# Uplink: transport -> model# ------------------------------------------------------------------@@ -627,6 +770,8 @@ class RealtimeAgent:await self.send(frame.data.get("text", ""))case ControlFrameType.USER_CONFIRM:await self.send(UserConfirmResultEvent(**frame.data))+ case ControlFrameType.EXTERNAL_EXECUTION_RESULT:+ await self.send(ExternalExecutionResultEvent(**frame.data))case ControlFrameType.INTERRUPT:await self.send(UserInterruptEvent(reply_id=self._reply_id))case _:@@ -647,9 +792,25 @@ class RealtimeAgent:the exit of :meth:`reply_stream`, so it is serialised; the losersfind the reply already closed and return."""+ if self.model.interaction_mode is InteractionMode.FULL_DUPLEX:+ await self._interrupt_live_speech()+ returnasync with self._barge_lock:await self._barge_in_locked()+ async def _interrupt_live_speech(self) -> None:+ """Stop Live speech without cancelling delegated backend work."""+ async with self._barge_lock:+ if self._transport is not None and self._audio_pending:+ await self._transport.clear_audio()+ self._audio_pending = False+ if self._connected:+ await self.model.interrupt_speech()+ self._finish_live_role(+ "assistant",+ ReplyFinishedReason.INTERRUPTED,+ )+async def _barge_in_locked(self) -> None:"""Body of :meth:`_barge_in`, run under the lock."""active_reply = self._reply@@ -685,6 +846,7 @@ class RealtimeAgent:self._mark_disconnected()self._reset_user_turns()self._finish_reply(ReplyFinishedReason.ERROR)+ self._finish_live_replies(ReplyFinishedReason.ERROR)logger.info("RealtimeAgent: model session ended; keep talking and it ""reconnects on the next audio.",@@ -692,6 +854,9 @@ class RealtimeAgent:async def _on_model_event(self, event: me.ModelEvent) -> None:"""Handle one model event."""+ if self.model.interaction_mode is InteractionMode.FULL_DUPLEX:+ await self._on_live_event(event)+ returnrate = self.model.output_sample_ratematch event:case me.SpeechStartedEvent():@@ -795,6 +960,904 @@ class RealtimeAgent:event.reason,)+ async def _on_live_event(self, event: me.ModelEvent) -> None:+ """Project one full-duplex timeline event into public events."""+ match event:+ case me.SessionStartedEvent():+ self._live_session_id = event.session_id+ self._last_live_checkpoint_at = None+ self._last_live_checkpoint_end_ms = None++ case me.TimelineAudioDeltaEvent():+ if self._transport is None:+ return+ reply = self._start_live_reply("assistant")+ playout_id = f"live:{self._live_session_id}"+ await self._transport.send_audio(event.pcm, playout_id)+ self._audio_pending = True+ self._emit_audio(reply, event.pcm, event.sample_rate)++ case me.TimelineTranscriptDeltaEvent():+ self._append_live_transcript(event)+ self._segment_live_text(event)++ case me.SessionClosedEvent():+ self._finish_live_replies(ReplyFinishedReason.COMPLETED)++ case me.ModelErrorEvent():+ logger.error(+ "RealtimeAgent: Live model error %s: %s",+ event.code,+ event.message,+ )+ self._finish_live_replies(ReplyFinishedReason.ERROR)++ case me.SessionEndedEvent():+ self._finish_live_replies(ReplyFinishedReason.ERROR)++ case me.DelegationCreatedEvent(target="client"):+ self._start_client_delegation(event.delegation_id)++ case me.DelegationCreatedEvent() | me.DelegatedResponseEvent():+ logger.debug(+ "RealtimeAgent: ignoring an unsupported delegated event",+ )++ case me.SessionUsageEvent():+ pass++ def _start_live_reply(self, role: str) -> _Reply:+ """Open one independent user or assistant Live transcript."""+ current = self._live_replies.get(role)+ if current is not None:+ return current+ reply_id = f"live:{role}:{uuid.uuid4().hex}"+ reply = _Reply(item_id=reply_id, reply_id=reply_id)+ self._live_replies[role] = reply+ self._live_message_ids.add(reply_id)+ self._emit(+ ReplyStartEvent(+ session_id=self.state.session_id,+ reply_id=reply_id,+ name="user" if role == "user" else self.name,+ role="user" if role == "user" else "assistant",+ ),+ )+ if role == "assistant":+ self._emit(+ ModelCallStartEvent(+ reply_id=reply_id,+ model_name=self.model.model,+ ),+ )+ return reply++ def _emit_live_text(+ self,+ role: str,+ reply: _Reply,+ delta: str,+ end_ms: int,+ ) -> None:+ """Append a timestamped Live transcript to its speaker's message."""+ reply.text_chars += len(delta)+ if not reply.text_started:+ reply.text_started = True+ self._emit(+ TextBlockStartEvent(+ reply_id=reply.reply_id,+ block_id=reply.text_block_id,+ ),+ )+ message = next(+ (+ item+ for item in reversed(self.state.context)+ if item.id == reply.reply_id+ ),+ None,+ )+ if message is None:+ message = Msg(+ id=reply.reply_id,+ role="user" if role == "user" else "assistant",+ name="user" if role == "user" else self.name,+ content=[TextBlock(id=reply.text_block_id, text=delta)],+ )+ self.state.context.append(message)+ else:+ blocks = message.get_content_blocks("text")+ if blocks:+ blocks[-1].text += delta+ else:+ message.content.append(+ TextBlock(id=reply.text_block_id, text=delta),+ )+ self._emit(+ TextBlockDeltaEvent(+ reply_id=reply.reply_id,+ block_id=reply.text_block_id,+ delta=delta,+ ),+ checkpoint=self._should_checkpoint_live_transcript(end_ms),+ )++ def _segment_live_text(+ self,+ event: me.TimelineTranscriptDeltaEvent,+ ) -> None:+ """Rotate bounded transcript segments without changing playback."""+ offset = 0+ while offset < len(event.delta):+ reply = self._start_live_reply(event.role)+ available = _LIVE_SEGMENT_MAX_TEXT_CHARS - reply.text_chars+ chunk = event.delta[offset : offset + available]+ self._prune_live_context(len(chunk))+ self._emit_live_text(event.role, reply, chunk, event.end_ms)+ offset += len(chunk)+ if reply.text_chars >= _LIVE_SEGMENT_MAX_TEXT_CHARS:+ # Persist the completed segment before any later event can+ # prune it from runtime context. History retains this copy.+ self._finish_live_role(+ event.role,+ ReplyFinishedReason.COMPLETED,+ )++ def _prune_live_context(self, incoming_chars: int) -> None:+ """Keep recent Live text in state and older segments in history."""+ live_messages = [+ msg+ for msg in self.state.context+ if msg.id.startswith(("live:user:", "live:assistant:"))+ ]+ total_chars = (+ sum(len(msg.get_text_content() or "") for msg in live_messages)+ + incoming_chars+ )+ count = len(live_messages)+ open_ids = {reply.reply_id for reply in self._live_replies.values()}+ removed: set[str] = set()+ for message in live_messages:+ if (+ total_chars <= _HISTORY_MAX_TEXT_CHARS+ and count < _HISTORY_MAX_MESSAGES+ ):+ break+ if message.id in open_ids:+ continue+ removed.add(message.id)+ total_chars -= len(message.get_text_content() or "")+ count -= 1+ if removed:+ self.state.context = [+ msg for msg in self.state.context if msg.id not in removed+ ]+ self._live_message_ids.difference_update(removed)++ def _should_checkpoint_live_transcript(self, end_ms: int) -> bool:+ """Sample state when the wall or Live timeline interval advances."""+ now = time.monotonic()+ interval_ms = int(_LIVE_CHECKPOINT_INTERVAL_SECS * 1_000)+ wall_due = (+ self._last_live_checkpoint_at is None+ or now - self._last_live_checkpoint_at+ >= _LIVE_CHECKPOINT_INTERVAL_SECS+ )+ timeline_due = (+ self._last_live_checkpoint_end_ms is not None+ and end_ms - self._last_live_checkpoint_end_ms >= interval_ms+ )+ if not wall_due and not timeline_due:+ return False+ self._last_live_checkpoint_at = now+ if self._last_live_checkpoint_end_ms is None:+ self._last_live_checkpoint_end_ms = end_ms+ else:+ self._last_live_checkpoint_end_ms = max(+ self._last_live_checkpoint_end_ms,+ end_ms,+ )+ return True++ def _finish_live_role(+ self,+ role: str,+ reason: ReplyFinishedReason,+ ) -> None:+ """Close one speaker's current Live presentation segment."""+ reply = self._live_replies.pop(role, None)+ if reply is None:+ return+ if reply.text_started:+ self._emit(+ TextBlockEndEvent(+ reply_id=reply.reply_id,+ block_id=reply.text_block_id,+ ),+ )+ if reply.audio_started:+ self._emit(+ DataBlockEndEvent(+ reply_id=reply.reply_id,+ block_id=reply.audio_block_id,+ ),+ )+ if role == "assistant":+ self._emit(+ ModelCallEndEvent(+ reply_id=reply.reply_id,+ input_tokens=0,+ output_tokens=0,+ ),+ )+ finished = ReplyEndEvent(+ session_id=self.state.session_id,+ reply_id=reply.reply_id,+ finished_reason=reason,+ )+ for message in reversed(self.state.context):+ if message.id == reply.reply_id:+ message.append_event(finished)+ break+ self._emit(finished)++ def _finish_live_replies(self, reason: ReplyFinishedReason) -> None:+ """Close both independently growing Live transcript streams."""+ self._finish_live_role("user", reason)+ self._finish_live_role("assistant", reason)++ def _start_client_delegation(self, delegation_id: str) -> None:+ """Queue one GPT-Live task behind the active delegation."""+ if (+ self._closing+ or not delegation_id+ or delegation_id in self._delegations+ ):+ return+ if self._delegation_agent_factory is None:+ self._schedule_delegation_failure(delegation_id)+ return+ prompt = UserMsg(+ name="user",+ content=self._build_delegation_prompt(),+ )+ delegation = _ClientDelegation(+ delegation_id=delegation_id,+ prompt=prompt,+ generation=self._connection_generation,+ )+ self._delegations[delegation_id] = delegation+ if self._active_delegation_id is None:+ self._activate_client_delegation(delegation)+ else:+ self._delegation_queue.append(delegation_id)++ def _activate_client_delegation(+ self,+ delegation: _ClientDelegation,+ ) -> None:+ """Create a backend from the latest state and start its task."""+ if self._delegation_agent_factory is None:+ self._delegations.pop(delegation.delegation_id, None)+ self._schedule_delegation_failure(delegation.delegation_id)+ self._activate_next_client_delegation()+ return+ try:+ agent = self._delegation_agent_factory(+ self._copy_delegation_state(),+ )+ if agent.name != self.name:+ raise ValueError(+ f"Delegation backend name {agent.name!r} must match "+ f"RealtimeAgent name {self.name!r}.",+ )+ except Exception: # noqa: BLE001+ logger.exception(+ "RealtimeAgent: failed to create delegation backend",+ )+ self._delegations.pop(delegation.delegation_id, None)+ self._schedule_delegation_failure(delegation.delegation_id)+ self._activate_next_client_delegation()+ return+ delegation.agent = agent+ delegation.prompt_id = delegation.prompt.id+ delegation.baseline_ids = {+ message.id for message in agent.state.context+ }+ self._active_delegation_id = delegation.delegation_id+ self._schedule_client_delegation(delegation, delegation.prompt)++ def _activate_next_client_delegation(self) -> None:+ """Start the oldest queued delegation, if one remains."""+ if self._closing or self._active_delegation_id is not None:+ return+ while self._delegation_queue:+ delegation_id = self._delegation_queue.popleft()+ delegation = self._delegations.get(delegation_id)+ if delegation is not None:+ self._activate_client_delegation(delegation)+ return++ def _schedule_client_delegation(+ self,+ delegation: _ClientDelegation,+ inputs: (+ Msg+ | UserConfirmResultEvent+ | ExternalExecutionResultEvent+ | UserInterruptEvent+ ),+ ) -> None:+ """Run or resume a delegated chat-agent reply in the background."""+ task = asyncio.create_task(+ self._drive_client_delegation(delegation, inputs),+ name=f"live-delegation-{delegation.delegation_id}",+ )+ self._tasks.add(task)+ task.add_done_callback(self._tasks.discard)++ def _schedule_delegation_timeout(+ self,+ delegation: _ClientDelegation,+ ) -> None:+ """Cancel a delegation that remains parked without input."""+ self._cancel_delegation_timeout(delegation)+ delegation.timeout_task = asyncio.create_task(+ self._expire_client_delegation(delegation),+ name=f"live-delegation-timeout-{delegation.delegation_id}",+ )++ def _cancel_delegation_timeout(+ self,+ delegation: _ClientDelegation,+ ) -> None:+ """Cancel the pending external-input deadline, if any."""+ task = delegation.timeout_task+ delegation.timeout_task = None+ if task is not None and task is not asyncio.current_task():+ task.cancel()++ async def _expire_client_delegation(+ self,+ delegation: _ClientDelegation,+ ) -> None:+ """Interrupt and finish a delegation after its input deadline."""+ try:+ await asyncio.sleep(_DELEGATION_INPUT_TIMEOUT_SECS)+ if (+ self._delegations.get(delegation.delegation_id)+ is not delegation+ or self._active_delegation_id != delegation.delegation_id+ or delegation.agent is None+ ):+ return+ delegation.expiring = True+ await self._drive_client_delegation(+ delegation,+ UserInterruptEvent(reply_id=delegation.agent.state.reply_id),+ result_override=(+ "The delegated task was cancelled because the required "+ "user input or external execution was not provided in "+ "time."+ ),+ )+ except asyncio.CancelledError:+ return++ def _schedule_delegation_failure(self, delegation_id: str) -> None:+ """Tell GPT-Live that an application backend is unavailable."""+ task = asyncio.create_task(+ self._send_delegation_result(+ delegation_id,+ "The delegated task could not be completed because the "+ "session chat agent is unavailable.",+ self._connection_generation,+ ),+ name=f"live-delegation-failure-{delegation_id}",+ )+ self._tasks.add(task)+ task.add_done_callback(self._tasks.discard)++ async def _send_delegation_result(+ self,+ delegation_id: str,+ content: str,+ generation: int,+ ) -> None:+ """Send results only to the Live connection that requested them."""+ if self._closing or generation != self._connection_generation:+ return+ await self.model.send_delegation_result(delegation_id, content)++ async def _drive_client_delegation(+ self,+ delegation: _ClientDelegation,+ inputs: (+ Msg+ | UserConfirmResultEvent+ | ExternalExecutionResultEvent+ | UserInterruptEvent+ ),+ result_override: str | None = None,+ ) -> None:+ """Run a chat agent and return its verified result to GPT-Live."""+ if delegation.generation != self._connection_generation:+ return+ async with self._delegation_lock:+ # Routing happened before this task waited for the lock.+ if (+ delegation.generation != self._connection_generation+ or self._delegations.get(delegation.delegation_id)+ is not delegation+ or self._active_delegation_id != delegation.delegation_id+ ):+ return+ if isinstance(+ inputs,+ (UserConfirmResultEvent, ExternalExecutionResultEvent),+ ):+ if (+ self._closing+ or delegation.expiring+ or delegation.agent is None+ or inputs.reply_id != delegation.agent.state.reply_id+ ):+ return+ awaiting = {+ tool_call.id: tool_call.state+ for tool_call in (+ delegation.agent.state.get_awaiting_tool_calls(+ self.name,+ )+ )+ }+ if isinstance(inputs, UserConfirmResultEvent):+ confirmations = [+ result+ for result in inputs.confirm_results+ if awaiting.get(result.tool_call.id)+ == ToolCallState.ASKING+ ]+ if not confirmations:+ return+ inputs = inputs.model_copy(+ update={"confirm_results": confirmations},+ )+ else:+ results = [+ result+ for result in inputs.execution_results+ if awaiting.get(result.id) == ToolCallState.SUBMITTED+ ]+ if not results:+ return+ inputs = inputs.model_copy(+ update={"execution_results": results},+ )+ # An earlier queued input may have parked and rearmed the timer.+ self._cancel_delegation_timeout(delegation)+ final: Msg | None = None+ waiting_for_external_input = False+ pending_result_end: ToolResultEndEvent | None = None+ try:+ if delegation.agent is None:+ raise RuntimeError(+ "The delegation backend is unavailable.",+ )+ async for item in delegation.agent.reply_stream(+ inputs,+ yield_final_msg=True,+ ):+ if pending_result_end is not None:+ self._checkpoint_delegation_tool_result(+ delegation,+ pending_result_end,+ )+ pending_result_end = None+ if isinstance(item, Msg):+ final = item+ continue+ if not delegation.visible and isinstance(+ item,+ (+ ReplyStartEvent,+ ToolCallStartEvent,+ ToolCallDeltaEvent,+ ToolCallEndEvent,+ ToolResultStartEvent,+ ToolResultTextDeltaEvent,+ ToolResultDataDeltaEvent,+ ToolResultEndEvent,+ ),+ ):+ delegation.buffered_events.append(item)+ if isinstance(item, ToolResultEndEvent):+ reply = delegation.agent.state.get_reply_message(+ self.name,+ )+ if (+ reply is not None+ and any(+ call.id == item.tool_call_id+ and call.state == ToolCallState.FINISHED+ for call in reply.get_content_blocks(+ "tool_call",+ )+ )+ and any(+ result.id == item.tool_call_id+ and result.state == item.state+ and result.state != ToolResultState.RUNNING+ for result in reply.get_content_blocks(+ "tool_result",+ )+ )+ ):+ # Local tools commit before END. Persist now,+ # before the next model request can block.+ self._checkpoint_delegation_tool_result(+ delegation,+ item,+ )+ else:+ # External and interrupted results may commit+ # after yielding END.+ pending_result_end = item+ continue+ if isinstance(+ item,+ (+ RequireUserConfirmEvent,+ RequireExternalExecutionEvent,+ ),+ ):+ self._sync_delegation_state(delegation)+ self._show_delegation_interaction(+ delegation,+ item,+ )+ waiting_for_external_input = True+ elif delegation.visible and isinstance(+ item,+ (+ ToolCallStartEvent,+ ToolCallDeltaEvent,+ ToolCallEndEvent,+ ToolResultStartEvent,+ ToolResultTextDeltaEvent,+ ToolResultDataDeltaEvent,+ ReplyEndEvent,+ ),+ ):+ if isinstance(item, ReplyEndEvent):+ self._sync_delegation_state(delegation)+ self._emit(item)++ self._sync_delegation_state(delegation)+ if pending_result_end is not None:+ self._checkpoint_delegation_tool_result(+ delegation,+ pending_result_end,+ )+ pending_result_end = None+ if waiting_for_external_input or (+ delegation.agent.state.has_awaiting_tool_calls(self.name)+ ):+ if not self._closing:+ self._schedule_delegation_timeout(delegation)+ return+ if final is None:+ raise RuntimeError(+ "The delegation backend ended without a final result.",+ )++ if self._closing:+ self._finish_client_delegation(delegation.delegation_id)+ return+ content = (+ result_override or (final.get_text_content() or "").strip()+ )+ if not content:+ content = (+ "The delegated task completed without a text result."+ )+ await self._send_delegation_result(+ delegation.delegation_id,+ content,+ delegation.generation,+ )+ self._finish_client_delegation(delegation.delegation_id)+ except Exception: # noqa: BLE001+ logger.exception(+ "RealtimeAgent: client delegation %s failed",+ delegation.delegation_id,+ )+ # Completed tools must survive a later backend failure.+ self._sync_delegation_state(delegation)+ if pending_result_end is not None:+ self._checkpoint_delegation_tool_result(+ delegation,+ pending_result_end,+ )+ if (+ not self._closing+ and not delegation.expiring+ and isinstance(inputs, ExternalExecutionResultEvent)+ and delegation.agent is not None+ and delegation.agent.state.has_awaiting_tool_calls(+ self.name,+ )+ ):+ # A rejected external answer leaves the backend parked.+ # Retain its route for retry after syncing valid results.+ awaiting_calls = (+ delegation.agent.state.get_awaiting_tool_calls(+ self.name,+ )+ )+ awaiting_ids = {call.id for call in awaiting_calls}+ rejected_ids = [+ result.id+ for result in inputs.execution_results+ if result.id in awaiting_ids+ ]+ if rejected_ids:+ self._emit(+ CustomEvent(+ name="external_execution_result_rejected",+ value={+ "request_id": inputs.id,+ "reply_id": (+ delegation.agent.state.reply_id+ ),+ "tool_call_ids": rejected_ids,+ "message": (+ "The submitted result was rejected. "+ "Please check your answer and submit "+ "again."+ ),+ },+ ),+ )+ self._schedule_delegation_timeout(delegation)+ return+ if self._closing:+ self._finish_client_delegation(delegation.delegation_id)+ return+ try:+ await self._send_delegation_result(+ delegation.delegation_id,+ "The delegated task failed. "+ "Ask the user to try again.",+ delegation.generation,+ )+ except Exception: # noqa: BLE001+ logger.exception(+ "RealtimeAgent: failed to report delegation failure",+ )+ self._finish_client_delegation(delegation.delegation_id)++ def _checkpoint_delegation_tool_result(+ self,+ delegation: _ClientDelegation,+ event: ToolResultEndEvent,+ ) -> None:+ """Persist committed results independently of tool-card visibility."""+ self._sync_delegation_state(delegation)+ if delegation.visible:+ self._emit(event)+ else:+ self._emit(+ CustomEvent(+ name="delegation_checkpoint",+ value={+ "delegation_id": delegation.delegation_id,+ "reply_id": event.reply_id,+ "tool_call_id": event.tool_call_id,+ },+ ),+ checkpoint=True,+ )++ def _build_delegation_prompt(self) -> str:+ """Build a bounded task prompt from the current Live transcript."""+ lines: list[str] = []+ for role, delta, _, _ in self._live_transcript:+ label = "User" if role == "user" else "Voice assistant"+ if lines and lines[-1].startswith(f"{label}: "):+ lines[-1] = f"{lines[-1]}{delta}"+ else:+ lines.append(f"{label}: {delta}")+ transcript = "\n".join(lines)+ if len(transcript) > _HISTORY_MAX_TEXT_CHARS:+ transcript = transcript[-_HISTORY_MAX_TEXT_CHARS:]+ if not transcript.strip():+ transcript = "No transcript text is available."+ return (+ f"You are the task backend for a live voice conversation. "+ f"Use the latest transcript, the session context, and the "+ f"available tools to complete the delegated task. Transcript "+ f"fragments may contain corrections; prefer the latest user "+ f"intent. Do not claim an action succeeded until a tool confirms "+ f"it. Return verified facts, current status, and the next step in "+ f"a concise result of at most 300 words for the voice assistant "+ f"to communicate.\n\n## Live transcript\n{transcript}"+ )++ def _show_delegation_interaction(+ self,+ delegation: _ClientDelegation,+ interaction: RequireUserConfirmEvent | RequireExternalExecutionEvent,+ ) -> None:+ """Expose a delegated external-input request through the UI."""+ if not delegation.visible:+ start = next(+ (+ event+ for event in delegation.buffered_events+ if isinstance(event, ReplyStartEvent)+ ),+ None,+ )+ if start is not None:+ self._emit(start)+ for event in delegation.buffered_events:+ if isinstance(+ event,+ (+ ToolCallStartEvent,+ ToolCallDeltaEvent,+ ToolCallEndEvent,+ ToolResultStartEvent,+ ToolResultTextDeltaEvent,+ ToolResultDataDeltaEvent,+ ToolResultEndEvent,+ ),+ ):+ self._emit(event)+ delegation.visible = True+ delegation.buffered_events.clear()+ self._emit(interaction)+ for tool_call in interaction.tool_calls:+ self._delegated_tool_calls[tool_call.id] = delegation.delegation_id++ def _copy_delegation_state(self) -> AgentState:+ """Copy session state while omitting duplicated Live messages."""+ state = self.state.model_copy()+ for field_name in type(self.state).model_fields:+ if field_name != "context":+ setattr(+ state,+ field_name,+ deepcopy(getattr(self.state, field_name)),+ )+ state.context = [+ message.model_copy(deep=True)+ for message in self.state.context+ if message.id not in self._live_message_ids+ ]+ return state++ def _sync_delegation_state(+ self,+ delegation: _ClientDelegation,+ ) -> None:+ """Persist delegated tool state without duplicating hidden prompts."""+ if delegation.agent is None:+ return+ backend_state = delegation.agent.state+ self.state.reply_context = backend_state.reply_context.model_copy(+ deep=True,+ )+ self.state.permission_context = (+ backend_state.permission_context.model_copy(deep=True)+ )+ self.state.tool_context = backend_state.tool_context.model_copy(+ deep=True,+ )+ self.state.tasks_context = backend_state.tasks_context.model_copy(+ deep=True,+ )+ self.state.middle_context = deepcopy(backend_state.middle_context)+ self.state.summary = deepcopy(backend_state.summary)+ projected: dict[str, Msg] = {}+ for message in backend_state.context:+ if message.id == delegation.prompt_id:+ continue+ if message.id in delegation.baseline_ids:+ projected[message.id] = message.model_copy(deep=True)+ continue+ kept = [+ block.model_copy(deep=True)+ for block in message.content+ if block.type in {"tool_call", "tool_result"}+ ]+ if not kept:+ continue+ synced = message.model_copy(deep=True)+ synced.content = kept+ projected[message.id] = synced++ # Only this backend's history may be replaced or compacted. Live+ # messages added or extended while it runs belong to the parent.+ owned_ids = delegation.baseline_ids | delegation.synced_ids+ delegation.synced_ids = projected.keys() - delegation.baseline_ids+ context = []+ for message in self.state.context:+ if message.id in owned_ids:+ replacement = projected.pop(message.id, None)+ if replacement is not None:+ context.append(replacement)+ else:+ context.append(message)+ context.extend(projected.values())+ self.state.context = context++ def _finish_client_delegation(self, delegation_id: str) -> None:+ """Drop routing state after a delegated task reaches a result."""+ delegation = self._delegations.pop(delegation_id, None)+ if delegation is not None:+ self._cancel_delegation_timeout(delegation)+ stale = [+ tool_call_id+ for tool_call_id, owner in self._delegated_tool_calls.items()+ if owner == delegation_id+ ]+ for tool_call_id in stale:+ self._delegated_tool_calls.pop(tool_call_id, None)+ if self._active_delegation_id == delegation_id:+ self._active_delegation_id = None+ self._activate_next_client_delegation()++ def _delegation_for_tool_calls(+ self,+ tool_call_ids: list[str],+ ) -> _ClientDelegation | None:+ """Resolve one active delegation owning every supplied tool call."""+ if not tool_call_ids:+ return None+ delegation_id = self._delegated_tool_calls.get(tool_call_ids[0])+ if delegation_id is None or any(+ self._delegated_tool_calls.get(tool_call_id) != delegation_id+ for tool_call_id in tool_call_ids+ ):+ return None+ if delegation_id != self._active_delegation_id:+ return None+ delegation = self._delegations.get(delegation_id)+ if (+ delegation is None+ or delegation.expiring+ or delegation.generation != self._connection_generation+ ):+ return None+ return delegation++ def _append_live_transcript(+ self,+ event: me.TimelineTranscriptDeltaEvent,+ ) -> None:+ """Keep a timestamped transcript window within the prompt budget."""+ fragment = (+ event.role,+ event.delta,+ event.start_ms,+ event.end_ms,+ )+ self._live_transcript.append(fragment)+ self._live_transcript_chars += len(fragment[1])+ while self._live_transcript_chars > _HISTORY_MAX_TEXT_CHARS:+ role, text, start_ms, end_ms = self._live_transcript.popleft()+ excess = self._live_transcript_chars - _HISTORY_MAX_TEXT_CHARS+ if not self._live_transcript and len(text) > excess:+ kept = text[excess:]+ self._live_transcript.appendleft(+ (role, kept, start_ms, end_ms),+ )+ self._live_transcript_chars -= excess+ break+ self._live_transcript_chars -= len(text)+def _start_user_turn(self, item_id: str = "") -> None:"""Open the user's turn as a reply of its own. A local VAD sees itbefore the provider creates an item, hence the generated id."""@@ -1131,19 +2194,25 @@ class RealtimeAgent:),)- def _emit(self, event: AgentEvent) -> None:+ def _emit(+ self,+ event: AgentEvent,+ *,+ checkpoint: bool = False,+ ) -> None:"""Queue one event for :meth:`reply_stream`."""- checkpoint = None- if isinstance(+ snapshot = None+ if checkpoint or isinstance(event,(ReplyEndEvent,+ RequireExternalExecutionEvent,RequireUserConfirmEvent,ToolResultEndEvent,),):- checkpoint = self.state.model_copy(deep=True)- self._out.put_nowait(_QueuedEvent(event, checkpoint))+ snapshot = self.state.model_copy(deep=True)+ self._out.put_nowait(_QueuedEvent(event, snapshot))def checkpoint_snapshot(self, event: AgentEvent) -> AgentState | None:"""Return the immutable state paired with the yielded event."""diff --git a/src/agentscope/app/_lifespan.py b/src/agentscope/app/_lifespan.pyindex 31e8a7e5c..e416090eb 100644--- a/src/agentscope/app/_lifespan.py+++ b/src/agentscope/app/_lifespan.py@@ -188,8 +188,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:background_task_manager=bg_manager,message_bus=message_bus,resource_access_service=resource_access_service,+ knowledge_base_manager=knowledge_base_manager,+ extra_agent_middlewares=app.state.extra_agent_middlewares,extra_agent_tools=app.state.extra_agent_tools,custom_subagent_templates=app.state.custom_subagent_templates,+ custom_agent_cls=app.state.custom_agent_cls,channel_clients=channel_clients,)diff --git a/src/agentscope/app/_router/_realtime.py b/src/agentscope/app/_router/_realtime.pyindex 69bba8ac2..2defc87f7 100644--- a/src/agentscope/app/_router/_realtime.py+++ b/src/agentscope/app/_router/_realtime.py@@ -123,7 +123,7 @@ async def _create_realtime_offer(realtime_service: RealtimeService,) -> RealtimeOfferResponse:"""Create one offer while this process serializes session starts."""- await access.resolve_agent(user_id, body.agent_id)+ agent_record = await access.resolve_agent(user_id, body.agent_id)session = await storage.get_session(user_id,body.agent_id,@@ -175,6 +175,28 @@ async def _create_realtime_offer(status_code=status.HTTP_400_BAD_REQUEST,detail=str(exc),) from exc+ card = getattr(model, "card", None)+ uses_session_chat = (+ getattr(card, "delegation_backend", "none") == "session_chat"+ )+ if uses_session_chat and session.state.has_awaiting_tool_calls(+ agent_record.data.name,+ ):+ raise HTTPException(+ status_code=status.HTTP_409_CONFLICT,+ detail=(+ "Complete the pending Chat confirmation or "+ "external execution before starting GPT Live."+ ),+ )+ if (+ uses_session_chat+ and getattr(session.config, "chat_model_config", None) is None+ ):+ raise HTTPException(+ status_code=status.HTTP_409_CONFLICT,+ detail="Configure a chat model before starting GPT Live.",+ )lock_key = MessageBusKeys.session_lock(session_id)if await message_bus.is_locked(lock_key):diff --git a/src/agentscope/app/_service/_realtime.py b/src/agentscope/app/_service/_realtime.pyindex 153e32838..b3b722cf9 100644--- a/src/agentscope/app/_service/_realtime.py+++ b/src/agentscope/app/_service/_realtime.py@@ -1,20 +1,32 @@# -*- coding: utf-8 -*-"""Runtime assembly for browser realtime agents."""+import inspectfrom typing import Literalfrom ..._logging import logger-from ...agent import RealtimeAgent+from ...agent import Agent, ModelConfig, RealtimeAgent+from ...middleware import MiddlewareBase, RAGMiddlewarefrom ...permission import AdditionalWorkingDirectory+from ...rag import KnowledgeBasefrom ...realtime import RealtimeModelBase+from ...state import AgentStatefrom .._manager import BackgroundTaskManager, SchedulerManager-from .._types import AgentToolFactory, SubAgentTemplate+from .._types import (+ AgentMiddlewareFactory,+ AgentToolFactory,+ SubAgentTemplate,+)from ...tool import ToolBase+from ...workspace import WorkspaceBasefrom ..channel import ChannelBase, ChannelClients, ChatKindfrom ..message_bus import MessageBus+from ..middleware import StateChangeMiddleware+from ..rag.knowledge_base_manager import KnowledgeBaseManagerBasefrom ..storage import ChannelOrigin, SessionRecord, StorageBasefrom ..workspace_manager import WorkspaceManagerBasefrom ._access import ResourceAccessService+from ._model import get_modelfrom ._toolkit import get_toolkit@@ -30,18 +42,70 @@ class RealtimeService:background_task_manager: BackgroundTaskManager,message_bus: MessageBus,resource_access_service: ResourceAccessService,+ knowledge_base_manager: KnowledgeBaseManagerBase | None = None,+ extra_agent_middlewares: AgentMiddlewareFactory | None = None,extra_agent_tools: AgentToolFactory | None = None,custom_subagent_templates: dict[str, SubAgentTemplate] | None = None,+ custom_agent_cls: type[Agent] | None = None,channel_clients: ChannelClients | None = None,) -> None:+ """Initialize the application-wide realtime Agent assembler.++ This service owns no running connection. A :class:`WebRTCSession`+ calls :meth:`create_agent` for each negotiated voice session.++ Args:+ storage (`StorageBase`):+ Persistent session and agent storage.+ workspace_manager (`WorkspaceManagerBase`):+ Resolves the workspace bound to each session.+ scheduler_manager (`SchedulerManager`):+ Provides schedule tools to assembled toolkits.+ background_task_manager (`BackgroundTaskManager`):+ Provides background-task tools and runtime state.+ message_bus (`MessageBus`):+ Shared bus used by session and team tools.+ resource_access_service (`ResourceAccessService`):+ Resolves caller-visible agents, credentials, and knowledge.+ knowledge_base_manager (`KnowledgeBaseManagerBase | None`,+ optional):+ Resolves knowledge handles for delegated Chat Agents.+ extra_agent_middlewares (`AgentMiddlewareFactory | None`,+ optional):+ Builds application-provided middlewares for delegation.+ extra_agent_tools (`AgentToolFactory | None`, optional):+ Builds application-provided tools for the session toolkit.+ custom_subagent_templates (`dict[str, SubAgentTemplate] | None`,+ optional):+ Sub-agent templates exposed through Agent creation tools.+ custom_agent_cls (`type[Agent] | None`, optional):+ Agent implementation used for delegated Chat tasks.+ channel_clients (`ChannelClients | None`, optional):+ Resolves tools and identity for channel-bound sessions.+ """self._storage = storageself._workspace_manager = workspace_managerself._scheduler_manager = scheduler_managerself._background_task_manager = background_task_managerself._message_bus = message_busself._access = resource_access_service+ self._knowledge_base_manager = knowledge_base_manager+ self._extra_agent_middlewares = extra_agent_middlewares+ self._middlewares_take_workspace = False+ if extra_agent_middlewares is not None:+ try:+ inspect.signature(extra_agent_middlewares).bind(+ "",+ "",+ "",+ None,+ )+ self._middlewares_take_workspace = True+ except (TypeError, ValueError):+ passself._extra_agent_tools = extra_agent_toolsself._subagent_templates = custom_subagent_templates or {}+ self._agent_cls = custom_agent_cls or Agentself._channel_clients = channel_clientsasync def create_agent(@@ -52,7 +116,34 @@ class RealtimeService:session_id: str,model: RealtimeModelBase,) -> RealtimeAgent:- """Load the locked session and assemble its realtime runtime."""+ """Load a locked session and assemble its realtime runtime.++ The caller must hold the distributed session lock before invoking+ this method so the loaded state cannot race another Chat or voice+ run. The returned Agent is not connected until its async context is+ entered.++ Args:+ user_id (`str`):+ Owner of the session being opened.+ agent_id (`str`):+ Agent configuration used by the session.+ session_id (`str`):+ Persisted session whose state the runtime will share.+ model (`RealtimeModelBase`):+ Provider adapter already resolved from the session config.++ Returns:+ `RealtimeAgent`:+ An assembled but disconnected realtime Agent.++ Raises:+ RuntimeError:+ If the session no longer exists.+ ValueError:+ If a client-delegating model has no Chat model configured+ or the session has a pending Chat interaction.+ """agent_record = await self._access.resolve_agent(user_id, agent_id)session = await self._storage.get_session(user_id,@@ -62,6 +153,18 @@ class RealtimeService:if session is None:raise RuntimeError(f"Session {session_id!r} no longer exists.")+ card = getattr(model, "card", None)+ uses_session_chat = (+ getattr(card, "delegation_backend", "none") == "session_chat"+ )+ if uses_session_chat and session.state.has_awaiting_tool_calls(+ agent_record.data.name,+ ):+ raise ValueError(+ "Complete the pending Chat confirmation or "+ "external execution before starting GPT Live.",+ )+workspace = await self._workspace_manager.get_workspace(user_id,agent_id,@@ -101,6 +204,17 @@ class RealtimeService:if channel is not None and channel_origin is not Noneelse [])+ middlewares = (+ await self._build_delegation_middlewares(+ user_id=user_id,+ agent_id=agent_id,+ session_id=session_id,+ session=session,+ workspace=workspace,+ )+ if uses_session_chat+ else []+ )toolkit = await get_toolkit(storage=self._storage,workspace=workspace,@@ -108,7 +222,7 @@ class RealtimeService:scheduler_manager=self._scheduler_manager,background_task_manager=self._background_task_manager,message_bus=self._message_bus,- middlewares=[],+ middlewares=middlewares,user_id=user_id,agent_record=agent_record,session_record=session,@@ -127,14 +241,113 @@ class RealtimeService:channel_tools,chat_kind,)+ delegation_agent_factory = None+ if uses_session_chat:+ chat_config = session.config.chat_model_config+ if chat_config is None:+ raise ValueError(+ "Configure a chat model before starting GPT Live.",+ )+ chat_model = await get_model(user_id, chat_config, self._access)+ fallback_config = session.config.fallback_chat_model_config+ fallback_model = (+ await get_model(user_id, fallback_config, self._access)+ if fallback_config is not None+ else None+ )++ def _create_delegation_agent(state: AgentState) -> Agent:+ """Build an isolated task agent from current session state."""+ config = agent_record.data.chat_config+ return self._agent_cls(+ name=agent_record.data.name,+ system_prompt=system_prompt,+ model=chat_model,+ toolkit=toolkit,+ model_config=ModelConfig(+ fallback_model=fallback_model,+ ),+ context_config=config.context_config,+ react_config=config.react_config,+ state=state,+ middlewares=middlewares,+ offloader=workspace,+ )++ delegation_agent_factory = _create_delegation_agent+return RealtimeAgent(name=agent_record.data.name,system_prompt=system_prompt,model=model,toolkit=toolkit,state=session.state,+ delegation_agent_factory=delegation_agent_factory,)+ async def _build_delegation_middlewares(+ self,+ *,+ user_id: str,+ agent_id: str,+ session_id: str,+ session: SessionRecord,+ workspace: WorkspaceBase,+ ) -> list[MiddlewareBase]:+ """Build Chat middlewares that are safe in a Live task backend."""+ middlewares: list[MiddlewareBase] = [+ StateChangeMiddleware(+ message_bus=self._message_bus,+ session_id=session_id,+ ),+ ]+ if self._extra_agent_middlewares is not None:+ factory_args: tuple = (user_id, agent_id, session_id)+ if self._middlewares_take_workspace:+ factory_args += (workspace,)+ middlewares.extend(+ await self._extra_agent_middlewares(*factory_args),+ )++ config = session.config.knowledge_config+ if (+ config is None+ or not config.knowledge_base_ids+ or self._knowledge_base_manager is None+ ):+ return middlewares++ knowledges: list[KnowledgeBase] = []+ for knowledge_id in config.knowledge_base_ids:+ try:+ record = await self._access.resolve_knowledge_base(+ user_id,+ knowledge_id,+ )+ knowledge = await self._knowledge_base_manager.get_knowledge(+ record.user_id,+ knowledge_id,+ )+ except Exception: # noqa: BLE001+ logger.exception(+ "Skipping knowledge base %r for session %r: failed "+ "to resolve runtime handle.",+ knowledge_id,+ session_id,+ )+ continue+ knowledges.append(knowledge)+ if knowledges:+ middlewares.append(+ RAGMiddleware(+ knowledge_bases=knowledges,+ parameters=RAGMiddleware.Parameters(+ **(config.parameters or {}),+ ),+ ),+ )+ return middlewares+async def _resolve_team_role(self,user_id: str,diff --git a/src/agentscope/app/_service/_webrtc_session.py b/src/agentscope/app/_service/_webrtc_session.pyindex 53a8b0442..15b17f98a 100644--- a/src/agentscope/app/_service/_webrtc_session.py+++ b/src/agentscope/app/_service/_webrtc_session.py@@ -2,13 +2,15 @@"""Lifecycle owner for one browser WebRTC realtime session."""import asyncio-from collections.abc import Awaitable, Callable+from collections.abc import AsyncGenerator, Awaitable, Callable+from contextlib import aclosingfrom aiortc import RTCPeerConnectionfrom ..._logging import loggerfrom ...agent import RealtimeAgentfrom ...event import (+ AgentEvent,DataBlockDeltaEvent,DataBlockEndEvent,DataBlockStartEvent,@@ -26,6 +28,12 @@ _CHECKPOINT_LOCK_TTL_SECS = ()+def _stream_id_key(entry_id: str) -> tuple[int, int]:+ """Return the sortable numeric components of a message-bus stream ID."""+ major, minor = entry_id.split("-", maxsplit=1)+ return int(major), int(minor)++class WebRTCSession:"""Run and persist one realtime agent over a peer connection."""@@ -42,6 +50,33 @@ class WebRTCSession:session_id: str,on_closed: Callable[["WebRTCSession"], None],) -> None:+ """Initialize one negotiated browser voice-session owner.++ The instance owns the peer connection, audio transport, Agent+ lifecycle, session lock, event publication, and state persistence.+ Calling :meth:`start` begins those activities in a background task.++ Args:+ peer_connection (`RTCPeerConnection`):+ Negotiated WebRTC connection to close with the session.+ transport (`WebRTCAudioTransport`):+ Audio and control-frame bridge for the peer connection.+ agent_factory (`Callable[[], Awaitable[RealtimeAgent]]`):+ Builds the Agent after the session lock is acquired.+ storage (`StorageBase`):+ Persists messages and the final Agent state.+ message_bus (`MessageBus`):+ Provides the session lock and publishes replayable events.+ user_id (`str`):+ Owner of the persisted session.+ agent_id (`str`):+ Agent associated with the persisted session.+ session_id (`str`):+ Session used for locking, events, and persistence.+ on_closed (`Callable[[WebRTCSession], None]`):+ Callback that removes this runner from the connection+ registry after all resources close.+ """self.peer_connection = peer_connectionself.transport = transportself._agent_factory = agent_factory@@ -150,7 +185,14 @@ class WebRTCSession:return Trueasync def _run(self) -> None:- """Hold the session lock while forwarding events to the SSE log."""+ """Own the locked realtime runtime until its transport ends.++ After acquiring the distributed session lock, this task creates and+ connects the Agent, forwards non-PCM events to the replayable SSE+ stream, persists event-bound checkpoints, and saves the latest state+ before releasing the lock. It always closes the transport and peer+ connection and invokes ``on_closed`` during cleanup.+ """if self._close_task is not None:returntry:@@ -166,11 +208,8 @@ class WebRTCSession:for message in self.agent.state.context}async with self.transport:- async with self.agent:- self._startup_complete = True- async for event in self.agent.reply_stream(- self.transport,- ):+ async with aclosing(self._agent_events()) as events:+ async for event in events:if isinstance(event, DataBlockStartEvent) and (event.media_type.startswith("audio/pcm")):@@ -209,9 +248,13 @@ class WebRTCSession:self._open_user_replies.discard(event.reply_id,)- if (- checkpoint is not None- and not self._open_user_replies+ if checkpoint is not None and (+ not self._open_user_replies+ or getattr(+ self.agent,+ "allows_open_user_reply_checkpoint",+ False,+ )):try:await self._checkpoint_state(@@ -273,6 +316,23 @@ class WebRTCSession:finally:await self._close_resources()+ async def _agent_events(self) -> AsyncGenerator[AgentEvent, None]:+ """Include terminal events emitted when the voice agent closes."""+ assert self.agent is not None+ try:+ async with self.agent:+ self._startup_complete = True+ async for event in self.agent.reply_stream(self.transport):+ yield event+ except Exception:+ # Publish cleanup events before the runner reports the error.+ # GeneratorExit and cancellation must not yield during close.+ for event in self.agent.drain_events():+ yield event+ raise+ for event in self.agent.drain_events():+ yield event+async def _checkpoint_state(self,entry_id: str | None,@@ -283,6 +343,13 @@ class WebRTCSession:MessageBusKeys.session_event_checkpoint_lock(self.session_id),ttl_secs=_CHECKPOINT_LOCK_TTL_SECS,):+ if (+ entry_id is not None+ and self._checkpoint_entry_id is not None+ and _stream_id_key(entry_id)+ <= _stream_id_key(self._checkpoint_entry_id)+ ):+ returnawait self._persist_state(state)if entry_id is not None:await self._set_checkpoint_cursor(entry_id)@@ -326,9 +393,12 @@ class WebRTCSession:)removed_ids = self._persisted_messages.keys() - current_messages.keys()for message_id in sorted(removed_ids):- await self.storage.delete_message(- self.user_id,- self.session_id,- message_id,- )+ # Completed Live segments leave the bounded runtime window but+ # remain readable in history. Drop their cache entries as well.+ if not message_id.startswith(("live:user:", "live:assistant:")):+ await self.storage.delete_message(+ self.user_id,+ self.session_id,+ message_id,+ )self._persisted_messages.pop(message_id, None)diff --git a/src/agentscope/credential/_openai.py b/src/agentscope/credential/_openai.pyindex 51a352e7a..eb5930aa7 100644--- a/src/agentscope/credential/_openai.py+++ b/src/agentscope/credential/_openai.py@@ -94,6 +94,6 @@ class OpenAICredential(CredentialBase):@classmethoddef get_realtime_model_classes(cls) -> list[Type["RealtimeModelBase"]]:"""Return the OpenAI realtime model classes."""- from ..realtime import OpenAIRealtimeModel+ from ..realtime import OpenAILiveModel, OpenAIRealtimeModel- return [OpenAIRealtimeModel]+ return [OpenAIRealtimeModel, OpenAILiveModel]diff --git a/src/agentscope/event/_event.py b/src/agentscope/event/_event.pyindex 0f90b2bcf..c72776ad2 100644--- a/src/agentscope/event/_event.py+++ b/src/agentscope/event/_event.py@@ -550,6 +550,13 @@ class CustomEvent(EventBase):changed during a tool call.- ``"team_updated"`` — team membership changed (memberadded / team created or dissolved).+ - ``"delegation_checkpoint"`` — a hidden Live tool result is+ ready to persist without displaying a tool card. ``value``+ contains ``delegation_id``, ``reply_id``, and ``tool_call_id``.+ - ``"external_execution_result_rejected"`` — a Live result+ submission needs retry. ``value`` contains ``request_id``+ (the submitted event ID), ``reply_id``, ``tool_call_ids``+ (only unaccepted results), and a safe ``message``.- ``"chat_input_started"`` — one or more FIFO-queued usermessages have started their agent turn.- ``"chat_input_failed"`` — a dequeued user turn failed beforediff --git a/src/agentscope/realtime/__init__.py b/src/agentscope/realtime/__init__.pyindex 6b910e840..ebdd68892 100644--- a/src/agentscope/realtime/__init__.py+++ b/src/agentscope/realtime/__init__.py@@ -5,28 +5,36 @@ The model side of a voice session: realtime models, their cards andevents, transports and VAD. The agent that ties them together is:class:`agentscope.agent.RealtimeAgent`."""-from ._base import ModelDisconnectedError, RealtimeModelBase+from ._base import InteractionMode, ModelDisconnectedError, RealtimeModelBasefrom ._dashscope import (DashScopeAudioRealtimeModel,DashScopeRealtimeModel,)from ._events import (AudioDeltaEvent,+ DelegatedResponseEvent,+ DelegationCreatedEvent,InputTranscriptionEvent,InputTranscriptionFailedEvent,ModelErrorEvent,ModelEvent,ResponseCreatedEvent,ResponseDoneEvent,+ SessionClosedEvent,SessionEndedEvent,+ SessionStartedEvent,+ SessionUsageEvent,SpeechEndedEvent,SpeechStartedEvent,+ TimelineAudioDeltaEvent,+ TimelineTranscriptDeltaEvent,ToolCallEvent,TranscriptDeltaEvent,)from ._gemini import GeminiRealtimeModelfrom ._model_card import RealtimeModelCardfrom ._openai import OpenAIRealtimeModel+from ._openai_live import OpenAILiveModelfrom ._playout import PlayoutPositionfrom ._transport import (AudioFrame,@@ -42,10 +50,12 @@ from ._xai import XAIRealtimeModel__all__ = [# Model"RealtimeModelBase",+ "InteractionMode","DashScopeRealtimeModel","DashScopeAudioRealtimeModel","GeminiRealtimeModel","OpenAIRealtimeModel",+ "OpenAILiveModel","XAIRealtimeModel","RealtimeModelCard","ModelDisconnectedError",@@ -63,13 +73,20 @@ __all__ = [# Model events"ModelEvent","SessionEndedEvent",+ "SessionStartedEvent",+ "SessionClosedEvent",+ "SessionUsageEvent","SpeechStartedEvent","SpeechEndedEvent","InputTranscriptionEvent","InputTranscriptionFailedEvent","ResponseCreatedEvent","AudioDeltaEvent",+ "TimelineAudioDeltaEvent","TranscriptDeltaEvent",+ "TimelineTranscriptDeltaEvent",+ "DelegationCreatedEvent",+ "DelegatedResponseEvent","ToolCallEvent","ResponseDoneEvent","ModelErrorEvent",diff --git a/src/agentscope/realtime/_base.py b/src/agentscope/realtime/_base.pyindex 58a2bd37a..2c30bac0b 100644--- a/src/agentscope/realtime/_base.py+++ b/src/agentscope/realtime/_base.py@@ -2,6 +2,7 @@"""The realtime model base class."""import inspectfrom abc import ABC, abstractmethod+from enum import StrEnumfrom pathlib import Pathfrom typing import Any, AsyncIterator, Sequence@@ -17,6 +18,13 @@ class ModelDisconnectedError(ConnectionError):"""The provider closed the session; the next user audio reconnects."""+class InteractionMode(StrEnum):+ """How audio input and model output share the conversation timeline."""++ TURN_BASED = "turn_based"+ FULL_DUPLEX = "full_duplex"++class RealtimeModelBase(ABC):"""A bidirectional realtime model: audio in, audio and tool calls out.@@ -43,6 +51,9 @@ class RealtimeModelBase(ABC):supports_history_replay: bool = False"""Whether completed messages can seed a newly opened session."""+ interaction_mode = InteractionMode.TURN_BASED+ """Whether the adapter uses turns or a continuous full-duplex stream."""+def __init__(self,model: str,@@ -121,6 +132,12 @@ class RealtimeModelBase(ABC):)cards = RealtimeModelCard.list_from_directory(yaml_dir, cls.Parameters)for card in cards:+ if card.interaction_mode != cls.interaction_mode:+ raise ValueError(+ f"Realtime model card {card.name!r} declares "+ f"{card.interaction_mode!r}, but {cls.__name__} uses "+ f"{cls.interaction_mode!r}.",+ )card.model_type = cls.typereturn cards@@ -235,3 +252,22 @@ class RealtimeModelBase(ABC):@abstractmethodasync def cancel_response(self) -> None:"""Stop the active reply without closing the session."""++ async def interrupt_speech(self) -> None:+ """Stop speech while preserving independent backend work.++ Turn-based adapters cancel their active response. Full-duplex+ adapters should override this method when speech and backend work+ have separate lifecycles.+ """+ await self.cancel_response()++ async def send_delegation_result(+ self,+ delegation_id: str,+ content: str,+ ) -> None:+ """Return an application-managed task result to the voice model."""+ raise NotImplementedError(+ f"{type(self).__name__} does not support client delegation.",+ )diff --git a/src/agentscope/realtime/_events.py b/src/agentscope/realtime/_events.pyindex ebfb4ad60..f65950f35 100644--- a/src/agentscope/realtime/_events.py+++ b/src/agentscope/realtime/_events.py@@ -5,6 +5,8 @@ These are internal to :mod:`agentscope.realtime`: adapters normalisevendor frames into them, and :class:`RealtimeAgent` translates them intothe public :class:`~agentscope.event.AgentEvent` stream."""+from typing import Any, Literal+from pydantic import BaseModel, Fieldfrom ..message import ToolCallBlock@@ -20,6 +22,27 @@ class SessionEndedEvent(ModelEvent):reason: str = ""+class SessionStartedEvent(ModelEvent):+ """A full-duplex provider session accepted its startup config."""++ session_id: str+++class SessionClosedEvent(ModelEvent):+ """A full-duplex provider session finalized normally."""++ session_id: str+ reason: str = ""+ usage_seconds: float | None = None+++class SessionUsageEvent(ModelEvent):+ """A cumulative full-duplex voice usage snapshot."""++ seconds: float+ context_usage_ratio: float | None = None++class SpeechStartedEvent(ModelEvent):"""The provider's own VAD detected the user starting to speak."""@@ -68,6 +91,40 @@ class TranscriptDeltaEvent(ModelEvent):delta: str+class TimelineTranscriptDeltaEvent(ModelEvent):+ """A timestamped transcript fragment in a full-duplex session."""++ role: Literal["user", "assistant"]+ delta: str+ start_ms: int+ end_ms: int+++class TimelineAudioDeltaEvent(ModelEvent):+ """An assistant audio fragment on the full-duplex session timeline."""++ pcm: bytes+ sample_rate: int+ start_ms: int | None = None+ end_ms: int | None = None+++class DelegationCreatedEvent(ModelEvent):+ """GPT-Live started an independent backend task."""++ delegation_id: str+ target: Literal["responses", "client"]+ response_id: str | None = None+ offset_ms: int = 0+++class DelegatedResponseEvent(ModelEvent):+ """One nested Responses event for a GPT-Live delegation."""++ delegation_id: str+ event: dict[str, Any]++class ToolCallEvent(ModelEvent):"""A complete tool call requested by the provider."""diff --git a/src/agentscope/realtime/_model_card.py b/src/agentscope/realtime/_model_card.pyindex d7189c4e6..73216c8e9 100644--- a/src/agentscope/realtime/_model_card.py+++ b/src/agentscope/realtime/_model_card.py@@ -64,6 +64,17 @@ class RealtimeModelCard(BaseModel):default=False,description="Whether this model accepts tool schemas.",)+ interaction_mode: Literal["turn_based", "full_duplex"] = Field(+ default="turn_based",+ description="Whether speech is turn-based or full duplex.",+ )+ delegation_backend: Literal["none", "session_chat"] = Field(+ default="none",+ description=(+ "Whether delegated tasks use the session's configured chat "+ "agent."+ ),+ )# Context limits. Providers use different units and several can apply# at once, so all are optional; ``None`` means unlimited or unknown.diff --git a/src/agentscope/realtime/_openai_live/__init__.py b/src/agentscope/realtime/_openai_live/__init__.pynew file mode 100644index 000000000..6fa34e616--- /dev/null+++ b/src/agentscope/realtime/_openai_live/__init__.py@@ -0,0 +1,5 @@+# -*- coding: utf-8 -*-+"""The OpenAI GPT-Live model."""+from ._model import OpenAILiveModel++__all__ = ["OpenAILiveModel"]diff --git a/src/agentscope/realtime/_openai_live/_model.py b/src/agentscope/realtime/_openai_live/_model.pynew file mode 100644index 000000000..b42a5c27a--- /dev/null+++ b/src/agentscope/realtime/_openai_live/_model.py@@ -0,0 +1,535 @@+# -*- coding: utf-8 -*-+"""The OpenAI GPT-Live WebSocket adapter."""+import asyncio+import base64+import json+import uuid+from typing import Any, AsyncIterator++from pydantic import Field++from .. import _events as me+from .._base import (+ InteractionMode,+ ModelDisconnectedError,+ RealtimeModelBase,+)+from .._model_card import RealtimeModelCard+from ..._logging import logger+from ...credential import OpenAICredential+from ...message import ToolResultBlock++_OPENAI_BASE_URL = "https://api.openai.com/v1"+_MAX_COMMENTARY_CHARS = 400+++class OpenAILiveModel(RealtimeModelBase):+ """GPT-Live over a server-side primary WebSocket.++ The adapter keeps Live audio, transcripts, and client-delegation metadata+ independent. It does not synthesize turn boundaries that the provider+ does not emit.+ """++ class Parameters(RealtimeModelBase.Parameters):+ """GPT-Live voice and connection settings."""++ voice: str = Field(+ default="marin",+ title="Voice",+ description="The voice used for generated audio.",+ )+ store: bool = Field(+ default=False,+ title="Store Session",+ description=(+ "Whether to request provider-side storage for the Live "+ "session."+ ),+ )+ startup_timeout_s: float = Field(+ default=15.0,+ gt=0,+ title="Startup Timeout",+ description=(+ "Maximum seconds to wait for the Live session to start."+ ),+ )+ close_timeout_s: float = Field(+ default=15.0,+ gt=0,+ title="Close Timeout",+ description=(+ "Maximum seconds to wait for graceful session closure."+ ),+ )++ type = "openai_live"+ """Adapter identifier stored in realtime model configurations."""++ interaction_mode = InteractionMode.FULL_DUPLEX+ """GPT-Live uses one continuous, full-duplex conversation timeline."""++ supports_text_input = False+ """Typed input is not exposed by the current GPT-Live adapter."""++ @property+ def input_transcription_enabled(self) -> bool:+ """GPT-Live always emits timestamped input transcript fragments."""+ return True++ def __init__(+ self,+ model: str,+ credential: OpenAICredential,+ parameters: "OpenAILiveModel.Parameters | None" = None,+ model_card: RealtimeModelCard | None = None,+ ) -> None:+ """Initialize the GPT-Live adapter.++ Args:+ model (`str`):+ GPT-Live model identifier, for example ``"gpt-live-1"``.+ credential (`OpenAICredential`):+ OpenAI credential used to authenticate the WebSocket.+ parameters (`OpenAILiveModel.Parameters | None`, optional):+ Voice, storage, and connection timeout settings. Defaults+ are used when omitted.+ model_card (`RealtimeModelCard | None`, optional):+ Model capabilities and audio formats. When omitted, the+ bundled card matching ``model`` is used.++ Raises:+ ValueError:+ If ``model_card`` is omitted and no bundled card matches+ ``model``.+ """+ super().__init__(model, credential, parameters, model_card)+ self.parameters: OpenAILiveModel.Parameters+ self._ws: Any = None+ self._reader: asyncio.Task | None = None+ self._queue: asyncio.Queue[me.ModelEvent | None] = asyncio.Queue()+ self._send_lock = asyncio.Lock()+ self._started: asyncio.Future[str] | None = None+ self._closed = asyncio.Event()+ self._session_id = ""+ self._finalized = False+ self._closing = False+ self._usage_seconds = 0.0+ self._context_usage_ratio: float | None = None++ @property+ def usage_seconds(self) -> float:+ """Return the latest cumulative voice usage snapshot."""+ return self._usage_seconds++ @property+ def context_usage_ratio(self) -> float | None:+ """Return the latest reported context-window usage ratio."""+ return self._context_usage_ratio++ async def connect(+ self,+ instructions: str,+ tools: list[dict] | None = None,+ **kwargs: Any,+ ) -> None:+ """Open the primary WebSocket and wait for ``session.started``.++ Args:+ instructions (`str`):+ System instructions sent in the immutable session startup+ configuration.+ tools (`list[dict] | None`, optional):+ Ignored because GPT-Live delegates tool work to the client+ session's Chat Agent.+ **kwargs (`Any`):+ Optional connection metadata. ``safety_identifier`` is sent+ as the ``OpenAI-Safety-Identifier`` header when present.++ Raises:+ asyncio.TimeoutError:+ If sending startup and receiving its acknowledgement+ together exceed ``startup_timeout_s``.+ ModelDisconnectedError:+ If the connection closes before startup completes.+ asyncio.CancelledError:+ If startup is cancelled, after releasing the connection.+ """+ import websockets++ await self.close()+ self._drain_queue()+ self._closed = asyncio.Event()+ self._session_id = ""+ self._finalized = False+ self._closing = False+ self._usage_seconds = 0.0+ self._context_usage_ratio = None+ self._started = asyncio.get_running_loop().create_future()++ credential: OpenAICredential = self.credential # type: ignore+ base_url = (credential.base_url or _OPENAI_BASE_URL).rstrip("/")+ url = base_url.replace("http", "ws", 1)+ headers = {+ "Authorization": (+ f"Bearer {credential.api_key.get_secret_value()}"+ ),+ }+ if credential.organization:+ headers["OpenAI-Organization"] = credential.organization+ safety_identifier = kwargs.get("safety_identifier")+ if safety_identifier:+ headers["OpenAI-Safety-Identifier"] = safety_identifier++ try:+ self._ws = await websockets.connect(+ f"{url}/live/sessions",+ additional_headers=headers,+ )+ self._reader = asyncio.create_task(+ self._read(),+ name="openai-live",+ )+ async with asyncio.timeout(self.parameters.startup_timeout_s):+ await self._send(+ self._session_start(instructions),+ )+ await asyncio.shield(self._started)+ except (Exception, asyncio.CancelledError):+ await self._abort()+ raise++ async def close(self) -> None:+ """Gracefully finalize usage, then release the WebSocket."""+ if self._ws is None and self._reader is None:+ return+ self._closing = True+ if self._ws is not None and self._session_id and not self._finalized:+ try:+ await self._send({"type": "session.close"})+ await asyncio.wait_for(+ self._closed.wait(),+ timeout=self.parameters.close_timeout_s,+ )+ except (asyncio.TimeoutError, ModelDisconnectedError):+ logger.warning(+ "OpenAILiveModel: session close was not finalized",+ )+ await self._abort()++ async def events(self) -> AsyncIterator[me.ModelEvent]:+ """Yield normalized Live events until the connection ends."""+ while True:+ event = await self._queue.get()+ if event is None:+ return+ yield event++ async def push_audio(self, pcm: bytes) -> None:+ """Append complete PCM16 samples to the Live input stream.++ Args:+ pcm (`bytes`):+ Little-endian PCM16 audio at :attr:`input_sample_rate`.++ Raises:+ ValueError:+ If ``pcm`` ends with an incomplete 16-bit sample.+ ModelDisconnectedError:+ If the Live session has not started or is disconnected.+ """+ if len(pcm) % 2:+ raise ValueError("PCM16 audio must contain complete samples.")+ if not self._session_id:+ raise ModelDisconnectedError("GPT-Live session is not started.")+ await self._send(+ {+ "type": "session.input_audio.append",+ "audio": base64.b64encode(pcm).decode("ascii"),+ },+ )++ async def push_text(self, text: str) -> None:+ """Reject typed input until its delegation semantics are exposed."""+ raise NotImplementedError(+ f"{type(self).__name__} does not expose typed input yet.",+ )++ async def push_tool_result(self, block: ToolResultBlock) -> None:+ """Reject direct tools because the session chat agent owns them."""+ raise NotImplementedError(+ f"{type(self).__name__} delegates tools to the session agent.",+ )++ async def commit_turn(self) -> None:+ """Reject turn commits because GPT-Live owns a continuous stream."""+ raise NotImplementedError("GPT-Live has no user turn commit.")++ async def request_response(self) -> None:+ """Reject Responses control because this adapter uses client mode."""+ raise NotImplementedError(+ f"{type(self).__name__} uses client delegation.",+ )++ async def cancel_response(self) -> None:+ """Reject Realtime-style cancellation for the Live frontend."""+ raise NotImplementedError(+ "GPT-Live speech and backend work have separate lifecycles.",+ )++ async def interrupt_speech(self) -> None:+ """Ask the Live frontend to stop speaking without cancelling work."""+ await self._send(+ {+ "type": "session.instructions.append",+ "event_id": self._event_id("interrupt"),+ "delegation_id": None,+ "content": "Stop speaking now and listen to the user.",+ },+ )++ async def send_delegation_result(+ self,+ delegation_id: str,+ content: str,+ ) -> None:+ """Append a verified client-delegation result for speech.++ Long results are split into ordered commentary commands before they+ are returned to the Live session.++ Args:+ delegation_id (`str`):+ Identifier from the corresponding delegation-created event.+ content (`str`):+ Verified task result for GPT-Live to communicate.++ Raises:+ ValueError:+ If either argument is empty.+ ModelDisconnectedError:+ If the Live WebSocket is not connected.+ """+ if not delegation_id:+ raise ValueError("A GPT-Live delegation id is required.")+ if not content.strip():+ raise ValueError("A GPT-Live delegation result is required.")+ for chunk in self._commentary_chunks(content):+ await self._send(+ {+ "type": "session.commentary.append",+ "event_id": self._event_id("delegation_result"),+ "delegation_id": delegation_id,+ "content": chunk,+ },+ )++ @staticmethod+ def _commentary_chunks(content: str) -> list[str]:+ """Split a result conservatively below Live's 500-token limit."""+ remaining = content.strip()+ chunks: list[str] = []+ while remaining:+ if len(remaining) <= _MAX_COMMENTARY_CHARS:+ chunks.append(remaining)+ break+ boundary = remaining.rfind(" ", 0, _MAX_COMMENTARY_CHARS + 1)+ if boundary < _MAX_COMMENTARY_CHARS // 2:+ boundary = _MAX_COMMENTARY_CHARS+ chunks.append(remaining[:boundary].strip())+ remaining = remaining[boundary:].strip()+ return chunks++ def _session_start(+ self,+ instructions: str,+ ) -> dict[str, Any]:+ """Build the immutable GPT-Live startup configuration."""+ session = {+ "model": self.model,+ "instructions": instructions,+ "input": [],+ "audio": {+ "format": {+ "type": "audio/pcm",+ "rate": self.input_sample_rate,+ },+ "output": {"voice": self.parameters.voice},+ },+ "delegation": {"type": "client"},+ "store": self.parameters.store,+ }+ return {+ "type": "session.start",+ "event_id": self._event_id("start"),+ "session": session,+ }++ async def _send(self, payload: dict[str, Any]) -> None:+ """Send one ordered JSON command."""+ from websockets.exceptions import ConnectionClosed++ if self._ws is None:+ raise ModelDisconnectedError("GPT-Live is not connected.")+ try:+ async with self._send_lock:+ await self._ws.send(+ json.dumps(payload, ensure_ascii=False),+ )+ except ConnectionClosed as exc:+ raise ModelDisconnectedError(str(exc.rcvd or exc)) from exc++ async def _read(self) -> None:+ """Run the connection's only receive loop."""+ try:+ async for raw in self._ws:+ if isinstance(raw, bytes):+ raw = raw.decode("utf-8")+ try:+ event = self._parse(json.loads(raw))+ except Exception: # noqa: BLE001+ logger.exception("OpenAILiveModel: invalid server frame")+ continue+ if event is not None:+ self._queue.put_nowait(event)+ except Exception as exc: # noqa: BLE001+ logger.error("OpenAILiveModel: connection lost: %s", exc)+ finally:+ if not self._finalized and not self._closing:+ self._queue.put_nowait(+ me.SessionEndedEvent(reason="connection_closed"),+ )+ if self._started is not None and not self._started.done():+ self._started.set_exception(+ ModelDisconnectedError(+ "Connection closed before session.started.",+ ),+ )+ self._closed.set()+ self._queue.put_nowait(None)++ # pylint: disable=too-many-return-statements+ def _parse(self, data: dict[str, Any]) -> me.ModelEvent | None:+ """Translate one GPT-Live server event."""+ kind = data.get("type", "")++ if kind == "session.started":+ session_id = data.get("session", {}).get("id", "")+ self._session_id = session_id+ if self._started is not None and not self._started.done():+ self._started.set_result(session_id)+ return me.SessionStartedEvent(session_id=session_id)++ if kind in {+ "session.input_transcript.delta",+ "session.output_transcript.delta",+ }:+ delta = data.get("delta", "")+ if not delta:+ return None+ role = "user" if ".input_" in kind else "assistant"+ return me.TimelineTranscriptDeltaEvent(+ role=role,+ delta=delta,+ start_ms=data.get("start_ms", 0),+ end_ms=data.get("end_ms", 0),+ )++ if kind == "session.output_audio.delta":+ delta = data.get("delta", "")+ if not delta:+ return None+ return me.TimelineAudioDeltaEvent(+ pcm=base64.b64decode(delta),+ sample_rate=self.output_sample_rate,+ start_ms=data.get("start_ms"),+ end_ms=data.get("end_ms"),+ )++ if kind == "session.delegation.created":+ delegation = data.get("delegation", {})+ return me.DelegationCreatedEvent(+ delegation_id=delegation.get("id", ""),+ target=delegation.get("target", "responses"),+ response_id=delegation.get("response_id"),+ offset_ms=data.get("offset_ms", 0),+ )++ if kind == "response.event":+ return me.DelegatedResponseEvent(+ delegation_id=data.get("delegation_id", ""),+ event=data.get("event", {}),+ )++ if kind == "session.usage.updated":+ usage = data.get("usage", {})+ self._usage_seconds = float(usage.get("seconds", 0.0))+ self._context_usage_ratio = data.get("context_window", {}).get(+ "usage_ratio",+ )+ return me.SessionUsageEvent(+ seconds=self._usage_seconds,+ context_usage_ratio=self._context_usage_ratio,+ )++ if kind == "session.closed":+ usage = data.get("usage", {})+ self._usage_seconds = float(+ usage.get("seconds", self._usage_seconds),+ )+ self._finalized = True+ self._closed.set()+ return me.SessionClosedEvent(+ session_id=data.get("session", {}).get(+ "id",+ self._session_id,+ ),+ reason=data.get("reason", ""),+ usage_seconds=self._usage_seconds,+ )++ if kind == "error":+ error = data.get("error", {})+ message = error.get("message", "")+ if self._started is not None and not self._started.done():+ self._started.set_exception(RuntimeError(message))+ return me.ModelErrorEvent(+ code=error.get("code") or error.get("type", ""),+ message=message,+ )++ logger.debug("OpenAILiveModel: ignoring %s", kind)+ return None++ async def _abort(self) -> None:+ """Release transport resources without waiting for finalization."""+ started = self._started+ self._started = None+ if started is not None:+ if not started.done():+ started.cancel()+ elif not started.cancelled():+ # Startup may have failed before connect began awaiting it.+ started.exception()+ ws = self._ws+ self._ws = None+ if ws is not None:+ await ws.close()+ reader = self._reader+ self._reader = None+ if reader is not None and reader is not asyncio.current_task():+ if not reader.done():+ reader.cancel()+ await asyncio.gather(reader, return_exceptions=True)+ self._session_id = ""++ def _drain_queue(self) -> None:+ """Discard terminal events left by a previous connection."""+ while not self._queue.empty():+ self._queue.get_nowait()++ @staticmethod+ def _event_id(prefix: str) -> str:+ """Build a unique client event identifier."""+ return f"{prefix}_{uuid.uuid4().hex}"diff --git a/src/agentscope/realtime/_openai_live/_models/gpt-live-1.yaml b/src/agentscope/realtime/_openai_live/_models/gpt-live-1.yamlnew file mode 100644index 000000000..aa1ed0301--- /dev/null+++ b/src/agentscope/realtime/_openai_live/_models/gpt-live-1.yaml@@ -0,0 +1,20 @@+type: realtime_model+name: gpt-live-1+label: GPT Live 1+status: active+interaction_mode: full_duplex+delegation_backend: session_chat++input_types:+ - audio/pcm+output_types:+ - audio/pcm+ - text/plain++input_sample_rate: 24000+output_sample_rate: 24000++# Tools run through the session's configured chat agent.+supports_tools: false++max_context_tokens: 128000diff --git a/src/agentscope/realtime/_transport/_base.py b/src/agentscope/realtime/_transport/_base.pyindex 425159a39..dc92b5ab4 100644--- a/src/agentscope/realtime/_transport/_base.py+++ b/src/agentscope/realtime/_transport/_base.py@@ -31,6 +31,9 @@ class ControlFrameType(StrEnum):USER_CONFIRM = "user_confirm""""The outcome of a tool permission prompt."""+ EXTERNAL_EXECUTION_RESULT = "external_execution_result"+ """The result returned by an external tool executor."""+INTERRUPT = "interrupt""""The user pressed stop, as opposed to speaking over the reply."""diff --git a/src/agentscope/state/_state.py b/src/agentscope/state/_state.pyindex e827e52aa..e14984102 100644--- a/src/agentscope/state/_state.py+++ b/src/agentscope/state/_state.py@@ -295,6 +295,29 @@ class AgentState(BaseModel):"""The context that allow the middlewares to store/get data acrossdifferent replies."""+ def get_reply_message(self, name: str) -> Msg | None:+ """Find this agent's current reply, even after later messages.++ Args:+ name (`str`):+ Author of the assistant reply to find.++ Returns:+ `Msg | None`:+ The assistant message matching both ``name`` and the current+ ``reply_id``, or ``None`` if it has not been created.+ """+ return next(+ (+ message+ for message in reversed(self.context)+ if message.id == self.reply_id+ and message.role == "assistant"+ and message.name == name+ ),+ None,+ )+def append_context(self,name: str,@@ -306,14 +329,9 @@ class AgentState(BaseModel):`reply_id`. If such message doesn't exist, a new assistant messagewith agent's name and current reply ID will be created."""- # If append to the latest message- if (- self.context- and self.context[-1].role == "assistant"- and self.context[-1].name == name- and self.context[-1].id == self.reply_id- ):- self.context[-1].content.extend(blocks)+ message = self.get_reply_message(name)+ if message is not None:+ message.content.extend(blocks)else:# Create a new assistant message with the current reply IDself.context.append(@@ -326,7 +344,7 @@ class AgentState(BaseModel):)def has_awaiting_tool_calls(self, name: str) -> bool:- """Whether the tail assistant message written by ``name`` has+ """Whether the current reply written by ``name`` hasany tool call still awaiting an outside response — an``ASKING`` user confirmation or a ``SUBMITTED`` externalexecution with no matching tool result yet.@@ -343,7 +361,7 @@ class AgentState(BaseModel):return bool(self.get_awaiting_tool_calls(name))def get_awaiting_tool_calls(self, name: str) -> list[ToolCallBlock]:- """Get the tail assistant message's tool calls still awaiting an+ """Get the current reply's tool calls still awaiting anoutside response — an ``ASKING`` user confirmation or a``SUBMITTED`` external execution with no matching tool result yet.@@ -356,10 +374,8 @@ class AgentState(BaseModel):`list[ToolCallBlock]`:The awaiting tool call blocks, empty if none."""- if not self.context:- return []- last_msg = self.context[-1]- if last_msg.role != "assistant" or last_msg.name != name:+ last_msg = self.get_reply_message(name)+ if last_msg is None:return []result_ids = {b.id for b in last_msg.get_content_blocks("tool_result")}return [@@ -386,14 +402,8 @@ class AgentState(BaseModel):`list[ToolCallBlock]`:The unfinished tool call blocks, empty if none."""- if not self.context:- return []- last_msg = self.context[-1]- if (- last_msg.role != "assistant"- or last_msg.name != name- or last_msg.id != self.reply_id- ):+ last_msg = self.get_reply_message(name)+ if last_msg is None:return []result_ids = {b.id for b in last_msg.get_content_blocks("tool_result")}return [diff --git a/tests/channel_gateway_test.py b/tests/channel_gateway_test.pyindex 5a006de00..ec86c8a5a 100644--- a/tests/channel_gateway_test.py+++ b/tests/channel_gateway_test.py@@ -655,6 +655,7 @@ class _AwaitingStorage:reply_id="reply-1",context=[Msg(+ id="reply-1",name="Friday",role="assistant",content=[diff --git a/tests/chat_input_queue_test.py b/tests/chat_input_queue_test.pyindex 168695d32..2c9625478 100644--- a/tests/chat_input_queue_test.py+++ b/tests/chat_input_queue_test.py@@ -564,6 +564,7 @@ class TestQueueFailureScope(IsolatedAsyncioTestCase):],),)+ session.state.reply_id = session.state.context[0].idstorage.get_session = AsyncMock(return_value=session)policy = SimpleNamespace(list_accessible=AsyncMock(@@ -648,6 +649,7 @@ class TestQueueFailureScope(IsolatedAsyncioTestCase):context=[AssistantMsg(name=name, content=blocks)],),)+ session.state.reply_id = session.state.context[0].idself.assertEqual(await queue._is_session_parked(session, "u", "a"),expected,diff --git a/tests/realtime_checkpoint_recovery_test.py b/tests/realtime_checkpoint_recovery_test.pynew file mode 100644index 000000000..ecd811194--- /dev/null+++ b/tests/realtime_checkpoint_recovery_test.py@@ -0,0 +1,579 @@+# -*- coding: utf-8 -*-+"""Recover parked Live delegations through a newly constructed Chat Agent."""+# pylint: disable=protected-access++import asyncio+from typing import Any+from unittest import IsolatedAsyncioTestCase, TestCase+from unittest.mock import AsyncMock, patch++from utils import MockModel++from agentscope.agent import Agent, RealtimeAgent+from agentscope.app._service._webrtc_audio_transport import (+ WebRTCAudioTransport,+)+from agentscope.app._service._webrtc_session import WebRTCSession+from agentscope.app.message_bus import InMemoryMessageBus, MessageBusKeys+from agentscope.credential import OpenAICredential+from agentscope.event import (+ ConfirmResult,+ ExternalExecutionResultEvent,+ RequireExternalExecutionEvent,+ RequireUserConfirmEvent,+ ToolResultEndEvent,+ UserConfirmResultEvent,+ UserInterruptEvent,+)+from agentscope.message import (+ AssistantMsg,+ TextBlock,+ ToolCallBlock,+ ToolResultBlock,+ ToolResultState,+ UserMsg,+)+from agentscope.model import ChatResponse, ChatUsage+from agentscope.realtime import OpenAILiveModel, TimelineTranscriptDeltaEvent+from agentscope.state import AgentState+from agentscope.tool import FunctionTool, Toolkit+++class RealtimeCheckpointRecoveryTest(IsolatedAsyncioTestCase):+ """Exercise serialized checkpoints rather than the in-memory backend."""++ async def test_interrupted_tool_end_checkpoint_has_its_result(+ self,+ ) -> None:+ """Restore interruption from END without relying on ReplyEnd."""++ def write() -> str:+ """Never execute a tool whose approval is interrupted."""+ self.fail("The interrupted tool must not execute.")++ toolkit = Toolkit(tools=[FunctionTool(write)])+ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[+ ToolCallBlock(id="write", name="write", input="{}"),+ ],+ is_last=True,+ ),+ ],+ )++ def factory(state: AgentState) -> Agent:+ """Create a real backend that waits for user approval."""+ return Agent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ toolkit=toolkit,+ )++ live_model = OpenAILiveModel(+ "gpt-live-1",+ OpenAICredential(api_key="test"),+ )+ live_model.send_delegation_result = AsyncMock()+ live = RealtimeAgent(+ "Friday",+ "Be brief.",+ live_model,+ delegation_agent_factory=factory,+ )+ self.addAsyncCleanup(live.close)+ live._start_client_delegation("task")+ await asyncio.gather(*list(live._tasks))+ delegation = live._delegations["task"]+ assert delegation.agent is not None+ list(live.drain_events())+ await live._drive_client_delegation(+ delegation,+ UserInterruptEvent(reply_id=delegation.agent.state.reply_id),+ )+ restored = None+ for event in live.drain_events():+ if isinstance(event, ToolResultEndEvent):+ self.assertIsNone(restored)+ snapshot = live.checkpoint_snapshot(event)+ assert snapshot is not None+ restored = AgentState.model_validate_json(+ snapshot.model_dump_json(),+ )+ assert restored is not None+ reply = restored.get_reply_message("Friday")+ assert reply is not None+ self.assertEqual(+ {+ "calls": [+ (block.id, block.state)+ for block in reply.get_content_blocks("tool_call")+ ],+ "results": [+ (block.id, block.state, block.output)+ for block in reply.get_content_blocks("tool_result")+ ],+ },+ {+ "calls": [("write", "finished")],+ "results": [+ (+ "write",+ "interrupted",+ "<system-reminder>The tool call has been interrupted "+ "by the user.</system-reminder>",+ ),+ ],+ },+ )++ expected = reply.model_dump(mode="json")+ model.set_responses(+ [ChatResponse(content=[TextBlock(text="Done")], is_last=True)],+ )+ recovered = factory(restored)+ await recovered.reply(UserMsg("user", "Continue."))+ self.assertEqual(+ restored.context[0].model_dump(mode="json"),+ expected,+ )++ async def test_hidden_tool_checkpoint_before_next_model_response(+ self,+ ) -> None:+ """Persist and restore completed work while reasoning is blocked."""+ waiting = asyncio.Event()+ persisted = asyncio.Event()+ executed: list[str] = []+ saved: list[str] = []++ def read() -> str:+ """Record one automatically allowed execution."""+ executed.append("read")+ return "contents"++ toolkit = Toolkit(tools=[FunctionTool(read, is_read_only=True)])+ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[+ ToolCallBlock(id="read", name="read", input="{}"),+ ],+ is_last=True,+ ),+ ],+ )+ call_api = model._call_api++ async def blocked_api(*args: Any, **kwargs: Any) -> Any:+ """Hold the response after tool execution until cancellation."""+ if model.cnt:+ waiting.set()+ await asyncio.Event().wait()+ return await call_api(*args, **kwargs)++ def factory(state: AgentState) -> Agent:+ """Use a real backend with a deterministic model boundary."""+ return Agent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ toolkit=toolkit,+ )++ async def save_state(**kwargs: Any) -> None:+ """Serialize at the actual WebRTC persistence boundary."""+ saved.append(kwargs["state"].model_dump_json())+ persisted.set()++ live_model = OpenAILiveModel(+ "gpt-live-1",+ OpenAICredential(api_key="test"),+ )+ live_model.send_delegation_result = AsyncMock()+ live = RealtimeAgent(+ "Friday",+ "Be brief.",+ live_model,+ delegation_agent_factory=factory,+ )+ transport = WebRTCAudioTransport(+ live_model.input_sample_rate,+ live_model.output_sample_rate,+ )+ storage = AsyncMock()+ storage.update_session_state.side_effect = save_state+ bus = InMemoryMessageBus()+ session = WebRTCSession(+ peer_connection=AsyncMock(),+ transport=transport,+ agent_factory=AsyncMock(return_value=live),+ storage=storage,+ message_bus=bus,+ user_id="user",+ agent_id="agent",+ session_id="session",+ on_closed=lambda _: None,+ )+ with (+ patch.object(model, "_call_api", side_effect=blocked_api),+ patch.object(live, "connect", AsyncMock()),+ patch.object(bus, "publish", wraps=bus.publish) as publish,+ ):+ session.start()+ try:+ self.assertTrue(await session.wait_until_lock_acquired(1))+ live._start_client_delegation("task")+ await asyncio.wait_for(waiting.wait(), timeout=1)+ await asyncio.wait_for(persisted.wait(), timeout=1)+ delegation = live._delegations["task"]+ assert delegation.agent is not None+ reply = delegation.agent.state.get_reply_message("Friday")+ assert reply is not None+ expected = reply.model_copy(deep=True)+ expected.content = [+ block+ for block in expected.content+ if block.type in {"tool_call", "tool_result"}+ ]+ restored = AgentState.model_validate_json(saved[0])+ self.assertEqual(+ {+ "visible": delegation.visible,+ "executed": executed,+ "context": restored.model_dump(mode="json")["context"],+ "unfinished": restored.get_unfinished_tool_calls(+ "Friday",+ ),+ "events": [+ (call.args[1]["name"], call.args[1]["value"])+ for call in publish.await_args_list+ ],+ },+ {+ "visible": False,+ "executed": ["read"],+ "context": [expected.model_dump(mode="json")],+ "unfinished": [],+ "events": [+ (+ "delegation_checkpoint",+ {+ "delegation_id": "task",+ "reply_id": reply.id,+ "tool_call_id": "read",+ },+ ),+ ],+ },+ )+ self.assertEqual(+ live.state.model_dump(mode="json")["context"],+ restored.model_dump(mode="json")["context"],+ )+ self.assertEqual(+ await bus.registry_get(+ MessageBusKeys.session_event_checkpoint("session"),+ MessageBusKeys.SESSION_EVENT_CURSOR_FIELD,+ ),+ publish.await_args.args[1]["_entry_id"],+ )+ self.assertFalse(session._task.done())+ live_model.send_delegation_result.assert_not_awaited()+ recovery_model = MockModel(context_size=10000)+ recovery_model.set_responses(+ [+ ChatResponse(+ content=[TextBlock(text="Done")],+ is_last=True,+ ),+ ],+ )+ recovered = Agent(+ "Friday",+ "Be brief.",+ recovery_model,+ state=restored,+ toolkit=toolkit,+ )+ await recovered.reply(UserMsg("user", "Continue."))+ self.assertEqual(executed, ["read"])+ self.assertEqual(+ restored.context[0].model_dump(mode="json"),+ expected.model_dump(mode="json"),+ )+ finally:+ await session.close()++ async def _checkpoint(+ self,+ external: bool,+ voice_role: str | None,+ ) -> tuple[AgentState, ToolCallBlock, Toolkit, list[str], dict]:+ """Park a real delegate, optionally append speech, then persist."""+ executed: list[str] = []+ backends: list[Agent] = []++ def write() -> str:+ """Record execution of the approved tool."""+ executed.append("write")+ return "written"++ tool = FunctionTool(write, is_read_only=external)+ tool.is_external_tool = external+ toolkit = Toolkit(tools=[tool])++ def factory(state: AgentState) -> Agent:+ """Create a deterministic backend that requests one tool."""+ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[+ ToolCallBlock(+ id="write",+ name="write",+ input="{}",+ ),+ ],+ is_last=True,+ ),+ ],+ )+ backend = Agent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ toolkit=toolkit,+ )+ backends.append(backend)+ return backend++ live_model = OpenAILiveModel(+ "gpt-live-1",+ OpenAICredential(api_key="test"),+ )+ live_model.send_delegation_result = AsyncMock()+ live = RealtimeAgent(+ "Friday",+ "Be brief.",+ live_model,+ delegation_agent_factory=factory,+ )+ self.addAsyncCleanup(live.close)+ live._start_client_delegation("task")+ while live._tasks:+ await asyncio.gather(*list(live._tasks))+ if voice_role is not None:+ await live._on_model_event(+ TimelineTranscriptDeltaEvent(+ role=voice_role,+ delta="Later speech",+ start_ms=0,+ end_ms=100,+ ),+ )+ snapshot = None+ call = None+ for event in live.drain_events():+ checkpoint = live.checkpoint_snapshot(event)+ if checkpoint is not None:+ snapshot = checkpoint+ if isinstance(+ event,+ (+ RequireUserConfirmEvent,+ RequireExternalExecutionEvent,+ ),+ ):+ call = event.tool_calls[0].model_copy(deep=True)+ assert snapshot is not None and call is not None+ reply_context = backends[0].state.reply_context.model_dump(mode="json")+ restored = AgentState.model_validate_json(snapshot.model_dump_json())+ await live.close()+ return restored, call, toolkit, executed, reply_context++ async def test_checkpoint_keeps_reply_context(self) -> None:+ """The persisted reply metadata must match the parked backend."""+ state, _, _, _, expected = await self._checkpoint(False, None)+ self.assertEqual(state.reply_context.model_dump(mode="json"), expected)++ async def test_chat_recovers_parked_tools_after_speech(self) -> None:+ """Resume each input kind on its original reply, preserving speech."""+ for action in (+ "accept",+ "deny",+ "external",+ "interrupt",+ "interrupt_external",+ ):+ for voice_role in (None, "user", "assistant"):+ with self.subTest(action=action, voice_role=voice_role):+ (+ state,+ call,+ toolkit,+ executed,+ reply_context,+ ) = await self._checkpoint(+ action in {"external", "interrupt_external"},+ voice_role,+ )+ reply_id = reply_context["reply_id"]+ original_ids = [msg.id for msg in state.context]+ speech = [+ msg.model_dump()+ for msg in state.context+ if msg.id != reply_id+ ]+ self.assertTrue(state.has_awaiting_tool_calls("Friday"))+ if action.startswith("interrupt"):+ event = UserInterruptEvent(reply_id=reply_id)+ expected_result = ToolResultState.INTERRUPTED+ elif action == "external":+ event = ExternalExecutionResultEvent(+ reply_id=reply_id,+ execution_results=[+ ToolResultBlock(+ id=call.id,+ name=call.name,+ output="written",+ state=ToolResultState.SUCCESS,+ ),+ ],+ )+ expected_result = ToolResultState.SUCCESS+ else:+ event = UserConfirmResultEvent(+ reply_id=reply_id,+ confirm_results=[+ ConfirmResult(+ confirmed=action == "accept",+ tool_call=call,+ ),+ ],+ )+ expected_result = (+ ToolResultState.SUCCESS+ if action == "accept"+ else ToolResultState.DENIED+ )+ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[TextBlock(text="Done")],+ is_last=True,+ usage=ChatUsage(+ input_tokens=10,+ output_tokens=2,+ time=0.1,+ ),+ ),+ ],+ )+ agent = Agent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ toolkit=toolkit,+ )+ events = [item async for item in agent.reply_stream(event)]+ reply = next(+ msg for msg in state.context if msg.id == reply_id+ )+ self.assertEqual(+ {+ "ids": [msg.id for msg in state.context],+ "speech": [+ msg.model_dump()+ for msg in state.context+ if msg.id != reply_id+ ],+ "calls": [+ (block.id, block.state)+ for block in reply.get_content_blocks(+ "tool_call",+ )+ ],+ "results": [+ (block.id, block.state)+ for block in reply.get_content_blocks(+ "tool_result",+ )+ ],+ "event_reply_ids": {+ item.reply_id+ for item in events+ if item.reply_id+ },+ "pending": state.get_awaiting_tool_calls("Friday"),+ "executed": executed,+ "usage": (+ reply.usage.input_tokens if reply.usage else 0+ ),+ },+ {+ "ids": original_ids,+ "speech": speech,+ "calls": [(call.id, "finished")],+ "results": [(call.id, expected_result)],+ "event_reply_ids": {reply_id},+ "pending": [],+ "executed": ["write"]+ if action == "accept"+ else [],+ "usage": 0+ if action.startswith("interrupt")+ else 10,+ },+ )+++class ReplyMessageLookupTest(TestCase):+ """Match both reply ID and author without reviving historical calls."""++ def test_current_reply_owns_its_blocks(self) -> None:+ """Append to the matching reply and ignore unrelated tool calls."""+ current = AssistantMsg("Friday", [], id="current")+ history = AssistantMsg(+ "Friday",+ [+ ToolCallBlock(+ id="stale",+ name="write",+ input="{}",+ state="asking",+ ),+ ],+ id="history",+ )+ other = AssistantMsg("Other", [], id="current")+ speech = UserMsg("user", "Later speech")+ state = AgentState(context=[history, current, other, speech])+ state.reply_id = "current"+ block = TextBlock(text="Done")+ state.append_context("Friday", [block])+ self.assertEqual(+ {+ "context": [msg.model_dump() for msg in state.context],+ "pending": state.get_awaiting_tool_calls("Friday"),+ },+ {+ "context": [+ history.model_dump(),+ {**current.model_dump(), "content": [block.model_dump()]},+ other.model_dump(),+ speech.model_dump(),+ ],+ "pending": [],+ },+ )diff --git a/tests/realtime_delegation_test.py b/tests/realtime_delegation_test.pynew file mode 100644index 000000000..f6836c1b8--- /dev/null+++ b/tests/realtime_delegation_test.py@@ -0,0 +1,610 @@+# -*- coding: utf-8 -*-+"""Test Live delegation deduplication, result validation, and recovery."""+# pylint: disable=protected-access++import asyncio+import json+from unittest import IsolatedAsyncioTestCase+from unittest.mock import AsyncMock, call, patch++from utils import MockModel++from agentscope.agent import Agent, RealtimeAgent+from agentscope.credential import OpenAICredential+from agentscope.event import (+ ConfirmResult,+ CustomEvent,+ ExternalExecutionResultEvent,+ ToolResultEndEvent,+ UserConfirmResultEvent,+)+from agentscope.message import TextBlock, ToolCallBlock, ToolResultBlock+from agentscope.model import ChatResponse+from agentscope.realtime import OpenAILiveModel+from agentscope.state import AgentState+from agentscope.tool import AskUser, FunctionTool, Toolkit+++def _create_live_agent(model: MockModel, toolkit: Toolkit) -> RealtimeAgent:+ """Build a Live agent backed by a real Agent and a deterministic model."""++ def factory(state: AgentState) -> Agent:+ """Use real permission checks and external-result validation."""+ return Agent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ toolkit=toolkit,+ )++ live_model = OpenAILiveModel(+ "gpt-live-1",+ OpenAICredential(api_key="test"),+ )+ live_model.send_delegation_result = AsyncMock()+ return RealtimeAgent(+ "Friday",+ "Be brief.",+ live_model,+ delegation_agent_factory=factory,+ )+++class RealtimeDelegationDedupTest(IsolatedAsyncioTestCase):+ """Exercise concurrent approvals and external results with real Agents."""++ async def _start(+ self,+ external: bool,+ call_ids: list[str],+ ) -> tuple[RealtimeAgent, Agent, list[str]]:+ """Create one parked delegation and record actual tool executions."""+ executed: list[str] = []++ def write(value: str) -> str:+ """Record each approved execution."""+ executed.append(value)+ return value++ tool = FunctionTool(write, is_read_only=external)+ tool.is_external_tool = external+ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[+ ToolCallBlock(+ id=call_id,+ name="write",+ input=json.dumps({"value": call_id}),+ )+ for call_id in call_ids+ ],+ is_last=True,+ ),+ ChatResponse(content=[TextBlock(text="Done.")], is_last=True),+ ],+ )++ live = _create_live_agent(model, Toolkit(tools=[tool]))+ self.addAsyncCleanup(live.close)+ live._start_client_delegation("task")+ await asyncio.gather(*list(live._tasks))+ backend = live._delegations["task"].agent+ assert backend is not None+ self.assertEqual(+ [tc.id for tc in backend.state.get_awaiting_tool_calls("Friday")],+ call_ids if external else call_ids[:1],+ )+ return live, backend, executed++ @staticmethod+ def _input(+ backend: Agent,+ external: bool,+ call_ids: list[str],+ ) -> UserConfirmResultEvent | ExternalExecutionResultEvent:+ """Build a response from the original calls, including stale ones."""+ reply = backend.state.get_reply_message("Friday")+ assert reply is not None+ calls = [+ tc.model_copy(deep=True)+ for tc in reply.get_content_blocks("tool_call")+ if tc.id in call_ids+ ]+ if external:+ return ExternalExecutionResultEvent(+ reply_id=reply.id,+ execution_results=[+ ToolResultBlock(+ id=tc.id,+ name=tc.name,+ output=tc.id,+ state="success",+ )+ for tc in calls+ ],+ )+ return UserConfirmResultEvent(+ reply_id=reply.id,+ confirm_results=[+ ConfirmResult(confirmed=True, tool_call=tc) for tc in calls+ ],+ )++ async def test_duplicates_during_execution_and_result_send(self) -> None:+ """A queued duplicate cannot send failure after the success result."""+ for external in (False, True):+ with self.subTest(external=external):+ live, backend, executed = await self._start(external, ["one"])+ sending = asyncio.Event()+ release = asyncio.Event()++ async def send_result(+ _delegation_id: str,+ _content: str,+ started: asyncio.Event = sending,+ finish: asyncio.Event = release,+ ) -> None:+ """Expose the window after execution and before removal."""+ started.set()+ await finish.wait()++ sender = AsyncMock(side_effect=send_result)+ live.model.send_delegation_result = sender+ event = self._input(backend, external, ["one"])+ with patch.object(+ backend,+ "reply_stream",+ wraps=backend.reply_stream,+ ) as reply_stream:+ try:+ await live.send(event)+ await live.send(event.model_copy(deep=True))+ await asyncio.wait_for(sending.wait(), timeout=1)+ await live.send(event.model_copy(deep=True))+ finally:+ release.set()+ await asyncio.gather(*list(live._tasks))+ self.assertEqual(reply_stream.call_count, 1)+ self.assertEqual(+ {+ "results": sender.await_args_list,+ "executed": executed,+ "delegations": live._delegations,+ "routes": live._delegated_tool_calls,+ },+ {+ "results": [call("task", "Done.")],+ "executed": [] if external else ["one"],+ "delegations": {},+ "routes": {},+ },+ )++ async def test_duplicates_preserve_other_pending_tools(self) -> None:+ """Skip only completed inputs, retaining new decisions in a batch."""+ for external in (False, True):+ with self.subTest(external=external):+ live, backend, executed = await self._start(+ external,+ ["one", "two"],+ )+ sender = live.model.send_delegation_result+ event = self._input(backend, external, ["one"])+ await live.send(event)+ await live.send(event.model_copy(deep=True))+ await asyncio.gather(*list(live._tasks))+ self.assertEqual(+ [+ tc.id+ for tc in live.state.get_awaiting_tool_calls("Friday")+ ],+ ["two"],+ )+ delegation = live._delegations["task"]+ self.assertIsNotNone(delegation.timeout_task)+ self.assertFalse(delegation.timeout_task.done())+ timeout = delegation.timeout_task+ await live.send(event.model_copy(deep=True))+ await asyncio.gather(*list(live._tasks))+ self.assertIs(delegation.timeout_task, timeout)+ self.assertFalse(timeout.done())+ sender.assert_not_awaited()+ await live.send(self._input(backend, external, ["one", "two"]))+ await asyncio.gather(*list(live._tasks))+ reply = live.state.get_reply_message("Friday")+ assert reply is not None+ self.assertEqual(+ {+ "results": sender.await_args_list,+ "executed": executed,+ "tool_results": [+ (block.id, block.state)+ for block in reply.get_content_blocks(+ "tool_result",+ )+ ],+ "pending": live.state.get_awaiting_tool_calls(+ "Friday",+ ),+ },+ {+ "results": [call("task", "Done.")],+ "executed": [] if external else ["one", "two"],+ "tool_results": [+ ("one", "success"),+ ("two", "success"),+ ],+ "pending": [],+ },+ )+++class RealtimeDelegationErrorTest(IsolatedAsyncioTestCase):+ """Exercise validation and cleanup with a real Agent and AskUser."""++ async def test_model_failure_preserves_completed_tool_history(+ self,+ ) -> None:+ """Keep successful tool records when subsequent reasoning fails."""+ executed: list[str] = []++ def write(value: str) -> str:+ """Record one execution before the model fails."""+ executed.append(value)+ return value++ for needs_confirmation in (False, True):+ with self.subTest(needs_confirmation=needs_confirmation):+ executed.clear()+ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[+ ToolCallBlock(+ id="write-1",+ name="write",+ input=json.dumps({"value": "saved"}),+ ),+ ],+ is_last=True,+ ),+ RuntimeError("Model failed after tool execution."),+ ],+ )+ live = _create_live_agent(+ model,+ Toolkit(+ tools=[+ FunctionTool(+ write,+ is_read_only=not needs_confirmation,+ ),+ ],+ ),+ )+ self.addAsyncCleanup(live.close)+ live._start_client_delegation("task")+ backend = live._delegations["task"].agent+ assert backend is not None+ if needs_confirmation:+ await asyncio.gather(*list(live._tasks))+ pending = backend.state.get_awaiting_tool_calls("Friday")+ await live.send(+ UserConfirmResultEvent(+ reply_id=backend.state.reply_id,+ confirm_results=[+ ConfirmResult(+ confirmed=True,+ tool_call=pending[0],+ ),+ ],+ ),+ )+ with self.assertLogs("as", level="ERROR"):+ await asyncio.gather(*list(live._tasks))+ reply = backend.state.get_reply_message("Friday")+ assert reply is not None+ self.assertEqual(+ {+ "executed": executed,+ "model_calls": model.cnt,+ "results": [+ (+ block.id,+ block.state,+ [+ item.text+ for item in block.output+ if isinstance(item, TextBlock)+ ],+ )+ for block in reply.get_content_blocks(+ "tool_result",+ )+ ],+ "active": live._active_delegation_id,+ "delegations": live._delegations,+ "routes": live._delegated_tool_calls,+ },+ {+ "executed": ["saved"],+ "model_calls": 2,+ "results": [("write-1", "success", ["saved"])],+ "active": None,+ "delegations": {},+ "routes": {},+ },+ )+ expected = reply.model_copy(deep=True)+ expected.content = [+ block+ for block in expected.content+ if block.type in {"tool_call", "tool_result"}+ ]+ expected_context = [expected.model_dump(mode="json")]+ self.assertEqual(+ live.state.model_dump(mode="json")["context"],+ expected_context,+ )+ live.model.send_delegation_result.assert_awaited_once_with(+ "task",+ "The delegated task failed. Ask the user to try again.",+ )+ await live.close()+ self.assertEqual(+ live.state.model_dump(mode="json")["context"],+ expected_context,+ )++ async def test_model_failure_publishes_accepted_external_result(+ self,+ ) -> None:+ """Publish a pending result END even when no next event is yielded."""+ live = await self._start_delegation(["last"])+ backend = live._delegations["task"].agent+ assert backend is not None+ backend.model.set_responses([RuntimeError("Model failed.")])+ result = self._result("last", True)+ with self.assertLogs("as", level="ERROR"):+ await live.send(+ ExternalExecutionResultEvent(+ reply_id=backend.state.reply_id,+ execution_results=[result],+ ),+ )+ await asyncio.gather(*list(live._tasks))+ self.assertEqual(+ [+ (event.tool_call_id, event.state)+ for event in live.drain_events()+ if isinstance(event, ToolResultEndEvent)+ ],+ [("last", "success")],+ )+ reply = live.state.get_reply_message("Friday")+ assert reply is not None+ self.assertEqual(+ reply.get_content_blocks("tool_result"),+ [result],+ )+ self.assertEqual(live._delegations, {})+ live.model.send_delegation_result.assert_awaited_once_with(+ "task",+ "The delegated task failed. Ask the user to try again.",+ )++ async def _start_delegation(self, call_ids: list[str]) -> RealtimeAgent:+ """Park the backend on one or more external questions."""+ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[+ ToolCallBlock(+ id=call_id,+ name="AskUser",+ input=json.dumps(+ {+ "questions": [+ {+ "question": f"Do {call_id}?",+ "header": "Approval",+ "options": [+ {+ "label": "Yes",+ "description": "Proceed",+ },+ {+ "label": "No",+ "description": "Stop",+ },+ ],+ },+ ],+ },+ ),+ )+ for call_id in call_ids+ ],+ is_last=True,+ ),+ ChatResponse(+ content=[TextBlock(text="Answered.")],+ is_last=True,+ ),+ ],+ )++ live = _create_live_agent(model, Toolkit(tools=[AskUser()]))+ self.addAsyncCleanup(live.close)+ live._start_client_delegation("task")+ await asyncio.gather(*list(live._tasks))+ self.assertEqual(+ [call.id for call in live.state.get_awaiting_tool_calls("Friday")],+ call_ids,+ )+ list(live.drain_events())+ return live++ @staticmethod+ def _result(call_id: str, valid: bool) -> ToolResultBlock:+ """Build an answer with or without the required metadata."""+ return ToolResultBlock(+ id=call_id,+ name="AskUser",+ output="Yes",+ state="success",+ metadata={+ "answers": [+ {"question": f"Do {call_id}?", "selected": ["Yes"]},+ ],+ }+ if valid+ else {},+ )++ async def test_invalid_results_can_retry_close_or_expire(self) -> None:+ """Rejected answers retain routing, timeout, and partial results."""+ for action in ("retry", "close", "expire"):+ for partial in (False, True):+ with self.subTest(action=action, partial=partial):+ call_ids = ["first", "last"] if partial else ["last"]+ live = await self._start_delegation(call_ids)+ send_result = live.model.send_delegation_result+ delegation = live._delegations["task"]+ assert delegation.agent is not None+ old_timeout = delegation.timeout_task+ results = [self._result("first", True)] if partial else []+ results.append(self._result("last", False))+ submission = ExternalExecutionResultEvent(+ reply_id=live.state.reply_id,+ execution_results=results,+ )+ with self.assertLogs("as", level="ERROR"):+ await live.send(submission)+ await asyncio.gather(*list(live._tasks))+ events = list(live.drain_events())+ self.assertEqual(+ {+ "active": live._active_delegation_id,+ "routes": live._delegated_tool_calls,+ "pending": [+ call.id+ for call in live.state.get_awaiting_tool_calls(+ "Friday",+ )+ ],+ "timeout_rearmed": (+ delegation.timeout_task is not None+ and delegation.timeout_task is not old_timeout+ and not delegation.timeout_task.done()+ ),+ "ends": [+ (event.tool_call_id, event.state)+ for event in events+ if isinstance(event, ToolResultEndEvent)+ ],+ "reported": send_result.await_count,+ "notifications": [+ (event.name, event.value)+ for event in events+ if isinstance(event, CustomEvent)+ ],+ },+ {+ "active": "task",+ "routes": dict.fromkeys(call_ids, "task"),+ "pending": ["last"],+ "timeout_rearmed": True,+ "ends": [("first", "success")] if partial else [],+ "reported": 0,+ "notifications": [+ (+ "external_execution_result_rejected",+ {+ "request_id": submission.id,+ "reply_id": submission.reply_id,+ "tool_call_ids": ["last"],+ "message": (+ "The submitted result was "+ "rejected. Please check your "+ "answer and submit again."+ ),+ },+ ),+ ],+ },+ )+ if action == "retry":+ await live.send(+ ExternalExecutionResultEvent(+ reply_id=live.state.reply_id,+ execution_results=[self._result("last", True)],+ ),+ )+ await asyncio.gather(*list(live._tasks))+ send_result.assert_awaited_once_with(+ "task",+ "Answered.",+ )+ elif action == "close":+ await live.close()+ send_result.assert_not_awaited()+ else:+ timeout = delegation.timeout_task+ live._cancel_delegation_timeout(delegation)+ await asyncio.gather(timeout, return_exceptions=True)+ with patch(+ "agentscope.agent._realtime._agent."+ "_DELEGATION_INPUT_TIMEOUT_SECS",+ 0,+ ):+ await live._expire_client_delegation(delegation)+ send_result.assert_awaited_once()+ reply = live.state.get_reply_message("Friday")+ assert reply is not None+ self.assertEqual(+ [+ event.name+ for event in live.drain_events()+ if isinstance(event, CustomEvent)+ ],+ [],+ )+ self.assertEqual(+ {+ "active": live._active_delegation_id,+ "delegations": live._delegations,+ "routes": live._delegated_tool_calls,+ "pending": live.state.get_awaiting_tool_calls(+ "Friday",+ ),+ "results": [+ (block.id, block.state)+ for block in reply.get_content_blocks(+ "tool_result",+ )+ ],+ },+ {+ "active": None,+ "delegations": {},+ "routes": {},+ "pending": [],+ "results": (+ [("first", "success")] if partial else []+ )+ + [+ (+ "last",+ "success"+ if action == "retry"+ else "interrupted",+ ),+ ],+ },+ )diff --git a/tests/realtime_openai_live_test.py b/tests/realtime_openai_live_test.pynew file mode 100644index 000000000..188f65829--- /dev/null+++ b/tests/realtime_openai_live_test.py@@ -0,0 +1,2017 @@+# -*- coding: utf-8 -*-+"""Tests for GPT-Live protocol and its minimal full-duplex agent path."""+# pylint: disable=protected-access+import asyncio+import base64+import json+import unittest+from typing import AsyncGenerator, AsyncIterator, Callable+from unittest.mock import AsyncMock, patch++from utils import MockModel++from agentscope.agent import Agent, ContextConfig, RealtimeAgent+from agentscope.agent._realtime._agent import _ClientDelegation+from agentscope.credential import OpenAICredential+from agentscope.event import (+ AgentEvent,+ ConfirmResult,+ CustomEvent,+ ExternalExecutionResultEvent,+ ReplyEndEvent,+ ReplyStartEvent,+ RequireExternalExecutionEvent,+ RequireUserConfirmEvent,+ TextBlockDeltaEvent,+ ToolCallEndEvent,+ ToolCallStartEvent,+ ToolResultDataDeltaEvent,+ ToolResultEndEvent,+ ToolResultStartEvent,+ ToolResultTextDeltaEvent,+ UserConfirmResultEvent,+ UserInterruptEvent,+)+from agentscope.message import (+ AssistantMsg,+ Msg,+ TextBlock,+ ToolCallBlock,+ ToolResultBlock,+ ToolResultState,+ UserMsg,+)+from agentscope.middleware import MiddlewareBase+from agentscope.model import ChatResponse, StructuredResponse+from agentscope.realtime import (+ ControlFrame,+ ControlFrameType,+ DelegatedResponseEvent,+ DelegationCreatedEvent,+ OpenAILiveModel,+ PlayoutPosition,+ SessionClosedEvent,+ SessionStartedEvent,+ SessionUsageEvent,+ TimelineAudioDeltaEvent,+ TimelineTranscriptDeltaEvent,+ TransportBase,+ TransportFrame,+)+from agentscope.state import AgentState+from agentscope.tool import FunctionTool, Toolkit++CRED = OpenAICredential(api_key="sk-x")+++def _model() -> OpenAILiveModel:+ """Build a Live model configured for client delegation."""+ return OpenAILiveModel("gpt-live-1", CRED)+++class _FakeWebSocket:+ """A controllable async WebSocket used by lifecycle tests."""++ def __init__(self) -> None:+ self.incoming: asyncio.Queue[str | None] = asyncio.Queue()+ self.sent: list[dict] = []+ self.closed = False++ def __aiter__(self) -> "_FakeWebSocket":+ return self++ async def __anext__(self) -> str:+ value = await self.incoming.get()+ if value is None:+ raise StopAsyncIteration+ return value++ async def send(self, raw: str) -> None:+ """Record a command and enqueue its simulated server response."""+ payload = json.loads(raw)+ self.sent.append(payload)+ if payload["type"] == "session.start":+ self.incoming.put_nowait(+ json.dumps(+ {+ "type": "session.started",+ "session": {"id": "live_1"},+ },+ ),+ )+ elif payload["type"] == "session.close":+ self.incoming.put_nowait(+ json.dumps(+ {+ "type": "session.closed",+ "reason": "close_requested",+ "session": {"id": "live_1"},+ "usage": {"seconds": 4.5},+ },+ ),+ )++ async def close(self) -> None:+ """End the fake server event stream."""+ self.closed = True+ self.incoming.put_nowait(None)+++class _PendingStartupWebSocket(_FakeWebSocket):+ """Pause startup either during send or while awaiting acknowledgement."""++ def __init__(self, block_send: bool) -> None:+ super().__init__()+ self.block_send = block_send+ self.startup_sent = asyncio.Event()+ self.reading = asyncio.Event()++ async def send(self, raw: str) -> None:+ """Record startup without automatically acknowledging it."""+ self.sent.append(json.loads(raw))+ self.startup_sent.set()+ if self.block_send:+ await asyncio.Event().wait()++ async def __anext__(self) -> str:+ self.reading.set()+ return await super().__anext__()+++class _FakeTransport(TransportBase):+ """Capture output audio without opening a real sound device."""++ input_sample_rate = 24000+ output_sample_rate = 24000++ def __init__(self) -> None:+ self.audio: list[tuple[bytes, str]] = []+ self.cleared = 0+ self.frames: asyncio.Queue[TransportFrame | None] = asyncio.Queue()++ async def start(self) -> None:+ """Start the in-memory transport."""++ async def close(self) -> None:+ """Close the in-memory transport."""++ async def incoming(self) -> AsyncIterator[TransportFrame]:+ """Yield explicitly queued transport frames."""+ while True:+ frame = await self.frames.get()+ if frame is None:+ return+ yield frame++ async def send_audio(self, pcm: bytes, item_id: str) -> None:+ """Capture one outgoing audio frame."""+ self.audio.append((pcm, item_id))++ async def clear_audio(self) -> PlayoutPosition:+ """Record one playback clear request."""+ self.cleared += 1+ return self.playout()++ def playout(self) -> PlayoutPosition:+ """Return the current item with no simulated playback progress."""+ return PlayoutPosition(item_id=self.audio[-1][1] if self.audio else "")+++class _FakeDelegationAgent:+ """Return one deterministic task result without a provider call."""++ name = "Friday"++ def __init__(self) -> None:+ self.state = AgentState()+ self.inputs: list[Msg] = []++ async def reply_stream(+ self,+ inputs: Msg,+ yield_final_msg: bool,+ ) -> AsyncIterator[Msg]:+ """Capture the generated transcript prompt and return a result."""+ self.inputs.append(inputs)+ self.state.context.append(inputs)+ result = AssistantMsg(name="Friday", content="Order 42 shipped.")+ self.state.context.append(result)+ if yield_final_msg:+ yield result+++class _ConfirmingDelegationAgent:+ """Pause once for permission, then return a completed result."""++ name = "Friday"++ def __init__(self) -> None:+ self.state = AgentState()+ self.tool_call = ToolCallBlock(+ id="tool_1",+ name="update_order",+ input='{"order_id":"42"}',+ state="asking",+ )++ async def reply_stream(+ self,+ inputs: Msg | UserConfirmResultEvent,+ yield_final_msg: bool,+ ) -> AsyncIterator[+ Msg+ | ReplyStartEvent+ | ToolCallStartEvent+ | ToolCallEndEvent+ | ToolResultStartEvent+ | ToolResultTextDeltaEvent+ | ToolResultEndEvent+ | RequireUserConfirmEvent+ ]:+ """Yield a confirmation on the first call and finish on resume."""+ if isinstance(inputs, Msg):+ self.state.reply_id = "backend_reply"+ self.state.append_context("Friday", [self.tool_call])+ yield ReplyStartEvent(+ session_id=self.state.session_id,+ reply_id="backend_reply",+ name="Friday",+ )+ yield ToolCallStartEvent(+ reply_id="backend_reply",+ tool_call_id=self.tool_call.id,+ tool_call_name=self.tool_call.name,+ )+ yield ToolCallEndEvent(+ reply_id="backend_reply",+ tool_call_id=self.tool_call.id,+ )+ yield RequireUserConfirmEvent(+ reply_id="backend_reply",+ tool_calls=[self.tool_call],+ )+ return+ yield ToolResultStartEvent(+ reply_id="backend_reply",+ tool_call_id=self.tool_call.id,+ tool_call_name=self.tool_call.name,+ )+ yield ToolResultTextDeltaEvent(+ reply_id="backend_reply",+ tool_call_id=self.tool_call.id,+ delta="Updated",+ )+ yield ToolResultEndEvent(+ reply_id="backend_reply",+ tool_call_id=self.tool_call.id,+ state=ToolResultState.SUCCESS,+ )+ self.tool_call.state = "finished"+ self.state.append_context(+ "Friday",+ [+ ToolResultBlock(+ id=self.tool_call.id,+ name=self.tool_call.name,+ output="Updated",+ state="success",+ ),+ ],+ )+ result = AssistantMsg(name="Friday", content="Order 42 updated.")+ if yield_final_msg:+ yield result+++class _ExternalDelegationAgent:+ """Pause for an external executor, then finish after its result."""++ name = "Friday"++ def __init__(self) -> None:+ self.state = AgentState()+ self.tool_call = ToolCallBlock(+ id="external_1",+ name="ask_user",+ input='{"question":"Which order?"}',+ state="submitted",+ )+ self.received: list[+ Msg | ExternalExecutionResultEvent | UserInterruptEvent+ ] = []++ async def reply_stream(+ self,+ inputs: Msg | ExternalExecutionResultEvent | UserInterruptEvent,+ yield_final_msg: bool,+ ) -> AsyncIterator[+ Msg+ | ReplyStartEvent+ | ToolCallStartEvent+ | ToolCallEndEvent+ | RequireExternalExecutionEvent+ ]:+ """Request external execution once and finish on resumption."""+ self.received.append(inputs)+ if isinstance(inputs, Msg):+ self.state.reply_id = "external_reply"+ self.state.append_context("Friday", [self.tool_call])+ yield ReplyStartEvent(+ session_id=self.state.session_id,+ reply_id="external_reply",+ name="Friday",+ )+ yield ToolCallStartEvent(+ reply_id="external_reply",+ tool_call_id=self.tool_call.id,+ tool_call_name=self.tool_call.name,+ )+ yield ToolCallEndEvent(+ reply_id="external_reply",+ tool_call_id=self.tool_call.id,+ )+ yield RequireExternalExecutionEvent(+ reply_id="external_reply",+ tool_calls=[self.tool_call],+ )+ return+ self.tool_call.state = "finished"+ if isinstance(inputs, UserInterruptEvent):+ result = AssistantMsg(name="Friday", content="Interrupted.")+ if yield_final_msg:+ yield result+ return+ result = AssistantMsg(name="Friday", content="Order 42 selected.")+ if yield_final_msg:+ yield result+++class _StateDelegationAgent:+ """Record one state mutation before returning a result."""++ name = "Friday"++ def __init__(self, state: AgentState, marker: str) -> None:+ self.state = state+ self.marker = marker++ async def reply_stream(+ self,+ inputs: Msg,+ yield_final_msg: bool,+ ) -> AsyncIterator[Msg]:+ """Append a marker that the next queued task must observe."""+ del inputs+ self.state.tool_context.activated_groups.append(self.marker)+ if yield_final_msg:+ yield AssistantMsg(name="Friday", content=self.marker)+++class OpenAILivePayloadTest(unittest.TestCase):+ """Model discovery, startup payloads, and server event parsing."""++ def test_card_and_credential_mapping(self) -> None:+ """The Live card resolves to its own protocol adapter."""+ cards = OpenAILiveModel.list_models()+ classes = {+ item.type: item for item in CRED.get_realtime_model_classes()+ }+ self.assertListEqual(+ [+ (+ card.name,+ card.model_type,+ card.interaction_mode,+ card.delegation_backend,+ card.supports_tools,+ classes[card.model_type].__name__,+ )+ for card in cards+ ],+ [+ (+ "gpt-live-1",+ "openai_live",+ "full_duplex",+ "session_chat",+ False,+ "OpenAILiveModel",+ ),+ ],+ )+ self.assertDictEqual(+ cards[0].parameter_schema,+ {+ "type": "object",+ "properties": {+ "voice": {+ "default": "marin",+ "description": "The voice used for generated audio.",+ "title": "Voice",+ "type": "string",+ },+ "store": {+ "default": False,+ "description": (+ "Whether to request provider-side storage for "+ "the Live session."+ ),+ "title": "Store Session",+ "type": "boolean",+ },+ "startup_timeout_s": {+ "default": 15.0,+ "description": (+ "Maximum seconds to wait for the Live session "+ "to start."+ ),+ "exclusiveMinimum": 0,+ "title": "Startup Timeout",+ "type": "number",+ },+ "close_timeout_s": {+ "default": 15.0,+ "description": (+ "Maximum seconds to wait for graceful session "+ "closure."+ ),+ "exclusiveMinimum": 0,+ "title": "Close Timeout",+ "type": "number",+ },+ },+ },+ )++ def test_session_start_payload(self) -> None:+ """Startup delegates backend tasks to the application."""+ model = _model()+ payload = model._session_start("Speak briefly.")+ event_id = payload.pop("event_id")+ self.assertTrue(event_id.startswith("start_"))+ self.assertDictEqual(+ payload,+ {+ "type": "session.start",+ "session": {+ "model": "gpt-live-1",+ "instructions": "Speak briefly.",+ "input": [],+ "audio": {+ "format": {+ "type": "audio/pcm",+ "rate": 24000,+ },+ "output": {"voice": "marin"},+ },+ "delegation": {"type": "client"},+ "store": False,+ },+ },+ )++ def test_server_event_mapping(self) -> None:+ """Live timelines and delegated Responses remain independent."""+ model = _model()+ encoded = base64.b64encode(b"\x01\x00").decode("ascii")+ frames = [+ {+ "type": "session.started",+ "session": {"id": "live_1"},+ },+ {+ "type": "session.input_transcript.delta",+ "delta": "hello",+ "start_ms": 10,+ "end_ms": 20,+ },+ {+ "type": "session.output_transcript.delta",+ "delta": "hi",+ "start_ms": 15,+ "end_ms": 25,+ },+ {"type": "session.output_audio.delta", "delta": encoded},+ {+ "type": "session.delegation.created",+ "offset_ms": 30,+ "delegation": {+ "id": "delegation_1",+ "target": "client",+ },+ },+ {+ "type": "response.event",+ "delegation_id": "delegation_1",+ "event": {+ "type": "response.output_text.delta",+ "delta": "working",+ },+ },+ {+ "type": "session.usage.updated",+ "usage": {"seconds": 3.25},+ "context_window": {"usage_ratio": 0.2},+ },+ {+ "type": "session.closed",+ "reason": "close_requested",+ "session": {"id": "live_1"},+ "usage": {"seconds": 4.5},+ },+ ]+ self.assertListEqual(+ [model._parse(frame) for frame in frames],+ [+ SessionStartedEvent(session_id="live_1"),+ TimelineTranscriptDeltaEvent(+ role="user",+ delta="hello",+ start_ms=10,+ end_ms=20,+ ),+ TimelineTranscriptDeltaEvent(+ role="assistant",+ delta="hi",+ start_ms=15,+ end_ms=25,+ ),+ TimelineAudioDeltaEvent(+ pcm=b"\x01\x00",+ sample_rate=24000,+ ),+ DelegationCreatedEvent(+ delegation_id="delegation_1",+ target="client",+ offset_ms=30,+ ),+ DelegatedResponseEvent(+ delegation_id="delegation_1",+ event={+ "type": "response.output_text.delta",+ "delta": "working",+ },+ ),+ SessionUsageEvent(+ seconds=3.25,+ context_usage_ratio=0.2,+ ),+ SessionClosedEvent(+ session_id="live_1",+ reason="close_requested",+ usage_seconds=4.5,+ ),+ ],+ )+ self.assertEqual(model.usage_seconds, 4.5)++ def test_long_commentary_is_split_at_a_safe_boundary(self) -> None:+ """Long client results are returned as bounded Live updates."""+ content = f"{'a' * 390} boundary {'b' * 20}"+ self.assertListEqual(+ OpenAILiveModel._commentary_chunks(content),+ [f"{'a' * 390} boundary", "b" * 20],+ )+++class OpenAILiveLifecycleTest(unittest.IsolatedAsyncioTestCase):+ """Connection startup, audio validation, and graceful close."""++ async def test_cancel_startup_releases_resources_and_reconnects(+ self,+ ) -> None:+ """Cancellation closes resources before propagating to the caller."""+ for block_send in (False, True):+ with self.subTest(block_send=block_send):+ ws = _PendingStartupWebSocket(block_send)+ replacement = _FakeWebSocket()+ model = _model()+ self.addAsyncCleanup(model.close)+ with patch(+ "websockets.connect",+ AsyncMock(side_effect=[ws, replacement]),+ ):+ connection = asyncio.create_task(model.connect("Brief."))+ await asyncio.wait_for(ws.startup_sent.wait(), timeout=1)+ await asyncio.wait_for(ws.reading.wait(), timeout=1)+ reader, started = model._reader, model._started+ assert reader is not None and started is not None+ self.assertFalse(reader.done())+ connection.cancel()+ with self.assertRaises(asyncio.CancelledError):+ await connection+ self.assertEqual(+ {+ "socket_closed": ws.closed,+ "socket": model._ws,+ "reader": model._reader,+ "reader_done": reader.done(),+ "startup_cancelled": started.cancelled(),+ "startup": model._started,+ "session_id": model._session_id,+ },+ {+ "socket_closed": True,+ "socket": None,+ "reader": None,+ "reader_done": True,+ "startup_cancelled": True,+ "startup": None,+ "session_id": "",+ },+ )+ await model.connect("Retry.")+ self.assertIs(model._ws, replacement)+ self.assertEqual(model._session_id, "live_1")+ self.assertEqual(+ await asyncio.wait_for(+ anext(model.events()),+ timeout=1,+ ),+ SessionStartedEvent(session_id="live_1"),+ )+ await model.close()+ self.assertTrue(replacement.closed)++ async def test_startup_timeout_settles_the_shielded_future(self) -> None:+ """Both startup phases time out and release their resources."""+ for block_send in (False, True):+ with self.subTest(block_send=block_send):+ ws = _PendingStartupWebSocket(block_send)+ model = _model()+ model.parameters.startup_timeout_s = 0.1+ self.addAsyncCleanup(model.close)+ resources: list[asyncio.Future | None] = []++ async def capture_startup(+ payload: dict,+ current: OpenAILiveModel = model,+ capture: Callable[+ [asyncio.Future | None],+ None,+ ] = resources.append,+ ) -> None:+ """Save references before startup can suspend or expire."""+ capture(current._reader)+ capture(current._started)+ await OpenAILiveModel._send(current, payload)++ with (+ patch("websockets.connect", AsyncMock(return_value=ws)),+ patch.object(model, "_send", capture_startup),+ ):+ connection = asyncio.create_task(model.connect("Brief."))+ with self.assertRaises(asyncio.TimeoutError):+ await asyncio.wait_for(connection, timeout=1)+ reader, started = resources+ assert reader is not None and started is not None+ self.assertEqual(+ {+ "guard_cancelled": connection.cancelled(),+ "socket_closed": ws.closed,+ "reader_done": reader.done(),+ "startup_cancelled": started.cancelled(),+ "socket": model._ws,+ "reader": model._reader,+ "startup": model._started,+ },+ {+ "guard_cancelled": False,+ "socket_closed": True,+ "reader_done": True,+ "startup_cancelled": True,+ "socket": None,+ "reader": None,+ "startup": None,+ },+ )++ async def test_startup_phases_share_one_timeout(self) -> None:+ """Sending must consume part of the acknowledgement deadline."""+ ws = _FakeWebSocket()+ model = _model()+ model.parameters.startup_timeout_s = 0.1+ self.addAsyncCleanup(model.close)+ acknowledgement: asyncio.TimerHandle | None = None++ async def delayed_send(raw: str) -> None:+ """Each phase fits alone, but their combined delay does not."""+ nonlocal acknowledgement+ ws.sent.append(json.loads(raw))+ await asyncio.sleep(0.06)+ acknowledgement = asyncio.get_running_loop().call_later(+ 0.06,+ ws.incoming.put_nowait,+ json.dumps(+ {"type": "session.started", "session": {"id": "late"}},+ ),+ )++ with (+ patch("websockets.connect", AsyncMock(return_value=ws)),+ patch.object(ws, "send", delayed_send),+ ):+ with self.assertRaises(asyncio.TimeoutError):+ try:+ await asyncio.wait_for(model.connect("Brief."), timeout=1)+ finally:+ if acknowledgement is not None:+ acknowledgement.cancel()+ self.assertEqual(+ (ws.closed, model._ws, model._reader, model._started),+ (True, None, None, None),+ )++ async def test_startup_error_is_preserved_after_cleanup(self) -> None:+ """Provider startup errors still reach the caller after cleanup."""+ ws = _PendingStartupWebSocket(False)+ model = _model()+ self.addAsyncCleanup(model.close)+ with patch("websockets.connect", AsyncMock(return_value=ws)):+ connection = asyncio.create_task(model.connect("Brief."))+ await asyncio.wait_for(ws.reading.wait(), timeout=1)+ reader = model._reader+ assert reader is not None+ ws.incoming.put_nowait(+ json.dumps(+ {+ "type": "error",+ "error": {"message": "Startup rejected"},+ },+ ),+ )+ with self.assertRaisesRegex(RuntimeError, "Startup rejected"):+ await connection+ self.assertEqual(+ (ws.closed, reader.done(), model._reader, model._started),+ (True, True, None, None),+ )++ async def test_connect_audio_and_close(self) -> None:+ """The adapter waits for startup and preserves final usage."""+ ws = _FakeWebSocket()+ connector = AsyncMock(return_value=ws)+ model = _model()+ with patch("websockets.connect", connector):+ await model.connect("Be brief.")+ await model.push_audio(b"\x01\x00")+ await model.send_delegation_result(+ "delegation_1",+ "The order shipped.",+ )+ with self.assertRaises(ValueError):+ await model.push_audio(b"\x01")+ await model.close()++ connector.assert_awaited_once()+ assert connector.await_args is not None+ args, kwargs = connector.await_args+ self.assertEqual(args, ("wss://api.openai.com/v1/live/sessions",))+ self.assertDictEqual(+ kwargs,+ {"additional_headers": {"Authorization": "Bearer sk-x"}},+ )+ self.assertListEqual(+ [item["type"] for item in ws.sent],+ [+ "session.start",+ "session.input_audio.append",+ "session.commentary.append",+ "session.close",+ ],+ )+ self.assertEqual(ws.sent[1]["audio"], "AQA=")+ commentary = ws.sent[2].copy()+ commentary_id = commentary.pop("event_id")+ self.assertTrue(commentary_id.startswith("delegation_result_"))+ self.assertDictEqual(+ commentary,+ {+ "type": "session.commentary.append",+ "delegation_id": "delegation_1",+ "content": "The order shipped.",+ },+ )+ self.assertEqual(model.usage_seconds, 4.5)+ self.assertTrue(ws.closed)+++class RealtimeAgentLiveTest(unittest.IsolatedAsyncioTestCase):+ """Minimal projection of overlapping Live transcript and audio."""++ async def test_mismatched_backend_name_fails_before_execution(+ self,+ ) -> None:+ """Reject an incompatible backend without leaving a parked task."""+ backend = Agent("TaskBot", "Be brief.", MockModel())+ model = _model()+ model.send_delegation_result = AsyncMock()+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ model,+ delegation_agent_factory=lambda _: backend,+ )+ self.addAsyncCleanup(agent.close)+ with (+ patch.object(backend, "reply_stream") as reply_stream,+ patch.object(agent, "_schedule_delegation_timeout") as timeout,+ ):+ with self.assertLogs("as", level="ERROR") as logs:+ agent._start_client_delegation("bad-name")+ await asyncio.gather(*list(agent._tasks))+ reply_stream.assert_not_called()+ timeout.assert_not_called()+ self.assertIn(+ "Delegation backend name 'TaskBot' must match "+ "RealtimeAgent name 'Friday'.",+ "\n".join(logs.output),+ )+ self.assertEqual(+ {+ "active": agent._active_delegation_id,+ "delegations": agent._delegations,+ "routes": agent._delegated_tool_calls,+ "queue": list(agent._delegation_queue),+ "tasks": agent._tasks,+ },+ {+ "active": None,+ "delegations": {},+ "routes": {},+ "queue": [],+ "tasks": set(),+ },+ )+ model.send_delegation_result.assert_awaited_once_with(+ "bad-name",+ "The delegated task could not be completed because the "+ "session chat agent is unavailable.",+ )++ async def test_interrupt_control_stops_speech_and_keeps_delegation(+ self,+ ) -> None:+ """Browser Stop clears speech without cancelling backend work."""+ model = _model()+ socket = _FakeWebSocket()+ model._ws = socket+ model.send_delegation_result = AsyncMock()+ backend = _ExternalDelegationAgent()+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ model,+ delegation_agent_factory=lambda _: backend,+ )+ self.addAsyncCleanup(agent.close)+ transport = _FakeTransport()+ agent._transport = transport+ agent._connected = True+ agent._start_client_delegation("task")+ await asyncio.gather(*list(agent._tasks))+ delegation = agent._delegations["task"]+ timeout = delegation.timeout_task+ await agent._on_model_event(SessionStartedEvent(session_id="live_1"))+ await agent._on_model_event(+ TimelineAudioDeltaEvent(pcm=b"\x01\x00", sample_rate=24000),+ )+ await agent._on_model_event(+ TimelineTranscriptDeltaEvent(+ role="assistant",+ delta="Sure thing",+ start_ms=10,+ end_ms=20,+ ),+ )+ reply_id = agent._live_replies["assistant"].reply_id+ list(agent.drain_events())++ await agent._on_control(ControlFrame(type=ControlFrameType.INTERRUPT))++ self.assertEqual(+ {+ "cleared": transport.cleared,+ "playout": transport.playout().model_dump(),+ "audio_pending": agent._audio_pending,+ "payloads": [+ {+ key: value+ for key, value in item.items()+ if key != "event_id"+ }+ for item in socket.sent+ ],+ "ends": [+ (event.reply_id, event.finished_reason)+ for event in agent.drain_events()+ if isinstance(event, ReplyEndEvent)+ ],+ "live_replies": agent._live_replies,+ "transcript": [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ if msg.id == reply_id+ ],+ "active": agent._active_delegation_id,+ "routes": agent._delegated_tool_calls,+ "backend_inputs": backend.received,+ },+ {+ "cleared": 1,+ "playout": {+ "item_id": "live:live_1",+ "played_ms": 0,+ "first_played_at": None,+ },+ "audio_pending": False,+ "payloads": [+ {+ "type": "session.instructions.append",+ "delegation_id": None,+ "content": "Stop speaking now and listen to the user.",+ },+ ],+ "ends": [(reply_id, "interrupted")],+ "live_replies": {},+ "transcript": [("assistant", "Sure thing")],+ "active": "task",+ "routes": {"external_1": "task"},+ "backend_inputs": [delegation.prompt],+ },+ )+ self.assertIs(agent._delegations["task"], delegation)+ self.assertIs(delegation.timeout_task, timeout)+ assert timeout is not None+ self.assertFalse(timeout.done())+ model.send_delegation_result.assert_not_awaited()++ async def test_overlapping_timeline_does_not_barge_in(self) -> None:+ """User transcript growth does not clear assistant playback."""+ model = _model()+ agent = RealtimeAgent("Friday", "Be brief.", model)+ transport = _FakeTransport()+ agent._transport = transport+ self.assertTrue(agent.allows_open_user_reply_checkpoint)++ await agent._on_model_event(SessionStartedEvent(session_id="live_1"))+ await agent._on_model_event(+ TimelineAudioDeltaEvent(+ pcm=b"\x01\x00",+ sample_rate=24000,+ ),+ )+ await agent._on_model_event(+ TimelineTranscriptDeltaEvent(+ role="assistant",+ delta="Sure",+ start_ms=10,+ end_ms=20,+ ),+ )+ await agent._on_model_event(+ TimelineTranscriptDeltaEvent(+ role="user",+ delta="Wait",+ start_ms=15,+ end_ms=25,+ ),+ )++ self.assertListEqual(+ transport.audio,+ [(b"\x01\x00", "live:live_1")],+ )+ self.assertEqual(transport.cleared, 0)+ self.assertListEqual(+ [+ (message.role, message.get_text_content())+ for message in agent.state.context+ ],+ [("assistant", "Sure"), ("user", "Wait")],+ )++ async def test_client_delegation_uses_live_transcript(self) -> None:+ """A client delegation runs the configured session chat agent."""+ model = _model()+ state = AgentState()+ state.context.append(+ UserMsg(+ id="live:user:historical",+ name="user",+ content="Earlier voice context.",+ ),+ )+ backend = _FakeDelegationAgent()++ def _factory(delegation_state: AgentState) -> _FakeDelegationAgent:+ backend.state = delegation_state+ return backend++ model.send_delegation_result = ( # type: ignore[method-assign]+ AsyncMock()+ )+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ delegation_agent_factory=_factory, # type: ignore[arg-type]+ )++ await agent._on_model_event(+ TimelineTranscriptDeltaEvent(+ role="user",+ delta="Check order 42.",+ start_ms=10,+ end_ms=20,+ ),+ )+ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id="delegation_1",+ target="client",+ ),+ )+ tasks = list(agent._tasks)+ await asyncio.gather(*tasks)++ self.assertEqual(len(backend.inputs), 1)+ prompt = backend.inputs[0].get_text_content()+ self.assertIsNotNone(prompt)+ assert prompt is not None+ self.assertIn("User: Check order 42.", prompt)+ backend_ids = {message.id for message in backend.state.context}+ self.assertDictEqual(+ {+ "historical_text": (+ backend.state.context[0].get_text_content()+ ),+ "current_live_ids": backend_ids & agent._live_message_ids,+ "backend_texts": [+ message.get_text_content()+ for message in backend.state.context+ ],+ },+ {+ "historical_text": "Earlier voice context.",+ "current_live_ids": set(),+ "backend_texts": [+ "Earlier voice context.",+ prompt,+ "Order 42 shipped.",+ ],+ },+ )+ model.send_delegation_result.assert_awaited_once_with(+ "delegation_1",+ "Order 42 shipped.",+ )+ self.assertDictEqual(agent._delegations, {})++ async def test_completed_results_replay_before_interaction(self) -> None:+ """Replay full hidden results with their persisted completion."""+ for interaction_type in (+ RequireUserConfirmEvent,+ RequireExternalExecutionEvent,+ ):+ with self.subTest(interaction=interaction_type.__name__):+ call = ToolCallBlock(id="read", name="Read", input="{}")+ result = ToolResultBlock(+ id=call.id,+ name=call.name,+ output="contents",+ state=ToolResultState.SUCCESS,+ )+ events = [+ ReplyStartEvent(+ session_id="session",+ reply_id="backend",+ name="Friday",+ ),+ ToolCallStartEvent(+ reply_id="backend",+ tool_call_id=call.id,+ tool_call_name=call.name,+ ),+ ToolCallEndEvent(+ reply_id="backend",+ tool_call_id=call.id,+ ),+ ToolResultStartEvent(+ reply_id="backend",+ tool_call_id=call.id,+ tool_call_name=call.name,+ ),+ ToolResultTextDeltaEvent(+ reply_id="backend",+ tool_call_id=call.id,+ delta="contents",+ ),+ ToolResultDataDeltaEvent(+ reply_id="backend",+ tool_call_id=call.id,+ media_type="image/png",+ data="aGVsbG8=",+ ),+ ToolResultEndEvent(+ reply_id="backend",+ tool_call_id=call.id,+ state=ToolResultState.SUCCESS,+ ),+ interaction_type(+ reply_id="backend",+ tool_calls=[+ ToolCallBlock(+ id="write",+ name="Write",+ input="{}",+ ),+ ],+ ),+ ]++ class Backend:+ """Persist a result after yielding its end event."""++ name = "Friday"++ def __init__(+ self,+ script: list[AgentEvent],+ completed: Msg,+ ) -> None:+ self.state = AgentState()+ self.script = script+ self.completed = completed++ # pylint: disable=unused-argument+ async def reply_stream(+ self,+ inputs: Msg,+ yield_final_msg: bool,+ ) -> AsyncIterator[AgentEvent]:+ """Complete Read before requesting outside input."""+ for event in self.script:+ yield event+ if isinstance(event, ToolResultEndEvent):+ self.state.context.append(self.completed)++ # pylint: enable=unused-argument++ backend = Backend(+ events,+ AssistantMsg(+ id="backend",+ name="Friday",+ content=[call, result],+ ),+ )+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ _model(),+ delegation_agent_factory=lambda _, current=backend: (+ current+ ),+ )+ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id="task",+ target="client",+ ),+ )+ await asyncio.gather(*list(agent._tasks))+ agent._cancel_delegation_timeout(agent._delegations["task"])+ published = []+ snapshots = []+ checkpoints = []+ for event in agent.drain_events():+ if isinstance(event, CustomEvent):+ checkpoints.append((event.name, event.value))+ snapshot = agent.checkpoint_snapshot(event)+ assert snapshot is not None+ self.assertEqual(+ snapshot.context[0].model_dump(),+ backend.state.context[0].model_dump(),+ )+ continue+ published.append(event.model_dump())+ if isinstance(event, ToolResultEndEvent):+ snapshot = agent.checkpoint_snapshot(event)+ assert snapshot is not None+ snapshots.append(snapshot.context[0].model_dump())+ self.assertEqual(+ {+ "events": published,+ "snapshots": snapshots,+ "checkpoints": checkpoints,+ },+ {+ "events": [event.model_dump() for event in events],+ "snapshots": [backend.state.context[0].model_dump()],+ "checkpoints": [+ (+ "delegation_checkpoint",+ {+ "delegation_id": "task",+ "reply_id": "backend",+ "tool_call_id": "read",+ },+ ),+ ],+ },+ )++ async def test_client_delegation_resumes_after_confirmation(self) -> None:+ """Realtime control routes approval back to the delegated agent."""+ model = _model()+ backend = _ConfirmingDelegationAgent()+ model.send_delegation_result = ( # type: ignore[method-assign]+ AsyncMock()+ )+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ model,+ delegation_agent_factory=lambda _state: backend,+ )++ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id="delegation_1",+ target="client",+ ),+ )+ await asyncio.gather(*list(agent._tasks))+ self.assertDictEqual(+ agent._delegated_tool_calls,+ {"tool_1": "delegation_1"},+ )+ delegation = agent._delegations["delegation_1"]+ self.assertListEqual(delegation.buffered_events, [])++ sync_calls = 0+ original_sync = agent._sync_delegation_state++ def _count_sync(current: object) -> None:+ nonlocal sync_calls+ sync_calls += 1+ original_sync(current) # type: ignore[arg-type]++ agent._sync_delegation_state = ( # type: ignore[method-assign]+ _count_sync+ )++ await agent.send(+ UserConfirmResultEvent(+ reply_id="backend_reply",+ confirm_results=[+ ConfirmResult(+ confirmed=True,+ tool_call=backend.tool_call,+ ),+ ],+ ),+ )+ await asyncio.gather(*list(agent._tasks))++ model.send_delegation_result.assert_awaited_once_with(+ "delegation_1",+ "Order 42 updated.",+ )+ self.assertDictEqual(agent._delegated_tool_calls, {})+ self.assertEqual(sync_calls, 2)++ async def test_delegations_preserve_middleware_state(self) -> None:+ """Carry middleware data across tasks without sharing nested state."""+ counts = []+ backends = []++ class Counter(MiddlewareBase):+ """Record the number of replies in persistent middleware data."""++ async def on_reply(+ self,+ agent: Agent,+ input_kwargs: dict,+ next_handler: Callable[..., AsyncGenerator],+ ) -> AsyncGenerator:+ """Increment state before running the actual Agent reply."""+ values = agent.state.middle_context.setdefault(+ "counter",+ {"count": 0, "history": []},+ )+ values["count"] += 1+ values["history"].append(values["count"])+ counts.append(values["count"])+ async for item in next_handler(**input_kwargs):+ yield item++ def create_backend(state: AgentState) -> Agent:+ """Build a real Agent with deterministic model output."""+ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[TextBlock(text="Done")],+ is_last=True,+ ),+ ],+ )+ backend = Agent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ middlewares=[Counter()],+ )+ backends.append(backend)+ return backend++ model = _model()+ model.send_delegation_result = AsyncMock()+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ model,+ delegation_agent_factory=create_backend,+ )+ for task_id in ("first", "second"):+ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id=task_id,+ target="client",+ ),+ )+ while agent._tasks:+ await asyncio.gather(*list(agent._tasks))++ backends[0].state.middle_context["counter"]["history"].append(99)+ backends[1].state.middle_context["counter"]["history"].append(100)+ self.assertEqual(+ {+ "counts": counts,+ "saved": agent.state.middle_context,+ "backend_states": [+ backend.state.middle_context for backend in backends+ ],+ },+ {+ "counts": [1, 2],+ "saved": {"counter": {"count": 2, "history": [1, 2]}},+ "backend_states": [+ {"counter": {"count": 1, "history": [1, 99]}},+ {"counter": {"count": 2, "history": [1, 2, 100]}},+ ],+ },+ )++ async def test_client_delegations_use_latest_serial_state(self) -> None:+ """Queued tasks are created only after the active task syncs state."""+ model = _model()+ model.send_delegation_result = ( # type: ignore[method-assign]+ AsyncMock()+ )+ state = AgentState()+ factory_snapshots: list[list[str]] = []++ def _factory(snapshot: AgentState) -> _StateDelegationAgent:+ factory_snapshots.append(+ list(snapshot.tool_context.activated_groups),+ )+ marker = f"task-{len(factory_snapshots)}"+ return _StateDelegationAgent(snapshot, marker)++ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ delegation_agent_factory=_factory, # type: ignore[arg-type]+ )++ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id="delegation_1",+ target="client",+ ),+ )+ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id="delegation_2",+ target="client",+ ),+ )+ while agent._tasks:+ await asyncio.gather(*list(agent._tasks))++ self.assertDictEqual(+ {+ "factory_snapshots": factory_snapshots,+ "shared_groups": state.tool_context.activated_groups,+ "delegations": agent._delegations,+ "queue": list(agent._delegation_queue),+ "active": agent._active_delegation_id,+ },+ {+ "factory_snapshots": [[], ["task-1"]],+ "shared_groups": ["task-1", "task-2"],+ "delegations": {},+ "queue": [],+ "active": None,+ },+ )++ async def test_client_delegation_resumes_external_execution(self) -> None:+ """External results route back to the parked delegated agent."""+ model = _model()+ backend = _ExternalDelegationAgent()+ model.send_delegation_result = ( # type: ignore[method-assign]+ AsyncMock()+ )+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ model,+ delegation_agent_factory=lambda _state: backend,+ )++ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id="delegation_1",+ target="client",+ ),+ )+ await asyncio.gather(*list(agent._tasks))+ model.send_delegation_result.assert_not_awaited()+ self.assertDictEqual(+ agent._delegated_tool_calls,+ {"external_1": "delegation_1"},+ )+ queued = []+ while not agent._out.empty():+ item = agent._out.get_nowait()+ queued.append(+ (type(item.event).__name__, item.checkpoint is not None),+ )+ self.assertIn(+ ("RequireExternalExecutionEvent", True),+ queued,+ )++ external_result = ExternalExecutionResultEvent(+ reply_id="external_reply",+ execution_results=[+ ToolResultBlock(+ id="external_1",+ name="ask_user",+ output=[TextBlock(text="Order 42")],+ state=ToolResultState.SUCCESS,+ ),+ ],+ )+ await agent._on_control(+ ControlFrame(+ type=ControlFrameType.EXTERNAL_EXECUTION_RESULT,+ data=external_result.model_dump(mode="json"),+ ),+ )+ await asyncio.gather(*list(agent._tasks))++ self.assertEqual(backend.received[-1], external_result)+ model.send_delegation_result.assert_awaited_once_with(+ "delegation_1",+ "Order 42 selected.",+ )+ self.assertDictEqual(agent._delegated_tool_calls, {})++ async def test_parked_delegation_intentionally_holds_fifo(self) -> None:+ """A parked task holds FIFO until its input deadline expires."""+ model = _model()+ first = _ExternalDelegationAgent()+ second = _FakeDelegationAgent()+ created: list[object] = []++ def _factory(state: AgentState) -> object:+ del state+ backend = first if not created else second+ created.append(backend)+ return backend++ model.send_delegation_result = ( # type: ignore[method-assign]+ AsyncMock()+ )+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ model,+ delegation_agent_factory=_factory, # type: ignore[arg-type]+ )++ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id="delegation_1",+ target="client",+ ),+ )+ await asyncio.gather(*list(agent._tasks))+ await agent._on_model_event(+ DelegationCreatedEvent(+ delegation_id="delegation_2",+ target="client",+ ),+ )++ self.assertDictEqual(+ {+ "created": created,+ "active": agent._active_delegation_id,+ "queue": list(agent._delegation_queue),+ },+ {+ "created": [first],+ "active": "delegation_1",+ "queue": ["delegation_2"],+ },+ )++ delegation = agent._delegations["delegation_1"]+ self.assertIsNotNone(delegation.timeout_task)+ agent._cancel_delegation_timeout(delegation)+ with patch(+ "agentscope.agent._realtime._agent."+ "_DELEGATION_INPUT_TIMEOUT_SECS",+ 0,+ ):+ await agent._expire_client_delegation(delegation)+ while agent._tasks:+ await asyncio.gather(*list(agent._tasks))++ self.assertDictEqual(+ {+ "first_received": [+ type(item).__name__ for item in first.received+ ],+ "created": created,+ "active": agent._active_delegation_id,+ "queue": list(agent._delegation_queue),+ "delegations": agent._delegations,+ "expired": delegation.expiring,+ },+ {+ "first_received": ["Msg", "UserInterruptEvent"],+ "created": [first, second],+ "active": None,+ "queue": [],+ "delegations": {},+ "expired": True,+ },+ )+ self.assertListEqual(+ model.send_delegation_result.await_args_list,+ [+ unittest.mock.call(+ "delegation_1",+ "The delegated task was cancelled because the required "+ "user input or external execution was not provided in "+ "time.",+ ),+ unittest.mock.call(+ "delegation_2",+ "Order 42 shipped.",+ ),+ ],+ )++ async def test_unmatched_external_result_is_logged(self) -> None:+ """An external result without a delegation is not silent."""+ agent = RealtimeAgent("Friday", "Be brief.", _model())+ event = ExternalExecutionResultEvent(+ reply_id="external_reply",+ execution_results=[+ ToolResultBlock(+ id="unknown_tool",+ name="ask_user",+ output=[TextBlock(text="Order 42")],+ state=ToolResultState.SUCCESS,+ ),+ ],+ )++ with self.assertLogs("as", level="WARNING") as captured:+ await agent.send(event)++ self.assertListEqual(+ captured.output,+ [+ "WARNING:as:RealtimeAgent: ignored external execution "+ "results without an active delegation: ['unknown_tool']",+ ],+ )++ async def test_live_transcript_samples_exact_snapshots(+ self,+ ) -> None:+ """Only sampled deltas carry their exact persistence boundary."""+ model = _model()+ agent = RealtimeAgent("Friday", "Be brief.", model)+ events = [+ TimelineTranscriptDeltaEvent(+ role="user",+ delta=delta,+ start_ms=start_ms,+ end_ms=end_ms,+ )+ for delta, start_ms, end_ms in [+ ("one", 0, 100),+ ("two", 100, 200),+ ("three", 2_000, 2_100),+ ("four", 2_100, 2_200),+ ]+ ]++ with patch(+ "agentscope.agent._realtime._agent.time.monotonic",+ return_value=1.0,+ ):+ for event in events:+ await agent._on_model_event(event)++ checkpoints = []+ while not agent._out.empty():+ queued = agent._out.get_nowait()+ if isinstance(queued.event, TextBlockDeltaEvent):+ checkpoints.append(+ {+ "delta": queued.event.delta,+ "checkpoint": (+ None+ if queued.checkpoint is None+ else queued.checkpoint.context[+ 0+ ].get_text_content()+ ),+ },+ )++ self.assertListEqual(+ checkpoints,+ [+ {"delta": "one", "checkpoint": "one"},+ {"delta": "two", "checkpoint": None},+ {"delta": "three", "checkpoint": "onetwothree"},+ {"delta": "four", "checkpoint": None},+ ],+ )++ async def test_live_transcript_keeps_a_timestamped_bounded_window(+ self,+ ) -> None:+ """Prompt history stays bounded while retaining timeline metadata."""+ model = _model()+ agent = RealtimeAgent("Friday", "Be brief.", model)+ events = [+ TimelineTranscriptDeltaEvent(+ role="user",+ delta="12345",+ start_ms=10,+ end_ms=20,+ ),+ TimelineTranscriptDeltaEvent(+ role="assistant",+ delta="abcde",+ start_ms=21,+ end_ms=30,+ ),+ TimelineTranscriptDeltaEvent(+ role="assistant",+ delta="fg",+ start_ms=31,+ end_ms=35,+ ),+ ]++ with patch(+ "agentscope.agent._realtime._agent._HISTORY_MAX_TEXT_CHARS",+ 8,+ ):+ for event in events:+ await agent._on_model_event(event)++ self.assertDictEqual(+ {+ "transcript": list(agent._live_transcript),+ "chars": agent._live_transcript_chars,+ },+ {+ "transcript": [+ ("assistant", "abcde", 21, 30),+ ("assistant", "fg", 31, 35),+ ],+ "chars": 7,+ },+ )+++class RealtimeDelegationLifecycleTest(unittest.IsolatedAsyncioTestCase):+ """Check delegation ownership across reconnects and compaction."""++ async def _reconnect(+ self,+ agent: RealtimeAgent,+ socket: _FakeWebSocket,+ ) -> None:+ """Wait for the provider disconnect before opening a new session."""+ await socket.close()++ async def wait_disconnected() -> None:+ while agent._connected:+ await asyncio.sleep(0)++ await asyncio.wait_for(wait_disconnected(), timeout=1)+ await agent.connect()++ async def test_reconnect_retires_parked_and_queued_delegations(+ self,+ ) -> None:+ """Keep completed work, interrupt HITL and discard old queue items."""+ sockets = [_FakeWebSocket(), _FakeWebSocket()]+ executed = []+ backends = []++ def read() -> str:+ """Record a completed read before the permission request."""+ executed.append("read")+ return "contents"++ def write() -> str:+ """Record a write only if the user approves it."""+ executed.append("write")+ return "written"++ def factory(state: AgentState) -> Agent:+ """Park the first task and let the next session finish normally."""+ model = MockModel(context_size=10000)+ responses = [+ ChatResponse(content=[TextBlock(text="Done")], is_last=True),+ ]+ if not backends:+ responses = [+ ChatResponse(+ content=[+ ToolCallBlock(+ id=name,+ name=name,+ input="{}",+ ),+ ],+ is_last=True,+ )+ for name in ("read", "write")+ ]+ model.set_responses(responses)+ backend = Agent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ toolkit=Toolkit(+ tools=[+ FunctionTool(read, is_read_only=True),+ FunctionTool(write),+ ],+ ),+ )+ backends.append(backend)+ return backend++ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ _model(),+ delegation_agent_factory=factory,+ )+ self.addAsyncCleanup(agent.close)+ with patch("websockets.connect", AsyncMock(side_effect=sockets)):+ await agent.connect()+ agent._start_client_delegation("old")+ while agent._tasks:+ await asyncio.gather(*list(agent._tasks))+ old = agent._delegations["old"]+ timeout = old.timeout_task+ agent._start_client_delegation("queued")+ await self._reconnect(agent, sockets[0])+ self.assertEqual(+ {+ "active": agent._active_delegation_id,+ "queue": list(agent._delegation_queue),+ "delegations": agent._delegations,+ "routes": agent._delegated_tool_calls,+ "timeout_done": timeout is not None and timeout.done(),+ "awaiting": backends[0].state.has_awaiting_tool_calls(+ "Friday",+ ),+ "executed": executed,+ "results": [+ (block.id, block.state)+ for message in agent.state.context+ for block in message.get_content_blocks("tool_result")+ ],+ },+ {+ "active": None,+ "queue": [],+ "delegations": {},+ "routes": {},+ "timeout_done": True,+ "awaiting": False,+ "executed": ["read"],+ "results": [+ ("read", ToolResultState.SUCCESS),+ ("write", ToolResultState.INTERRUPTED),+ ],+ },+ )+ # A late callback must not revive a task from the old connection.+ await agent._drive_client_delegation(old, old.prompt)+ agent._start_client_delegation("new")+ while agent._tasks:+ await asyncio.gather(*list(agent._tasks))+ self.assertEqual(+ {+ "backends": len(backends),+ "commentary": [+ (item["delegation_id"], item["content"])+ for item in sockets[1].sent+ if item["type"] == "session.commentary.append"+ ],+ },+ {"backends": 2, "commentary": [("new", "Done")]},+ )++ async def test_reconnect_cancels_running_backend_before_new_session(+ self,+ ) -> None:+ """Do not send an old task's late result on the new connection."""+ started = asyncio.Event()+ cancelled = asyncio.Event()++ class Backend(_FakeDelegationAgent):+ """Keep a task running until reconnect cancels its execution."""++ async def reply_stream(+ self,+ inputs: Msg,+ yield_final_msg: bool,+ ) -> AsyncIterator[Msg]:+ """Return late output after receiving cancellation."""+ self.state.middle_context["completed_operation"] = True+ started.set()+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ cancelled.set()+ async for item in super().reply_stream(+ inputs,+ yield_final_msg,+ ):+ yield item++ backend = Backend()+ sockets = [_FakeWebSocket(), _FakeWebSocket()]+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ _model(),+ delegation_agent_factory=lambda _: backend,+ )+ self.addAsyncCleanup(agent.close)+ with patch("websockets.connect", AsyncMock(side_effect=sockets)):+ await agent.connect()+ agent._start_client_delegation("old")+ await asyncio.wait_for(started.wait(), timeout=1)+ await self._reconnect(agent, sockets[0])+ self.assertEqual(+ {+ "cancelled": cancelled.is_set(),+ "tasks": len(agent._tasks),+ "active": agent._active_delegation_id,+ "state": agent.state.middle_context,+ "commentary": [+ item+ for item in sockets[1].sent+ if item["type"] == "session.commentary.append"+ ],+ },+ {+ "cancelled": True,+ "tasks": 0,+ "active": None,+ "state": {"completed_operation": True},+ "commentary": [],+ },+ )++ async def test_compression_prunes_only_delegated_history(self) -> None:+ """Sync real compression while retaining concurrent Live messages."""+ recent = UserMsg("user", "3" * 40, id="recent")+ parent = RealtimeAgent(+ "Friday",+ "Be brief.",+ _model(),+ state=AgentState(+ summary="old summary",+ context=[+ UserMsg("user", "1" * 120, id="old1"),+ AssistantMsg("Friday", "2" * 40, id="old2"),+ recent,+ ],+ ),+ )+ model = MockModel(context_size=100)+ model.set_structured_response(+ StructuredResponse(+ content={+ "task_overview": "Compressed",+ "current_state": "Done",+ "important_discoveries": "Facts",+ "next_steps": "Continue",+ "context_to_preserve": "Keep this",+ },+ ),+ )+ backend = Agent(+ "Friday",+ "0" * 80,+ model,+ toolkit=Toolkit(),+ state=parent._copy_delegation_state(),+ context_config=ContextConfig(trigger_ratio=0.7, reserve_ratio=0.4),+ )+ delegation = _ClientDelegation(+ delegation_id="task",+ prompt=UserMsg("user", "Task"),+ generation=parent._connection_generation,+ agent=backend,+ baseline_ids={msg.id for msg in backend.state.context},+ )+ backend.state.context.insert(+ 2,+ AssistantMsg(+ "Friday",+ id="tool_reply",+ content=[+ ToolResultBlock(+ id="read",+ name="Read",+ output="contents " * 100,+ state=ToolResultState.SUCCESS,+ ),+ ],+ ),+ )+ parent._sync_delegation_state(delegation)+ self.assertIn("tool_reply", [msg.id for msg in parent.state.context])+ await parent._on_model_event(+ TimelineTranscriptDeltaEvent(+ role="user",+ delta="Concurrent speech",+ start_ms=0,+ end_ms=10,+ ),+ )+ await backend.compress_context()+ self.assertEqual([msg.id for msg in backend.state.context], ["recent"])+ self.assertNotEqual(backend.state.summary, "old summary")+ await parent._on_model_event(+ TimelineTranscriptDeltaEvent(+ role="user",+ delta=" continues",+ start_ms=10,+ end_ms=20,+ ),+ )+ live = parent.state.context[-1].model_dump()+ parent._sync_delegation_state(delegation)+ # Repeated checkpoints must not reintroduce compacted tool messages.+ parent._sync_delegation_state(delegation)+ self.assertEqual(+ {+ "summary": parent.state.summary,+ "messages": [msg.model_dump() for msg in parent.state.context],+ "next_messages": [+ msg.model_dump()+ for msg in parent._copy_delegation_state().context+ ],+ },+ {+ "summary": backend.state.summary,+ "messages": [recent.model_dump(), live],+ "next_messages": [recent.model_dump()],+ },+ )+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/realtime_openai_test.py b/tests/realtime_openai_test.pyindex d589fc1a5..6af825114 100644--- a/tests/realtime_openai_test.py+++ b/tests/realtime_openai_test.py@@ -85,6 +85,7 @@ class OpenAICardsTest(unittest.TestCase):"gpt-realtime-2": "OpenAIRealtimeModel","gpt-realtime-2.1": "OpenAIRealtimeModel","gpt-realtime-2.1-mini": "OpenAIRealtimeModel",+ "gpt-live-1": "OpenAILiveModel",},)diff --git a/tests/realtime_service_test.py b/tests/realtime_service_test.pyindex 2ddbd10a5..c12eac179 100644--- a/tests/realtime_service_test.py+++ b/tests/realtime_service_test.py@@ -1,21 +1,35 @@# -*- coding: utf-8 -*-"""Tests for realtime service runtime assembly."""+# pylint: disable=protected-accessimport unittestfrom types import SimpleNamespacefrom typing import castfrom unittest.mock import AsyncMock, patch+from utils import MockModel++from agentscope.agent import Agentfrom agentscope.app._service._realtime import RealtimeServicefrom agentscope.app.channel import ChatKind+from agentscope.app.message_bus import MessageBusKeys+from agentscope.app.middleware import StateChangeMiddlewarefrom agentscope.app.storage import (AgentData,AgentRecord,+ ChatModelConfig,ChannelOrigin,SessionConfig,+ SessionKnowledgeConfig,SessionRecord,)-from agentscope.realtime import RealtimeModelBase+from agentscope.middleware import RAGMiddleware+from agentscope.message import AssistantMsg, TextBlock, ToolCallBlock, UserMsg+from agentscope.model import ChatResponse+from agentscope.permission import PermissionMode+from agentscope.realtime import InteractionMode, RealtimeModelBase+from agentscope.state import AgentState+from agentscope.tool import FunctionTool, TaskCreate, Toolkitclass RealtimeServiceTest(unittest.IsolatedAsyncioTestCase):@@ -75,7 +89,10 @@ class RealtimeServiceTest(unittest.IsolatedAsyncioTestCase):extra_factory = AsyncMock()templates = {"researcher": object()}toolkit = object()- model = object()+ model = SimpleNamespace(+ interaction_mode=InteractionMode.TURN_BASED,+ card=SimpleNamespace(delegation_backend="none"),+ )service = RealtimeService(storage=storage,workspace_manager=workspace_manager,@@ -220,7 +237,13 @@ class RealtimeServiceTest(unittest.IsolatedAsyncioTestCase):),)toolkit = object()- model = cast(RealtimeModelBase, object())+ model = cast(+ RealtimeModelBase,+ SimpleNamespace(+ interaction_mode=InteractionMode.TURN_BASED,+ card=SimpleNamespace(delegation_backend="none"),+ ),+ )with patch("agentscope.app._service._realtime.get_toolkit",@@ -260,6 +283,418 @@ class RealtimeServiceTest(unittest.IsolatedAsyncioTestCase):channel.chat_kind.assert_awaited_once_with("chat-1")channel.chat_name.assert_not_awaited()+ async def test_gpt_live_rejects_pending_chat_interactions(self) -> None:+ """Check freshly loaded state before allocating runtime resources."""+ record = AgentRecord(+ id="agent",+ user_id="user",+ data=AgentData(name="Friday"),+ )+ session = SessionRecord(+ id="session",+ user_id="user",+ agent_id="agent",+ config=SessionConfig(workspace_id="workspace"),+ )+ workspace_manager = SimpleNamespace(get_workspace=AsyncMock())+ service = RealtimeService(+ storage=SimpleNamespace(+ get_session=AsyncMock(return_value=session),+ ),+ workspace_manager=workspace_manager,+ scheduler_manager=object(),+ background_task_manager=object(),+ message_bus=object(),+ resource_access_service=SimpleNamespace(+ resolve_agent=AsyncMock(return_value=record),+ ),+ )+ model = cast(+ RealtimeModelBase,+ SimpleNamespace(+ card=SimpleNamespace(delegation_backend="session_chat"),+ ),+ )+ for state in ("asking", "submitted"):+ with self.subTest(state=state):+ session.state.context = [+ AssistantMsg(+ name="Friday",+ id=session.state.reply_id,+ content=[+ ToolCallBlock(+ id="old_tool",+ name="Write",+ input="{}",+ state=state,+ ),+ ],+ ),+ UserMsg("user", "Later speech"),+ AssistantMsg("Friday", "Voice response"),+ ]+ original = session.state.model_dump()+ with self.assertRaises(ValueError) as raised:+ await service.create_agent(+ user_id="user",+ agent_id="agent",+ session_id="session",+ model=model,+ )+ self.assertEqual(+ {+ "error": str(raised.exception),+ "state": session.state.model_dump(),+ "workspace_calls": (+ workspace_manager.get_workspace.await_count+ ),+ },+ {+ "error": (+ "Complete the pending Chat confirmation or "+ "external execution before starting GPT Live."+ ),+ "state": original,+ "workspace_calls": 0,+ },+ )++ async def test_gpt_live_reuses_the_session_chat_agent(self) -> None:+ """Client delegation follows the current session chat model."""+ user_id = "user-1"+ agent_id = "agent-1"+ session_id = "session-1"+ agent_record = AgentRecord(+ id=agent_id,+ user_id=user_id,+ data=AgentData(name="Friday", system_prompt="Be concise."),+ )+ chat_config = ChatModelConfig(+ type="openai_chat",+ credential_id="credential-1",+ model="chat-model",+ parameters={},+ )+ session = SessionRecord(+ id=session_id,+ user_id=user_id,+ agent_id=agent_id,+ config=SessionConfig(+ workspace_id="workspace-1",+ chat_model_config=chat_config,+ ),+ )+ session.state.context = [+ UserMsg(+ id="kept-message",+ name="user",+ content="Persisted chat context.",+ ),+ UserMsg(+ id="live:user:current",+ name="user",+ content="Duplicated Live transcript.",+ ),+ ]+ storage = SimpleNamespace(+ get_session=AsyncMock(return_value=session),+ )+ workspace = SimpleNamespace(workdir="/workspace/session-1")+ middleware = SimpleNamespace(+ is_implemented=lambda name: name == "on_reply",+ )+ middleware_calls: list[tuple] = []++ async def _extra_middlewares(*args: object) -> list[object]:+ middleware_calls.append(args)+ return [middleware]++ service = RealtimeService(+ storage=storage,+ workspace_manager=SimpleNamespace(+ get_workspace=AsyncMock(return_value=workspace),+ ),+ scheduler_manager=object(),+ background_task_manager=object(),+ message_bus=object(),+ resource_access_service=SimpleNamespace(+ resolve_agent=AsyncMock(return_value=agent_record),+ ),+ extra_agent_middlewares=(+ _extra_middlewares # type: ignore[arg-type]+ ),+ )+ realtime_model = cast(+ RealtimeModelBase,+ SimpleNamespace(+ interaction_mode=InteractionMode.FULL_DUPLEX,+ card=SimpleNamespace(+ delegation_backend="session_chat",+ ),+ ),+ )+ toolkit = object()+ backend_model = object()++ with (+ patch(+ "agentscope.app._service._realtime.get_toolkit",+ new=AsyncMock(return_value=toolkit),+ ) as get_toolkit_mock,+ patch(+ "agentscope.app._service._realtime.get_model",+ new=AsyncMock(return_value=backend_model),+ ) as get_model_mock,+ ):+ realtime_agent = await service.create_agent(+ user_id=user_id,+ agent_id=agent_id,+ session_id=session_id,+ model=realtime_model,+ )++ factory = realtime_agent._delegation_agent_factory+ self.assertIsNotNone(factory)+ assert factory is not None+ delegated_state = realtime_agent._copy_delegation_state()+ delegated_agent = factory(delegated_state)+ notifier = delegated_agent._reply_middlewares[0]+ self.assertIsInstance(notifier, StateChangeMiddleware)+ self.assertDictEqual(+ {+ "model": delegated_agent.model,+ "toolkit": delegated_agent.toolkit,+ "system_prompt": delegated_agent._system_prompt,+ "state_texts": [+ message.get_text_content()+ for message in delegated_agent.state.context+ ],+ "state_isolated": delegated_agent.state is not session.state,+ "message_isolated": (+ delegated_agent.state.context[0]+ is not session.state.context[0]+ ),+ "reply_middlewares": delegated_agent._reply_middlewares,+ "middleware_calls": middleware_calls,+ },+ {+ "model": backend_model,+ "toolkit": toolkit,+ "system_prompt": (+ "Be concise.\n\n<system-notification>You're within a "+ "session (id=session-1).</system-notification>"+ ),+ "state_texts": [+ "Persisted chat context.",+ "Duplicated Live transcript.",+ ],+ "state_isolated": True,+ "message_isolated": True,+ "reply_middlewares": [notifier, middleware],+ "middleware_calls": [+ (user_id, agent_id, session_id, workspace),+ ],+ },+ )+ self.assertListEqual(+ get_toolkit_mock.await_args.kwargs["middlewares"],+ [notifier, middleware],+ )+ get_model_mock.assert_awaited_once_with(+ user_id,+ chat_config,+ service._access,+ )++ async def test_delegation_publishes_state_and_team_changes(self) -> None:+ """Assembled middleware publishes tool changes to the session bus."""+ bus = SimpleNamespace(+ log_append=AsyncMock(return_value="1-0"),+ publish=AsyncMock(),+ )+ service = RealtimeService(+ storage=object(),+ workspace_manager=object(),+ scheduler_manager=object(),+ background_task_manager=object(),+ message_bus=bus,+ resource_access_service=object(),+ )+ session = SessionRecord(+ id="session",+ user_id="user",+ agent_id="agent",+ config=SessionConfig(workspace_id="workspace"),+ )+ middlewares = await service._build_delegation_middlewares(+ user_id="user",+ agent_id="agent",+ session_id=session.id,+ session=session,+ workspace=object(),+ )+ state = AgentState()+ initial_permission = state.permission_context.model_dump(mode="json")++ def update_permission() -> str:+ """Simulate a tool changing session permission state."""+ state.permission_context.mode = PermissionMode.ACCEPT_EDITS+ return "Updated"++ def create_team() -> str:+ """Simulate a successful team tool without external storage."""+ return "Created"++ model = MockModel(context_size=10000)+ model.set_responses(+ [+ ChatResponse(+ content=[+ ToolCallBlock(+ id="task",+ name="TaskCreate",+ input=(+ '{"subject":"Test",'+ '"description":"Notify UI"}'+ ),+ ),+ ],+ is_last=True,+ ),+ ChatResponse(+ content=[+ ToolCallBlock(+ id="permission",+ name="update_permission",+ input="{}",+ ),+ ],+ is_last=True,+ ),+ ChatResponse(+ content=[+ ToolCallBlock(+ id="team",+ name="TeamCreate",+ input="{}",+ ),+ ],+ is_last=True,+ ),+ ChatResponse(content=[TextBlock(text="Done")], is_last=True),+ ],+ )+ agent = Agent(+ "Friday",+ "Be brief.",+ model,+ state=state,+ middlewares=middlewares,+ toolkit=Toolkit(+ tools=[+ TaskCreate(),+ FunctionTool(update_permission, is_read_only=True),+ FunctionTool(+ create_team,+ name="TeamCreate",+ is_read_only=True,+ ),+ ],+ ),+ )+ async for _ in agent.reply_stream(+ UserMsg(name="user", content="Create a task and team."),+ ):+ pass++ final_state = {+ "tasks_context": state.tasks_context.model_dump(mode="json"),+ "permission_context": state.permission_context.model_dump(+ mode="json",+ ),+ }+ key = MessageBusKeys.session_events(session.id)+ self.assertEqual(+ {+ "task_count": len(state.tasks_context.tasks),+ "notifications": [+ (call.args[0], call.args[1]["name"], call.args[1]["value"])+ for call in bus.publish.call_args_list+ ],+ },+ {+ "task_count": 1,+ "notifications": [+ (+ key,+ "state_updated",+ {+ **final_state,+ "permission_context": initial_permission,+ },+ ),+ (key, "state_updated", final_state),+ (key, "team_updated", {}),+ (key, "state_updated", final_state),+ ],+ },+ )++ async def test_gpt_live_builds_knowledge_middleware(self) -> None:+ """Attach configured session knowledge to delegated Chat agents."""+ knowledge = object()+ manager = SimpleNamespace(+ get_knowledge=AsyncMock(return_value=knowledge),+ )+ access = SimpleNamespace(+ resolve_knowledge_base=AsyncMock(+ return_value=SimpleNamespace(user_id="owner-1"),+ ),+ )+ service = RealtimeService(+ storage=object(), # type: ignore[arg-type]+ workspace_manager=object(), # type: ignore[arg-type]+ scheduler_manager=object(), # type: ignore[arg-type]+ background_task_manager=object(), # type: ignore[arg-type]+ message_bus=object(), # type: ignore[arg-type]+ resource_access_service=access, # type: ignore[arg-type]+ knowledge_base_manager=manager, # type: ignore[arg-type]+ )+ session = SessionRecord(+ id="session-1",+ user_id="user-1",+ agent_id="agent-1",+ config=SessionConfig(+ workspace_id="workspace-1",+ knowledge_config=SessionKnowledgeConfig(+ knowledge_base_ids=["knowledge-1"],+ parameters={"top_k": 3},+ ),+ ),+ )++ middlewares = await service._build_delegation_middlewares(+ user_id="user-1",+ agent_id="agent-1",+ session_id="session-1",+ session=session,+ workspace=object(),+ )++ self.assertEqual(len(middlewares), 2)+ self.assertIsInstance(middlewares[0], StateChangeMiddleware)+ self.assertIsInstance(middlewares[1], RAGMiddleware)+ self.assertListEqual(middlewares[1]._knowledge_bases, [knowledge])+ self.assertEqual(middlewares[1]._parameters.top_k, 3)+ access.resolve_knowledge_base.assert_awaited_once_with(+ "user-1",+ "knowledge-1",+ )+ manager.get_knowledge.assert_awaited_once_with(+ "owner-1",+ "knowledge-1",+ )+if __name__ == "__main__":unittest.main()diff --git a/tests/service_team_failure_report_test.py b/tests/service_team_failure_report_test.pyindex 7c63d5446..571c584e9 100644--- a/tests/service_team_failure_report_test.py+++ b/tests/service_team_failure_report_test.py@@ -456,6 +456,7 @@ class TeamFailureReportTest(IsolatedAsyncioTestCase):self.state.context.append(AssistantMsg(name=self.name,+ id=self.state.reply_id,content=[ToolCallBlock(id="call-1",diff --git a/tests/service_wakeup_dispatcher_test.py b/tests/service_wakeup_dispatcher_test.pyindex 0c83bed76..a6cbcc6e6 100644--- a/tests/service_wakeup_dispatcher_test.py+++ b/tests/service_wakeup_dispatcher_test.py@@ -90,7 +90,10 @@ class _FakeStorage:],),]- return SimpleNamespace(state=AgentState(context=context))+ state = AgentState(context=context)+ if context:+ state.reply_id = context[0].id+ return SimpleNamespace(state=state)async def get_agent(self, _user_id: str, _agent_id: str) -> object:"""Return the agent name used by canonical HITL detection."""diff --git a/tests/webrtc_audio_transport_test.py b/tests/webrtc_audio_transport_test.pyindex de57699da..62121a6c0 100644--- a/tests/webrtc_audio_transport_test.py+++ b/tests/webrtc_audio_transport_test.py@@ -5,7 +5,7 @@import asyncioimport jsonimport unittest-from collections.abc import Callable+from collections.abc import Callable, Iteratorfrom contextlib import asynccontextmanagerfrom fractions import Fractionfrom itertools import count@@ -34,6 +34,7 @@ from agentscope.app._service._webrtc_session import WebRTCSessionfrom agentscope.app.message_bus import InMemoryMessageBus, MessageBusKeysfrom agentscope.app._bus_ops import enqueue_chat_inputfrom agentscope.app.storage import CredentialRecord, RealtimeModelConfig+from agentscope.credential import OpenAICredentialfrom agentscope.event import (DataBlockDeltaEvent,DataBlockEndEvent,@@ -46,15 +47,23 @@ from agentscope.event import (TextBlockStartEvent,ToolResultEndEvent,)-from agentscope.message import ToolCallBlock, ToolResultState, UserMsg+from agentscope.message import (+ AssistantMsg,+ ToolCallBlock,+ ToolResultState,+ UserMsg,+)from agentscope.realtime import (AudioFrame,DashScopeAudioRealtimeModel,GeminiRealtimeModel,+ OpenAILiveModel,RealtimeModelBase,RealtimeModelCard,+ TimelineTranscriptDeltaEvent,)from agentscope.realtime import _events as me+from agentscope.state import AgentStatefrom agentscope.tool import FunctionTool, Toolkit@@ -154,6 +163,10 @@ class _FakeAgent:for event in self.events:yield event+ def drain_events(self) -> Iterator[Any]:+ """No additional cleanup events for the default fixture."""+ return iter(())+def checkpoint_snapshot(self, event: Any) -> object | None:"""Return state for the same boundaries as the real agent."""if isinstance(@@ -168,6 +181,44 @@ class _FakeAgent:return None+class _FailingClosingAgent(_FakeAgent):+ """Fail after starting a reply and enqueue terminal events on exit."""++ def __init__(self) -> None:+ super().__init__({"id": "reply-1"}, [])+ self.failure = RuntimeError("stream failed")+ self.started = ReplyStartEvent(+ session_id="session-1",+ reply_id="reply-1",+ name="Friday",+ )+ self.terminal = [+ ToolResultEndEvent(+ reply_id="reply-1",+ tool_call_id="tool-1",+ state=ToolResultState.INTERRUPTED,+ ),+ ReplyEndEvent(session_id="session-1", reply_id="reply-1"),+ ]++ async def __aexit__(self, *exc: object) -> None:+ self.events.extend(self.terminal)++ async def reply_stream(+ self,+ transport: object,+ ) -> AsyncIterator[Any]:+ """Start the reply before simulating a provider stream error."""+ del transport+ yield self.started+ raise self.failure++ def drain_events(self) -> Iterator[Any]:+ """Consume the events produced by context-manager cleanup."""+ events, self.events = self.events, []+ return iter(events)++class _UnacknowledgedSetupSocket:"""Keep a real Gemini connect waiting for its setup acknowledgement."""@@ -922,6 +973,59 @@ class RealtimeOfferTest(unittest.IsolatedAsyncioTestCase):},)+ async def test_gpt_live_offer_rejects_pending_chat(self) -> None:+ """Reject parked Chat interactions before WebRTC negotiation."""+ self.model.card = SimpleNamespace(delegation_backend="session_chat")+ self.session.config.chat_model_config = object()+ self.access.resolve_agent.return_value = SimpleNamespace(+ data=SimpleNamespace(name="Friday"),+ )+ for state in ("asking", "submitted"):+ with self.subTest(state=state):+ self.session.state = AgentState(+ context=[+ AssistantMsg(+ name="Friday",+ content=[+ ToolCallBlock(+ id="old_tool",+ name="Write",+ input="{}",+ state=state,+ ),+ ],+ ),+ ],+ )+ self.session.state.reply_id = self.session.state.context[0].id+ self.session.state.context.extend(+ [+ UserMsg("user", "Later speech"),+ AssistantMsg("Friday", "Voice response"),+ ],+ )+ with self.assertRaises(HTTPException) as raised:+ await self._offer()+ self.assertEqual(+ {+ "status": raised.exception.status_code,+ "detail": raised.exception.detail,+ "negotiation_calls": (+ self.peer.setRemoteDescription.await_count+ ),+ "connections": self.connections,+ },+ {+ "status": 409,+ "detail": (+ "Complete the pending Chat confirmation or "+ "external execution before starting GPT Live."+ ),+ "negotiation_calls": 0,+ "connections": {},+ },+ )+async def test_offer_replaces_the_previous_connection(self) -> None:"""A reconnect closes the previous local runner before reload."""previous = SimpleNamespace(close=AsyncMock())@@ -995,9 +1099,100 @@ class RealtimeOfferTest(unittest.IsolatedAsyncioTestCase):self.peer.close.assert_awaited_once_with()+# pylint: disable-next=too-many-public-methodsclass WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):"""Verify that a WebRTC run keeps the normal session contract."""+ async def test_sampled_live_checkpoint_persists_immediately(self) -> None:+ """Persist a sampled transcript snapshot at its published cursor."""++ class _LiveFakeAgent(_FakeAgent):+ allows_open_user_reply_checkpoint = True++ def checkpoint_snapshot(self, event: Any) -> object | None:+ if isinstance(event, TextBlockDeltaEvent):+ return self.state+ return super().checkpoint_snapshot(event)++ events = [+ ReplyStartEvent(+ session_id="session-1",+ reply_id="user-1",+ name="user",+ role="user",+ ),+ TextBlockDeltaEvent(+ reply_id="user-1",+ block_id="text-1",+ delta="hello",+ ),+ ]+ agent = _LiveFakeAgent(+ {"id": "user-1", "role": "user"},+ events,+ )+ message_bus = _FakeMessageBus(sequence_entries=True)+ storage = _FakeStorage()+ closed = asyncio.Event()+ session = _make_session(+ agent_factory=AsyncMock(return_value=agent),+ storage=storage,+ message_bus=message_bus,+ on_closed=lambda _: closed.set(),+ )++ session.start()+ await asyncio.wait_for(closed.wait(), timeout=1)++ self.assertDictEqual(+ {+ "storage_methods": [item["method"] for item in storage.calls],+ "cursor_values": [+ item["value"]+ for item in message_bus.calls+ if item["method"] == "registry_set"+ ],+ },+ {+ "storage_methods": [+ "update_session_state",+ "update_session_state",+ ],+ "cursor_values": ["2-0"],+ },+ )++ async def test_older_checkpoint_cannot_regress_cursor(self) -> None:+ """A delayed flush cannot overwrite a newer checkpoint cursor."""+ message_bus = _FakeMessageBus()+ session = _make_session(+ agent_factory=AsyncMock(),+ storage=_FakeStorage(),+ message_bus=message_bus,+ on_closed=lambda _: None,+ )+ session._checkpoint_entry_id = "2-0"+ session._persist_state = AsyncMock() # type: ignore[method-assign]++ await session._checkpoint_state("1-0", cast(Any, object()))++ self.assertDictEqual(+ {+ "persist_calls": session._persist_state.await_args_list,+ "checkpoint_entry_id": session._checkpoint_entry_id,+ "registry_calls": [+ item+ for item in message_bus.calls+ if item["method"] == "registry_set"+ ],+ },+ {+ "persist_calls": [],+ "checkpoint_entry_id": "2-0",+ "registry_calls": [],+ },+ )+async def test_transcript_between_responses_preserves_full_reply(self,) -> None:@@ -1536,6 +1731,193 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):},)+ async def test_pruned_live_segments_remain_in_message_history(+ self,+ ) -> None:+ """Checkpoint pruning bounds state without erasing voice history."""+ agent = RealtimeAgent(+ "Friday",+ "Be brief.",+ OpenAILiveModel(+ "gpt-live-1",+ OpenAICredential(api_key="fake"),+ ),+ )+ storage = _FakeStorage()+ session = _make_session(+ agent_factory=AsyncMock(),+ storage=storage,+ message_bus=_FakeMessageBus(),+ on_closed=lambda _: None,+ )+ session.agent = agent+ for index in range(50):+ for role in ("user", "assistant"):+ await agent._on_model_event(+ TimelineTranscriptDeltaEvent(+ role=role,+ delta=role[0] * 1_000,+ start_ms=index * 2_100,+ end_ms=index * 2_100 + 100,+ ),+ )+ for event in agent.drain_events():+ snapshot = agent.checkpoint_snapshot(event)+ if snapshot is not None:+ await session._persist_state(snapshot)+ await agent.close()+ for event in agent.drain_events():+ snapshot = agent.checkpoint_snapshot(event)+ if snapshot is not None:+ await session._persist_state(snapshot)+ await session._persist_state()+ self.assertEqual(+ {+ "history": {+ role: "".join(+ msg.get_text_content() or ""+ for msg in storage.messages.values()+ if msg.role == role+ )+ for role in ("user", "assistant")+ },+ "deletions": [+ call+ for call in storage.calls+ if call["method"] == "delete_message"+ ],+ "cached_ids": set(session._persisted_messages),+ },+ {+ "history": {+ "user": "u" * 50_000,+ "assistant": "a" * 50_000,+ },+ "deletions": [],+ "cached_ids": {msg.id for msg in agent.state.context},+ },+ )+ self.assertLessEqual(+ sum(+ len(msg.get_text_content() or "")+ for msg in agent.state.context+ ),+ 32_000,+ )+ self.assertLess(len(agent.state.context), len(storage.messages))++ async def test_close_events_are_published_before_final_persistence(+ self,+ ) -> None:+ """Events emitted by Agent exit are not stranded in its queue."""+ ended = ReplyEndEvent(+ session_id="session-1",+ reply_id="reply-1",+ )++ class ClosingAgent(_FakeAgent):+ """Emit the final reply only after the runtime closes."""++ def drain_events(self) -> Iterator[Any]:+ return iter([ended])++ agent = ClosingAgent({"id": "reply-1"}, [])+ bus = _FakeMessageBus()+ storage = _FakeStorage()+ session = _make_session(+ agent_factory=AsyncMock(return_value=agent),+ storage=storage,+ message_bus=bus,+ on_closed=lambda _: None,+ )+ await session._run()+ self.assertEqual(session._last_entry_id, "1-0")+ self.assertEqual(session._checkpoint_entry_id, "1-0")+ self.assertEqual(session.transport.errors, [])++ async def test_failed_stream_publishes_close_events(self) -> None:+ """Publish terminal events and their checkpoint before reporting."""+ agent = _FailingClosingAgent()+ bus = _FakeMessageBus(sequence_entries=True)+ storage = _FakeStorage()+ session = _make_session(+ agent_factory=AsyncMock(return_value=agent),+ storage=storage,+ message_bus=bus,+ on_closed=lambda _: None,+ )+ with self.assertLogs("as", level="ERROR"):+ await session._run()+ self.assertEqual(+ {+ "published": [+ call["event"]+ for call in bus.calls+ if call["method"] == "publish"+ ],+ "cursor": session._checkpoint_entry_id,+ "errors": session.transport.errors,+ "transport_closed": session.transport.closed,+ "peer_closed": session.peer_connection.closed,+ "remaining": list(agent.drain_events()),+ },+ {+ "published": [+ {+ **event.model_dump(mode="json"),+ "_entry_id": f"{index}-0",+ }+ for index, event in enumerate(+ [agent.started, *agent.terminal],+ 1,+ )+ ],+ "cursor": "3-0",+ "errors": ["stream failed"],+ "transport_closed": True,+ "peer_closed": True,+ "remaining": [],+ },+ )++ async def test_failed_stream_preserves_original_exception(self) -> None:+ """Drain closing events before propagating the same exception."""+ agent = _FailingClosingAgent()+ session = _make_session(+ agent_factory=AsyncMock(return_value=agent),+ storage=_FakeStorage(),+ message_bus=_FakeMessageBus(),+ on_closed=lambda _: None,+ )+ session.agent = agent+ events = []+ with self.assertRaises(RuntimeError) as raised:+ async for event in session._agent_events():+ events.append(event)+ self.assertIs(raised.exception, agent.failure)+ self.assertEqual(events, [agent.started, *agent.terminal])++ async def test_event_generator_close_does_not_yield(self) -> None:+ """Explicit generator close and cancellation must not yield."""+ for cancel in (False, True):+ with self.subTest(cancel=cancel):+ agent = _FailingClosingAgent()+ session = _make_session(+ agent_factory=AsyncMock(return_value=agent),+ storage=_FakeStorage(),+ message_bus=_FakeMessageBus(),+ on_closed=lambda _: None,+ )+ session.agent = agent+ events = session._agent_events()+ self.assertEqual(await anext(events), agent.started)+ if cancel:+ with self.assertRaises(asyncio.CancelledError):+ await events.athrow(asyncio.CancelledError())+ else:+ await events.aclose()+ self.assertEqual(list(agent.drain_events()), agent.terminal)+async def test_pcm_events_are_not_published_to_session_events(self,) -> None:@@ -1994,6 +2376,7 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):agent = AsyncMock()agent.state = SimpleNamespace(context=[])agent.__aenter__.side_effect = RuntimeError("Setup failed.")+ agent.drain_events = MagicMock(return_value=iter(()))storage = BlockingStorage(message_bus)session = _make_session(agent_factory=AsyncMock(return_value=agent),@@ -2026,6 +2409,7 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):self.assertEqual(message_bus.held_locks, set())self.assertEqual(closed_sessions, [session])agent.close.assert_awaited_once_with()+ self.assertEqual(session.transport.errors, ["Setup failed."])finally:release_write.set()if session._task is not None:@@ -2066,7 +2450,7 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):transport = _FakeTransport()peer_connection = _FakePeerConnection()storage = _FakeStorage()- message_bus = _FakeMessageBus()+ message_bus = _FakeMessageBus(sequence_entries=True)closed = asyncio.Event()async def _create_agent() -> _FakeAgent:@@ -2100,7 +2484,7 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):{"storage_methods": ["update_session_state"] * 3,"published_event_types": [event.type for event in events],- "cursor_values": ["1-0"] * 2,+ "cursor_values": ["2-0", "3-0"],},)