Skip to content

Build and run agents you can see, understand and trust.

feat(realtime): add GPT Live Voice Integration with Chat Agent Delegation (#3272)

qbc692066f370b1
diff --git a/examples/realtime/README.md b/examples/realtime/README.md
index eac9be9f8..2a973f4f8 100644
--- a/examples/realtime/README.md
+++ b/examples/realtime/README.md
@@ -40,6 +40,22 @@ python examples/realtime/local_mic.py
Speak, hear the reply, and speak over it to interrupt. `Ctrl-Q` quits.
+### GPT-Live prototype
+
+The GPT-Live integration uses a server-side WebSocket and client delegation.
+The local example builds a chat agent from the selected backend model; the Web
+UI automatically reuses the session's current Chat model and tools:
+
+```bash
+export OPENAI_API_KEY=sk-...
+export OPENAI_LIVE_BACKEND_MODEL=your-responses-model
+python examples/realtime/openai_live_local.py
+```
+
+This integration supports continuous full-duplex audio, independent user and
+assistant captions, and application-managed task execution through the normal
+AgentScope chat agent.
+
The default model is `qwen-audio-3.0-realtime-plus`. Any card listed for the
credential works; the class is looked up from the card, the same way the
service layer does it:
diff --git a/examples/realtime/openai_live_local.py b/examples/realtime/openai_live_local.py
new file mode 100644
index 000000000..2fce7ad1a
--- /dev/null
+++ b/examples/realtime/openai_live_local.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""Talk to GPT-Live through the local microphone.
+
+Set ``OPENAI_API_KEY`` and ``OPENAI_LIVE_BACKEND_MODEL`` before running.
+Use headphones because the local transport does not cancel acoustic echo.
+"""
+import asyncio
+import os
+
+from agentscope.agent import Agent, RealtimeAgent
+from agentscope.credential import OpenAICredential
+from agentscope.model import OpenAIChatModel
+from agentscope.realtime import LocalAudioTransport, OpenAILiveModel
+from agentscope.state import AgentState
+from agentscope.tui import launch_realtime_ui
+
+
+async def main() -> None:
+ """Run one local GPT-Live voice session."""
+ api_key = os.environ.get("OPENAI_API_KEY")
+ backend_model = os.environ.get("OPENAI_LIVE_BACKEND_MODEL")
+ if not api_key:
+ raise SystemExit("Set OPENAI_API_KEY first.")
+ if not backend_model:
+ raise SystemExit("Set OPENAI_LIVE_BACKEND_MODEL first.")
+
+ credential = OpenAICredential(api_key=api_key)
+ live_model = OpenAILiveModel(
+ "gpt-live-1",
+ credential,
+ )
+ backend = OpenAIChatModel(credential=credential, model=backend_model)
+
+ def _create_backend_agent(state: AgentState) -> Agent:
+ """Build the task agent used for one client delegation."""
+ return Agent(
+ name="Friday",
+ system_prompt=(
+ "Answer accurately and keep task results concise for speech."
+ ),
+ model=backend,
+ state=state,
+ )
+
+ agent = RealtimeAgent(
+ name="Friday",
+ system_prompt=(
+ "Speak briefly. Listen while the user is talking. "
+ "Delegate questions that need deeper reasoning."
+ ),
+ model=live_model,
+ delegation_agent_factory=_create_backend_agent,
+ )
+ transport = LocalAudioTransport(
+ input_sample_rate=live_model.input_sample_rate,
+ output_sample_rate=live_model.output_sample_rate,
+ input_device=_device("REALTIME_INPUT_DEVICE"),
+ output_device=_device("REALTIME_OUTPUT_DEVICE"),
+ )
+
+ async with agent, transport:
+ await launch_realtime_ui(agent, transport)
+
+
+def _device(env: str) -> int | str | None:
+ """Return a sounddevice index or name from an environment variable."""
+ value = os.environ.get(env)
+ if value is None:
+ return None
+ return int(value) if value.isdigit() else value
+
+
+if __name__ == "__main__":
+ try:
+ asyncio.run(main())
+ except KeyboardInterrupt:
+ print("\nbye")
diff --git a/examples/web_ui/frontend/src/api/types.ts b/examples/web_ui/frontend/src/api/types.ts
index 4774350e8..48453b255 100644
--- a/examples/web_ui/frontend/src/api/types.ts
+++ b/examples/web_ui/frontend/src/api/types.ts
@@ -888,6 +888,8 @@ export interface RealtimeModelCard {
input_sample_rate: number;
output_sample_rate: number;
supports_tools: boolean;
+ interaction_mode: 'turn_based' | 'full_duplex';
+ delegation_backend: 'none' | 'session_chat';
max_context_tokens: number | null;
max_audio_turns: number | null;
max_audio_duration_s: number | null;
diff --git a/examples/web_ui/frontend/src/components/chat/ChatContent.tsx b/examples/web_ui/frontend/src/components/chat/ChatContent.tsx
index 9cd39663b..775cfde84 100644
--- a/examples/web_ui/frontend/src/components/chat/ChatContent.tsx
+++ b/examples/web_ui/frontend/src/components/chat/ChatContent.tsx
@@ -1,7 +1,9 @@
-import { ReplyFinishedReason } from '@agentscope-ai/agentscope/event';
+import {
+ type ExternalExecutionResultEvent,
+ ReplyFinishedReason,
+} from '@agentscope-ai/agentscope/event';
import {
type ContentBlock,
- getContentBlocks,
type Msg,
type TextBlock,
type ToolCallBlock,
@@ -13,6 +15,7 @@ import type { GitStatus } from '@/api';
import type { ChatQueueItem } from '@/api/chat';
import { ASMessageBubble, SteeringMessageBubble } from '@/components/chat/ASMessageBubble.tsx';
import { ConfirmCard } from '@/components/chat/ConfirmCard.tsx';
+import { ExternalExecutionCard } from '@/components/chat/ExternalExecutionCard.tsx';
import { FlipCard } from '@/components/chat/FlipCard.tsx';
import { QueuedMessages } from '@/components/chat/QueuedMessages.tsx';
import { TextInput } from '@/components/chat/TextInput.tsx';
@@ -34,6 +37,7 @@ import type { ReplyPhase } from '@/hooks/useMessages';
import { useTranslation } from '@/i18n/useI18n';
import type { RealtimeASRConfig } from '@/lib/browserRealtimeASRTransport';
import type { RealtimeConnectionState } from '@/lib/browserWebRTCTransport';
+import { getPendingHitlCalls } from '@/lib/pendingToolCalls';
import { cn } from '@/lib/utils';
/** How long a load may run before it is worth showing a spinner. */
@@ -90,6 +94,7 @@ interface ChatContentProps {
replyId: string,
rules?: ToolCallBlock['suggested_rules'],
) => Promise<void>;
+ onExternalExecutionResult: (event: ExternalExecutionResultEvent) => Promise<void>;
autoComplete?: (input: string) => string | null;
className?: string;
/** Called when the user clicks the stop button. */
@@ -151,6 +156,7 @@ const ChatContentComponent: React.FC<ChatContentProps> = ({
disabled,
onSend,
onUserConfirm,
+ onExternalExecutionResult,
autoComplete,
className,
onInterrupt,
@@ -217,14 +223,9 @@ const ChatContentComponent: React.FC<ChatContentProps> = ({
return () => clearTimeout(timer);
}, [loading]);
- const toConfirmedToolCalls = useMemo(() => {
- if (msgs.length === 0) return [];
-
- const lastMsg = msgs[msgs.length - 1];
- return getContentBlocks(lastMsg, 'tool_call')
- .filter((tc) => tc.state === 'asking')
- .map((tc) => ({ replyId: lastMsg.id, toolCall: tc }));
- }, [msgs]);
+ const pendingHitlCalls = useMemo(() => getPendingHitlCalls(msgs), [msgs]);
+ const toConfirmedToolCalls = pendingHitlCalls.filter((call) => call.kind === 'confirmation');
+ const externalToolCalls = pendingHitlCalls.filter((call) => call.kind === 'external_execution');
// On an empty session the prompt and the input centre together, so every box
// down to the message list shrinks to its content instead of filling.
@@ -342,7 +343,11 @@ const ChatContentComponent: React.FC<ChatContentProps> = ({
{loading ? null : (
<div className="relative min-w-full max-w-full w-full">
<FlipCard
- visible={toConfirmedToolCalls.length > 0 || footerSlot !== null}
+ visible={
+ toConfirmedToolCalls.length > 0 ||
+ externalToolCalls.length > 0 ||
+ footerSlot !== null
+ }
className="absolute bottom-full left-0 right-0 mb-2 z-50"
>
{toConfirmedToolCalls.length > 0 ? (
@@ -358,6 +363,13 @@ const ChatContentComponent: React.FC<ChatContentProps> = ({
)
}
/>
+ ) : externalToolCalls.length > 0 ? (
+ <ExternalExecutionCard
+ key={`${externalToolCalls[0].replyId}:${externalToolCalls[0].toolCall.id}`}
+ replyId={externalToolCalls[0].replyId}
+ toolCall={externalToolCalls[0].toolCall}
+ onResult={onExternalExecutionResult}
+ />
) : (
footerSlot
)}
diff --git a/examples/web_ui/frontend/src/components/chat/ExternalExecutionCard.tsx b/examples/web_ui/frontend/src/components/chat/ExternalExecutionCard.tsx
new file mode 100644
index 000000000..a1dc33d4d
--- /dev/null
+++ b/examples/web_ui/frontend/src/components/chat/ExternalExecutionCard.tsx
@@ -0,0 +1,283 @@
+import { EventType, type ExternalExecutionResultEvent } from '@agentscope-ai/agentscope/event';
+import type { ToolCallBlock } from '@agentscope-ai/agentscope/message';
+import { Check, CircleHelp, PlugZap, Send } from 'lucide-react';
+import { useMemo, useRef, useState } from 'react';
+
+import { Button } from '@/components/ui/button';
+import { Input } from '@/components/ui/input';
+import { Spinner } from '@/components/ui/spinner';
+import { useTranslation } from '@/i18n/useI18n';
+import { cn } from '@/lib/utils';
+
+interface AskUserOption {
+ label: string;
+ description: string;
+ preview?: string | null;
+}
+
+interface AskUserQuestion {
+ question: string;
+ header: string;
+ context?: string | null;
+ options: AskUserOption[];
+ multi_select?: boolean;
+}
+
+interface AskUserParams {
+ questions: AskUserQuestion[];
+}
+
+interface AnswerDraft {
+ selected: string[];
+ other: string;
+ usesOther: boolean;
+}
+
+function parseAskUserParams(input: string): AskUserParams | null {
+ try {
+ const value = JSON.parse(input) as Partial<AskUserParams>;
+ if (!Array.isArray(value.questions) || value.questions.length === 0) return null;
+ if (
+ !value.questions.every(
+ (question) =>
+ typeof question?.question === 'string' &&
+ typeof question.header === 'string' &&
+ Array.isArray(question.options) &&
+ question.options.length > 0 &&
+ question.options.every(
+ (option) =>
+ typeof option?.label === 'string' &&
+ typeof option.description === 'string',
+ ),
+ )
+ ) {
+ return null;
+ }
+ return value as AskUserParams;
+ } catch {
+ return null;
+ }
+}
+
+function AskUserCard({
+ replyId,
+ toolCall,
+ onResult,
+}: {
+ replyId: string;
+ toolCall: ToolCallBlock;
+ onResult: (event: ExternalExecutionResultEvent) => Promise<void>;
+}) {
+ const { t } = useTranslation();
+ const params = useMemo(() => parseAskUserParams(toolCall.input), [toolCall.input]);
+ const [answers, setAnswers] = useState<AnswerDraft[]>(() =>
+ (params?.questions ?? []).map(() => ({ selected: [], other: '', usesOther: false })),
+ );
+ const [submitting, setSubmitting] = useState(false);
+ const submittingRef = useRef(false);
+
+ if (!params) {
+ return (
+ <div className="rounded-[28px] bg-muted p-5 ring ring-border">
+ <div className="flex items-center gap-2 text-sm font-medium">
+ <CircleHelp className="size-4" />
+ {t('chat.askUser.invalid')}
+ </div>
+ </div>
+ );
+ }
+
+ const updateAnswer = (index: number, update: (answer: AnswerDraft) => AnswerDraft) => {
+ setAnswers((current) =>
+ current.map((answer, answerIndex) => (answerIndex === index ? update(answer) : answer)),
+ );
+ };
+
+ const complete = answers.every(
+ (answer) => answer.selected.length > 0 || answer.other.trim().length > 0,
+ );
+
+ const submit = async () => {
+ if (!complete || submittingRef.current) return;
+ submittingRef.current = true;
+ setSubmitting(true);
+ const submittedAnswers = params.questions.map((question, index) => ({
+ question: question.question,
+ selected: answers[index].usesOther ? [] : answers[index].selected,
+ other: answers[index].usesOther ? answers[index].other.trim() : null,
+ }));
+ const output = submittedAnswers
+ .map((answer) => `${answer.question}\n${answer.other ?? answer.selected.join(', ')}`)
+ .join('\n\n');
+ const createdAt = new Date().toISOString();
+ try {
+ await onResult({
+ type: EventType.EXTERNAL_EXECUTION_RESULT,
+ id: crypto.randomUUID(),
+ created_at: createdAt,
+ reply_id: replyId,
+ execution_results: [
+ {
+ type: 'tool_result',
+ id: toolCall.id,
+ name: toolCall.name,
+ output,
+ state: 'success',
+ metadata: { answers: submittedAnswers },
+ created_at: createdAt,
+ finished_at: createdAt,
+ },
+ ],
+ });
+ } catch {
+ submittingRef.current = false;
+ setSubmitting(false);
+ }
+ };
+
+ return (
+ <div className="max-h-[min(70vh,560px)] w-full space-y-5 overflow-y-auto rounded-[28px] bg-muted px-6 py-5 text-sm ring ring-border">
+ <div className="flex items-center gap-2 text-secondary-foreground">
+ <CircleHelp className="size-4 shrink-0" />
+ <strong>{t('chat.askUser.title')}</strong>
+ </div>
+ {params.questions.map((question, questionIndex) => {
+ const answer = answers[questionIndex];
+ const preview = question.options.find(
+ (option) => answer.selected.includes(option.label) && option.preview,
+ )?.preview;
+ return (
+ <section key={question.question} className="space-y-3">
+ <div className="space-y-1">
+ <span className="text-xs font-medium uppercase tracking-wide text-muted-foreground">
+ {question.header}
+ </span>
+ <p className="font-medium text-foreground">{question.question}</p>
+ {question.context ? (
+ <p className="whitespace-pre-wrap text-muted-foreground">
+ {question.context}
+ </p>
+ ) : null}
+ </div>
+ <div className="grid gap-2 sm:grid-cols-2">
+ {question.options.map((option) => {
+ const selected = answer.selected.includes(option.label);
+ return (
+ <Button
+ key={option.label}
+ type="button"
+ variant={selected ? 'secondary' : 'outline'}
+ className="h-auto min-h-14 justify-start gap-3 whitespace-normal px-3 py-2 text-left"
+ aria-pressed={selected}
+ disabled={submitting}
+ onClick={() =>
+ updateAnswer(questionIndex, (current) => ({
+ selected: question.multi_select
+ ? selected
+ ? current.selected.filter(
+ (label) => label !== option.label,
+ )
+ : [...current.selected, option.label]
+ : [option.label],
+ other: '',
+ usesOther: false,
+ }))
+ }
+ >
+ <span
+ className={cn(
+ 'flex size-5 shrink-0 items-center justify-center rounded-full border',
+ selected &&
+ 'border-primary bg-primary text-primary-foreground',
+ )}
+ >
+ {selected ? <Check className="size-3" /> : null}
+ </span>
+ <span className="min-w-0">
+ <span className="block font-medium">
+ {option.label}
+ </span>
+ <span className="block text-xs text-muted-foreground">
+ {option.description}
+ </span>
+ </span>
+ </Button>
+ );
+ })}
+ </div>
+ <div className="space-y-2">
+ <Button
+ type="button"
+ variant={answer.usesOther ? 'secondary' : 'ghost'}
+ size="sm"
+ disabled={submitting}
+ onClick={() =>
+ updateAnswer(questionIndex, (current) => ({
+ selected: [],
+ other: current.usesOther ? '' : current.other,
+ usesOther: !current.usesOther,
+ }))
+ }
+ >
+ {t('chat.askUser.other')}
+ </Button>
+ {answer.usesOther ? (
+ <Input
+ autoFocus
+ value={answer.other}
+ placeholder={t('chat.askUser.otherPlaceholder')}
+ disabled={submitting}
+ onChange={(event) =>
+ updateAnswer(questionIndex, (current) => ({
+ ...current,
+ other: event.target.value,
+ }))
+ }
+ />
+ ) : null}
+ </div>
+ {preview ? (
+ <pre className="max-h-44 overflow-auto whitespace-pre-wrap rounded-lg border bg-background p-3 font-mono text-xs">
+ {preview}
+ </pre>
+ ) : null}
+ </section>
+ );
+ })}
+ <div className="flex justify-end border-t pt-4">
+ <Button
+ type="button"
+ disabled={!complete || submitting}
+ onClick={() => void submit()}
+ >
+ {submitting ? <Spinner className="size-4" /> : <Send className="size-4" />}
+ {t('chat.askUser.submit')}
+ </Button>
+ </div>
+ </div>
+ );
+}
+
+export function ExternalExecutionCard({
+ replyId,
+ toolCall,
+ onResult,
+}: {
+ replyId: string;
+ toolCall: ToolCallBlock;
+ onResult: (event: ExternalExecutionResultEvent) => Promise<void>;
+}) {
+ const { t } = useTranslation();
+ if (toolCall.name === 'AskUser') {
+ return <AskUserCard replyId={replyId} toolCall={toolCall} onResult={onResult} />;
+ }
+ return (
+ <div className="rounded-[28px] bg-muted px-6 py-5 text-sm ring ring-border">
+ <div className="flex items-center gap-2 font-medium text-secondary-foreground">
+ <PlugZap className="size-4 shrink-0" />
+ {t('chat.externalExecution.title', { name: toolCall.name })}
+ </div>
+ <p className="mt-2 text-muted-foreground">{t('chat.externalExecution.description')}</p>
+ </div>
+ );
+}
diff --git a/examples/web_ui/frontend/src/hooks/useMessages.ts b/examples/web_ui/frontend/src/hooks/useMessages.ts
index 9a9269d22..068a59f13 100644
--- a/examples/web_ui/frontend/src/hooks/useMessages.ts
+++ b/examples/web_ui/frontend/src/hooks/useMessages.ts
@@ -5,7 +5,9 @@ import type {
DataBlockStartEvent,
DataBlockDeltaEvent,
DataBlockEndEvent,
+ ExternalExecutionResultEvent,
ReplyStartEvent,
+ ToolResultEndEvent,
UserConfirmResultEvent,
} from '@agentscope-ai/agentscope/event';
import { appendEvent, AssistantMsg, UserMsg } from '@agentscope-ai/agentscope/message';
@@ -24,6 +26,7 @@ import {
useChatQueue,
} from '@/hooks/useChatQueue';
import { useTranslation } from '@/i18n/useI18n';
+import { getPendingHitlCalls } from '@/lib/pendingToolCalls';
/**
* One pending subagent HITL request, projected from a team *member*
@@ -43,23 +46,6 @@ export type SubagentHitlEntry = {
created_at: string;
};
-/**
- * Return true if ``msg`` is an assistant reply currently parked on a
- * pending tool_call (awaiting user confirmation or an external
- * execution result). Used both to detect the "in-flight reply on page
- * load" case and as the SDK-gap workaround that hides the confirm
- * card once the paired tool_result lands.
- */
-const hasPendingToolCall = (msg: Msg | undefined): boolean => {
- if (!msg || msg.role !== 'assistant') return false;
- for (const block of msg.content) {
- if (block.type !== 'tool_call') continue;
- const state = (block as ToolCallBlock).state;
- if (state === 'asking' || state === 'submitted') return true;
- }
- return false;
-};
-
const hitlKey = (e: { worker_session_id: string; reply_id: string }) =>
`${e.worker_session_id}:${e.reply_id}`;
@@ -144,16 +130,36 @@ export function useMessages(
isRealtimeAudioActive?: () => boolean;
/** Send a tool confirmation over the active realtime transport. */
sendRealtimeUserConfirm?: (event: UserConfirmResultEvent) => Promise<void>;
+ /** Return an external tool result over the realtime transport. */
+ sendRealtimeExternalExecutionResult?: (
+ event: ExternalExecutionResultEvent,
+ ) => Promise<void>;
},
) {
const { t } = useTranslation();
// A new identity for each visit also rejects late responses after A -> B -> A.
const session = useMemo(() => ({ agentId, sessionId }), [agentId, sessionId]);
+ const externalSubmissionsRef = useRef(
+ new Map<
+ string,
+ {
+ replyId: string;
+ toolCallIds: Set<string>;
+ resolve: () => void;
+ reject: (error: Error) => void;
+ }
+ >(),
+ );
const activeSessionRef = useRef<typeof session | null>(null);
useLayoutEffect(() => {
activeSessionRef.current = session;
+ const externalSubmissions = externalSubmissionsRef.current;
return () => {
activeSessionRef.current = null;
+ for (const pending of externalSubmissions.values()) {
+ pending.reject(new Error('The session closed before the result was acknowledged.'));
+ }
+ externalSubmissions.clear();
};
}, [session]);
const [msgs, setMsgs] = useState<Msg[]>([]);
@@ -225,7 +231,23 @@ export function useMessages(
// reply content — route them to callbacks and skip appendEvent.
if (event.type === EventType.CUSTOM) {
const custom = event as CustomEvent;
- if (custom.name === 'tts_error') {
+ if (custom.name === 'external_execution_result_rejected') {
+ const failure = custom.value as {
+ request_id: string;
+ reply_id: string;
+ tool_call_ids: string[];
+ };
+ const pending = externalSubmissionsRef.current.get(failure.request_id);
+ if (
+ pending?.replyId === failure.reply_id &&
+ failure.tool_call_ids.some((id) => pending.toolCallIds.has(id))
+ ) {
+ externalSubmissionsRef.current.delete(failure.request_id);
+ const error = new Error(t('chat.externalExecution.rejected'));
+ pending.reject(error);
+ toast.error(error.message);
+ }
+ } else if (custom.name === 'tts_error') {
const failure = custom.value as { stage?: string; reply_id?: string };
const sessionKey = JSON.stringify([agentId, sessionId]);
if (failure.stage === 'configuration') {
@@ -298,6 +320,23 @@ export function useMessages(
}
return;
}
+ if (event.type === EventType.TOOL_RESULT_END) {
+ const result = event as ToolResultEndEvent;
+ for (const [requestId, pending] of externalSubmissionsRef.current) {
+ if (pending.replyId !== result.reply_id) continue;
+ pending.toolCallIds.delete(result.tool_call_id);
+ if (pending.toolCallIds.size === 0) {
+ externalSubmissionsRef.current.delete(requestId);
+ pending.resolve();
+ }
+ }
+ } else if (event.type === EventType.REPLY_END) {
+ for (const [requestId, pending] of externalSubmissionsRef.current) {
+ if (pending.replyId !== event.reply_id) continue;
+ externalSubmissionsRef.current.delete(requestId);
+ pending.reject(new Error(t('chat.externalExecution.rejected')));
+ }
+ }
if (event.type === EventType.REPLY_START) {
const e = event as ReplyStartEvent;
// A continuation (the run resuming after a confirmation or an
@@ -441,25 +480,27 @@ export function useMessages(
if (cancelled) return;
eventCursor = event_cursor;
msgsRef.current = messages;
- // If a reply is in flight (running on a worker) OR the
- // tail msg is parked on a pending tool_call (awaiting
+ // If a reply is in flight (running on a worker) OR any
+ // persisted msg is parked on a pending tool_call (awaiting
// user confirmation / external execution), initialise the
// phase to ``streaming`` so the interrupt button is
// available immediately — otherwise a fresh page load
// while parked leaves the UI stuck on ``idle`` with no
// way to abort.
const tail = messages[messages.length - 1];
- const pendingToolCall = hasPendingToolCall(tail);
- if (is_running || pendingToolCall) {
+ const pendingHitl = getPendingHitlCalls(messages)[0];
+ if (is_running || pendingHitl) {
setPhase('streaming');
- if (is_running && !pendingToolCall && tail?.role === 'assistant') {
+ if (is_running && !pendingHitl && tail?.role === 'assistant') {
setActiveReplyId(tail.id);
}
- if (pendingToolCall) {
+ if (pendingHitl) {
// Prime the ref so continuation events (which
// arrive without a fresh REPLY_START) apply to
// the right msg.
- currentReplyRef.current = tail ?? null;
+ currentReplyRef.current =
+ messages.find((message) => message.id === pendingHitl.replyId) ??
+ null;
}
}
// Published synchronously, not through `scheduleUpdate`:
@@ -615,6 +656,47 @@ export function useMessages(
[agentId, sessionId],
);
+ const onExternalExecutionResult = useCallback(
+ async (event: ExternalExecutionResultEvent) => {
+ if (!agentId || !sessionId || activeSessionRef.current !== session) return;
+ try {
+ if (
+ optionsRef.current?.isRealtimeAudioActive?.() &&
+ optionsRef.current.sendRealtimeExternalExecutionResult
+ ) {
+ // Sending a DataChannel frame does not acknowledge its result.
+ // Register first so even an immediate SSE response is matched.
+ const receipt = new Promise<void>((resolve, reject) => {
+ externalSubmissionsRef.current.set(event.id, {
+ replyId: event.reply_id,
+ toolCallIds: new Set(
+ event.execution_results.map((result) => result.id),
+ ),
+ resolve,
+ reject,
+ });
+ });
+ await Promise.all([
+ receipt,
+ optionsRef.current.sendRealtimeExternalExecutionResult(event),
+ ]);
+ } else {
+ await chatApi.trigger({
+ agent_id: agentId,
+ session_id: sessionId,
+ input: event,
+ });
+ }
+ } catch (e) {
+ if (activeSessionRef.current === session) setError(e as Error);
+ throw e;
+ } finally {
+ externalSubmissionsRef.current.delete(event.id);
+ }
+ },
+ [agentId, sessionId, session],
+ );
+
/** Abort the current SSE connection. */
const abort = useCallback(() => {
abortRef.current?.abort();
@@ -744,6 +826,7 @@ export function useMessages(
steerQueued,
send,
onUserConfirm,
+ onExternalExecutionResult,
onSubagentConfirm,
subagentHitl: ownsConversation ? subagentHitl : [],
abort,
diff --git a/examples/web_ui/frontend/src/hooks/useRealtimeVoice.ts b/examples/web_ui/frontend/src/hooks/useRealtimeVoice.ts
index da08717c2..5b79a87ed 100644
--- a/examples/web_ui/frontend/src/hooks/useRealtimeVoice.ts
+++ b/examples/web_ui/frontend/src/hooks/useRealtimeVoice.ts
@@ -1,4 +1,7 @@
-import type { UserConfirmResultEvent } from '@agentscope-ai/agentscope/event';
+import type {
+ ExternalExecutionResultEvent,
+ UserConfirmResultEvent,
+} from '@agentscope-ai/agentscope/event';
import { useCallback, useEffect, useRef, useState } from 'react';
import { toast } from 'sonner';
@@ -65,5 +68,11 @@ export function useRealtimeVoice(agentId: string | null, sessionId: string | nul
transport.userConfirm(event);
}, []);
- return { state, toggle, userConfirm };
+ const externalExecutionResult = useCallback(async (event: ExternalExecutionResultEvent) => {
+ const transport = transportRef.current;
+ if (!transport) throw new Error('Realtime voice mode is not active.');
+ transport.externalExecutionResult(event);
+ }, []);
+
+ return { state, toggle, userConfirm, externalExecutionResult };
}
diff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.json
index 0253ef626..f5b2b9ea5 100644
--- a/examples/web_ui/frontend/src/i18n/locales/en.json
+++ b/examples/web_ui/frontend/src/i18n/locales/en.json
@@ -175,6 +175,18 @@
"inputPlaceholder": "Type a message...",
"confirmToolCall": "Allow this action?",
"subagentConfirmTitle": "{{name}} requests confirmation",
+ "askUser": {
+ "title": "Your input is needed",
+ "other": "Other",
+ "otherPlaceholder": "Enter your answer",
+ "submit": "Submit answers",
+ "invalid": "This question could not be displayed."
+ },
+ "externalExecution": {
+ "title": "{{name}} is waiting for an external executor",
+ "rejected": "The result was not accepted. Please check your answer and submit again.",
+ "description": "This tool must be completed by an integration provided by the host application."
+ },
"newSession": "New Session",
"greeting": "Where should we start?",
"maxItersExceeded": {
@@ -485,6 +497,9 @@
"noModels": "No realtime voice models are available.",
"disable": "Disable voice model",
"selectFirst": "Realtime voice is off. Select a model from Voice model (the waveform icon) at the top.",
+ "backendFollows": "Task backend follows Chat model · {{model}}",
+ "backendMissing": "Select a Chat model before starting GPT Live",
+ "pendingChatInteraction": "Complete the pending Chat confirmation or external execution before starting GPT Live.",
"start": "Start voice conversation",
"stop": "End voice conversation"
},
diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.json
index ccc4eab42..1021743e1 100644
--- a/examples/web_ui/frontend/src/i18n/locales/zh.json
+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json
@@ -175,6 +175,18 @@
"inputPlaceholder": "输入消息...",
"confirmToolCall": "允许此操作?",
"subagentConfirmTitle": "{{name}} 请求确认",
+ "askUser": {
+ "title": "需要你的选择",
+ "other": "其他",
+ "otherPlaceholder": "输入你的回答",
+ "submit": "提交回答",
+ "invalid": "无法显示这个问题。"
+ },
+ "externalExecution": {
+ "title": "{{name}} 正在等待外部执行器",
+ "rejected": "提交的结果未被接受,请检查回答后重新提交。",
+ "description": "该工具需要由宿主应用提供的集成完成。"
+ },
"newSession": "新会话",
"greeting": "我们从哪儿开始?",
"maxItersExceeded": {
@@ -485,6 +497,9 @@
"noModels": "暂无可用的实时语音模型。",
"disable": "停用语音模型",
"selectFirst": "实时语音未开启。请在顶部的「语音模型」(声波图标)中选择模型。",
+ "backendFollows": "任务后端跟随 Chat 模型 · {{model}}",
+ "backendMissing": "请先选择 Chat 模型,再启动 GPT Live",
+ "pendingChatInteraction": "请先完成 Chat 中待处理的工具确认或外部执行,再启动 GPT Live。",
"start": "开始语音对话",
"stop": "结束语音对话"
},
diff --git a/examples/web_ui/frontend/src/lib/browserWebRTCTransport.ts b/examples/web_ui/frontend/src/lib/browserWebRTCTransport.ts
index eb66ffc0b..d7f470873 100644
--- a/examples/web_ui/frontend/src/lib/browserWebRTCTransport.ts
+++ b/examples/web_ui/frontend/src/lib/browserWebRTCTransport.ts
@@ -1,4 +1,7 @@
-import type { UserConfirmResultEvent } from '@agentscope-ai/agentscope/event';
+import type {
+ ExternalExecutionResultEvent,
+ UserConfirmResultEvent,
+} from '@agentscope-ai/agentscope/event';
import { realtimeApi } from '@/api';
@@ -180,6 +183,19 @@ export class BrowserWebRTCTransport {
);
}
+ externalExecutionResult(event: ExternalExecutionResultEvent): void {
+ if (this.controlChannel?.readyState !== 'open') {
+ throw new Error('The realtime control channel is not open.');
+ }
+ this.controlChannel.send(
+ JSON.stringify({
+ type: 'control',
+ control: 'external_execution_result',
+ data: event,
+ }),
+ );
+ }
+
private attachRemoteTrack(track: MediaStreamTrack): void {
if (track.kind !== 'audio') return;
this.clearPlayoutFence();
diff --git a/examples/web_ui/frontend/src/lib/pendingToolCalls.ts b/examples/web_ui/frontend/src/lib/pendingToolCalls.ts
new file mode 100644
index 000000000..05f7cccb8
--- /dev/null
+++ b/examples/web_ui/frontend/src/lib/pendingToolCalls.ts
@@ -0,0 +1,30 @@
+import { getContentBlocks, type Msg, type ToolCallBlock } from '@agentscope-ai/agentscope/message';
+
+export type PendingHitlCall = {
+ kind: 'confirmation' | 'external_execution';
+ replyId: string;
+ toolCall: ToolCallBlock;
+};
+
+/** Return unresolved HITL calls, newest message first. */
+export function getPendingHitlCalls(messages: readonly Msg[]): PendingHitlCall[] {
+ const pending: PendingHitlCall[] = [];
+ for (let index = messages.length - 1; index >= 0; index -= 1) {
+ const message = messages[index];
+ if (message.role !== 'assistant') continue;
+ const resultIds = new Set(
+ getContentBlocks(message, 'tool_result')
+ .filter((result) => result.state !== 'running')
+ .map((result) => result.id),
+ );
+ for (const toolCall of getContentBlocks(message, 'tool_call')) {
+ if (resultIds.has(toolCall.id)) continue;
+ if (toolCall.state === 'asking') {
+ pending.push({ kind: 'confirmation', replyId: message.id, toolCall });
+ } else if (toolCall.state === 'submitted') {
+ pending.push({ kind: 'external_execution', replyId: message.id, toolCall });
+ }
+ }
+ }
+ return pending;
+}
diff --git a/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx b/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx
index 85decd4ef..2f5cd5318 100644
--- a/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx
+++ b/examples/web_ui/frontend/src/pages/chat/ChatViewport.tsx
@@ -3,6 +3,7 @@ import type { TaskContext } from '@agentscope-ai/agentscope/state';
import {
Activity,
BookText,
+ BrainCircuit,
ChevronDown,
Database,
ListTodo,
@@ -13,6 +14,7 @@ import {
X,
} from 'lucide-react';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { toast } from 'sonner';
import type {
ASRModelConfig,
@@ -70,6 +72,7 @@ import { useSessions } from '@/hooks/useSessions';
import { useWorkspace } from '@/hooks/useWorkspace.ts';
import { useWorkspaceStatus } from '@/hooks/useWorkspaceStatus';
import { useTranslation } from '@/i18n/useI18n';
+import { getPendingHitlCalls } from '@/lib/pendingToolCalls';
const REALTIME_EXCLUDED_ENUM_VALUES: Record<string, unknown[]> = {
turn_detection: ['none'],
@@ -319,6 +322,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
moveQueued,
reorderQueued,
onUserConfirm,
+ onExternalExecutionResult,
onSubagentConfirm,
subagentHitl,
interrupt,
@@ -328,6 +332,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
onSessionUpdated: handleSessionUpdated,
isRealtimeAudioActive: () => voiceBusy,
sendRealtimeUserConfirm: realtimeVoice.userConfirm,
+ sendRealtimeExternalExecutionResult: realtimeVoice.externalExecutionResult,
});
const {
mcps,
@@ -665,7 +670,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
if (card) return card;
}
return null;
- }, [groups, selectedModel?.type, selectedModel?.model, selectedModel?.model_class]);
+ }, [groups, selectedModel]);
const selectedRealtimeModelCard = useMemo(() => {
if (!selectedRealtimeModel) return null;
@@ -682,6 +687,8 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
}
return null;
}, [realtimeGroups, selectedRealtimeModel]);
+ const realtimeUsesSessionChat =
+ selectedRealtimeModelCard?.delegation_backend === 'session_chat';
const selectedASRModelCard = useMemo(() => {
if (!selectedASRModel) return null;
@@ -985,7 +992,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
value={selectedModel}
onChange={handleLlmChange}
onAddCredential={() => setCredentialOpen(true)}
- disabled={configPending}
+ disabled={configPending || voiceBusy}
/>
<ModelParametersPopover
selectedModel={selectedModel}
@@ -997,7 +1004,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
onTTSChange={handleTTSChange}
selectedASRModel={selectedASRModel}
onASRChange={handleASRChange}
- disabled={configPending}
+ disabled={configPending || voiceBusy}
/>
<PermissionModeSelect
id="tour-permission-mode"
@@ -1060,6 +1067,23 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
</DropdownMenu>
</div>
</div>
+ {realtimeUsesSessionChat && (
+ <div
+ className={
+ 'flex min-h-7 items-center justify-end gap-1.5 px-2 ' +
+ 'text-xs text-muted-foreground'
+ }
+ >
+ <BrainCircuit className="size-3.5 shrink-0" />
+ <span className="max-w-full truncate">
+ {selectedModel
+ ? t('realtime.backendFollows', {
+ model: selectedModel.model,
+ })
+ : t('realtime.backendMissing')}
+ </span>
+ </div>
+ )}
<div className="flex flex-1 justify-center min-h-0 overflow-hidden relative [--chat-content-w:48rem]">
<ChatContent
className={'max-w-[var(--chat-content-w)] w-full'}
@@ -1079,6 +1103,7 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
disabled={selectedModel === null || voiceBusy}
onSend={send}
onUserConfirm={onUserConfirm}
+ onExternalExecutionResult={onExternalExecutionResult}
onInterrupt={interrupt}
onUpdateQueued={updateQueued}
onDeleteQueued={deleteQueued}
@@ -1086,8 +1111,23 @@ export function ChatViewport({ agentId, sessionId, onSessionsChanged }: ChatView
onMoveQueued={moveQueued}
onReorderQueued={reorderQueued}
voiceState={realtimeVoice.state}
- onVoiceToggle={() => void realtimeVoice.toggle()}
- voiceDisabled={!selectedRealtimeModel || !agentId || !sessionId}
+ onVoiceToggle={() => {
+ if (
+ !voiceBusy &&
+ realtimeUsesSessionChat &&
+ getPendingHitlCalls(msgs).length > 0
+ ) {
+ toast.error(t('realtime.pendingChatInteraction'));
+ return;
+ }
+ void realtimeVoice.toggle();
+ }}
+ voiceDisabled={
+ !selectedRealtimeModel ||
+ !agentId ||
+ !sessionId ||
+ (realtimeUsesSessionChat && !selectedModel)
+ }
asrInputTypes={selectedASRModelCard?.input_types ?? []}
onTranscribe={handleTranscribe}
asrRealtime={!!selectedASRModelCard?.realtime}
diff --git a/pyproject.toml b/pyproject.toml
index e82d54589..043ff6eb9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -113,7 +113,7 @@ tools = [
]
# ------------ Realtime voice ------------
realtime = [
- "websockets>=13",
+ "websockets>=14",
]
realtime-local = ["sounddevice"]
# ------------ Terminal UI ------------
diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.py
index 012358e48..71a38bc1d 100644
--- a/src/agentscope/agent/_agent.py
+++ b/src/agentscope/agent/_agent.py
@@ -741,10 +741,12 @@ class Agent:
and self._get_reply_usage() is None
):
self.state.append_context(self.name, [])
- self.state.context[-1].usage = current_reply_usage
+ reply = self.state.get_reply_message(self.name)
+ assert reply is not None
+ reply.usage = current_reply_usage
# The compression call is not covered by the model call events,
- # so record its cost on the context tail to keep it in the token
+ # so record its cost in the context to keep it in the token
# accounting
if res is not None and res.usage is not None:
if not self.state.context:
@@ -752,7 +754,11 @@ class Agent:
# empty message, which is skipped by the formatters
self.state.append_context(self.name, [])
- self.state.context[-1].append_usage(
+ usage_message = (
+ self.state.get_reply_message(self.name)
+ or self.state.context[-1]
+ )
+ usage_message.append_usage(
Usage(
input_tokens=res.usage.input_tokens,
output_tokens=res.usage.output_tokens,
@@ -970,11 +976,8 @@ class Agent:
]:
"""Close the unfinished tool calls on interruption, so the next
input will be handled normally."""
- if not self.state.context:
- return
-
- last_msg = self.state.context[-1]
- if last_msg.role != "assistant" or last_msg.name != self.name:
+ last_msg = self.state.get_reply_message(self.name)
+ if last_msg is None:
return
# Searching for tool calls that requires user confirmation or external
@@ -1954,7 +1957,8 @@ class Agent:
The events generated during the handling of the incoming event.
"""
# Return directly if no event
- if event is None or len(self.state.context) == 0:
+ last_msg = self.state.get_reply_message(self.name)
+ if event is None or last_msg is None:
return
if isinstance(event, UserConfirmResultEvent):
@@ -1964,7 +1968,6 @@ class Agent:
}
# Update the state with the confirmed tool calls
- last_msg = self.state.context[-1]
for tool_call in last_msg.get_content_blocks("tool_call"):
if len(confirmed_tool_calls) == 0:
break
@@ -3489,10 +3492,8 @@ class Agent:
state (`ToolCallState`):
The new state of the tool call.
"""
- if len(self.state.context) == 0:
- return
- last_msg = self.state.context[-1]
- if last_msg.role != "assistant" or last_msg.name != self.name:
+ last_msg = self.state.get_reply_message(self.name)
+ if last_msg is None:
return
for block in last_msg.get_content_blocks():
if isinstance(block, ToolCallBlock) and block.id == tool_call_id:
@@ -3549,16 +3550,13 @@ class Agent:
self.state.append_context(self.name, persisted_blocks)
if msg_usage is not None:
- self.state.context[-1].append_usage(msg_usage)
+ reply = self.state.get_reply_message(self.name)
+ assert reply is not None
+ reply.append_usage(msg_usage)
def _get_last_msg(self) -> Msg | None:
- """Get the last message in the context that belongs to this agent."""
- if len(self.state.context) == 0:
- return None
- last_msg = self.state.context[-1]
- if last_msg.role == "assistant" and last_msg.name == self.name:
- return last_msg
- return None
+ """Get the current reply without assuming it is the context tail."""
+ return self.state.get_reply_message(self.name)
def _get_reply_usage(self) -> Usage | None:
"""Get a copy of the accumulated usage for the current reply."""
diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.py
index 25653d128..ee335dc40 100644
--- a/src/agentscope/agent/_realtime/_agent.py
+++ b/src/agentscope/agent/_realtime/_agent.py
@@ -4,12 +4,19 @@ import asyncio
import base64
import time
import uuid
+from collections import deque
+from collections.abc import Callable, Iterator
+from copy import deepcopy
from dataclasses import dataclass, field
-from typing import Any, AsyncIterator
+from typing import Any, AsyncIterator, TYPE_CHECKING
from ...realtime import _events as me
from ._aggregator import TurnAggregator
-from ...realtime._base import ModelDisconnectedError, RealtimeModelBase
+from ...realtime._base import (
+ InteractionMode,
+ ModelDisconnectedError,
+ RealtimeModelBase,
+)
from ._metrics import TurnMetrics
from ...realtime._transport._base import (
AudioFrame,
@@ -23,6 +30,7 @@ from ..._utils._common import _json_loads_with_repair
from ...event import (
AgentEvent,
ConfirmResult,
+ CustomEvent,
DataBlockDeltaEvent,
DataBlockEndEvent,
DataBlockStartEvent,
@@ -30,6 +38,7 @@ from ...event import (
ModelCallStartEvent,
ReplyEndEvent,
ReplyStartEvent,
+ RequireExternalExecutionEvent,
RequireUserConfirmEvent,
TextBlockDeltaEvent,
TextBlockEndEvent,
@@ -38,8 +47,10 @@ from ...event import (
ToolCallEndEvent,
ToolCallStartEvent,
ToolResultEndEvent,
+ ToolResultDataDeltaEvent,
ToolResultStartEvent,
ToolResultTextDeltaEvent,
+ ExternalExecutionResultEvent,
UserConfirmResultEvent,
UserInterruptEvent,
)
@@ -59,10 +70,16 @@ from ...state import AgentState
from ...tool import ToolChunk, ToolResponse, Toolkit
from ...types import ReplyFinishedReason
+if TYPE_CHECKING:
+ from .._agent import Agent
+
# Audio buffered while the model is being reconnected: 10 s at 100 ms chunks.
_BACKLOG_FRAMES = 100
_HISTORY_MAX_MESSAGES = 100
_HISTORY_MAX_TEXT_CHARS = 32_000
+_LIVE_SEGMENT_MAX_TEXT_CHARS = 4_000
+_LIVE_CHECKPOINT_INTERVAL_SECS = 2.0
+_DELEGATION_INPUT_TIMEOUT_SECS = 300.0
@dataclass
@@ -79,6 +96,7 @@ class _Reply:
audio_block_id: str = field(default_factory=lambda: uuid.uuid4().hex)
text_started: bool = False
audio_started: bool = False
+ text_chars: int = 0
@dataclass
@@ -99,6 +117,23 @@ class _QueuedEvent:
checkpoint: AgentState | None = None
+@dataclass
+class _ClientDelegation:
+ """One application-managed task requested by a Live session."""
+
+ delegation_id: str
+ prompt: Msg
+ generation: int
+ agent: "Agent | None" = None
+ prompt_id: str = ""
+ baseline_ids: set[str] = field(default_factory=set)
+ synced_ids: set[str] = field(default_factory=set)
+ buffered_events: list[AgentEvent] = field(default_factory=list)
+ visible: bool = False
+ timeout_task: asyncio.Task[None] | None = None
+ expiring: bool = False
+
+
class RealtimeAgent:
"""A voice agent: a realtime model on one side, a transport on the
other, and the turn-taking state machine in between.
@@ -139,6 +174,9 @@ class RealtimeAgent:
state: AgentState | None = None,
vad: VADBase | None = None,
aggregator: TurnAggregator | None = None,
+ delegation_agent_factory: (
+ Callable[[AgentState], "Agent"] | None
+ ) = None,
) -> None:
"""Initialize the realtime agent.
@@ -173,6 +211,15 @@ class RealtimeAgent:
acknowledgements. Subclass it to change what counts as a
turn. A default with no backchannel list is used if
omitted.
+ delegation_agent_factory (`Callable[[AgentState], Agent] | None`,
+ optional):
+ Builds a chat agent for each client-delegated Live task from
+ the supplied isolated session state. Its model, toolkit, and
+ permission state normally come from the current persisted
+ chat session. The returned agent must have the same name
+ as this realtime agent so tool routing and checkpoint
+ recovery use the same assistant identity. A mismatched
+ name fails the delegation before backend execution.
"""
self.name = name
self.system_prompt = system_prompt
@@ -181,6 +228,14 @@ class RealtimeAgent:
self.state = state or AgentState()
self.vad = vad
self.aggregator = aggregator or TurnAggregator()
+ self._delegation_agent_factory = delegation_agent_factory
+ if (
+ self.model.interaction_mode is InteractionMode.FULL_DUPLEX
+ and self.vad is not None
+ ):
+ raise ValueError(
+ "A full-duplex model cannot use client-side turn VAD.",
+ )
self._engine = PermissionEngine(self.state.permission_context)
self._transport: TransportBase | None = None
@@ -200,6 +255,19 @@ class RealtimeAgent:
self._confirmations: dict[str, asyncio.Future[ConfirmResult]] = {}
self._tasks: set[asyncio.Task] = set()
self._barge_lock = asyncio.Lock()
+ self._live_session_id = ""
+ self._live_replies: dict[str, _Reply] = {}
+ self._live_message_ids: set[str] = set()
+ self._live_transcript: deque[tuple[str, str, int, int]] = deque()
+ self._live_transcript_chars = 0
+ self._last_live_checkpoint_at: float | None = None
+ self._last_live_checkpoint_end_ms: int | None = None
+ self._delegations: dict[str, _ClientDelegation] = {}
+ self._delegation_queue: deque[str] = deque()
+ self._active_delegation_id: str | None = None
+ self._delegated_tool_calls: dict[str, str] = {}
+ self._delegation_lock = asyncio.Lock()
+ self._closing = False
# Model session: connected flag, the downlink pump that outlives
# any transport, and reconnect bookkeeping.
@@ -229,11 +297,16 @@ class RealtimeAgent:
closed it; a no-op while connected."""
if self._connected:
return
-
if self._connection_generation:
async with self._barge_lock:
self._reset_user_turns()
self._finish_reply(ReplyFinishedReason.ERROR)
+ self._finish_live_replies(ReplyFinishedReason.ERROR)
+ if self.model.interaction_mode is InteractionMode.FULL_DUPLEX:
+ # A new Live session cannot accept old delegation IDs.
+ # Close and persist its tasks before opening the new socket.
+ await self.close()
+ self._closing = False
self._connection_generation += 1
instructions = self.system_prompt
@@ -328,20 +401,45 @@ class RealtimeAgent:
async def close(self) -> None:
"""Cancel everything in flight and close the model session."""
+ self._closing = True
+ if self._downlink is not None:
+ self._downlink.cancel()
+ await asyncio.gather(self._downlink, return_exceptions=True)
+ self._downlink = None
for future in self._confirmations.values():
future.cancel()
self._confirmations.clear()
+ timeout_tasks = [
+ delegation.timeout_task
+ for delegation in self._delegations.values()
+ if delegation.timeout_task is not None
+ ]
+ for task in timeout_tasks:
+ task.cancel()
for task in list(self._tasks):
task.cancel()
if self._tasks:
await asyncio.gather(*self._tasks, return_exceptions=True)
- if self._downlink is not None:
- self._downlink.cancel()
- await asyncio.gather(self._downlink, return_exceptions=True)
- self._downlink = None
+ if timeout_tasks:
+ await asyncio.gather(*timeout_tasks, return_exceptions=True)
+ for delegation in list(self._delegations.values()):
+ if delegation.agent is not None:
+ if delegation.agent.state.has_awaiting_tool_calls(self.name):
+ await self._drive_client_delegation(
+ delegation,
+ UserInterruptEvent(
+ reply_id=delegation.agent.state.reply_id,
+ ),
+ )
+ self._sync_delegation_state(delegation)
+ self._delegations.clear()
+ self._delegation_queue.clear()
+ self._delegated_tool_calls.clear()
+ self._active_delegation_id = None
self._connected = False
self._connected_event.clear()
await self.model.close()
+ self._finish_live_replies(ReplyFinishedReason.COMPLETED)
def _on_downlink_done(self, task: asyncio.Task) -> None:
"""Surface a crashed downlink instead of a silently dead session."""
@@ -460,19 +558,13 @@ class RealtimeAgent:
# the model stops talking to nobody, then hand the caller the
# events that produced (the interrupted ReplyEnd) before the
# stream ends rather than leaking them into the next run.
- await self._barge_in()
+ if self.model.interaction_mode is InteractionMode.FULL_DUPLEX:
+ await self._interrupt_live_speech()
+ else:
+ await self._barge_in()
self._retire_unbound_user_turns()
- while not self._out.empty():
- queued = self._out.get_nowait()
- self._yielded_checkpoint = (
- (queued.event.id, queued.checkpoint)
- if queued.checkpoint is not None
- else None
- )
- try:
- yield queued.event
- finally:
- self._yielded_checkpoint = None
+ for event in self.drain_events():
+ yield event
# A transport failure must not look like a clean disconnect.
if not uplink.cancelled() and uplink.exception() is not None:
raise uplink.exception() # type: ignore[misc]
@@ -489,13 +581,38 @@ class RealtimeAgent:
self._retire_unbound_user_turns()
self._transport = None
+ def drain_events(self) -> Iterator[AgentEvent]:
+ """Yield pending events, including cleanup emitted by ``close``.
+
+ Each yielded event retains its checkpoint until iteration resumes.
+ Session owners should drain after closing to publish final tool and
+ transcript states before persisting the final session snapshot.
+ """
+ while not self._out.empty():
+ queued = self._out.get_nowait()
+ self._yielded_checkpoint = (
+ (queued.event.id, queued.checkpoint)
+ if queued.checkpoint is not None
+ else None
+ )
+ try:
+ yield queued.event
+ finally:
+ self._yielded_checkpoint = None
+
# ------------------------------------------------------------------
# Discrete input
# ------------------------------------------------------------------
async def send(
self,
- inputs: (str | Msg | UserConfirmResultEvent | UserInterruptEvent),
+ inputs: (
+ str
+ | Msg
+ | UserConfirmResultEvent
+ | ExternalExecutionResultEvent
+ | UserInterruptEvent
+ ),
) -> None:
"""Feed the agent anything that is not audio.
@@ -514,11 +631,32 @@ class RealtimeAgent:
await self._barge_in()
case UserConfirmResultEvent():
+ tool_call_ids = [
+ result.tool_call.id for result in inputs.confirm_results
+ ]
+ delegation = self._delegation_for_tool_calls(tool_call_ids)
+ if delegation is not None:
+ self._schedule_client_delegation(delegation, inputs)
+ return
for result in inputs.confirm_results:
future = self._confirmations.get(result.tool_call.id)
if future and not future.done():
future.set_result(result)
+ case ExternalExecutionResultEvent():
+ tool_call_ids = [
+ result.id for result in inputs.execution_results
+ ]
+ delegation = self._delegation_for_tool_calls(tool_call_ids)
+ if delegation is not None:
+ self._schedule_client_delegation(delegation, inputs)
+ return
+ # pylint: disable-next=logging-fstring-interpolation
+ logger.warning(
+ f"RealtimeAgent: ignored external execution results "
+ f"without an active delegation: {tool_call_ids}",
+ )
+
case str() | Msg():
if not self.model.supports_text_input:
raise NotImplementedError(
@@ -553,6 +691,11 @@ class RealtimeAgent:
"""Latency breakdown of the most recent turn."""
return self._metrics
+ @property
+ def allows_open_user_reply_checkpoint(self) -> bool:
+ """Whether an incomplete user transcript is safe to persist."""
+ return self.model.interaction_mode is InteractionMode.FULL_DUPLEX
+
# ------------------------------------------------------------------
# Uplink: transport -> model
# ------------------------------------------------------------------
@@ -627,6 +770,8 @@ class RealtimeAgent:
await self.send(frame.data.get("text", ""))
case ControlFrameType.USER_CONFIRM:
await self.send(UserConfirmResultEvent(**frame.data))
+ case ControlFrameType.EXTERNAL_EXECUTION_RESULT:
+ await self.send(ExternalExecutionResultEvent(**frame.data))
case ControlFrameType.INTERRUPT:
await self.send(UserInterruptEvent(reply_id=self._reply_id))
case _:
@@ -647,9 +792,25 @@ class RealtimeAgent:
the exit of :meth:`reply_stream`, so it is serialised; the losers
find the reply already closed and return.
"""
+ if self.model.interaction_mode is InteractionMode.FULL_DUPLEX:
+ await self._interrupt_live_speech()
+ return
async with self._barge_lock:
await self._barge_in_locked()
+ async def _interrupt_live_speech(self) -> None:
+ """Stop Live speech without cancelling delegated backend work."""
+ async with self._barge_lock:
+ if self._transport is not None and self._audio_pending:
+ await self._transport.clear_audio()
+ self._audio_pending = False
+ if self._connected:
+ await self.model.interrupt_speech()
+ self._finish_live_role(
+ "assistant",
+ ReplyFinishedReason.INTERRUPTED,
+ )
+
async def _barge_in_locked(self) -> None:
"""Body of :meth:`_barge_in`, run under the lock."""
active_reply = self._reply
@@ -685,6 +846,7 @@ class RealtimeAgent:
self._mark_disconnected()
self._reset_user_turns()
self._finish_reply(ReplyFinishedReason.ERROR)
+ self._finish_live_replies(ReplyFinishedReason.ERROR)
logger.info(
"RealtimeAgent: model session ended; keep talking and it "
"reconnects on the next audio.",
@@ -692,6 +854,9 @@ class RealtimeAgent:
async def _on_model_event(self, event: me.ModelEvent) -> None:
"""Handle one model event."""
+ if self.model.interaction_mode is InteractionMode.FULL_DUPLEX:
+ await self._on_live_event(event)
+ return
rate = self.model.output_sample_rate
match event:
case me.SpeechStartedEvent():
@@ -795,6 +960,904 @@ class RealtimeAgent:
event.reason,
)
+ async def _on_live_event(self, event: me.ModelEvent) -> None:
+ """Project one full-duplex timeline event into public events."""
+ match event:
+ case me.SessionStartedEvent():
+ self._live_session_id = event.session_id
+ self._last_live_checkpoint_at = None
+ self._last_live_checkpoint_end_ms = None
+
+ case me.TimelineAudioDeltaEvent():
+ if self._transport is None:
+ return
+ reply = self._start_live_reply("assistant")
+ playout_id = f"live:{self._live_session_id}"
+ await self._transport.send_audio(event.pcm, playout_id)
+ self._audio_pending = True
+ self._emit_audio(reply, event.pcm, event.sample_rate)
+
+ case me.TimelineTranscriptDeltaEvent():
+ self._append_live_transcript(event)
+ self._segment_live_text(event)
+
+ case me.SessionClosedEvent():
+ self._finish_live_replies(ReplyFinishedReason.COMPLETED)
+
+ case me.ModelErrorEvent():
+ logger.error(
+ "RealtimeAgent: Live model error %s: %s",
+ event.code,
+ event.message,
+ )
+ self._finish_live_replies(ReplyFinishedReason.ERROR)
+
+ case me.SessionEndedEvent():
+ self._finish_live_replies(ReplyFinishedReason.ERROR)
+
+ case me.DelegationCreatedEvent(target="client"):
+ self._start_client_delegation(event.delegation_id)
+
+ case me.DelegationCreatedEvent() | me.DelegatedResponseEvent():
+ logger.debug(
+ "RealtimeAgent: ignoring an unsupported delegated event",
+ )
+
+ case me.SessionUsageEvent():
+ pass
+
+ def _start_live_reply(self, role: str) -> _Reply:
+ """Open one independent user or assistant Live transcript."""
+ current = self._live_replies.get(role)
+ if current is not None:
+ return current
+ reply_id = f"live:{role}:{uuid.uuid4().hex}"
+ reply = _Reply(item_id=reply_id, reply_id=reply_id)
+ self._live_replies[role] = reply
+ self._live_message_ids.add(reply_id)
+ self._emit(
+ ReplyStartEvent(
+ session_id=self.state.session_id,
+ reply_id=reply_id,
+ name="user" if role == "user" else self.name,
+ role="user" if role == "user" else "assistant",
+ ),
+ )
+ if role == "assistant":
+ self._emit(
+ ModelCallStartEvent(
+ reply_id=reply_id,
+ model_name=self.model.model,
+ ),
+ )
+ return reply
+
+ def _emit_live_text(
+ self,
+ role: str,
+ reply: _Reply,
+ delta: str,
+ end_ms: int,
+ ) -> None:
+ """Append a timestamped Live transcript to its speaker's message."""
+ reply.text_chars += len(delta)
+ if not reply.text_started:
+ reply.text_started = True
+ self._emit(
+ TextBlockStartEvent(
+ reply_id=reply.reply_id,
+ block_id=reply.text_block_id,
+ ),
+ )
+ message = next(
+ (
+ item
+ for item in reversed(self.state.context)
+ if item.id == reply.reply_id
+ ),
+ None,
+ )
+ if message is None:
+ message = Msg(
+ id=reply.reply_id,
+ role="user" if role == "user" else "assistant",
+ name="user" if role == "user" else self.name,
+ content=[TextBlock(id=reply.text_block_id, text=delta)],
+ )
+ self.state.context.append(message)
+ else:
+ blocks = message.get_content_blocks("text")
+ if blocks:
+ blocks[-1].text += delta
+ else:
+ message.content.append(
+ TextBlock(id=reply.text_block_id, text=delta),
+ )
+ self._emit(
+ TextBlockDeltaEvent(
+ reply_id=reply.reply_id,
+ block_id=reply.text_block_id,
+ delta=delta,
+ ),
+ checkpoint=self._should_checkpoint_live_transcript(end_ms),
+ )
+
+ def _segment_live_text(
+ self,
+ event: me.TimelineTranscriptDeltaEvent,
+ ) -> None:
+ """Rotate bounded transcript segments without changing playback."""
+ offset = 0
+ while offset < len(event.delta):
+ reply = self._start_live_reply(event.role)
+ available = _LIVE_SEGMENT_MAX_TEXT_CHARS - reply.text_chars
+ chunk = event.delta[offset : offset + available]
+ self._prune_live_context(len(chunk))
+ self._emit_live_text(event.role, reply, chunk, event.end_ms)
+ offset += len(chunk)
+ if reply.text_chars >= _LIVE_SEGMENT_MAX_TEXT_CHARS:
+ # Persist the completed segment before any later event can
+ # prune it from runtime context. History retains this copy.
+ self._finish_live_role(
+ event.role,
+ ReplyFinishedReason.COMPLETED,
+ )
+
+ def _prune_live_context(self, incoming_chars: int) -> None:
+ """Keep recent Live text in state and older segments in history."""
+ live_messages = [
+ msg
+ for msg in self.state.context
+ if msg.id.startswith(("live:user:", "live:assistant:"))
+ ]
+ total_chars = (
+ sum(len(msg.get_text_content() or "") for msg in live_messages)
+ + incoming_chars
+ )
+ count = len(live_messages)
+ open_ids = {reply.reply_id for reply in self._live_replies.values()}
+ removed: set[str] = set()
+ for message in live_messages:
+ if (
+ total_chars <= _HISTORY_MAX_TEXT_CHARS
+ and count < _HISTORY_MAX_MESSAGES
+ ):
+ break
+ if message.id in open_ids:
+ continue
+ removed.add(message.id)
+ total_chars -= len(message.get_text_content() or "")
+ count -= 1
+ if removed:
+ self.state.context = [
+ msg for msg in self.state.context if msg.id not in removed
+ ]
+ self._live_message_ids.difference_update(removed)
+
+ def _should_checkpoint_live_transcript(self, end_ms: int) -> bool:
+ """Sample state when the wall or Live timeline interval advances."""
+ now = time.monotonic()
+ interval_ms = int(_LIVE_CHECKPOINT_INTERVAL_SECS * 1_000)
+ wall_due = (
+ self._last_live_checkpoint_at is None
+ or now - self._last_live_checkpoint_at
+ >= _LIVE_CHECKPOINT_INTERVAL_SECS
+ )
+ timeline_due = (
+ self._last_live_checkpoint_end_ms is not None
+ and end_ms - self._last_live_checkpoint_end_ms >= interval_ms
+ )
+ if not wall_due and not timeline_due:
+ return False
+ self._last_live_checkpoint_at = now
+ if self._last_live_checkpoint_end_ms is None:
+ self._last_live_checkpoint_end_ms = end_ms
+ else:
+ self._last_live_checkpoint_end_ms = max(
+ self._last_live_checkpoint_end_ms,
+ end_ms,
+ )
+ return True
+
+ def _finish_live_role(
+ self,
+ role: str,
+ reason: ReplyFinishedReason,
+ ) -> None:
+ """Close one speaker's current Live presentation segment."""
+ reply = self._live_replies.pop(role, None)
+ if reply is None:
+ return
+ if reply.text_started:
+ self._emit(
+ TextBlockEndEvent(
+ reply_id=reply.reply_id,
+ block_id=reply.text_block_id,
+ ),
+ )
+ if reply.audio_started:
+ self._emit(
+ DataBlockEndEvent(
+ reply_id=reply.reply_id,
+ block_id=reply.audio_block_id,
+ ),
+ )
+ if role == "assistant":
+ self._emit(
+ ModelCallEndEvent(
+ reply_id=reply.reply_id,
+ input_tokens=0,
+ output_tokens=0,
+ ),
+ )
+ finished = ReplyEndEvent(
+ session_id=self.state.session_id,
+ reply_id=reply.reply_id,
+ finished_reason=reason,
+ )
+ for message in reversed(self.state.context):
+ if message.id == reply.reply_id:
+ message.append_event(finished)
+ break
+ self._emit(finished)
+
+ def _finish_live_replies(self, reason: ReplyFinishedReason) -> None:
+ """Close both independently growing Live transcript streams."""
+ self._finish_live_role("user", reason)
+ self._finish_live_role("assistant", reason)
+
+ def _start_client_delegation(self, delegation_id: str) -> None:
+ """Queue one GPT-Live task behind the active delegation."""
+ if (
+ self._closing
+ or not delegation_id
+ or delegation_id in self._delegations
+ ):
+ return
+ if self._delegation_agent_factory is None:
+ self._schedule_delegation_failure(delegation_id)
+ return
+ prompt = UserMsg(
+ name="user",
+ content=self._build_delegation_prompt(),
+ )
+ delegation = _ClientDelegation(
+ delegation_id=delegation_id,
+ prompt=prompt,
+ generation=self._connection_generation,
+ )
+ self._delegations[delegation_id] = delegation
+ if self._active_delegation_id is None:
+ self._activate_client_delegation(delegation)
+ else:
+ self._delegation_queue.append(delegation_id)
+
+ def _activate_client_delegation(
+ self,
+ delegation: _ClientDelegation,
+ ) -> None:
+ """Create a backend from the latest state and start its task."""
+ if self._delegation_agent_factory is None:
+ self._delegations.pop(delegation.delegation_id, None)
+ self._schedule_delegation_failure(delegation.delegation_id)
+ self._activate_next_client_delegation()
+ return
+ try:
+ agent = self._delegation_agent_factory(
+ self._copy_delegation_state(),
+ )
+ if agent.name != self.name:
+ raise ValueError(
+ f"Delegation backend name {agent.name!r} must match "
+ f"RealtimeAgent name {self.name!r}.",
+ )
+ except Exception: # noqa: BLE001
+ logger.exception(
+ "RealtimeAgent: failed to create delegation backend",
+ )
+ self._delegations.pop(delegation.delegation_id, None)
+ self._schedule_delegation_failure(delegation.delegation_id)
+ self._activate_next_client_delegation()
+ return
+ delegation.agent = agent
+ delegation.prompt_id = delegation.prompt.id
+ delegation.baseline_ids = {
+ message.id for message in agent.state.context
+ }
+ self._active_delegation_id = delegation.delegation_id
+ self._schedule_client_delegation(delegation, delegation.prompt)
+
+ def _activate_next_client_delegation(self) -> None:
+ """Start the oldest queued delegation, if one remains."""
+ if self._closing or self._active_delegation_id is not None:
+ return
+ while self._delegation_queue:
+ delegation_id = self._delegation_queue.popleft()
+ delegation = self._delegations.get(delegation_id)
+ if delegation is not None:
+ self._activate_client_delegation(delegation)
+ return
+
+ def _schedule_client_delegation(
+ self,
+ delegation: _ClientDelegation,
+ inputs: (
+ Msg
+ | UserConfirmResultEvent
+ | ExternalExecutionResultEvent
+ | UserInterruptEvent
+ ),
+ ) -> None:
+ """Run or resume a delegated chat-agent reply in the background."""
+ task = asyncio.create_task(
+ self._drive_client_delegation(delegation, inputs),
+ name=f"live-delegation-{delegation.delegation_id}",
+ )
+ self._tasks.add(task)
+ task.add_done_callback(self._tasks.discard)
+
+ def _schedule_delegation_timeout(
+ self,
+ delegation: _ClientDelegation,
+ ) -> None:
+ """Cancel a delegation that remains parked without input."""
+ self._cancel_delegation_timeout(delegation)
+ delegation.timeout_task = asyncio.create_task(
+ self._expire_client_delegation(delegation),
+ name=f"live-delegation-timeout-{delegation.delegation_id}",
+ )
+
+ def _cancel_delegation_timeout(
+ self,
+ delegation: _ClientDelegation,
+ ) -> None:
+ """Cancel the pending external-input deadline, if any."""
+ task = delegation.timeout_task
+ delegation.timeout_task = None
+ if task is not None and task is not asyncio.current_task():
+ task.cancel()
+
+ async def _expire_client_delegation(
+ self,
+ delegation: _ClientDelegation,
+ ) -> None:
+ """Interrupt and finish a delegation after its input deadline."""
+ try:
+ await asyncio.sleep(_DELEGATION_INPUT_TIMEOUT_SECS)
+ if (
+ self._delegations.get(delegation.delegation_id)
+ is not delegation
+ or self._active_delegation_id != delegation.delegation_id
+ or delegation.agent is None
+ ):
+ return
+ delegation.expiring = True
+ await self._drive_client_delegation(
+ delegation,
+ UserInterruptEvent(reply_id=delegation.agent.state.reply_id),
+ result_override=(
+ "The delegated task was cancelled because the required "
+ "user input or external execution was not provided in "
+ "time."
+ ),
+ )
+ except asyncio.CancelledError:
+ return
+
+ def _schedule_delegation_failure(self, delegation_id: str) -> None:
+ """Tell GPT-Live that an application backend is unavailable."""
+ task = asyncio.create_task(
+ self._send_delegation_result(
+ delegation_id,
+ "The delegated task could not be completed because the "
+ "session chat agent is unavailable.",
+ self._connection_generation,
+ ),
+ name=f"live-delegation-failure-{delegation_id}",
+ )
+ self._tasks.add(task)
+ task.add_done_callback(self._tasks.discard)
+
+ async def _send_delegation_result(
+ self,
+ delegation_id: str,
+ content: str,
+ generation: int,
+ ) -> None:
+ """Send results only to the Live connection that requested them."""
+ if self._closing or generation != self._connection_generation:
+ return
+ await self.model.send_delegation_result(delegation_id, content)
+
+ async def _drive_client_delegation(
+ self,
+ delegation: _ClientDelegation,
+ inputs: (
+ Msg
+ | UserConfirmResultEvent
+ | ExternalExecutionResultEvent
+ | UserInterruptEvent
+ ),
+ result_override: str | None = None,
+ ) -> None:
+ """Run a chat agent and return its verified result to GPT-Live."""
+ if delegation.generation != self._connection_generation:
+ return
+ async with self._delegation_lock:
+ # Routing happened before this task waited for the lock.
+ if (
+ delegation.generation != self._connection_generation
+ or self._delegations.get(delegation.delegation_id)
+ is not delegation
+ or self._active_delegation_id != delegation.delegation_id
+ ):
+ return
+ if isinstance(
+ inputs,
+ (UserConfirmResultEvent, ExternalExecutionResultEvent),
+ ):
+ if (
+ self._closing
+ or delegation.expiring
+ or delegation.agent is None
+ or inputs.reply_id != delegation.agent.state.reply_id
+ ):
+ return
+ awaiting = {
+ tool_call.id: tool_call.state
+ for tool_call in (
+ delegation.agent.state.get_awaiting_tool_calls(
+ self.name,
+ )
+ )
+ }
+ if isinstance(inputs, UserConfirmResultEvent):
+ confirmations = [
+ result
+ for result in inputs.confirm_results
+ if awaiting.get(result.tool_call.id)
+ == ToolCallState.ASKING
+ ]
+ if not confirmations:
+ return
+ inputs = inputs.model_copy(
+ update={"confirm_results": confirmations},
+ )
+ else:
+ results = [
+ result
+ for result in inputs.execution_results
+ if awaiting.get(result.id) == ToolCallState.SUBMITTED
+ ]
+ if not results:
+ return
+ inputs = inputs.model_copy(
+ update={"execution_results": results},
+ )
+ # An earlier queued input may have parked and rearmed the timer.
+ self._cancel_delegation_timeout(delegation)
+ final: Msg | None = None
+ waiting_for_external_input = False
+ pending_result_end: ToolResultEndEvent | None = None
+ try:
+ if delegation.agent is None:
+ raise RuntimeError(
+ "The delegation backend is unavailable.",
+ )
+ async for item in delegation.agent.reply_stream(
+ inputs,
+ yield_final_msg=True,
+ ):
+ if pending_result_end is not None:
+ self._checkpoint_delegation_tool_result(
+ delegation,
+ pending_result_end,
+ )
+ pending_result_end = None
+ if isinstance(item, Msg):
+ final = item
+ continue
+ if not delegation.visible and isinstance(
+ item,
+ (
+ ReplyStartEvent,
+ ToolCallStartEvent,
+ ToolCallDeltaEvent,
+ ToolCallEndEvent,
+ ToolResultStartEvent,
+ ToolResultTextDeltaEvent,
+ ToolResultDataDeltaEvent,
+ ToolResultEndEvent,
+ ),
+ ):
+ delegation.buffered_events.append(item)
+ if isinstance(item, ToolResultEndEvent):
+ reply = delegation.agent.state.get_reply_message(
+ self.name,
+ )
+ if (
+ reply is not None
+ and any(
+ call.id == item.tool_call_id
+ and call.state == ToolCallState.FINISHED
+ for call in reply.get_content_blocks(
+ "tool_call",
+ )
+ )
+ and any(
+ result.id == item.tool_call_id
+ and result.state == item.state
+ and result.state != ToolResultState.RUNNING
+ for result in reply.get_content_blocks(
+ "tool_result",
+ )
+ )
+ ):
+ # Local tools commit before END. Persist now,
+ # before the next model request can block.
+ self._checkpoint_delegation_tool_result(
+ delegation,
+ item,
+ )
+ else:
+ # External and interrupted results may commit
+ # after yielding END.
+ pending_result_end = item
+ continue
+ if isinstance(
+ item,
+ (
+ RequireUserConfirmEvent,
+ RequireExternalExecutionEvent,
+ ),
+ ):
+ self._sync_delegation_state(delegation)
+ self._show_delegation_interaction(
+ delegation,
+ item,
+ )
+ waiting_for_external_input = True
+ elif delegation.visible and isinstance(
+ item,
+ (
+ ToolCallStartEvent,
+ ToolCallDeltaEvent,
+ ToolCallEndEvent,
+ ToolResultStartEvent,
+ ToolResultTextDeltaEvent,
+ ToolResultDataDeltaEvent,
+ ReplyEndEvent,
+ ),
+ ):
+ if isinstance(item, ReplyEndEvent):
+ self._sync_delegation_state(delegation)
+ self._emit(item)
+
+ self._sync_delegation_state(delegation)
+ if pending_result_end is not None:
+ self._checkpoint_delegation_tool_result(
+ delegation,
+ pending_result_end,
+ )
+ pending_result_end = None
+ if waiting_for_external_input or (
+ delegation.agent.state.has_awaiting_tool_calls(self.name)
+ ):
+ if not self._closing:
+ self._schedule_delegation_timeout(delegation)
+ return
+ if final is None:
+ raise RuntimeError(
+ "The delegation backend ended without a final result.",
+ )
+
+ if self._closing:
+ self._finish_client_delegation(delegation.delegation_id)
+ return
+ content = (
+ result_override or (final.get_text_content() or "").strip()
+ )
+ if not content:
+ content = (
+ "The delegated task completed without a text result."
+ )
+ await self._send_delegation_result(
+ delegation.delegation_id,
+ content,
+ delegation.generation,
+ )
+ self._finish_client_delegation(delegation.delegation_id)
+ except Exception: # noqa: BLE001
+ logger.exception(
+ "RealtimeAgent: client delegation %s failed",
+ delegation.delegation_id,
+ )
+ # Completed tools must survive a later backend failure.
+ self._sync_delegation_state(delegation)
+ if pending_result_end is not None:
+ self._checkpoint_delegation_tool_result(
+ delegation,
+ pending_result_end,
+ )
+ if (
+ not self._closing
+ and not delegation.expiring
+ and isinstance(inputs, ExternalExecutionResultEvent)
+ and delegation.agent is not None
+ and delegation.agent.state.has_awaiting_tool_calls(
+ self.name,
+ )
+ ):
+ # A rejected external answer leaves the backend parked.
+ # Retain its route for retry after syncing valid results.
+ awaiting_calls = (
+ delegation.agent.state.get_awaiting_tool_calls(
+ self.name,
+ )
+ )
+ awaiting_ids = {call.id for call in awaiting_calls}
+ rejected_ids = [
+ result.id
+ for result in inputs.execution_results
+ if result.id in awaiting_ids
+ ]
+ if rejected_ids:
+ self._emit(
+ CustomEvent(
+ name="external_execution_result_rejected",
+ value={
+ "request_id": inputs.id,
+ "reply_id": (
+ delegation.agent.state.reply_id
+ ),
+ "tool_call_ids": rejected_ids,
+ "message": (
+ "The submitted result was rejected. "
+ "Please check your answer and submit "
+ "again."
+ ),
+ },
+ ),
+ )
+ self._schedule_delegation_timeout(delegation)
+ return
+ if self._closing:
+ self._finish_client_delegation(delegation.delegation_id)
+ return
+ try:
+ await self._send_delegation_result(
+ delegation.delegation_id,
+ "The delegated task failed. "
+ "Ask the user to try again.",
+ delegation.generation,
+ )
+ except Exception: # noqa: BLE001
+ logger.exception(
+ "RealtimeAgent: failed to report delegation failure",
+ )
+ self._finish_client_delegation(delegation.delegation_id)
+
+ def _checkpoint_delegation_tool_result(
+ self,
+ delegation: _ClientDelegation,
+ event: ToolResultEndEvent,
+ ) -> None:
+ """Persist committed results independently of tool-card visibility."""
+ self._sync_delegation_state(delegation)
+ if delegation.visible:
+ self._emit(event)
+ else:
+ self._emit(
+ CustomEvent(
+ name="delegation_checkpoint",
+ value={
+ "delegation_id": delegation.delegation_id,
+ "reply_id": event.reply_id,
+ "tool_call_id": event.tool_call_id,
+ },
+ ),
+ checkpoint=True,
+ )
+
+ def _build_delegation_prompt(self) -> str:
+ """Build a bounded task prompt from the current Live transcript."""
+ lines: list[str] = []
+ for role, delta, _, _ in self._live_transcript:
+ label = "User" if role == "user" else "Voice assistant"
+ if lines and lines[-1].startswith(f"{label}: "):
+ lines[-1] = f"{lines[-1]}{delta}"
+ else:
+ lines.append(f"{label}: {delta}")
+ transcript = "\n".join(lines)
+ if len(transcript) > _HISTORY_MAX_TEXT_CHARS:
+ transcript = transcript[-_HISTORY_MAX_TEXT_CHARS:]
+ if not transcript.strip():
+ transcript = "No transcript text is available."
+ return (
+ f"You are the task backend for a live voice conversation. "
+ f"Use the latest transcript, the session context, and the "
+ f"available tools to complete the delegated task. Transcript "
+ f"fragments may contain corrections; prefer the latest user "
+ f"intent. Do not claim an action succeeded until a tool confirms "
+ f"it. Return verified facts, current status, and the next step in "
+ f"a concise result of at most 300 words for the voice assistant "
+ f"to communicate.\n\n## Live transcript\n{transcript}"
+ )
+
+ def _show_delegation_interaction(
+ self,
+ delegation: _ClientDelegation,
+ interaction: RequireUserConfirmEvent | RequireExternalExecutionEvent,
+ ) -> None:
+ """Expose a delegated external-input request through the UI."""
+ if not delegation.visible:
+ start = next(
+ (
+ event
+ for event in delegation.buffered_events
+ if isinstance(event, ReplyStartEvent)
+ ),
+ None,
+ )
+ if start is not None:
+ self._emit(start)
+ for event in delegation.buffered_events:
+ if isinstance(
+ event,
+ (
+ ToolCallStartEvent,
+ ToolCallDeltaEvent,
+ ToolCallEndEvent,
+ ToolResultStartEvent,
+ ToolResultTextDeltaEvent,
+ ToolResultDataDeltaEvent,
+ ToolResultEndEvent,
+ ),
+ ):
+ self._emit(event)
+ delegation.visible = True
+ delegation.buffered_events.clear()
+ self._emit(interaction)
+ for tool_call in interaction.tool_calls:
+ self._delegated_tool_calls[tool_call.id] = delegation.delegation_id
+
+ def _copy_delegation_state(self) -> AgentState:
+ """Copy session state while omitting duplicated Live messages."""
+ state = self.state.model_copy()
+ for field_name in type(self.state).model_fields:
+ if field_name != "context":
+ setattr(
+ state,
+ field_name,
+ deepcopy(getattr(self.state, field_name)),
+ )
+ state.context = [
+ message.model_copy(deep=True)
+ for message in self.state.context
+ if message.id not in self._live_message_ids
+ ]
+ return state
+
+ def _sync_delegation_state(
+ self,
+ delegation: _ClientDelegation,
+ ) -> None:
+ """Persist delegated tool state without duplicating hidden prompts."""
+ if delegation.agent is None:
+ return
+ backend_state = delegation.agent.state
+ self.state.reply_context = backend_state.reply_context.model_copy(
+ deep=True,
+ )
+ self.state.permission_context = (
+ backend_state.permission_context.model_copy(deep=True)
+ )
+ self.state.tool_context = backend_state.tool_context.model_copy(
+ deep=True,
+ )
+ self.state.tasks_context = backend_state.tasks_context.model_copy(
+ deep=True,
+ )
+ self.state.middle_context = deepcopy(backend_state.middle_context)
+ self.state.summary = deepcopy(backend_state.summary)
+ projected: dict[str, Msg] = {}
+ for message in backend_state.context:
+ if message.id == delegation.prompt_id:
+ continue
+ if message.id in delegation.baseline_ids:
+ projected[message.id] = message.model_copy(deep=True)
+ continue
+ kept = [
+ block.model_copy(deep=True)
+ for block in message.content
+ if block.type in {"tool_call", "tool_result"}
+ ]
+ if not kept:
+ continue
+ synced = message.model_copy(deep=True)
+ synced.content = kept
+ projected[message.id] = synced
+
+ # Only this backend's history may be replaced or compacted. Live
+ # messages added or extended while it runs belong to the parent.
+ owned_ids = delegation.baseline_ids | delegation.synced_ids
+ delegation.synced_ids = projected.keys() - delegation.baseline_ids
+ context = []
+ for message in self.state.context:
+ if message.id in owned_ids:
+ replacement = projected.pop(message.id, None)
+ if replacement is not None:
+ context.append(replacement)
+ else:
+ context.append(message)
+ context.extend(projected.values())
+ self.state.context = context
+
+ def _finish_client_delegation(self, delegation_id: str) -> None:
+ """Drop routing state after a delegated task reaches a result."""
+ delegation = self._delegations.pop(delegation_id, None)
+ if delegation is not None:
+ self._cancel_delegation_timeout(delegation)
+ stale = [
+ tool_call_id
+ for tool_call_id, owner in self._delegated_tool_calls.items()
+ if owner == delegation_id
+ ]
+ for tool_call_id in stale:
+ self._delegated_tool_calls.pop(tool_call_id, None)
+ if self._active_delegation_id == delegation_id:
+ self._active_delegation_id = None
+ self._activate_next_client_delegation()
+
+ def _delegation_for_tool_calls(
+ self,
+ tool_call_ids: list[str],
+ ) -> _ClientDelegation | None:
+ """Resolve one active delegation owning every supplied tool call."""
+ if not tool_call_ids:
+ return None
+ delegation_id = self._delegated_tool_calls.get(tool_call_ids[0])
+ if delegation_id is None or any(
+ self._delegated_tool_calls.get(tool_call_id) != delegation_id
+ for tool_call_id in tool_call_ids
+ ):
+ return None
+ if delegation_id != self._active_delegation_id:
+ return None
+ delegation = self._delegations.get(delegation_id)
+ if (
+ delegation is None
+ or delegation.expiring
+ or delegation.generation != self._connection_generation
+ ):
+ return None
+ return delegation
+
+ def _append_live_transcript(
+ self,
+ event: me.TimelineTranscriptDeltaEvent,
+ ) -> None:
+ """Keep a timestamped transcript window within the prompt budget."""
+ fragment = (
+ event.role,
+ event.delta,
+ event.start_ms,
+ event.end_ms,
+ )
+ self._live_transcript.append(fragment)
+ self._live_transcript_chars += len(fragment[1])
+ while self._live_transcript_chars > _HISTORY_MAX_TEXT_CHARS:
+ role, text, start_ms, end_ms = self._live_transcript.popleft()
+ excess = self._live_transcript_chars - _HISTORY_MAX_TEXT_CHARS
+ if not self._live_transcript and len(text) > excess:
+ kept = text[excess:]
+ self._live_transcript.appendleft(
+ (role, kept, start_ms, end_ms),
+ )
+ self._live_transcript_chars -= excess
+ break
+ self._live_transcript_chars -= len(text)
+
def _start_user_turn(self, item_id: str = "") -> None:
"""Open the user's turn as a reply of its own. A local VAD sees it
before the provider creates an item, hence the generated id."""
@@ -1131,19 +2194,25 @@ class RealtimeAgent:
),
)
- def _emit(self, event: AgentEvent) -> None:
+ def _emit(
+ self,
+ event: AgentEvent,
+ *,
+ checkpoint: bool = False,
+ ) -> None:
"""Queue one event for :meth:`reply_stream`."""
- checkpoint = None
- if isinstance(
+ snapshot = None
+ if checkpoint or isinstance(
event,
(
ReplyEndEvent,
+ RequireExternalExecutionEvent,
RequireUserConfirmEvent,
ToolResultEndEvent,
),
):
- checkpoint = self.state.model_copy(deep=True)
- self._out.put_nowait(_QueuedEvent(event, checkpoint))
+ snapshot = self.state.model_copy(deep=True)
+ self._out.put_nowait(_QueuedEvent(event, snapshot))
def checkpoint_snapshot(self, event: AgentEvent) -> AgentState | None:
"""Return the immutable state paired with the yielded event."""
diff --git a/src/agentscope/app/_lifespan.py b/src/agentscope/app/_lifespan.py
index 31e8a7e5c..e416090eb 100644
--- a/src/agentscope/app/_lifespan.py
+++ b/src/agentscope/app/_lifespan.py
@@ -188,8 +188,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
background_task_manager=bg_manager,
message_bus=message_bus,
resource_access_service=resource_access_service,
+ knowledge_base_manager=knowledge_base_manager,
+ extra_agent_middlewares=app.state.extra_agent_middlewares,
extra_agent_tools=app.state.extra_agent_tools,
custom_subagent_templates=app.state.custom_subagent_templates,
+ custom_agent_cls=app.state.custom_agent_cls,
channel_clients=channel_clients,
)
diff --git a/src/agentscope/app/_router/_realtime.py b/src/agentscope/app/_router/_realtime.py
index 69bba8ac2..2defc87f7 100644
--- a/src/agentscope/app/_router/_realtime.py
+++ b/src/agentscope/app/_router/_realtime.py
@@ -123,7 +123,7 @@ async def _create_realtime_offer(
realtime_service: RealtimeService,
) -> RealtimeOfferResponse:
"""Create one offer while this process serializes session starts."""
- await access.resolve_agent(user_id, body.agent_id)
+ agent_record = await access.resolve_agent(user_id, body.agent_id)
session = await storage.get_session(
user_id,
body.agent_id,
@@ -175,6 +175,28 @@ async def _create_realtime_offer(
status_code=status.HTTP_400_BAD_REQUEST,
detail=str(exc),
) from exc
+ card = getattr(model, "card", None)
+ uses_session_chat = (
+ getattr(card, "delegation_backend", "none") == "session_chat"
+ )
+ if uses_session_chat and session.state.has_awaiting_tool_calls(
+ agent_record.data.name,
+ ):
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail=(
+ "Complete the pending Chat confirmation or "
+ "external execution before starting GPT Live."
+ ),
+ )
+ if (
+ uses_session_chat
+ and getattr(session.config, "chat_model_config", None) is None
+ ):
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail="Configure a chat model before starting GPT Live.",
+ )
lock_key = MessageBusKeys.session_lock(session_id)
if await message_bus.is_locked(lock_key):
diff --git a/src/agentscope/app/_service/_realtime.py b/src/agentscope/app/_service/_realtime.py
index 153e32838..b3b722cf9 100644
--- a/src/agentscope/app/_service/_realtime.py
+++ b/src/agentscope/app/_service/_realtime.py
@@ -1,20 +1,32 @@
# -*- coding: utf-8 -*-
"""Runtime assembly for browser realtime agents."""
+import inspect
from typing import Literal
from ..._logging import logger
-from ...agent import RealtimeAgent
+from ...agent import Agent, ModelConfig, RealtimeAgent
+from ...middleware import MiddlewareBase, RAGMiddleware
from ...permission import AdditionalWorkingDirectory
+from ...rag import KnowledgeBase
from ...realtime import RealtimeModelBase
+from ...state import AgentState
from .._manager import BackgroundTaskManager, SchedulerManager
-from .._types import AgentToolFactory, SubAgentTemplate
+from .._types import (
+ AgentMiddlewareFactory,
+ AgentToolFactory,
+ SubAgentTemplate,
+)
from ...tool import ToolBase
+from ...workspace import WorkspaceBase
from ..channel import ChannelBase, ChannelClients, ChatKind
from ..message_bus import MessageBus
+from ..middleware import StateChangeMiddleware
+from ..rag.knowledge_base_manager import KnowledgeBaseManagerBase
from ..storage import ChannelOrigin, SessionRecord, StorageBase
from ..workspace_manager import WorkspaceManagerBase
from ._access import ResourceAccessService
+from ._model import get_model
from ._toolkit import get_toolkit
@@ -30,18 +42,70 @@ class RealtimeService:
background_task_manager: BackgroundTaskManager,
message_bus: MessageBus,
resource_access_service: ResourceAccessService,
+ knowledge_base_manager: KnowledgeBaseManagerBase | None = None,
+ extra_agent_middlewares: AgentMiddlewareFactory | None = None,
extra_agent_tools: AgentToolFactory | None = None,
custom_subagent_templates: dict[str, SubAgentTemplate] | None = None,
+ custom_agent_cls: type[Agent] | None = None,
channel_clients: ChannelClients | None = None,
) -> None:
+ """Initialize the application-wide realtime Agent assembler.
+
+ This service owns no running connection. A :class:`WebRTCSession`
+ calls :meth:`create_agent` for each negotiated voice session.
+
+ Args:
+ storage (`StorageBase`):
+ Persistent session and agent storage.
+ workspace_manager (`WorkspaceManagerBase`):
+ Resolves the workspace bound to each session.
+ scheduler_manager (`SchedulerManager`):
+ Provides schedule tools to assembled toolkits.
+ background_task_manager (`BackgroundTaskManager`):
+ Provides background-task tools and runtime state.
+ message_bus (`MessageBus`):
+ Shared bus used by session and team tools.
+ resource_access_service (`ResourceAccessService`):
+ Resolves caller-visible agents, credentials, and knowledge.
+ knowledge_base_manager (`KnowledgeBaseManagerBase | None`,
+ optional):
+ Resolves knowledge handles for delegated Chat Agents.
+ extra_agent_middlewares (`AgentMiddlewareFactory | None`,
+ optional):
+ Builds application-provided middlewares for delegation.
+ extra_agent_tools (`AgentToolFactory | None`, optional):
+ Builds application-provided tools for the session toolkit.
+ custom_subagent_templates (`dict[str, SubAgentTemplate] | None`,
+ optional):
+ Sub-agent templates exposed through Agent creation tools.
+ custom_agent_cls (`type[Agent] | None`, optional):
+ Agent implementation used for delegated Chat tasks.
+ channel_clients (`ChannelClients | None`, optional):
+ Resolves tools and identity for channel-bound sessions.
+ """
self._storage = storage
self._workspace_manager = workspace_manager
self._scheduler_manager = scheduler_manager
self._background_task_manager = background_task_manager
self._message_bus = message_bus
self._access = resource_access_service
+ self._knowledge_base_manager = knowledge_base_manager
+ self._extra_agent_middlewares = extra_agent_middlewares
+ self._middlewares_take_workspace = False
+ if extra_agent_middlewares is not None:
+ try:
+ inspect.signature(extra_agent_middlewares).bind(
+ "",
+ "",
+ "",
+ None,
+ )
+ self._middlewares_take_workspace = True
+ except (TypeError, ValueError):
+ pass
self._extra_agent_tools = extra_agent_tools
self._subagent_templates = custom_subagent_templates or {}
+ self._agent_cls = custom_agent_cls or Agent
self._channel_clients = channel_clients
async def create_agent(
@@ -52,7 +116,34 @@ class RealtimeService:
session_id: str,
model: RealtimeModelBase,
) -> RealtimeAgent:
- """Load the locked session and assemble its realtime runtime."""
+ """Load a locked session and assemble its realtime runtime.
+
+ The caller must hold the distributed session lock before invoking
+ this method so the loaded state cannot race another Chat or voice
+ run. The returned Agent is not connected until its async context is
+ entered.
+
+ Args:
+ user_id (`str`):
+ Owner of the session being opened.
+ agent_id (`str`):
+ Agent configuration used by the session.
+ session_id (`str`):
+ Persisted session whose state the runtime will share.
+ model (`RealtimeModelBase`):
+ Provider adapter already resolved from the session config.
+
+ Returns:
+ `RealtimeAgent`:
+ An assembled but disconnected realtime Agent.
+
+ Raises:
+ RuntimeError:
+ If the session no longer exists.
+ ValueError:
+ If a client-delegating model has no Chat model configured
+ or the session has a pending Chat interaction.
+ """
agent_record = await self._access.resolve_agent(user_id, agent_id)
session = await self._storage.get_session(
user_id,
@@ -62,6 +153,18 @@ class RealtimeService:
if session is None:
raise RuntimeError(f"Session {session_id!r} no longer exists.")
+ card = getattr(model, "card", None)
+ uses_session_chat = (
+ getattr(card, "delegation_backend", "none") == "session_chat"
+ )
+ if uses_session_chat and session.state.has_awaiting_tool_calls(
+ agent_record.data.name,
+ ):
+ raise ValueError(
+ "Complete the pending Chat confirmation or "
+ "external execution before starting GPT Live.",
+ )
+
workspace = await self._workspace_manager.get_workspace(
user_id,
agent_id,
@@ -101,6 +204,17 @@ class RealtimeService:
if channel is not None and channel_origin is not None
else []
)
+ middlewares = (
+ await self._build_delegation_middlewares(
+ user_id=user_id,
+ agent_id=agent_id,
+ session_id=session_id,
+ session=session,
+ workspace=workspace,
+ )
+ if uses_session_chat
+ else []
+ )
toolkit = await get_toolkit(
storage=self._storage,
workspace=workspace,
@@ -108,7 +222,7 @@ class RealtimeService:
scheduler_manager=self._scheduler_manager,
background_task_manager=self._background_task_manager,
message_bus=self._message_bus,
- middlewares=[],
+ middlewares=middlewares,
user_id=user_id,
agent_record=agent_record,
session_record=session,
@@ -127,14 +241,113 @@ class RealtimeService:
channel_tools,
chat_kind,
)
+ delegation_agent_factory = None
+ if uses_session_chat:
+ chat_config = session.config.chat_model_config
+ if chat_config is None:
+ raise ValueError(
+ "Configure a chat model before starting GPT Live.",
+ )
+ chat_model = await get_model(user_id, chat_config, self._access)
+ fallback_config = session.config.fallback_chat_model_config
+ fallback_model = (
+ await get_model(user_id, fallback_config, self._access)
+ if fallback_config is not None
+ else None
+ )
+
+ def _create_delegation_agent(state: AgentState) -> Agent:
+ """Build an isolated task agent from current session state."""
+ config = agent_record.data.chat_config
+ return self._agent_cls(
+ name=agent_record.data.name,
+ system_prompt=system_prompt,
+ model=chat_model,
+ toolkit=toolkit,
+ model_config=ModelConfig(
+ fallback_model=fallback_model,
+ ),
+ context_config=config.context_config,
+ react_config=config.react_config,
+ state=state,
+ middlewares=middlewares,
+ offloader=workspace,
+ )
+
+ delegation_agent_factory = _create_delegation_agent
+
return RealtimeAgent(
name=agent_record.data.name,
system_prompt=system_prompt,
model=model,
toolkit=toolkit,
state=session.state,
+ delegation_agent_factory=delegation_agent_factory,
)
+ async def _build_delegation_middlewares(
+ self,
+ *,
+ user_id: str,
+ agent_id: str,
+ session_id: str,
+ session: SessionRecord,
+ workspace: WorkspaceBase,
+ ) -> list[MiddlewareBase]:
+ """Build Chat middlewares that are safe in a Live task backend."""
+ middlewares: list[MiddlewareBase] = [
+ StateChangeMiddleware(
+ message_bus=self._message_bus,
+ session_id=session_id,
+ ),
+ ]
+ if self._extra_agent_middlewares is not None:
+ factory_args: tuple = (user_id, agent_id, session_id)
+ if self._middlewares_take_workspace:
+ factory_args += (workspace,)
+ middlewares.extend(
+ await self._extra_agent_middlewares(*factory_args),
+ )
+
+ config = session.config.knowledge_config
+ if (
+ config is None
+ or not config.knowledge_base_ids
+ or self._knowledge_base_manager is None
+ ):
+ return middlewares
+
+ knowledges: list[KnowledgeBase] = []
+ for knowledge_id in config.knowledge_base_ids:
+ try:
+ record = await self._access.resolve_knowledge_base(
+ user_id,
+ knowledge_id,
+ )
+ knowledge = await self._knowledge_base_manager.get_knowledge(
+ record.user_id,
+ knowledge_id,
+ )
+ except Exception: # noqa: BLE001
+ logger.exception(
+ "Skipping knowledge base %r for session %r: failed "
+ "to resolve runtime handle.",
+ knowledge_id,
+ session_id,
+ )
+ continue
+ knowledges.append(knowledge)
+ if knowledges:
+ middlewares.append(
+ RAGMiddleware(
+ knowledge_bases=knowledges,
+ parameters=RAGMiddleware.Parameters(
+ **(config.parameters or {}),
+ ),
+ ),
+ )
+ return middlewares
+
async def _resolve_team_role(
self,
user_id: str,
diff --git a/src/agentscope/app/_service/_webrtc_session.py b/src/agentscope/app/_service/_webrtc_session.py
index 53a8b0442..15b17f98a 100644
--- a/src/agentscope/app/_service/_webrtc_session.py
+++ b/src/agentscope/app/_service/_webrtc_session.py
@@ -2,13 +2,15 @@
"""Lifecycle owner for one browser WebRTC realtime session."""
import asyncio
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncGenerator, Awaitable, Callable
+from contextlib import aclosing
from aiortc import RTCPeerConnection
from ..._logging import logger
from ...agent import RealtimeAgent
from ...event import (
+ AgentEvent,
DataBlockDeltaEvent,
DataBlockEndEvent,
DataBlockStartEvent,
@@ -26,6 +28,12 @@ _CHECKPOINT_LOCK_TTL_SECS = (
)
+def _stream_id_key(entry_id: str) -> tuple[int, int]:
+ """Return the sortable numeric components of a message-bus stream ID."""
+ major, minor = entry_id.split("-", maxsplit=1)
+ return int(major), int(minor)
+
+
class WebRTCSession:
"""Run and persist one realtime agent over a peer connection."""
@@ -42,6 +50,33 @@ class WebRTCSession:
session_id: str,
on_closed: Callable[["WebRTCSession"], None],
) -> None:
+ """Initialize one negotiated browser voice-session owner.
+
+ The instance owns the peer connection, audio transport, Agent
+ lifecycle, session lock, event publication, and state persistence.
+ Calling :meth:`start` begins those activities in a background task.
+
+ Args:
+ peer_connection (`RTCPeerConnection`):
+ Negotiated WebRTC connection to close with the session.
+ transport (`WebRTCAudioTransport`):
+ Audio and control-frame bridge for the peer connection.
+ agent_factory (`Callable[[], Awaitable[RealtimeAgent]]`):
+ Builds the Agent after the session lock is acquired.
+ storage (`StorageBase`):
+ Persists messages and the final Agent state.
+ message_bus (`MessageBus`):
+ Provides the session lock and publishes replayable events.
+ user_id (`str`):
+ Owner of the persisted session.
+ agent_id (`str`):
+ Agent associated with the persisted session.
+ session_id (`str`):
+ Session used for locking, events, and persistence.
+ on_closed (`Callable[[WebRTCSession], None]`):
+ Callback that removes this runner from the connection
+ registry after all resources close.
+ """
self.peer_connection = peer_connection
self.transport = transport
self._agent_factory = agent_factory
@@ -150,7 +185,14 @@ class WebRTCSession:
return True
async def _run(self) -> None:
- """Hold the session lock while forwarding events to the SSE log."""
+ """Own the locked realtime runtime until its transport ends.
+
+ After acquiring the distributed session lock, this task creates and
+ connects the Agent, forwards non-PCM events to the replayable SSE
+ stream, persists event-bound checkpoints, and saves the latest state
+ before releasing the lock. It always closes the transport and peer
+ connection and invokes ``on_closed`` during cleanup.
+ """
if self._close_task is not None:
return
try:
@@ -166,11 +208,8 @@ class WebRTCSession:
for message in self.agent.state.context
}
async with self.transport:
- async with self.agent:
- self._startup_complete = True
- async for event in self.agent.reply_stream(
- self.transport,
- ):
+ async with aclosing(self._agent_events()) as events:
+ async for event in events:
if isinstance(event, DataBlockStartEvent) and (
event.media_type.startswith("audio/pcm")
):
@@ -209,9 +248,13 @@ class WebRTCSession:
self._open_user_replies.discard(
event.reply_id,
)
- if (
- checkpoint is not None
- and not self._open_user_replies
+ if checkpoint is not None and (
+ not self._open_user_replies
+ or getattr(
+ self.agent,
+ "allows_open_user_reply_checkpoint",
+ False,
+ )
):
try:
await self._checkpoint_state(
@@ -273,6 +316,23 @@ class WebRTCSession:
finally:
await self._close_resources()
+ async def _agent_events(self) -> AsyncGenerator[AgentEvent, None]:
+ """Include terminal events emitted when the voice agent closes."""
+ assert self.agent is not None
+ try:
+ async with self.agent:
+ self._startup_complete = True
+ async for event in self.agent.reply_stream(self.transport):
+ yield event
+ except Exception:
+ # Publish cleanup events before the runner reports the error.
+ # GeneratorExit and cancellation must not yield during close.
+ for event in self.agent.drain_events():
+ yield event
+ raise
+ for event in self.agent.drain_events():
+ yield event
+
async def _checkpoint_state(
self,
entry_id: str | None,
@@ -283,6 +343,13 @@ class WebRTCSession:
MessageBusKeys.session_event_checkpoint_lock(self.session_id),
ttl_secs=_CHECKPOINT_LOCK_TTL_SECS,
):
+ if (
+ entry_id is not None
+ and self._checkpoint_entry_id is not None
+ and _stream_id_key(entry_id)
+ <= _stream_id_key(self._checkpoint_entry_id)
+ ):
+ return
await self._persist_state(state)
if entry_id is not None:
await self._set_checkpoint_cursor(entry_id)
@@ -326,9 +393,12 @@ class WebRTCSession:
)
removed_ids = self._persisted_messages.keys() - current_messages.keys()
for message_id in sorted(removed_ids):
- await self.storage.delete_message(
- self.user_id,
- self.session_id,
- message_id,
- )
+ # Completed Live segments leave the bounded runtime window but
+ # remain readable in history. Drop their cache entries as well.
+ if not message_id.startswith(("live:user:", "live:assistant:")):
+ await self.storage.delete_message(
+ self.user_id,
+ self.session_id,
+ message_id,
+ )
self._persisted_messages.pop(message_id, None)
diff --git a/src/agentscope/credential/_openai.py b/src/agentscope/credential/_openai.py
index 51a352e7a..eb5930aa7 100644
--- a/src/agentscope/credential/_openai.py
+++ b/src/agentscope/credential/_openai.py
@@ -94,6 +94,6 @@ class OpenAICredential(CredentialBase):
@classmethod
def get_realtime_model_classes(cls) -> list[Type["RealtimeModelBase"]]:
"""Return the OpenAI realtime model classes."""
- from ..realtime import OpenAIRealtimeModel
+ from ..realtime import OpenAILiveModel, OpenAIRealtimeModel
- return [OpenAIRealtimeModel]
+ return [OpenAIRealtimeModel, OpenAILiveModel]
diff --git a/src/agentscope/event/_event.py b/src/agentscope/event/_event.py
index 0f90b2bcf..c72776ad2 100644
--- a/src/agentscope/event/_event.py
+++ b/src/agentscope/event/_event.py
@@ -550,6 +550,13 @@ class CustomEvent(EventBase):
changed during a tool call.
- ``"team_updated"`` — team membership changed (member
added / team created or dissolved).
+ - ``"delegation_checkpoint"`` — a hidden Live tool result is
+ ready to persist without displaying a tool card. ``value``
+ contains ``delegation_id``, ``reply_id``, and ``tool_call_id``.
+ - ``"external_execution_result_rejected"`` — a Live result
+ submission needs retry. ``value`` contains ``request_id``
+ (the submitted event ID), ``reply_id``, ``tool_call_ids``
+ (only unaccepted results), and a safe ``message``.
- ``"chat_input_started"`` — one or more FIFO-queued user
messages have started their agent turn.
- ``"chat_input_failed"`` — a dequeued user turn failed before
diff --git a/src/agentscope/realtime/__init__.py b/src/agentscope/realtime/__init__.py
index 6b910e840..ebdd68892 100644
--- a/src/agentscope/realtime/__init__.py
+++ b/src/agentscope/realtime/__init__.py
@@ -5,28 +5,36 @@ The model side of a voice session: realtime models, their cards and
events, transports and VAD. The agent that ties them together is
:class:`agentscope.agent.RealtimeAgent`.
"""
-from ._base import ModelDisconnectedError, RealtimeModelBase
+from ._base import InteractionMode, ModelDisconnectedError, RealtimeModelBase
from ._dashscope import (
DashScopeAudioRealtimeModel,
DashScopeRealtimeModel,
)
from ._events import (
AudioDeltaEvent,
+ DelegatedResponseEvent,
+ DelegationCreatedEvent,
InputTranscriptionEvent,
InputTranscriptionFailedEvent,
ModelErrorEvent,
ModelEvent,
ResponseCreatedEvent,
ResponseDoneEvent,
+ SessionClosedEvent,
SessionEndedEvent,
+ SessionStartedEvent,
+ SessionUsageEvent,
SpeechEndedEvent,
SpeechStartedEvent,
+ TimelineAudioDeltaEvent,
+ TimelineTranscriptDeltaEvent,
ToolCallEvent,
TranscriptDeltaEvent,
)
from ._gemini import GeminiRealtimeModel
from ._model_card import RealtimeModelCard
from ._openai import OpenAIRealtimeModel
+from ._openai_live import OpenAILiveModel
from ._playout import PlayoutPosition
from ._transport import (
AudioFrame,
@@ -42,10 +50,12 @@ from ._xai import XAIRealtimeModel
__all__ = [
# Model
"RealtimeModelBase",
+ "InteractionMode",
"DashScopeRealtimeModel",
"DashScopeAudioRealtimeModel",
"GeminiRealtimeModel",
"OpenAIRealtimeModel",
+ "OpenAILiveModel",
"XAIRealtimeModel",
"RealtimeModelCard",
"ModelDisconnectedError",
@@ -63,13 +73,20 @@ __all__ = [
# Model events
"ModelEvent",
"SessionEndedEvent",
+ "SessionStartedEvent",
+ "SessionClosedEvent",
+ "SessionUsageEvent",
"SpeechStartedEvent",
"SpeechEndedEvent",
"InputTranscriptionEvent",
"InputTranscriptionFailedEvent",
"ResponseCreatedEvent",
"AudioDeltaEvent",
+ "TimelineAudioDeltaEvent",
"TranscriptDeltaEvent",
+ "TimelineTranscriptDeltaEvent",
+ "DelegationCreatedEvent",
+ "DelegatedResponseEvent",
"ToolCallEvent",
"ResponseDoneEvent",
"ModelErrorEvent",
diff --git a/src/agentscope/realtime/_base.py b/src/agentscope/realtime/_base.py
index 58a2bd37a..2c30bac0b 100644
--- a/src/agentscope/realtime/_base.py
+++ b/src/agentscope/realtime/_base.py
@@ -2,6 +2,7 @@
"""The realtime model base class."""
import inspect
from abc import ABC, abstractmethod
+from enum import StrEnum
from pathlib import Path
from typing import Any, AsyncIterator, Sequence
@@ -17,6 +18,13 @@ class ModelDisconnectedError(ConnectionError):
"""The provider closed the session; the next user audio reconnects."""
+class InteractionMode(StrEnum):
+ """How audio input and model output share the conversation timeline."""
+
+ TURN_BASED = "turn_based"
+ FULL_DUPLEX = "full_duplex"
+
+
class RealtimeModelBase(ABC):
"""A bidirectional realtime model: audio in, audio and tool calls out.
@@ -43,6 +51,9 @@ class RealtimeModelBase(ABC):
supports_history_replay: bool = False
"""Whether completed messages can seed a newly opened session."""
+ interaction_mode = InteractionMode.TURN_BASED
+ """Whether the adapter uses turns or a continuous full-duplex stream."""
+
def __init__(
self,
model: str,
@@ -121,6 +132,12 @@ class RealtimeModelBase(ABC):
)
cards = RealtimeModelCard.list_from_directory(yaml_dir, cls.Parameters)
for card in cards:
+ if card.interaction_mode != cls.interaction_mode:
+ raise ValueError(
+ f"Realtime model card {card.name!r} declares "
+ f"{card.interaction_mode!r}, but {cls.__name__} uses "
+ f"{cls.interaction_mode!r}.",
+ )
card.model_type = cls.type
return cards
@@ -235,3 +252,22 @@ class RealtimeModelBase(ABC):
@abstractmethod
async def cancel_response(self) -> None:
"""Stop the active reply without closing the session."""
+
+ async def interrupt_speech(self) -> None:
+ """Stop speech while preserving independent backend work.
+
+ Turn-based adapters cancel their active response. Full-duplex
+ adapters should override this method when speech and backend work
+ have separate lifecycles.
+ """
+ await self.cancel_response()
+
+ async def send_delegation_result(
+ self,
+ delegation_id: str,
+ content: str,
+ ) -> None:
+ """Return an application-managed task result to the voice model."""
+ raise NotImplementedError(
+ f"{type(self).__name__} does not support client delegation.",
+ )
diff --git a/src/agentscope/realtime/_events.py b/src/agentscope/realtime/_events.py
index ebfb4ad60..f65950f35 100644
--- a/src/agentscope/realtime/_events.py
+++ b/src/agentscope/realtime/_events.py
@@ -5,6 +5,8 @@ These are internal to :mod:`agentscope.realtime`: adapters normalise
vendor frames into them, and :class:`RealtimeAgent` translates them into
the public :class:`~agentscope.event.AgentEvent` stream.
"""
+from typing import Any, Literal
+
from pydantic import BaseModel, Field
from ..message import ToolCallBlock
@@ -20,6 +22,27 @@ class SessionEndedEvent(ModelEvent):
reason: str = ""
+class SessionStartedEvent(ModelEvent):
+ """A full-duplex provider session accepted its startup config."""
+
+ session_id: str
+
+
+class SessionClosedEvent(ModelEvent):
+ """A full-duplex provider session finalized normally."""
+
+ session_id: str
+ reason: str = ""
+ usage_seconds: float | None = None
+
+
+class SessionUsageEvent(ModelEvent):
+ """A cumulative full-duplex voice usage snapshot."""
+
+ seconds: float
+ context_usage_ratio: float | None = None
+
+
class SpeechStartedEvent(ModelEvent):
"""The provider's own VAD detected the user starting to speak."""
@@ -68,6 +91,40 @@ class TranscriptDeltaEvent(ModelEvent):
delta: str
+class TimelineTranscriptDeltaEvent(ModelEvent):
+ """A timestamped transcript fragment in a full-duplex session."""
+
+ role: Literal["user", "assistant"]
+ delta: str
+ start_ms: int
+ end_ms: int
+
+
+class TimelineAudioDeltaEvent(ModelEvent):
+ """An assistant audio fragment on the full-duplex session timeline."""
+
+ pcm: bytes
+ sample_rate: int
+ start_ms: int | None = None
+ end_ms: int | None = None
+
+
+class DelegationCreatedEvent(ModelEvent):
+ """GPT-Live started an independent backend task."""
+
+ delegation_id: str
+ target: Literal["responses", "client"]
+ response_id: str | None = None
+ offset_ms: int = 0
+
+
+class DelegatedResponseEvent(ModelEvent):
+ """One nested Responses event for a GPT-Live delegation."""
+
+ delegation_id: str
+ event: dict[str, Any]
+
+
class ToolCallEvent(ModelEvent):
"""A complete tool call requested by the provider."""
diff --git a/src/agentscope/realtime/_model_card.py b/src/agentscope/realtime/_model_card.py
index d7189c4e6..73216c8e9 100644
--- a/src/agentscope/realtime/_model_card.py
+++ b/src/agentscope/realtime/_model_card.py
@@ -64,6 +64,17 @@ class RealtimeModelCard(BaseModel):
default=False,
description="Whether this model accepts tool schemas.",
)
+ interaction_mode: Literal["turn_based", "full_duplex"] = Field(
+ default="turn_based",
+ description="Whether speech is turn-based or full duplex.",
+ )
+ delegation_backend: Literal["none", "session_chat"] = Field(
+ default="none",
+ description=(
+ "Whether delegated tasks use the session's configured chat "
+ "agent."
+ ),
+ )
# Context limits. Providers use different units and several can apply
# at once, so all are optional; ``None`` means unlimited or unknown.
diff --git a/src/agentscope/realtime/_openai_live/__init__.py b/src/agentscope/realtime/_openai_live/__init__.py
new file mode 100644
index 000000000..6fa34e616
--- /dev/null
+++ b/src/agentscope/realtime/_openai_live/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+"""The OpenAI GPT-Live model."""
+from ._model import OpenAILiveModel
+
+__all__ = ["OpenAILiveModel"]
diff --git a/src/agentscope/realtime/_openai_live/_model.py b/src/agentscope/realtime/_openai_live/_model.py
new file mode 100644
index 000000000..b42a5c27a
--- /dev/null
+++ b/src/agentscope/realtime/_openai_live/_model.py
@@ -0,0 +1,535 @@
+# -*- coding: utf-8 -*-
+"""The OpenAI GPT-Live WebSocket adapter."""
+import asyncio
+import base64
+import json
+import uuid
+from typing import Any, AsyncIterator
+
+from pydantic import Field
+
+from .. import _events as me
+from .._base import (
+ InteractionMode,
+ ModelDisconnectedError,
+ RealtimeModelBase,
+)
+from .._model_card import RealtimeModelCard
+from ..._logging import logger
+from ...credential import OpenAICredential
+from ...message import ToolResultBlock
+
+_OPENAI_BASE_URL = "https://api.openai.com/v1"
+_MAX_COMMENTARY_CHARS = 400
+
+
+class OpenAILiveModel(RealtimeModelBase):
+ """GPT-Live over a server-side primary WebSocket.
+
+ The adapter keeps Live audio, transcripts, and client-delegation metadata
+ independent. It does not synthesize turn boundaries that the provider
+ does not emit.
+ """
+
+ class Parameters(RealtimeModelBase.Parameters):
+ """GPT-Live voice and connection settings."""
+
+ voice: str = Field(
+ default="marin",
+ title="Voice",
+ description="The voice used for generated audio.",
+ )
+ store: bool = Field(
+ default=False,
+ title="Store Session",
+ description=(
+ "Whether to request provider-side storage for the Live "
+ "session."
+ ),
+ )
+ startup_timeout_s: float = Field(
+ default=15.0,
+ gt=0,
+ title="Startup Timeout",
+ description=(
+ "Maximum seconds to wait for the Live session to start."
+ ),
+ )
+ close_timeout_s: float = Field(
+ default=15.0,
+ gt=0,
+ title="Close Timeout",
+ description=(
+ "Maximum seconds to wait for graceful session closure."
+ ),
+ )
+
+ type = "openai_live"
+ """Adapter identifier stored in realtime model configurations."""
+
+ interaction_mode = InteractionMode.FULL_DUPLEX
+ """GPT-Live uses one continuous, full-duplex conversation timeline."""
+
+ supports_text_input = False
+ """Typed input is not exposed by the current GPT-Live adapter."""
+
+ @property
+ def input_transcription_enabled(self) -> bool:
+ """GPT-Live always emits timestamped input transcript fragments."""
+ return True
+
+ def __init__(
+ self,
+ model: str,
+ credential: OpenAICredential,
+ parameters: "OpenAILiveModel.Parameters | None" = None,
+ model_card: RealtimeModelCard | None = None,
+ ) -> None:
+ """Initialize the GPT-Live adapter.
+
+ Args:
+ model (`str`):
+ GPT-Live model identifier, for example ``"gpt-live-1"``.
+ credential (`OpenAICredential`):
+ OpenAI credential used to authenticate the WebSocket.
+ parameters (`OpenAILiveModel.Parameters | None`, optional):
+ Voice, storage, and connection timeout settings. Defaults
+ are used when omitted.
+ model_card (`RealtimeModelCard | None`, optional):
+ Model capabilities and audio formats. When omitted, the
+ bundled card matching ``model`` is used.
+
+ Raises:
+ ValueError:
+ If ``model_card`` is omitted and no bundled card matches
+ ``model``.
+ """
+ super().__init__(model, credential, parameters, model_card)
+ self.parameters: OpenAILiveModel.Parameters
+ self._ws: Any = None
+ self._reader: asyncio.Task | None = None
+ self._queue: asyncio.Queue[me.ModelEvent | None] = asyncio.Queue()
+ self._send_lock = asyncio.Lock()
+ self._started: asyncio.Future[str] | None = None
+ self._closed = asyncio.Event()
+ self._session_id = ""
+ self._finalized = False
+ self._closing = False
+ self._usage_seconds = 0.0
+ self._context_usage_ratio: float | None = None
+
+ @property
+ def usage_seconds(self) -> float:
+ """Return the latest cumulative voice usage snapshot."""
+ return self._usage_seconds
+
+ @property
+ def context_usage_ratio(self) -> float | None:
+ """Return the latest reported context-window usage ratio."""
+ return self._context_usage_ratio
+
+ async def connect(
+ self,
+ instructions: str,
+ tools: list[dict] | None = None,
+ **kwargs: Any,
+ ) -> None:
+ """Open the primary WebSocket and wait for ``session.started``.
+
+ Args:
+ instructions (`str`):
+ System instructions sent in the immutable session startup
+ configuration.
+ tools (`list[dict] | None`, optional):
+ Ignored because GPT-Live delegates tool work to the client
+ session's Chat Agent.
+ **kwargs (`Any`):
+ Optional connection metadata. ``safety_identifier`` is sent
+ as the ``OpenAI-Safety-Identifier`` header when present.
+
+ Raises:
+ asyncio.TimeoutError:
+ If sending startup and receiving its acknowledgement
+ together exceed ``startup_timeout_s``.
+ ModelDisconnectedError:
+ If the connection closes before startup completes.
+ asyncio.CancelledError:
+ If startup is cancelled, after releasing the connection.
+ """
+ import websockets
+
+ await self.close()
+ self._drain_queue()
+ self._closed = asyncio.Event()
+ self._session_id = ""
+ self._finalized = False
+ self._closing = False
+ self._usage_seconds = 0.0
+ self._context_usage_ratio = None
+ self._started = asyncio.get_running_loop().create_future()
+
+ credential: OpenAICredential = self.credential # type: ignore
+ base_url = (credential.base_url or _OPENAI_BASE_URL).rstrip("/")
+ url = base_url.replace("http", "ws", 1)
+ headers = {
+ "Authorization": (
+ f"Bearer {credential.api_key.get_secret_value()}"
+ ),
+ }
+ if credential.organization:
+ headers["OpenAI-Organization"] = credential.organization
+ safety_identifier = kwargs.get("safety_identifier")
+ if safety_identifier:
+ headers["OpenAI-Safety-Identifier"] = safety_identifier
+
+ try:
+ self._ws = await websockets.connect(
+ f"{url}/live/sessions",
+ additional_headers=headers,
+ )
+ self._reader = asyncio.create_task(
+ self._read(),
+ name="openai-live",
+ )
+ async with asyncio.timeout(self.parameters.startup_timeout_s):
+ await self._send(
+ self._session_start(instructions),
+ )
+ await asyncio.shield(self._started)
+ except (Exception, asyncio.CancelledError):
+ await self._abort()
+ raise
+
+ async def close(self) -> None:
+ """Gracefully finalize usage, then release the WebSocket."""
+ if self._ws is None and self._reader is None:
+ return
+ self._closing = True
+ if self._ws is not None and self._session_id and not self._finalized:
+ try:
+ await self._send({"type": "session.close"})
+ await asyncio.wait_for(
+ self._closed.wait(),
+ timeout=self.parameters.close_timeout_s,
+ )
+ except (asyncio.TimeoutError, ModelDisconnectedError):
+ logger.warning(
+ "OpenAILiveModel: session close was not finalized",
+ )
+ await self._abort()
+
+ async def events(self) -> AsyncIterator[me.ModelEvent]:
+ """Yield normalized Live events until the connection ends."""
+ while True:
+ event = await self._queue.get()
+ if event is None:
+ return
+ yield event
+
+ async def push_audio(self, pcm: bytes) -> None:
+ """Append complete PCM16 samples to the Live input stream.
+
+ Args:
+ pcm (`bytes`):
+ Little-endian PCM16 audio at :attr:`input_sample_rate`.
+
+ Raises:
+ ValueError:
+ If ``pcm`` ends with an incomplete 16-bit sample.
+ ModelDisconnectedError:
+ If the Live session has not started or is disconnected.
+ """
+ if len(pcm) % 2:
+ raise ValueError("PCM16 audio must contain complete samples.")
+ if not self._session_id:
+ raise ModelDisconnectedError("GPT-Live session is not started.")
+ await self._send(
+ {
+ "type": "session.input_audio.append",
+ "audio": base64.b64encode(pcm).decode("ascii"),
+ },
+ )
+
+ async def push_text(self, text: str) -> None:
+ """Reject typed input until its delegation semantics are exposed."""
+ raise NotImplementedError(
+ f"{type(self).__name__} does not expose typed input yet.",
+ )
+
+ async def push_tool_result(self, block: ToolResultBlock) -> None:
+ """Reject direct tools because the session chat agent owns them."""
+ raise NotImplementedError(
+ f"{type(self).__name__} delegates tools to the session agent.",
+ )
+
+ async def commit_turn(self) -> None:
+ """Reject turn commits because GPT-Live owns a continuous stream."""
+ raise NotImplementedError("GPT-Live has no user turn commit.")
+
+ async def request_response(self) -> None:
+ """Reject Responses control because this adapter uses client mode."""
+ raise NotImplementedError(
+ f"{type(self).__name__} uses client delegation.",
+ )
+
+ async def cancel_response(self) -> None:
+ """Reject Realtime-style cancellation for the Live frontend."""
+ raise NotImplementedError(
+ "GPT-Live speech and backend work have separate lifecycles.",
+ )
+
+ async def interrupt_speech(self) -> None:
+ """Ask the Live frontend to stop speaking without cancelling work."""
+ await self._send(
+ {
+ "type": "session.instructions.append",
+ "event_id": self._event_id("interrupt"),
+ "delegation_id": None,
+ "content": "Stop speaking now and listen to the user.",
+ },
+ )
+
+ async def send_delegation_result(
+ self,
+ delegation_id: str,
+ content: str,
+ ) -> None:
+ """Append a verified client-delegation result for speech.
+
+ Long results are split into ordered commentary commands before they
+ are returned to the Live session.
+
+ Args:
+ delegation_id (`str`):
+ Identifier from the corresponding delegation-created event.
+ content (`str`):
+ Verified task result for GPT-Live to communicate.
+
+ Raises:
+ ValueError:
+ If either argument is empty.
+ ModelDisconnectedError:
+ If the Live WebSocket is not connected.
+ """
+ if not delegation_id:
+ raise ValueError("A GPT-Live delegation id is required.")
+ if not content.strip():
+ raise ValueError("A GPT-Live delegation result is required.")
+ for chunk in self._commentary_chunks(content):
+ await self._send(
+ {
+ "type": "session.commentary.append",
+ "event_id": self._event_id("delegation_result"),
+ "delegation_id": delegation_id,
+ "content": chunk,
+ },
+ )
+
+ @staticmethod
+ def _commentary_chunks(content: str) -> list[str]:
+ """Split a result conservatively below Live's 500-token limit."""
+ remaining = content.strip()
+ chunks: list[str] = []
+ while remaining:
+ if len(remaining) <= _MAX_COMMENTARY_CHARS:
+ chunks.append(remaining)
+ break
+ boundary = remaining.rfind(" ", 0, _MAX_COMMENTARY_CHARS + 1)
+ if boundary < _MAX_COMMENTARY_CHARS // 2:
+ boundary = _MAX_COMMENTARY_CHARS
+ chunks.append(remaining[:boundary].strip())
+ remaining = remaining[boundary:].strip()
+ return chunks
+
+ def _session_start(
+ self,
+ instructions: str,
+ ) -> dict[str, Any]:
+ """Build the immutable GPT-Live startup configuration."""
+ session = {
+ "model": self.model,
+ "instructions": instructions,
+ "input": [],
+ "audio": {
+ "format": {
+ "type": "audio/pcm",
+ "rate": self.input_sample_rate,
+ },
+ "output": {"voice": self.parameters.voice},
+ },
+ "delegation": {"type": "client"},
+ "store": self.parameters.store,
+ }
+ return {
+ "type": "session.start",
+ "event_id": self._event_id("start"),
+ "session": session,
+ }
+
+ async def _send(self, payload: dict[str, Any]) -> None:
+ """Send one ordered JSON command."""
+ from websockets.exceptions import ConnectionClosed
+
+ if self._ws is None:
+ raise ModelDisconnectedError("GPT-Live is not connected.")
+ try:
+ async with self._send_lock:
+ await self._ws.send(
+ json.dumps(payload, ensure_ascii=False),
+ )
+ except ConnectionClosed as exc:
+ raise ModelDisconnectedError(str(exc.rcvd or exc)) from exc
+
+ async def _read(self) -> None:
+ """Run the connection's only receive loop."""
+ try:
+ async for raw in self._ws:
+ if isinstance(raw, bytes):
+ raw = raw.decode("utf-8")
+ try:
+ event = self._parse(json.loads(raw))
+ except Exception: # noqa: BLE001
+ logger.exception("OpenAILiveModel: invalid server frame")
+ continue
+ if event is not None:
+ self._queue.put_nowait(event)
+ except Exception as exc: # noqa: BLE001
+ logger.error("OpenAILiveModel: connection lost: %s", exc)
+ finally:
+ if not self._finalized and not self._closing:
+ self._queue.put_nowait(
+ me.SessionEndedEvent(reason="connection_closed"),
+ )
+ if self._started is not None and not self._started.done():
+ self._started.set_exception(
+ ModelDisconnectedError(
+ "Connection closed before session.started.",
+ ),
+ )
+ self._closed.set()
+ self._queue.put_nowait(None)
+
+ # pylint: disable=too-many-return-statements
+ def _parse(self, data: dict[str, Any]) -> me.ModelEvent | None:
+ """Translate one GPT-Live server event."""
+ kind = data.get("type", "")
+
+ if kind == "session.started":
+ session_id = data.get("session", {}).get("id", "")
+ self._session_id = session_id
+ if self._started is not None and not self._started.done():
+ self._started.set_result(session_id)
+ return me.SessionStartedEvent(session_id=session_id)
+
+ if kind in {
+ "session.input_transcript.delta",
+ "session.output_transcript.delta",
+ }:
+ delta = data.get("delta", "")
+ if not delta:
+ return None
+ role = "user" if ".input_" in kind else "assistant"
+ return me.TimelineTranscriptDeltaEvent(
+ role=role,
+ delta=delta,
+ start_ms=data.get("start_ms", 0),
+ end_ms=data.get("end_ms", 0),
+ )
+
+ if kind == "session.output_audio.delta":
+ delta = data.get("delta", "")
+ if not delta:
+ return None
+ return me.TimelineAudioDeltaEvent(
+ pcm=base64.b64decode(delta),
+ sample_rate=self.output_sample_rate,
+ start_ms=data.get("start_ms"),
+ end_ms=data.get("end_ms"),
+ )
+
+ if kind == "session.delegation.created":
+ delegation = data.get("delegation", {})
+ return me.DelegationCreatedEvent(
+ delegation_id=delegation.get("id", ""),
+ target=delegation.get("target", "responses"),
+ response_id=delegation.get("response_id"),
+ offset_ms=data.get("offset_ms", 0),
+ )
+
+ if kind == "response.event":
+ return me.DelegatedResponseEvent(
+ delegation_id=data.get("delegation_id", ""),
+ event=data.get("event", {}),
+ )
+
+ if kind == "session.usage.updated":
+ usage = data.get("usage", {})
+ self._usage_seconds = float(usage.get("seconds", 0.0))
+ self._context_usage_ratio = data.get("context_window", {}).get(
+ "usage_ratio",
+ )
+ return me.SessionUsageEvent(
+ seconds=self._usage_seconds,
+ context_usage_ratio=self._context_usage_ratio,
+ )
+
+ if kind == "session.closed":
+ usage = data.get("usage", {})
+ self._usage_seconds = float(
+ usage.get("seconds", self._usage_seconds),
+ )
+ self._finalized = True
+ self._closed.set()
+ return me.SessionClosedEvent(
+ session_id=data.get("session", {}).get(
+ "id",
+ self._session_id,
+ ),
+ reason=data.get("reason", ""),
+ usage_seconds=self._usage_seconds,
+ )
+
+ if kind == "error":
+ error = data.get("error", {})
+ message = error.get("message", "")
+ if self._started is not None and not self._started.done():
+ self._started.set_exception(RuntimeError(message))
+ return me.ModelErrorEvent(
+ code=error.get("code") or error.get("type", ""),
+ message=message,
+ )
+
+ logger.debug("OpenAILiveModel: ignoring %s", kind)
+ return None
+
+ async def _abort(self) -> None:
+ """Release transport resources without waiting for finalization."""
+ started = self._started
+ self._started = None
+ if started is not None:
+ if not started.done():
+ started.cancel()
+ elif not started.cancelled():
+ # Startup may have failed before connect began awaiting it.
+ started.exception()
+ ws = self._ws
+ self._ws = None
+ if ws is not None:
+ await ws.close()
+ reader = self._reader
+ self._reader = None
+ if reader is not None and reader is not asyncio.current_task():
+ if not reader.done():
+ reader.cancel()
+ await asyncio.gather(reader, return_exceptions=True)
+ self._session_id = ""
+
+ def _drain_queue(self) -> None:
+ """Discard terminal events left by a previous connection."""
+ while not self._queue.empty():
+ self._queue.get_nowait()
+
+ @staticmethod
+ def _event_id(prefix: str) -> str:
+ """Build a unique client event identifier."""
+ return f"{prefix}_{uuid.uuid4().hex}"
diff --git a/src/agentscope/realtime/_openai_live/_models/gpt-live-1.yaml b/src/agentscope/realtime/_openai_live/_models/gpt-live-1.yaml
new file mode 100644
index 000000000..aa1ed0301
--- /dev/null
+++ b/src/agentscope/realtime/_openai_live/_models/gpt-live-1.yaml
@@ -0,0 +1,20 @@
+type: realtime_model
+name: gpt-live-1
+label: GPT Live 1
+status: active
+interaction_mode: full_duplex
+delegation_backend: session_chat
+
+input_types:
+ - audio/pcm
+output_types:
+ - audio/pcm
+ - text/plain
+
+input_sample_rate: 24000
+output_sample_rate: 24000
+
+# Tools run through the session's configured chat agent.
+supports_tools: false
+
+max_context_tokens: 128000
diff --git a/src/agentscope/realtime/_transport/_base.py b/src/agentscope/realtime/_transport/_base.py
index 425159a39..dc92b5ab4 100644
--- a/src/agentscope/realtime/_transport/_base.py
+++ b/src/agentscope/realtime/_transport/_base.py
@@ -31,6 +31,9 @@ class ControlFrameType(StrEnum):
USER_CONFIRM = "user_confirm"
"""The outcome of a tool permission prompt."""
+ EXTERNAL_EXECUTION_RESULT = "external_execution_result"
+ """The result returned by an external tool executor."""
+
INTERRUPT = "interrupt"
"""The user pressed stop, as opposed to speaking over the reply."""
diff --git a/src/agentscope/state/_state.py b/src/agentscope/state/_state.py
index e827e52aa..e14984102 100644
--- a/src/agentscope/state/_state.py
+++ b/src/agentscope/state/_state.py
@@ -295,6 +295,29 @@ class AgentState(BaseModel):
"""The context that allow the middlewares to store/get data across
different replies."""
+ def get_reply_message(self, name: str) -> Msg | None:
+ """Find this agent's current reply, even after later messages.
+
+ Args:
+ name (`str`):
+ Author of the assistant reply to find.
+
+ Returns:
+ `Msg | None`:
+ The assistant message matching both ``name`` and the current
+ ``reply_id``, or ``None`` if it has not been created.
+ """
+ return next(
+ (
+ message
+ for message in reversed(self.context)
+ if message.id == self.reply_id
+ and message.role == "assistant"
+ and message.name == name
+ ),
+ None,
+ )
+
def append_context(
self,
name: str,
@@ -306,14 +329,9 @@ class AgentState(BaseModel):
`reply_id`. If such message doesn't exist, a new assistant message
with agent's name and current reply ID will be created.
"""
- # If append to the latest message
- if (
- self.context
- and self.context[-1].role == "assistant"
- and self.context[-1].name == name
- and self.context[-1].id == self.reply_id
- ):
- self.context[-1].content.extend(blocks)
+ message = self.get_reply_message(name)
+ if message is not None:
+ message.content.extend(blocks)
else:
# Create a new assistant message with the current reply ID
self.context.append(
@@ -326,7 +344,7 @@ class AgentState(BaseModel):
)
def has_awaiting_tool_calls(self, name: str) -> bool:
- """Whether the tail assistant message written by ``name`` has
+ """Whether the current reply written by ``name`` has
any tool call still awaiting an outside response — an
``ASKING`` user confirmation or a ``SUBMITTED`` external
execution with no matching tool result yet.
@@ -343,7 +361,7 @@ class AgentState(BaseModel):
return bool(self.get_awaiting_tool_calls(name))
def get_awaiting_tool_calls(self, name: str) -> list[ToolCallBlock]:
- """Get the tail assistant message's tool calls still awaiting an
+ """Get the current reply's tool calls still awaiting an
outside response — an ``ASKING`` user confirmation or a
``SUBMITTED`` external execution with no matching tool result yet.
@@ -356,10 +374,8 @@ class AgentState(BaseModel):
`list[ToolCallBlock]`:
The awaiting tool call blocks, empty if none.
"""
- if not self.context:
- return []
- last_msg = self.context[-1]
- if last_msg.role != "assistant" or last_msg.name != name:
+ last_msg = self.get_reply_message(name)
+ if last_msg is None:
return []
result_ids = {b.id for b in last_msg.get_content_blocks("tool_result")}
return [
@@ -386,14 +402,8 @@ class AgentState(BaseModel):
`list[ToolCallBlock]`:
The unfinished tool call blocks, empty if none.
"""
- if not self.context:
- return []
- last_msg = self.context[-1]
- if (
- last_msg.role != "assistant"
- or last_msg.name != name
- or last_msg.id != self.reply_id
- ):
+ last_msg = self.get_reply_message(name)
+ if last_msg is None:
return []
result_ids = {b.id for b in last_msg.get_content_blocks("tool_result")}
return [
diff --git a/tests/channel_gateway_test.py b/tests/channel_gateway_test.py
index 5a006de00..ec86c8a5a 100644
--- a/tests/channel_gateway_test.py
+++ b/tests/channel_gateway_test.py
@@ -655,6 +655,7 @@ class _AwaitingStorage:
reply_id="reply-1",
context=[
Msg(
+ id="reply-1",
name="Friday",
role="assistant",
content=[
diff --git a/tests/chat_input_queue_test.py b/tests/chat_input_queue_test.py
index 168695d32..2c9625478 100644
--- a/tests/chat_input_queue_test.py
+++ b/tests/chat_input_queue_test.py
@@ -564,6 +564,7 @@ class TestQueueFailureScope(IsolatedAsyncioTestCase):
],
),
)
+ session.state.reply_id = session.state.context[0].id
storage.get_session = AsyncMock(return_value=session)
policy = SimpleNamespace(
list_accessible=AsyncMock(
@@ -648,6 +649,7 @@ class TestQueueFailureScope(IsolatedAsyncioTestCase):
context=[AssistantMsg(name=name, content=blocks)],
),
)
+ session.state.reply_id = session.state.context[0].id
self.assertEqual(
await queue._is_session_parked(session, "u", "a"),
expected,
diff --git a/tests/realtime_checkpoint_recovery_test.py b/tests/realtime_checkpoint_recovery_test.py
new file mode 100644
index 000000000..ecd811194
--- /dev/null
+++ b/tests/realtime_checkpoint_recovery_test.py
@@ -0,0 +1,579 @@
+# -*- coding: utf-8 -*-
+"""Recover parked Live delegations through a newly constructed Chat Agent."""
+# pylint: disable=protected-access
+
+import asyncio
+from typing import Any
+from unittest import IsolatedAsyncioTestCase, TestCase
+from unittest.mock import AsyncMock, patch
+
+from utils import MockModel
+
+from agentscope.agent import Agent, RealtimeAgent
+from agentscope.app._service._webrtc_audio_transport import (
+ WebRTCAudioTransport,
+)
+from agentscope.app._service._webrtc_session import WebRTCSession
+from agentscope.app.message_bus import InMemoryMessageBus, MessageBusKeys
+from agentscope.credential import OpenAICredential
+from agentscope.event import (
+ ConfirmResult,
+ ExternalExecutionResultEvent,
+ RequireExternalExecutionEvent,
+ RequireUserConfirmEvent,
+ ToolResultEndEvent,
+ UserConfirmResultEvent,
+ UserInterruptEvent,
+)
+from agentscope.message import (
+ AssistantMsg,
+ TextBlock,
+ ToolCallBlock,
+ ToolResultBlock,
+ ToolResultState,
+ UserMsg,
+)
+from agentscope.model import ChatResponse, ChatUsage
+from agentscope.realtime import OpenAILiveModel, TimelineTranscriptDeltaEvent
+from agentscope.state import AgentState
+from agentscope.tool import FunctionTool, Toolkit
+
+
+class RealtimeCheckpointRecoveryTest(IsolatedAsyncioTestCase):
+ """Exercise serialized checkpoints rather than the in-memory backend."""
+
+ async def test_interrupted_tool_end_checkpoint_has_its_result(
+ self,
+ ) -> None:
+ """Restore interruption from END without relying on ReplyEnd."""
+
+ def write() -> str:
+ """Never execute a tool whose approval is interrupted."""
+ self.fail("The interrupted tool must not execute.")
+
+ toolkit = Toolkit(tools=[FunctionTool(write)])
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[
+ ToolCallBlock(id="write", name="write", input="{}"),
+ ],
+ is_last=True,
+ ),
+ ],
+ )
+
+ def factory(state: AgentState) -> Agent:
+ """Create a real backend that waits for user approval."""
+ return Agent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ toolkit=toolkit,
+ )
+
+ live_model = OpenAILiveModel(
+ "gpt-live-1",
+ OpenAICredential(api_key="test"),
+ )
+ live_model.send_delegation_result = AsyncMock()
+ live = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ live_model,
+ delegation_agent_factory=factory,
+ )
+ self.addAsyncCleanup(live.close)
+ live._start_client_delegation("task")
+ await asyncio.gather(*list(live._tasks))
+ delegation = live._delegations["task"]
+ assert delegation.agent is not None
+ list(live.drain_events())
+ await live._drive_client_delegation(
+ delegation,
+ UserInterruptEvent(reply_id=delegation.agent.state.reply_id),
+ )
+ restored = None
+ for event in live.drain_events():
+ if isinstance(event, ToolResultEndEvent):
+ self.assertIsNone(restored)
+ snapshot = live.checkpoint_snapshot(event)
+ assert snapshot is not None
+ restored = AgentState.model_validate_json(
+ snapshot.model_dump_json(),
+ )
+ assert restored is not None
+ reply = restored.get_reply_message("Friday")
+ assert reply is not None
+ self.assertEqual(
+ {
+ "calls": [
+ (block.id, block.state)
+ for block in reply.get_content_blocks("tool_call")
+ ],
+ "results": [
+ (block.id, block.state, block.output)
+ for block in reply.get_content_blocks("tool_result")
+ ],
+ },
+ {
+ "calls": [("write", "finished")],
+ "results": [
+ (
+ "write",
+ "interrupted",
+ "<system-reminder>The tool call has been interrupted "
+ "by the user.</system-reminder>",
+ ),
+ ],
+ },
+ )
+
+ expected = reply.model_dump(mode="json")
+ model.set_responses(
+ [ChatResponse(content=[TextBlock(text="Done")], is_last=True)],
+ )
+ recovered = factory(restored)
+ await recovered.reply(UserMsg("user", "Continue."))
+ self.assertEqual(
+ restored.context[0].model_dump(mode="json"),
+ expected,
+ )
+
+ async def test_hidden_tool_checkpoint_before_next_model_response(
+ self,
+ ) -> None:
+ """Persist and restore completed work while reasoning is blocked."""
+ waiting = asyncio.Event()
+ persisted = asyncio.Event()
+ executed: list[str] = []
+ saved: list[str] = []
+
+ def read() -> str:
+ """Record one automatically allowed execution."""
+ executed.append("read")
+ return "contents"
+
+ toolkit = Toolkit(tools=[FunctionTool(read, is_read_only=True)])
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[
+ ToolCallBlock(id="read", name="read", input="{}"),
+ ],
+ is_last=True,
+ ),
+ ],
+ )
+ call_api = model._call_api
+
+ async def blocked_api(*args: Any, **kwargs: Any) -> Any:
+ """Hold the response after tool execution until cancellation."""
+ if model.cnt:
+ waiting.set()
+ await asyncio.Event().wait()
+ return await call_api(*args, **kwargs)
+
+ def factory(state: AgentState) -> Agent:
+ """Use a real backend with a deterministic model boundary."""
+ return Agent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ toolkit=toolkit,
+ )
+
+ async def save_state(**kwargs: Any) -> None:
+ """Serialize at the actual WebRTC persistence boundary."""
+ saved.append(kwargs["state"].model_dump_json())
+ persisted.set()
+
+ live_model = OpenAILiveModel(
+ "gpt-live-1",
+ OpenAICredential(api_key="test"),
+ )
+ live_model.send_delegation_result = AsyncMock()
+ live = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ live_model,
+ delegation_agent_factory=factory,
+ )
+ transport = WebRTCAudioTransport(
+ live_model.input_sample_rate,
+ live_model.output_sample_rate,
+ )
+ storage = AsyncMock()
+ storage.update_session_state.side_effect = save_state
+ bus = InMemoryMessageBus()
+ session = WebRTCSession(
+ peer_connection=AsyncMock(),
+ transport=transport,
+ agent_factory=AsyncMock(return_value=live),
+ storage=storage,
+ message_bus=bus,
+ user_id="user",
+ agent_id="agent",
+ session_id="session",
+ on_closed=lambda _: None,
+ )
+ with (
+ patch.object(model, "_call_api", side_effect=blocked_api),
+ patch.object(live, "connect", AsyncMock()),
+ patch.object(bus, "publish", wraps=bus.publish) as publish,
+ ):
+ session.start()
+ try:
+ self.assertTrue(await session.wait_until_lock_acquired(1))
+ live._start_client_delegation("task")
+ await asyncio.wait_for(waiting.wait(), timeout=1)
+ await asyncio.wait_for(persisted.wait(), timeout=1)
+ delegation = live._delegations["task"]
+ assert delegation.agent is not None
+ reply = delegation.agent.state.get_reply_message("Friday")
+ assert reply is not None
+ expected = reply.model_copy(deep=True)
+ expected.content = [
+ block
+ for block in expected.content
+ if block.type in {"tool_call", "tool_result"}
+ ]
+ restored = AgentState.model_validate_json(saved[0])
+ self.assertEqual(
+ {
+ "visible": delegation.visible,
+ "executed": executed,
+ "context": restored.model_dump(mode="json")["context"],
+ "unfinished": restored.get_unfinished_tool_calls(
+ "Friday",
+ ),
+ "events": [
+ (call.args[1]["name"], call.args[1]["value"])
+ for call in publish.await_args_list
+ ],
+ },
+ {
+ "visible": False,
+ "executed": ["read"],
+ "context": [expected.model_dump(mode="json")],
+ "unfinished": [],
+ "events": [
+ (
+ "delegation_checkpoint",
+ {
+ "delegation_id": "task",
+ "reply_id": reply.id,
+ "tool_call_id": "read",
+ },
+ ),
+ ],
+ },
+ )
+ self.assertEqual(
+ live.state.model_dump(mode="json")["context"],
+ restored.model_dump(mode="json")["context"],
+ )
+ self.assertEqual(
+ await bus.registry_get(
+ MessageBusKeys.session_event_checkpoint("session"),
+ MessageBusKeys.SESSION_EVENT_CURSOR_FIELD,
+ ),
+ publish.await_args.args[1]["_entry_id"],
+ )
+ self.assertFalse(session._task.done())
+ live_model.send_delegation_result.assert_not_awaited()
+ recovery_model = MockModel(context_size=10000)
+ recovery_model.set_responses(
+ [
+ ChatResponse(
+ content=[TextBlock(text="Done")],
+ is_last=True,
+ ),
+ ],
+ )
+ recovered = Agent(
+ "Friday",
+ "Be brief.",
+ recovery_model,
+ state=restored,
+ toolkit=toolkit,
+ )
+ await recovered.reply(UserMsg("user", "Continue."))
+ self.assertEqual(executed, ["read"])
+ self.assertEqual(
+ restored.context[0].model_dump(mode="json"),
+ expected.model_dump(mode="json"),
+ )
+ finally:
+ await session.close()
+
+ async def _checkpoint(
+ self,
+ external: bool,
+ voice_role: str | None,
+ ) -> tuple[AgentState, ToolCallBlock, Toolkit, list[str], dict]:
+ """Park a real delegate, optionally append speech, then persist."""
+ executed: list[str] = []
+ backends: list[Agent] = []
+
+ def write() -> str:
+ """Record execution of the approved tool."""
+ executed.append("write")
+ return "written"
+
+ tool = FunctionTool(write, is_read_only=external)
+ tool.is_external_tool = external
+ toolkit = Toolkit(tools=[tool])
+
+ def factory(state: AgentState) -> Agent:
+ """Create a deterministic backend that requests one tool."""
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[
+ ToolCallBlock(
+ id="write",
+ name="write",
+ input="{}",
+ ),
+ ],
+ is_last=True,
+ ),
+ ],
+ )
+ backend = Agent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ toolkit=toolkit,
+ )
+ backends.append(backend)
+ return backend
+
+ live_model = OpenAILiveModel(
+ "gpt-live-1",
+ OpenAICredential(api_key="test"),
+ )
+ live_model.send_delegation_result = AsyncMock()
+ live = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ live_model,
+ delegation_agent_factory=factory,
+ )
+ self.addAsyncCleanup(live.close)
+ live._start_client_delegation("task")
+ while live._tasks:
+ await asyncio.gather(*list(live._tasks))
+ if voice_role is not None:
+ await live._on_model_event(
+ TimelineTranscriptDeltaEvent(
+ role=voice_role,
+ delta="Later speech",
+ start_ms=0,
+ end_ms=100,
+ ),
+ )
+ snapshot = None
+ call = None
+ for event in live.drain_events():
+ checkpoint = live.checkpoint_snapshot(event)
+ if checkpoint is not None:
+ snapshot = checkpoint
+ if isinstance(
+ event,
+ (
+ RequireUserConfirmEvent,
+ RequireExternalExecutionEvent,
+ ),
+ ):
+ call = event.tool_calls[0].model_copy(deep=True)
+ assert snapshot is not None and call is not None
+ reply_context = backends[0].state.reply_context.model_dump(mode="json")
+ restored = AgentState.model_validate_json(snapshot.model_dump_json())
+ await live.close()
+ return restored, call, toolkit, executed, reply_context
+
+ async def test_checkpoint_keeps_reply_context(self) -> None:
+ """The persisted reply metadata must match the parked backend."""
+ state, _, _, _, expected = await self._checkpoint(False, None)
+ self.assertEqual(state.reply_context.model_dump(mode="json"), expected)
+
+ async def test_chat_recovers_parked_tools_after_speech(self) -> None:
+ """Resume each input kind on its original reply, preserving speech."""
+ for action in (
+ "accept",
+ "deny",
+ "external",
+ "interrupt",
+ "interrupt_external",
+ ):
+ for voice_role in (None, "user", "assistant"):
+ with self.subTest(action=action, voice_role=voice_role):
+ (
+ state,
+ call,
+ toolkit,
+ executed,
+ reply_context,
+ ) = await self._checkpoint(
+ action in {"external", "interrupt_external"},
+ voice_role,
+ )
+ reply_id = reply_context["reply_id"]
+ original_ids = [msg.id for msg in state.context]
+ speech = [
+ msg.model_dump()
+ for msg in state.context
+ if msg.id != reply_id
+ ]
+ self.assertTrue(state.has_awaiting_tool_calls("Friday"))
+ if action.startswith("interrupt"):
+ event = UserInterruptEvent(reply_id=reply_id)
+ expected_result = ToolResultState.INTERRUPTED
+ elif action == "external":
+ event = ExternalExecutionResultEvent(
+ reply_id=reply_id,
+ execution_results=[
+ ToolResultBlock(
+ id=call.id,
+ name=call.name,
+ output="written",
+ state=ToolResultState.SUCCESS,
+ ),
+ ],
+ )
+ expected_result = ToolResultState.SUCCESS
+ else:
+ event = UserConfirmResultEvent(
+ reply_id=reply_id,
+ confirm_results=[
+ ConfirmResult(
+ confirmed=action == "accept",
+ tool_call=call,
+ ),
+ ],
+ )
+ expected_result = (
+ ToolResultState.SUCCESS
+ if action == "accept"
+ else ToolResultState.DENIED
+ )
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[TextBlock(text="Done")],
+ is_last=True,
+ usage=ChatUsage(
+ input_tokens=10,
+ output_tokens=2,
+ time=0.1,
+ ),
+ ),
+ ],
+ )
+ agent = Agent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ toolkit=toolkit,
+ )
+ events = [item async for item in agent.reply_stream(event)]
+ reply = next(
+ msg for msg in state.context if msg.id == reply_id
+ )
+ self.assertEqual(
+ {
+ "ids": [msg.id for msg in state.context],
+ "speech": [
+ msg.model_dump()
+ for msg in state.context
+ if msg.id != reply_id
+ ],
+ "calls": [
+ (block.id, block.state)
+ for block in reply.get_content_blocks(
+ "tool_call",
+ )
+ ],
+ "results": [
+ (block.id, block.state)
+ for block in reply.get_content_blocks(
+ "tool_result",
+ )
+ ],
+ "event_reply_ids": {
+ item.reply_id
+ for item in events
+ if item.reply_id
+ },
+ "pending": state.get_awaiting_tool_calls("Friday"),
+ "executed": executed,
+ "usage": (
+ reply.usage.input_tokens if reply.usage else 0
+ ),
+ },
+ {
+ "ids": original_ids,
+ "speech": speech,
+ "calls": [(call.id, "finished")],
+ "results": [(call.id, expected_result)],
+ "event_reply_ids": {reply_id},
+ "pending": [],
+ "executed": ["write"]
+ if action == "accept"
+ else [],
+ "usage": 0
+ if action.startswith("interrupt")
+ else 10,
+ },
+ )
+
+
+class ReplyMessageLookupTest(TestCase):
+ """Match both reply ID and author without reviving historical calls."""
+
+ def test_current_reply_owns_its_blocks(self) -> None:
+ """Append to the matching reply and ignore unrelated tool calls."""
+ current = AssistantMsg("Friday", [], id="current")
+ history = AssistantMsg(
+ "Friday",
+ [
+ ToolCallBlock(
+ id="stale",
+ name="write",
+ input="{}",
+ state="asking",
+ ),
+ ],
+ id="history",
+ )
+ other = AssistantMsg("Other", [], id="current")
+ speech = UserMsg("user", "Later speech")
+ state = AgentState(context=[history, current, other, speech])
+ state.reply_id = "current"
+ block = TextBlock(text="Done")
+ state.append_context("Friday", [block])
+ self.assertEqual(
+ {
+ "context": [msg.model_dump() for msg in state.context],
+ "pending": state.get_awaiting_tool_calls("Friday"),
+ },
+ {
+ "context": [
+ history.model_dump(),
+ {**current.model_dump(), "content": [block.model_dump()]},
+ other.model_dump(),
+ speech.model_dump(),
+ ],
+ "pending": [],
+ },
+ )
diff --git a/tests/realtime_delegation_test.py b/tests/realtime_delegation_test.py
new file mode 100644
index 000000000..f6836c1b8
--- /dev/null
+++ b/tests/realtime_delegation_test.py
@@ -0,0 +1,610 @@
+# -*- coding: utf-8 -*-
+"""Test Live delegation deduplication, result validation, and recovery."""
+# pylint: disable=protected-access
+
+import asyncio
+import json
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, call, patch
+
+from utils import MockModel
+
+from agentscope.agent import Agent, RealtimeAgent
+from agentscope.credential import OpenAICredential
+from agentscope.event import (
+ ConfirmResult,
+ CustomEvent,
+ ExternalExecutionResultEvent,
+ ToolResultEndEvent,
+ UserConfirmResultEvent,
+)
+from agentscope.message import TextBlock, ToolCallBlock, ToolResultBlock
+from agentscope.model import ChatResponse
+from agentscope.realtime import OpenAILiveModel
+from agentscope.state import AgentState
+from agentscope.tool import AskUser, FunctionTool, Toolkit
+
+
+def _create_live_agent(model: MockModel, toolkit: Toolkit) -> RealtimeAgent:
+ """Build a Live agent backed by a real Agent and a deterministic model."""
+
+ def factory(state: AgentState) -> Agent:
+ """Use real permission checks and external-result validation."""
+ return Agent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ toolkit=toolkit,
+ )
+
+ live_model = OpenAILiveModel(
+ "gpt-live-1",
+ OpenAICredential(api_key="test"),
+ )
+ live_model.send_delegation_result = AsyncMock()
+ return RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ live_model,
+ delegation_agent_factory=factory,
+ )
+
+
+class RealtimeDelegationDedupTest(IsolatedAsyncioTestCase):
+ """Exercise concurrent approvals and external results with real Agents."""
+
+ async def _start(
+ self,
+ external: bool,
+ call_ids: list[str],
+ ) -> tuple[RealtimeAgent, Agent, list[str]]:
+ """Create one parked delegation and record actual tool executions."""
+ executed: list[str] = []
+
+ def write(value: str) -> str:
+ """Record each approved execution."""
+ executed.append(value)
+ return value
+
+ tool = FunctionTool(write, is_read_only=external)
+ tool.is_external_tool = external
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[
+ ToolCallBlock(
+ id=call_id,
+ name="write",
+ input=json.dumps({"value": call_id}),
+ )
+ for call_id in call_ids
+ ],
+ is_last=True,
+ ),
+ ChatResponse(content=[TextBlock(text="Done.")], is_last=True),
+ ],
+ )
+
+ live = _create_live_agent(model, Toolkit(tools=[tool]))
+ self.addAsyncCleanup(live.close)
+ live._start_client_delegation("task")
+ await asyncio.gather(*list(live._tasks))
+ backend = live._delegations["task"].agent
+ assert backend is not None
+ self.assertEqual(
+ [tc.id for tc in backend.state.get_awaiting_tool_calls("Friday")],
+ call_ids if external else call_ids[:1],
+ )
+ return live, backend, executed
+
+ @staticmethod
+ def _input(
+ backend: Agent,
+ external: bool,
+ call_ids: list[str],
+ ) -> UserConfirmResultEvent | ExternalExecutionResultEvent:
+ """Build a response from the original calls, including stale ones."""
+ reply = backend.state.get_reply_message("Friday")
+ assert reply is not None
+ calls = [
+ tc.model_copy(deep=True)
+ for tc in reply.get_content_blocks("tool_call")
+ if tc.id in call_ids
+ ]
+ if external:
+ return ExternalExecutionResultEvent(
+ reply_id=reply.id,
+ execution_results=[
+ ToolResultBlock(
+ id=tc.id,
+ name=tc.name,
+ output=tc.id,
+ state="success",
+ )
+ for tc in calls
+ ],
+ )
+ return UserConfirmResultEvent(
+ reply_id=reply.id,
+ confirm_results=[
+ ConfirmResult(confirmed=True, tool_call=tc) for tc in calls
+ ],
+ )
+
+ async def test_duplicates_during_execution_and_result_send(self) -> None:
+ """A queued duplicate cannot send failure after the success result."""
+ for external in (False, True):
+ with self.subTest(external=external):
+ live, backend, executed = await self._start(external, ["one"])
+ sending = asyncio.Event()
+ release = asyncio.Event()
+
+ async def send_result(
+ _delegation_id: str,
+ _content: str,
+ started: asyncio.Event = sending,
+ finish: asyncio.Event = release,
+ ) -> None:
+ """Expose the window after execution and before removal."""
+ started.set()
+ await finish.wait()
+
+ sender = AsyncMock(side_effect=send_result)
+ live.model.send_delegation_result = sender
+ event = self._input(backend, external, ["one"])
+ with patch.object(
+ backend,
+ "reply_stream",
+ wraps=backend.reply_stream,
+ ) as reply_stream:
+ try:
+ await live.send(event)
+ await live.send(event.model_copy(deep=True))
+ await asyncio.wait_for(sending.wait(), timeout=1)
+ await live.send(event.model_copy(deep=True))
+ finally:
+ release.set()
+ await asyncio.gather(*list(live._tasks))
+ self.assertEqual(reply_stream.call_count, 1)
+ self.assertEqual(
+ {
+ "results": sender.await_args_list,
+ "executed": executed,
+ "delegations": live._delegations,
+ "routes": live._delegated_tool_calls,
+ },
+ {
+ "results": [call("task", "Done.")],
+ "executed": [] if external else ["one"],
+ "delegations": {},
+ "routes": {},
+ },
+ )
+
+ async def test_duplicates_preserve_other_pending_tools(self) -> None:
+ """Skip only completed inputs, retaining new decisions in a batch."""
+ for external in (False, True):
+ with self.subTest(external=external):
+ live, backend, executed = await self._start(
+ external,
+ ["one", "two"],
+ )
+ sender = live.model.send_delegation_result
+ event = self._input(backend, external, ["one"])
+ await live.send(event)
+ await live.send(event.model_copy(deep=True))
+ await asyncio.gather(*list(live._tasks))
+ self.assertEqual(
+ [
+ tc.id
+ for tc in live.state.get_awaiting_tool_calls("Friday")
+ ],
+ ["two"],
+ )
+ delegation = live._delegations["task"]
+ self.assertIsNotNone(delegation.timeout_task)
+ self.assertFalse(delegation.timeout_task.done())
+ timeout = delegation.timeout_task
+ await live.send(event.model_copy(deep=True))
+ await asyncio.gather(*list(live._tasks))
+ self.assertIs(delegation.timeout_task, timeout)
+ self.assertFalse(timeout.done())
+ sender.assert_not_awaited()
+ await live.send(self._input(backend, external, ["one", "two"]))
+ await asyncio.gather(*list(live._tasks))
+ reply = live.state.get_reply_message("Friday")
+ assert reply is not None
+ self.assertEqual(
+ {
+ "results": sender.await_args_list,
+ "executed": executed,
+ "tool_results": [
+ (block.id, block.state)
+ for block in reply.get_content_blocks(
+ "tool_result",
+ )
+ ],
+ "pending": live.state.get_awaiting_tool_calls(
+ "Friday",
+ ),
+ },
+ {
+ "results": [call("task", "Done.")],
+ "executed": [] if external else ["one", "two"],
+ "tool_results": [
+ ("one", "success"),
+ ("two", "success"),
+ ],
+ "pending": [],
+ },
+ )
+
+
+class RealtimeDelegationErrorTest(IsolatedAsyncioTestCase):
+ """Exercise validation and cleanup with a real Agent and AskUser."""
+
+ async def test_model_failure_preserves_completed_tool_history(
+ self,
+ ) -> None:
+ """Keep successful tool records when subsequent reasoning fails."""
+ executed: list[str] = []
+
+ def write(value: str) -> str:
+ """Record one execution before the model fails."""
+ executed.append(value)
+ return value
+
+ for needs_confirmation in (False, True):
+ with self.subTest(needs_confirmation=needs_confirmation):
+ executed.clear()
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[
+ ToolCallBlock(
+ id="write-1",
+ name="write",
+ input=json.dumps({"value": "saved"}),
+ ),
+ ],
+ is_last=True,
+ ),
+ RuntimeError("Model failed after tool execution."),
+ ],
+ )
+ live = _create_live_agent(
+ model,
+ Toolkit(
+ tools=[
+ FunctionTool(
+ write,
+ is_read_only=not needs_confirmation,
+ ),
+ ],
+ ),
+ )
+ self.addAsyncCleanup(live.close)
+ live._start_client_delegation("task")
+ backend = live._delegations["task"].agent
+ assert backend is not None
+ if needs_confirmation:
+ await asyncio.gather(*list(live._tasks))
+ pending = backend.state.get_awaiting_tool_calls("Friday")
+ await live.send(
+ UserConfirmResultEvent(
+ reply_id=backend.state.reply_id,
+ confirm_results=[
+ ConfirmResult(
+ confirmed=True,
+ tool_call=pending[0],
+ ),
+ ],
+ ),
+ )
+ with self.assertLogs("as", level="ERROR"):
+ await asyncio.gather(*list(live._tasks))
+ reply = backend.state.get_reply_message("Friday")
+ assert reply is not None
+ self.assertEqual(
+ {
+ "executed": executed,
+ "model_calls": model.cnt,
+ "results": [
+ (
+ block.id,
+ block.state,
+ [
+ item.text
+ for item in block.output
+ if isinstance(item, TextBlock)
+ ],
+ )
+ for block in reply.get_content_blocks(
+ "tool_result",
+ )
+ ],
+ "active": live._active_delegation_id,
+ "delegations": live._delegations,
+ "routes": live._delegated_tool_calls,
+ },
+ {
+ "executed": ["saved"],
+ "model_calls": 2,
+ "results": [("write-1", "success", ["saved"])],
+ "active": None,
+ "delegations": {},
+ "routes": {},
+ },
+ )
+ expected = reply.model_copy(deep=True)
+ expected.content = [
+ block
+ for block in expected.content
+ if block.type in {"tool_call", "tool_result"}
+ ]
+ expected_context = [expected.model_dump(mode="json")]
+ self.assertEqual(
+ live.state.model_dump(mode="json")["context"],
+ expected_context,
+ )
+ live.model.send_delegation_result.assert_awaited_once_with(
+ "task",
+ "The delegated task failed. Ask the user to try again.",
+ )
+ await live.close()
+ self.assertEqual(
+ live.state.model_dump(mode="json")["context"],
+ expected_context,
+ )
+
+ async def test_model_failure_publishes_accepted_external_result(
+ self,
+ ) -> None:
+ """Publish a pending result END even when no next event is yielded."""
+ live = await self._start_delegation(["last"])
+ backend = live._delegations["task"].agent
+ assert backend is not None
+ backend.model.set_responses([RuntimeError("Model failed.")])
+ result = self._result("last", True)
+ with self.assertLogs("as", level="ERROR"):
+ await live.send(
+ ExternalExecutionResultEvent(
+ reply_id=backend.state.reply_id,
+ execution_results=[result],
+ ),
+ )
+ await asyncio.gather(*list(live._tasks))
+ self.assertEqual(
+ [
+ (event.tool_call_id, event.state)
+ for event in live.drain_events()
+ if isinstance(event, ToolResultEndEvent)
+ ],
+ [("last", "success")],
+ )
+ reply = live.state.get_reply_message("Friday")
+ assert reply is not None
+ self.assertEqual(
+ reply.get_content_blocks("tool_result"),
+ [result],
+ )
+ self.assertEqual(live._delegations, {})
+ live.model.send_delegation_result.assert_awaited_once_with(
+ "task",
+ "The delegated task failed. Ask the user to try again.",
+ )
+
+ async def _start_delegation(self, call_ids: list[str]) -> RealtimeAgent:
+ """Park the backend on one or more external questions."""
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[
+ ToolCallBlock(
+ id=call_id,
+ name="AskUser",
+ input=json.dumps(
+ {
+ "questions": [
+ {
+ "question": f"Do {call_id}?",
+ "header": "Approval",
+ "options": [
+ {
+ "label": "Yes",
+ "description": "Proceed",
+ },
+ {
+ "label": "No",
+ "description": "Stop",
+ },
+ ],
+ },
+ ],
+ },
+ ),
+ )
+ for call_id in call_ids
+ ],
+ is_last=True,
+ ),
+ ChatResponse(
+ content=[TextBlock(text="Answered.")],
+ is_last=True,
+ ),
+ ],
+ )
+
+ live = _create_live_agent(model, Toolkit(tools=[AskUser()]))
+ self.addAsyncCleanup(live.close)
+ live._start_client_delegation("task")
+ await asyncio.gather(*list(live._tasks))
+ self.assertEqual(
+ [call.id for call in live.state.get_awaiting_tool_calls("Friday")],
+ call_ids,
+ )
+ list(live.drain_events())
+ return live
+
+ @staticmethod
+ def _result(call_id: str, valid: bool) -> ToolResultBlock:
+ """Build an answer with or without the required metadata."""
+ return ToolResultBlock(
+ id=call_id,
+ name="AskUser",
+ output="Yes",
+ state="success",
+ metadata={
+ "answers": [
+ {"question": f"Do {call_id}?", "selected": ["Yes"]},
+ ],
+ }
+ if valid
+ else {},
+ )
+
+ async def test_invalid_results_can_retry_close_or_expire(self) -> None:
+ """Rejected answers retain routing, timeout, and partial results."""
+ for action in ("retry", "close", "expire"):
+ for partial in (False, True):
+ with self.subTest(action=action, partial=partial):
+ call_ids = ["first", "last"] if partial else ["last"]
+ live = await self._start_delegation(call_ids)
+ send_result = live.model.send_delegation_result
+ delegation = live._delegations["task"]
+ assert delegation.agent is not None
+ old_timeout = delegation.timeout_task
+ results = [self._result("first", True)] if partial else []
+ results.append(self._result("last", False))
+ submission = ExternalExecutionResultEvent(
+ reply_id=live.state.reply_id,
+ execution_results=results,
+ )
+ with self.assertLogs("as", level="ERROR"):
+ await live.send(submission)
+ await asyncio.gather(*list(live._tasks))
+ events = list(live.drain_events())
+ self.assertEqual(
+ {
+ "active": live._active_delegation_id,
+ "routes": live._delegated_tool_calls,
+ "pending": [
+ call.id
+ for call in live.state.get_awaiting_tool_calls(
+ "Friday",
+ )
+ ],
+ "timeout_rearmed": (
+ delegation.timeout_task is not None
+ and delegation.timeout_task is not old_timeout
+ and not delegation.timeout_task.done()
+ ),
+ "ends": [
+ (event.tool_call_id, event.state)
+ for event in events
+ if isinstance(event, ToolResultEndEvent)
+ ],
+ "reported": send_result.await_count,
+ "notifications": [
+ (event.name, event.value)
+ for event in events
+ if isinstance(event, CustomEvent)
+ ],
+ },
+ {
+ "active": "task",
+ "routes": dict.fromkeys(call_ids, "task"),
+ "pending": ["last"],
+ "timeout_rearmed": True,
+ "ends": [("first", "success")] if partial else [],
+ "reported": 0,
+ "notifications": [
+ (
+ "external_execution_result_rejected",
+ {
+ "request_id": submission.id,
+ "reply_id": submission.reply_id,
+ "tool_call_ids": ["last"],
+ "message": (
+ "The submitted result was "
+ "rejected. Please check your "
+ "answer and submit again."
+ ),
+ },
+ ),
+ ],
+ },
+ )
+ if action == "retry":
+ await live.send(
+ ExternalExecutionResultEvent(
+ reply_id=live.state.reply_id,
+ execution_results=[self._result("last", True)],
+ ),
+ )
+ await asyncio.gather(*list(live._tasks))
+ send_result.assert_awaited_once_with(
+ "task",
+ "Answered.",
+ )
+ elif action == "close":
+ await live.close()
+ send_result.assert_not_awaited()
+ else:
+ timeout = delegation.timeout_task
+ live._cancel_delegation_timeout(delegation)
+ await asyncio.gather(timeout, return_exceptions=True)
+ with patch(
+ "agentscope.agent._realtime._agent."
+ "_DELEGATION_INPUT_TIMEOUT_SECS",
+ 0,
+ ):
+ await live._expire_client_delegation(delegation)
+ send_result.assert_awaited_once()
+ reply = live.state.get_reply_message("Friday")
+ assert reply is not None
+ self.assertEqual(
+ [
+ event.name
+ for event in live.drain_events()
+ if isinstance(event, CustomEvent)
+ ],
+ [],
+ )
+ self.assertEqual(
+ {
+ "active": live._active_delegation_id,
+ "delegations": live._delegations,
+ "routes": live._delegated_tool_calls,
+ "pending": live.state.get_awaiting_tool_calls(
+ "Friday",
+ ),
+ "results": [
+ (block.id, block.state)
+ for block in reply.get_content_blocks(
+ "tool_result",
+ )
+ ],
+ },
+ {
+ "active": None,
+ "delegations": {},
+ "routes": {},
+ "pending": [],
+ "results": (
+ [("first", "success")] if partial else []
+ )
+ + [
+ (
+ "last",
+ "success"
+ if action == "retry"
+ else "interrupted",
+ ),
+ ],
+ },
+ )
diff --git a/tests/realtime_openai_live_test.py b/tests/realtime_openai_live_test.py
new file mode 100644
index 000000000..188f65829
--- /dev/null
+++ b/tests/realtime_openai_live_test.py
@@ -0,0 +1,2017 @@
+# -*- coding: utf-8 -*-
+"""Tests for GPT-Live protocol and its minimal full-duplex agent path."""
+# pylint: disable=protected-access
+import asyncio
+import base64
+import json
+import unittest
+from typing import AsyncGenerator, AsyncIterator, Callable
+from unittest.mock import AsyncMock, patch
+
+from utils import MockModel
+
+from agentscope.agent import Agent, ContextConfig, RealtimeAgent
+from agentscope.agent._realtime._agent import _ClientDelegation
+from agentscope.credential import OpenAICredential
+from agentscope.event import (
+ AgentEvent,
+ ConfirmResult,
+ CustomEvent,
+ ExternalExecutionResultEvent,
+ ReplyEndEvent,
+ ReplyStartEvent,
+ RequireExternalExecutionEvent,
+ RequireUserConfirmEvent,
+ TextBlockDeltaEvent,
+ ToolCallEndEvent,
+ ToolCallStartEvent,
+ ToolResultDataDeltaEvent,
+ ToolResultEndEvent,
+ ToolResultStartEvent,
+ ToolResultTextDeltaEvent,
+ UserConfirmResultEvent,
+ UserInterruptEvent,
+)
+from agentscope.message import (
+ AssistantMsg,
+ Msg,
+ TextBlock,
+ ToolCallBlock,
+ ToolResultBlock,
+ ToolResultState,
+ UserMsg,
+)
+from agentscope.middleware import MiddlewareBase
+from agentscope.model import ChatResponse, StructuredResponse
+from agentscope.realtime import (
+ ControlFrame,
+ ControlFrameType,
+ DelegatedResponseEvent,
+ DelegationCreatedEvent,
+ OpenAILiveModel,
+ PlayoutPosition,
+ SessionClosedEvent,
+ SessionStartedEvent,
+ SessionUsageEvent,
+ TimelineAudioDeltaEvent,
+ TimelineTranscriptDeltaEvent,
+ TransportBase,
+ TransportFrame,
+)
+from agentscope.state import AgentState
+from agentscope.tool import FunctionTool, Toolkit
+
+CRED = OpenAICredential(api_key="sk-x")
+
+
+def _model() -> OpenAILiveModel:
+ """Build a Live model configured for client delegation."""
+ return OpenAILiveModel("gpt-live-1", CRED)
+
+
+class _FakeWebSocket:
+ """A controllable async WebSocket used by lifecycle tests."""
+
+ def __init__(self) -> None:
+ self.incoming: asyncio.Queue[str | None] = asyncio.Queue()
+ self.sent: list[dict] = []
+ self.closed = False
+
+ def __aiter__(self) -> "_FakeWebSocket":
+ return self
+
+ async def __anext__(self) -> str:
+ value = await self.incoming.get()
+ if value is None:
+ raise StopAsyncIteration
+ return value
+
+ async def send(self, raw: str) -> None:
+ """Record a command and enqueue its simulated server response."""
+ payload = json.loads(raw)
+ self.sent.append(payload)
+ if payload["type"] == "session.start":
+ self.incoming.put_nowait(
+ json.dumps(
+ {
+ "type": "session.started",
+ "session": {"id": "live_1"},
+ },
+ ),
+ )
+ elif payload["type"] == "session.close":
+ self.incoming.put_nowait(
+ json.dumps(
+ {
+ "type": "session.closed",
+ "reason": "close_requested",
+ "session": {"id": "live_1"},
+ "usage": {"seconds": 4.5},
+ },
+ ),
+ )
+
+ async def close(self) -> None:
+ """End the fake server event stream."""
+ self.closed = True
+ self.incoming.put_nowait(None)
+
+
+class _PendingStartupWebSocket(_FakeWebSocket):
+ """Pause startup either during send or while awaiting acknowledgement."""
+
+ def __init__(self, block_send: bool) -> None:
+ super().__init__()
+ self.block_send = block_send
+ self.startup_sent = asyncio.Event()
+ self.reading = asyncio.Event()
+
+ async def send(self, raw: str) -> None:
+ """Record startup without automatically acknowledging it."""
+ self.sent.append(json.loads(raw))
+ self.startup_sent.set()
+ if self.block_send:
+ await asyncio.Event().wait()
+
+ async def __anext__(self) -> str:
+ self.reading.set()
+ return await super().__anext__()
+
+
+class _FakeTransport(TransportBase):
+ """Capture output audio without opening a real sound device."""
+
+ input_sample_rate = 24000
+ output_sample_rate = 24000
+
+ def __init__(self) -> None:
+ self.audio: list[tuple[bytes, str]] = []
+ self.cleared = 0
+ self.frames: asyncio.Queue[TransportFrame | None] = asyncio.Queue()
+
+ async def start(self) -> None:
+ """Start the in-memory transport."""
+
+ async def close(self) -> None:
+ """Close the in-memory transport."""
+
+ async def incoming(self) -> AsyncIterator[TransportFrame]:
+ """Yield explicitly queued transport frames."""
+ while True:
+ frame = await self.frames.get()
+ if frame is None:
+ return
+ yield frame
+
+ async def send_audio(self, pcm: bytes, item_id: str) -> None:
+ """Capture one outgoing audio frame."""
+ self.audio.append((pcm, item_id))
+
+ async def clear_audio(self) -> PlayoutPosition:
+ """Record one playback clear request."""
+ self.cleared += 1
+ return self.playout()
+
+ def playout(self) -> PlayoutPosition:
+ """Return the current item with no simulated playback progress."""
+ return PlayoutPosition(item_id=self.audio[-1][1] if self.audio else "")
+
+
+class _FakeDelegationAgent:
+ """Return one deterministic task result without a provider call."""
+
+ name = "Friday"
+
+ def __init__(self) -> None:
+ self.state = AgentState()
+ self.inputs: list[Msg] = []
+
+ async def reply_stream(
+ self,
+ inputs: Msg,
+ yield_final_msg: bool,
+ ) -> AsyncIterator[Msg]:
+ """Capture the generated transcript prompt and return a result."""
+ self.inputs.append(inputs)
+ self.state.context.append(inputs)
+ result = AssistantMsg(name="Friday", content="Order 42 shipped.")
+ self.state.context.append(result)
+ if yield_final_msg:
+ yield result
+
+
+class _ConfirmingDelegationAgent:
+ """Pause once for permission, then return a completed result."""
+
+ name = "Friday"
+
+ def __init__(self) -> None:
+ self.state = AgentState()
+ self.tool_call = ToolCallBlock(
+ id="tool_1",
+ name="update_order",
+ input='{"order_id":"42"}',
+ state="asking",
+ )
+
+ async def reply_stream(
+ self,
+ inputs: Msg | UserConfirmResultEvent,
+ yield_final_msg: bool,
+ ) -> AsyncIterator[
+ Msg
+ | ReplyStartEvent
+ | ToolCallStartEvent
+ | ToolCallEndEvent
+ | ToolResultStartEvent
+ | ToolResultTextDeltaEvent
+ | ToolResultEndEvent
+ | RequireUserConfirmEvent
+ ]:
+ """Yield a confirmation on the first call and finish on resume."""
+ if isinstance(inputs, Msg):
+ self.state.reply_id = "backend_reply"
+ self.state.append_context("Friday", [self.tool_call])
+ yield ReplyStartEvent(
+ session_id=self.state.session_id,
+ reply_id="backend_reply",
+ name="Friday",
+ )
+ yield ToolCallStartEvent(
+ reply_id="backend_reply",
+ tool_call_id=self.tool_call.id,
+ tool_call_name=self.tool_call.name,
+ )
+ yield ToolCallEndEvent(
+ reply_id="backend_reply",
+ tool_call_id=self.tool_call.id,
+ )
+ yield RequireUserConfirmEvent(
+ reply_id="backend_reply",
+ tool_calls=[self.tool_call],
+ )
+ return
+ yield ToolResultStartEvent(
+ reply_id="backend_reply",
+ tool_call_id=self.tool_call.id,
+ tool_call_name=self.tool_call.name,
+ )
+ yield ToolResultTextDeltaEvent(
+ reply_id="backend_reply",
+ tool_call_id=self.tool_call.id,
+ delta="Updated",
+ )
+ yield ToolResultEndEvent(
+ reply_id="backend_reply",
+ tool_call_id=self.tool_call.id,
+ state=ToolResultState.SUCCESS,
+ )
+ self.tool_call.state = "finished"
+ self.state.append_context(
+ "Friday",
+ [
+ ToolResultBlock(
+ id=self.tool_call.id,
+ name=self.tool_call.name,
+ output="Updated",
+ state="success",
+ ),
+ ],
+ )
+ result = AssistantMsg(name="Friday", content="Order 42 updated.")
+ if yield_final_msg:
+ yield result
+
+
+class _ExternalDelegationAgent:
+ """Pause for an external executor, then finish after its result."""
+
+ name = "Friday"
+
+ def __init__(self) -> None:
+ self.state = AgentState()
+ self.tool_call = ToolCallBlock(
+ id="external_1",
+ name="ask_user",
+ input='{"question":"Which order?"}',
+ state="submitted",
+ )
+ self.received: list[
+ Msg | ExternalExecutionResultEvent | UserInterruptEvent
+ ] = []
+
+ async def reply_stream(
+ self,
+ inputs: Msg | ExternalExecutionResultEvent | UserInterruptEvent,
+ yield_final_msg: bool,
+ ) -> AsyncIterator[
+ Msg
+ | ReplyStartEvent
+ | ToolCallStartEvent
+ | ToolCallEndEvent
+ | RequireExternalExecutionEvent
+ ]:
+ """Request external execution once and finish on resumption."""
+ self.received.append(inputs)
+ if isinstance(inputs, Msg):
+ self.state.reply_id = "external_reply"
+ self.state.append_context("Friday", [self.tool_call])
+ yield ReplyStartEvent(
+ session_id=self.state.session_id,
+ reply_id="external_reply",
+ name="Friday",
+ )
+ yield ToolCallStartEvent(
+ reply_id="external_reply",
+ tool_call_id=self.tool_call.id,
+ tool_call_name=self.tool_call.name,
+ )
+ yield ToolCallEndEvent(
+ reply_id="external_reply",
+ tool_call_id=self.tool_call.id,
+ )
+ yield RequireExternalExecutionEvent(
+ reply_id="external_reply",
+ tool_calls=[self.tool_call],
+ )
+ return
+ self.tool_call.state = "finished"
+ if isinstance(inputs, UserInterruptEvent):
+ result = AssistantMsg(name="Friday", content="Interrupted.")
+ if yield_final_msg:
+ yield result
+ return
+ result = AssistantMsg(name="Friday", content="Order 42 selected.")
+ if yield_final_msg:
+ yield result
+
+
+class _StateDelegationAgent:
+ """Record one state mutation before returning a result."""
+
+ name = "Friday"
+
+ def __init__(self, state: AgentState, marker: str) -> None:
+ self.state = state
+ self.marker = marker
+
+ async def reply_stream(
+ self,
+ inputs: Msg,
+ yield_final_msg: bool,
+ ) -> AsyncIterator[Msg]:
+ """Append a marker that the next queued task must observe."""
+ del inputs
+ self.state.tool_context.activated_groups.append(self.marker)
+ if yield_final_msg:
+ yield AssistantMsg(name="Friday", content=self.marker)
+
+
+class OpenAILivePayloadTest(unittest.TestCase):
+ """Model discovery, startup payloads, and server event parsing."""
+
+ def test_card_and_credential_mapping(self) -> None:
+ """The Live card resolves to its own protocol adapter."""
+ cards = OpenAILiveModel.list_models()
+ classes = {
+ item.type: item for item in CRED.get_realtime_model_classes()
+ }
+ self.assertListEqual(
+ [
+ (
+ card.name,
+ card.model_type,
+ card.interaction_mode,
+ card.delegation_backend,
+ card.supports_tools,
+ classes[card.model_type].__name__,
+ )
+ for card in cards
+ ],
+ [
+ (
+ "gpt-live-1",
+ "openai_live",
+ "full_duplex",
+ "session_chat",
+ False,
+ "OpenAILiveModel",
+ ),
+ ],
+ )
+ self.assertDictEqual(
+ cards[0].parameter_schema,
+ {
+ "type": "object",
+ "properties": {
+ "voice": {
+ "default": "marin",
+ "description": "The voice used for generated audio.",
+ "title": "Voice",
+ "type": "string",
+ },
+ "store": {
+ "default": False,
+ "description": (
+ "Whether to request provider-side storage for "
+ "the Live session."
+ ),
+ "title": "Store Session",
+ "type": "boolean",
+ },
+ "startup_timeout_s": {
+ "default": 15.0,
+ "description": (
+ "Maximum seconds to wait for the Live session "
+ "to start."
+ ),
+ "exclusiveMinimum": 0,
+ "title": "Startup Timeout",
+ "type": "number",
+ },
+ "close_timeout_s": {
+ "default": 15.0,
+ "description": (
+ "Maximum seconds to wait for graceful session "
+ "closure."
+ ),
+ "exclusiveMinimum": 0,
+ "title": "Close Timeout",
+ "type": "number",
+ },
+ },
+ },
+ )
+
+ def test_session_start_payload(self) -> None:
+ """Startup delegates backend tasks to the application."""
+ model = _model()
+ payload = model._session_start("Speak briefly.")
+ event_id = payload.pop("event_id")
+ self.assertTrue(event_id.startswith("start_"))
+ self.assertDictEqual(
+ payload,
+ {
+ "type": "session.start",
+ "session": {
+ "model": "gpt-live-1",
+ "instructions": "Speak briefly.",
+ "input": [],
+ "audio": {
+ "format": {
+ "type": "audio/pcm",
+ "rate": 24000,
+ },
+ "output": {"voice": "marin"},
+ },
+ "delegation": {"type": "client"},
+ "store": False,
+ },
+ },
+ )
+
+ def test_server_event_mapping(self) -> None:
+ """Live timelines and delegated Responses remain independent."""
+ model = _model()
+ encoded = base64.b64encode(b"\x01\x00").decode("ascii")
+ frames = [
+ {
+ "type": "session.started",
+ "session": {"id": "live_1"},
+ },
+ {
+ "type": "session.input_transcript.delta",
+ "delta": "hello",
+ "start_ms": 10,
+ "end_ms": 20,
+ },
+ {
+ "type": "session.output_transcript.delta",
+ "delta": "hi",
+ "start_ms": 15,
+ "end_ms": 25,
+ },
+ {"type": "session.output_audio.delta", "delta": encoded},
+ {
+ "type": "session.delegation.created",
+ "offset_ms": 30,
+ "delegation": {
+ "id": "delegation_1",
+ "target": "client",
+ },
+ },
+ {
+ "type": "response.event",
+ "delegation_id": "delegation_1",
+ "event": {
+ "type": "response.output_text.delta",
+ "delta": "working",
+ },
+ },
+ {
+ "type": "session.usage.updated",
+ "usage": {"seconds": 3.25},
+ "context_window": {"usage_ratio": 0.2},
+ },
+ {
+ "type": "session.closed",
+ "reason": "close_requested",
+ "session": {"id": "live_1"},
+ "usage": {"seconds": 4.5},
+ },
+ ]
+ self.assertListEqual(
+ [model._parse(frame) for frame in frames],
+ [
+ SessionStartedEvent(session_id="live_1"),
+ TimelineTranscriptDeltaEvent(
+ role="user",
+ delta="hello",
+ start_ms=10,
+ end_ms=20,
+ ),
+ TimelineTranscriptDeltaEvent(
+ role="assistant",
+ delta="hi",
+ start_ms=15,
+ end_ms=25,
+ ),
+ TimelineAudioDeltaEvent(
+ pcm=b"\x01\x00",
+ sample_rate=24000,
+ ),
+ DelegationCreatedEvent(
+ delegation_id="delegation_1",
+ target="client",
+ offset_ms=30,
+ ),
+ DelegatedResponseEvent(
+ delegation_id="delegation_1",
+ event={
+ "type": "response.output_text.delta",
+ "delta": "working",
+ },
+ ),
+ SessionUsageEvent(
+ seconds=3.25,
+ context_usage_ratio=0.2,
+ ),
+ SessionClosedEvent(
+ session_id="live_1",
+ reason="close_requested",
+ usage_seconds=4.5,
+ ),
+ ],
+ )
+ self.assertEqual(model.usage_seconds, 4.5)
+
+ def test_long_commentary_is_split_at_a_safe_boundary(self) -> None:
+ """Long client results are returned as bounded Live updates."""
+ content = f"{'a' * 390} boundary {'b' * 20}"
+ self.assertListEqual(
+ OpenAILiveModel._commentary_chunks(content),
+ [f"{'a' * 390} boundary", "b" * 20],
+ )
+
+
+class OpenAILiveLifecycleTest(unittest.IsolatedAsyncioTestCase):
+ """Connection startup, audio validation, and graceful close."""
+
+ async def test_cancel_startup_releases_resources_and_reconnects(
+ self,
+ ) -> None:
+ """Cancellation closes resources before propagating to the caller."""
+ for block_send in (False, True):
+ with self.subTest(block_send=block_send):
+ ws = _PendingStartupWebSocket(block_send)
+ replacement = _FakeWebSocket()
+ model = _model()
+ self.addAsyncCleanup(model.close)
+ with patch(
+ "websockets.connect",
+ AsyncMock(side_effect=[ws, replacement]),
+ ):
+ connection = asyncio.create_task(model.connect("Brief."))
+ await asyncio.wait_for(ws.startup_sent.wait(), timeout=1)
+ await asyncio.wait_for(ws.reading.wait(), timeout=1)
+ reader, started = model._reader, model._started
+ assert reader is not None and started is not None
+ self.assertFalse(reader.done())
+ connection.cancel()
+ with self.assertRaises(asyncio.CancelledError):
+ await connection
+ self.assertEqual(
+ {
+ "socket_closed": ws.closed,
+ "socket": model._ws,
+ "reader": model._reader,
+ "reader_done": reader.done(),
+ "startup_cancelled": started.cancelled(),
+ "startup": model._started,
+ "session_id": model._session_id,
+ },
+ {
+ "socket_closed": True,
+ "socket": None,
+ "reader": None,
+ "reader_done": True,
+ "startup_cancelled": True,
+ "startup": None,
+ "session_id": "",
+ },
+ )
+ await model.connect("Retry.")
+ self.assertIs(model._ws, replacement)
+ self.assertEqual(model._session_id, "live_1")
+ self.assertEqual(
+ await asyncio.wait_for(
+ anext(model.events()),
+ timeout=1,
+ ),
+ SessionStartedEvent(session_id="live_1"),
+ )
+ await model.close()
+ self.assertTrue(replacement.closed)
+
+ async def test_startup_timeout_settles_the_shielded_future(self) -> None:
+ """Both startup phases time out and release their resources."""
+ for block_send in (False, True):
+ with self.subTest(block_send=block_send):
+ ws = _PendingStartupWebSocket(block_send)
+ model = _model()
+ model.parameters.startup_timeout_s = 0.1
+ self.addAsyncCleanup(model.close)
+ resources: list[asyncio.Future | None] = []
+
+ async def capture_startup(
+ payload: dict,
+ current: OpenAILiveModel = model,
+ capture: Callable[
+ [asyncio.Future | None],
+ None,
+ ] = resources.append,
+ ) -> None:
+ """Save references before startup can suspend or expire."""
+ capture(current._reader)
+ capture(current._started)
+ await OpenAILiveModel._send(current, payload)
+
+ with (
+ patch("websockets.connect", AsyncMock(return_value=ws)),
+ patch.object(model, "_send", capture_startup),
+ ):
+ connection = asyncio.create_task(model.connect("Brief."))
+ with self.assertRaises(asyncio.TimeoutError):
+ await asyncio.wait_for(connection, timeout=1)
+ reader, started = resources
+ assert reader is not None and started is not None
+ self.assertEqual(
+ {
+ "guard_cancelled": connection.cancelled(),
+ "socket_closed": ws.closed,
+ "reader_done": reader.done(),
+ "startup_cancelled": started.cancelled(),
+ "socket": model._ws,
+ "reader": model._reader,
+ "startup": model._started,
+ },
+ {
+ "guard_cancelled": False,
+ "socket_closed": True,
+ "reader_done": True,
+ "startup_cancelled": True,
+ "socket": None,
+ "reader": None,
+ "startup": None,
+ },
+ )
+
+ async def test_startup_phases_share_one_timeout(self) -> None:
+ """Sending must consume part of the acknowledgement deadline."""
+ ws = _FakeWebSocket()
+ model = _model()
+ model.parameters.startup_timeout_s = 0.1
+ self.addAsyncCleanup(model.close)
+ acknowledgement: asyncio.TimerHandle | None = None
+
+ async def delayed_send(raw: str) -> None:
+ """Each phase fits alone, but their combined delay does not."""
+ nonlocal acknowledgement
+ ws.sent.append(json.loads(raw))
+ await asyncio.sleep(0.06)
+ acknowledgement = asyncio.get_running_loop().call_later(
+ 0.06,
+ ws.incoming.put_nowait,
+ json.dumps(
+ {"type": "session.started", "session": {"id": "late"}},
+ ),
+ )
+
+ with (
+ patch("websockets.connect", AsyncMock(return_value=ws)),
+ patch.object(ws, "send", delayed_send),
+ ):
+ with self.assertRaises(asyncio.TimeoutError):
+ try:
+ await asyncio.wait_for(model.connect("Brief."), timeout=1)
+ finally:
+ if acknowledgement is not None:
+ acknowledgement.cancel()
+ self.assertEqual(
+ (ws.closed, model._ws, model._reader, model._started),
+ (True, None, None, None),
+ )
+
+ async def test_startup_error_is_preserved_after_cleanup(self) -> None:
+ """Provider startup errors still reach the caller after cleanup."""
+ ws = _PendingStartupWebSocket(False)
+ model = _model()
+ self.addAsyncCleanup(model.close)
+ with patch("websockets.connect", AsyncMock(return_value=ws)):
+ connection = asyncio.create_task(model.connect("Brief."))
+ await asyncio.wait_for(ws.reading.wait(), timeout=1)
+ reader = model._reader
+ assert reader is not None
+ ws.incoming.put_nowait(
+ json.dumps(
+ {
+ "type": "error",
+ "error": {"message": "Startup rejected"},
+ },
+ ),
+ )
+ with self.assertRaisesRegex(RuntimeError, "Startup rejected"):
+ await connection
+ self.assertEqual(
+ (ws.closed, reader.done(), model._reader, model._started),
+ (True, True, None, None),
+ )
+
+ async def test_connect_audio_and_close(self) -> None:
+ """The adapter waits for startup and preserves final usage."""
+ ws = _FakeWebSocket()
+ connector = AsyncMock(return_value=ws)
+ model = _model()
+ with patch("websockets.connect", connector):
+ await model.connect("Be brief.")
+ await model.push_audio(b"\x01\x00")
+ await model.send_delegation_result(
+ "delegation_1",
+ "The order shipped.",
+ )
+ with self.assertRaises(ValueError):
+ await model.push_audio(b"\x01")
+ await model.close()
+
+ connector.assert_awaited_once()
+ assert connector.await_args is not None
+ args, kwargs = connector.await_args
+ self.assertEqual(args, ("wss://api.openai.com/v1/live/sessions",))
+ self.assertDictEqual(
+ kwargs,
+ {"additional_headers": {"Authorization": "Bearer sk-x"}},
+ )
+ self.assertListEqual(
+ [item["type"] for item in ws.sent],
+ [
+ "session.start",
+ "session.input_audio.append",
+ "session.commentary.append",
+ "session.close",
+ ],
+ )
+ self.assertEqual(ws.sent[1]["audio"], "AQA=")
+ commentary = ws.sent[2].copy()
+ commentary_id = commentary.pop("event_id")
+ self.assertTrue(commentary_id.startswith("delegation_result_"))
+ self.assertDictEqual(
+ commentary,
+ {
+ "type": "session.commentary.append",
+ "delegation_id": "delegation_1",
+ "content": "The order shipped.",
+ },
+ )
+ self.assertEqual(model.usage_seconds, 4.5)
+ self.assertTrue(ws.closed)
+
+
+class RealtimeAgentLiveTest(unittest.IsolatedAsyncioTestCase):
+ """Minimal projection of overlapping Live transcript and audio."""
+
+ async def test_mismatched_backend_name_fails_before_execution(
+ self,
+ ) -> None:
+ """Reject an incompatible backend without leaving a parked task."""
+ backend = Agent("TaskBot", "Be brief.", MockModel())
+ model = _model()
+ model.send_delegation_result = AsyncMock()
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ model,
+ delegation_agent_factory=lambda _: backend,
+ )
+ self.addAsyncCleanup(agent.close)
+ with (
+ patch.object(backend, "reply_stream") as reply_stream,
+ patch.object(agent, "_schedule_delegation_timeout") as timeout,
+ ):
+ with self.assertLogs("as", level="ERROR") as logs:
+ agent._start_client_delegation("bad-name")
+ await asyncio.gather(*list(agent._tasks))
+ reply_stream.assert_not_called()
+ timeout.assert_not_called()
+ self.assertIn(
+ "Delegation backend name 'TaskBot' must match "
+ "RealtimeAgent name 'Friday'.",
+ "\n".join(logs.output),
+ )
+ self.assertEqual(
+ {
+ "active": agent._active_delegation_id,
+ "delegations": agent._delegations,
+ "routes": agent._delegated_tool_calls,
+ "queue": list(agent._delegation_queue),
+ "tasks": agent._tasks,
+ },
+ {
+ "active": None,
+ "delegations": {},
+ "routes": {},
+ "queue": [],
+ "tasks": set(),
+ },
+ )
+ model.send_delegation_result.assert_awaited_once_with(
+ "bad-name",
+ "The delegated task could not be completed because the "
+ "session chat agent is unavailable.",
+ )
+
+ async def test_interrupt_control_stops_speech_and_keeps_delegation(
+ self,
+ ) -> None:
+ """Browser Stop clears speech without cancelling backend work."""
+ model = _model()
+ socket = _FakeWebSocket()
+ model._ws = socket
+ model.send_delegation_result = AsyncMock()
+ backend = _ExternalDelegationAgent()
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ model,
+ delegation_agent_factory=lambda _: backend,
+ )
+ self.addAsyncCleanup(agent.close)
+ transport = _FakeTransport()
+ agent._transport = transport
+ agent._connected = True
+ agent._start_client_delegation("task")
+ await asyncio.gather(*list(agent._tasks))
+ delegation = agent._delegations["task"]
+ timeout = delegation.timeout_task
+ await agent._on_model_event(SessionStartedEvent(session_id="live_1"))
+ await agent._on_model_event(
+ TimelineAudioDeltaEvent(pcm=b"\x01\x00", sample_rate=24000),
+ )
+ await agent._on_model_event(
+ TimelineTranscriptDeltaEvent(
+ role="assistant",
+ delta="Sure thing",
+ start_ms=10,
+ end_ms=20,
+ ),
+ )
+ reply_id = agent._live_replies["assistant"].reply_id
+ list(agent.drain_events())
+
+ await agent._on_control(ControlFrame(type=ControlFrameType.INTERRUPT))
+
+ self.assertEqual(
+ {
+ "cleared": transport.cleared,
+ "playout": transport.playout().model_dump(),
+ "audio_pending": agent._audio_pending,
+ "payloads": [
+ {
+ key: value
+ for key, value in item.items()
+ if key != "event_id"
+ }
+ for item in socket.sent
+ ],
+ "ends": [
+ (event.reply_id, event.finished_reason)
+ for event in agent.drain_events()
+ if isinstance(event, ReplyEndEvent)
+ ],
+ "live_replies": agent._live_replies,
+ "transcript": [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ if msg.id == reply_id
+ ],
+ "active": agent._active_delegation_id,
+ "routes": agent._delegated_tool_calls,
+ "backend_inputs": backend.received,
+ },
+ {
+ "cleared": 1,
+ "playout": {
+ "item_id": "live:live_1",
+ "played_ms": 0,
+ "first_played_at": None,
+ },
+ "audio_pending": False,
+ "payloads": [
+ {
+ "type": "session.instructions.append",
+ "delegation_id": None,
+ "content": "Stop speaking now and listen to the user.",
+ },
+ ],
+ "ends": [(reply_id, "interrupted")],
+ "live_replies": {},
+ "transcript": [("assistant", "Sure thing")],
+ "active": "task",
+ "routes": {"external_1": "task"},
+ "backend_inputs": [delegation.prompt],
+ },
+ )
+ self.assertIs(agent._delegations["task"], delegation)
+ self.assertIs(delegation.timeout_task, timeout)
+ assert timeout is not None
+ self.assertFalse(timeout.done())
+ model.send_delegation_result.assert_not_awaited()
+
+ async def test_overlapping_timeline_does_not_barge_in(self) -> None:
+ """User transcript growth does not clear assistant playback."""
+ model = _model()
+ agent = RealtimeAgent("Friday", "Be brief.", model)
+ transport = _FakeTransport()
+ agent._transport = transport
+ self.assertTrue(agent.allows_open_user_reply_checkpoint)
+
+ await agent._on_model_event(SessionStartedEvent(session_id="live_1"))
+ await agent._on_model_event(
+ TimelineAudioDeltaEvent(
+ pcm=b"\x01\x00",
+ sample_rate=24000,
+ ),
+ )
+ await agent._on_model_event(
+ TimelineTranscriptDeltaEvent(
+ role="assistant",
+ delta="Sure",
+ start_ms=10,
+ end_ms=20,
+ ),
+ )
+ await agent._on_model_event(
+ TimelineTranscriptDeltaEvent(
+ role="user",
+ delta="Wait",
+ start_ms=15,
+ end_ms=25,
+ ),
+ )
+
+ self.assertListEqual(
+ transport.audio,
+ [(b"\x01\x00", "live:live_1")],
+ )
+ self.assertEqual(transport.cleared, 0)
+ self.assertListEqual(
+ [
+ (message.role, message.get_text_content())
+ for message in agent.state.context
+ ],
+ [("assistant", "Sure"), ("user", "Wait")],
+ )
+
+ async def test_client_delegation_uses_live_transcript(self) -> None:
+ """A client delegation runs the configured session chat agent."""
+ model = _model()
+ state = AgentState()
+ state.context.append(
+ UserMsg(
+ id="live:user:historical",
+ name="user",
+ content="Earlier voice context.",
+ ),
+ )
+ backend = _FakeDelegationAgent()
+
+ def _factory(delegation_state: AgentState) -> _FakeDelegationAgent:
+ backend.state = delegation_state
+ return backend
+
+ model.send_delegation_result = ( # type: ignore[method-assign]
+ AsyncMock()
+ )
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ delegation_agent_factory=_factory, # type: ignore[arg-type]
+ )
+
+ await agent._on_model_event(
+ TimelineTranscriptDeltaEvent(
+ role="user",
+ delta="Check order 42.",
+ start_ms=10,
+ end_ms=20,
+ ),
+ )
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id="delegation_1",
+ target="client",
+ ),
+ )
+ tasks = list(agent._tasks)
+ await asyncio.gather(*tasks)
+
+ self.assertEqual(len(backend.inputs), 1)
+ prompt = backend.inputs[0].get_text_content()
+ self.assertIsNotNone(prompt)
+ assert prompt is not None
+ self.assertIn("User: Check order 42.", prompt)
+ backend_ids = {message.id for message in backend.state.context}
+ self.assertDictEqual(
+ {
+ "historical_text": (
+ backend.state.context[0].get_text_content()
+ ),
+ "current_live_ids": backend_ids & agent._live_message_ids,
+ "backend_texts": [
+ message.get_text_content()
+ for message in backend.state.context
+ ],
+ },
+ {
+ "historical_text": "Earlier voice context.",
+ "current_live_ids": set(),
+ "backend_texts": [
+ "Earlier voice context.",
+ prompt,
+ "Order 42 shipped.",
+ ],
+ },
+ )
+ model.send_delegation_result.assert_awaited_once_with(
+ "delegation_1",
+ "Order 42 shipped.",
+ )
+ self.assertDictEqual(agent._delegations, {})
+
+ async def test_completed_results_replay_before_interaction(self) -> None:
+ """Replay full hidden results with their persisted completion."""
+ for interaction_type in (
+ RequireUserConfirmEvent,
+ RequireExternalExecutionEvent,
+ ):
+ with self.subTest(interaction=interaction_type.__name__):
+ call = ToolCallBlock(id="read", name="Read", input="{}")
+ result = ToolResultBlock(
+ id=call.id,
+ name=call.name,
+ output="contents",
+ state=ToolResultState.SUCCESS,
+ )
+ events = [
+ ReplyStartEvent(
+ session_id="session",
+ reply_id="backend",
+ name="Friday",
+ ),
+ ToolCallStartEvent(
+ reply_id="backend",
+ tool_call_id=call.id,
+ tool_call_name=call.name,
+ ),
+ ToolCallEndEvent(
+ reply_id="backend",
+ tool_call_id=call.id,
+ ),
+ ToolResultStartEvent(
+ reply_id="backend",
+ tool_call_id=call.id,
+ tool_call_name=call.name,
+ ),
+ ToolResultTextDeltaEvent(
+ reply_id="backend",
+ tool_call_id=call.id,
+ delta="contents",
+ ),
+ ToolResultDataDeltaEvent(
+ reply_id="backend",
+ tool_call_id=call.id,
+ media_type="image/png",
+ data="aGVsbG8=",
+ ),
+ ToolResultEndEvent(
+ reply_id="backend",
+ tool_call_id=call.id,
+ state=ToolResultState.SUCCESS,
+ ),
+ interaction_type(
+ reply_id="backend",
+ tool_calls=[
+ ToolCallBlock(
+ id="write",
+ name="Write",
+ input="{}",
+ ),
+ ],
+ ),
+ ]
+
+ class Backend:
+ """Persist a result after yielding its end event."""
+
+ name = "Friday"
+
+ def __init__(
+ self,
+ script: list[AgentEvent],
+ completed: Msg,
+ ) -> None:
+ self.state = AgentState()
+ self.script = script
+ self.completed = completed
+
+ # pylint: disable=unused-argument
+ async def reply_stream(
+ self,
+ inputs: Msg,
+ yield_final_msg: bool,
+ ) -> AsyncIterator[AgentEvent]:
+ """Complete Read before requesting outside input."""
+ for event in self.script:
+ yield event
+ if isinstance(event, ToolResultEndEvent):
+ self.state.context.append(self.completed)
+
+ # pylint: enable=unused-argument
+
+ backend = Backend(
+ events,
+ AssistantMsg(
+ id="backend",
+ name="Friday",
+ content=[call, result],
+ ),
+ )
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ _model(),
+ delegation_agent_factory=lambda _, current=backend: (
+ current
+ ),
+ )
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id="task",
+ target="client",
+ ),
+ )
+ await asyncio.gather(*list(agent._tasks))
+ agent._cancel_delegation_timeout(agent._delegations["task"])
+ published = []
+ snapshots = []
+ checkpoints = []
+ for event in agent.drain_events():
+ if isinstance(event, CustomEvent):
+ checkpoints.append((event.name, event.value))
+ snapshot = agent.checkpoint_snapshot(event)
+ assert snapshot is not None
+ self.assertEqual(
+ snapshot.context[0].model_dump(),
+ backend.state.context[0].model_dump(),
+ )
+ continue
+ published.append(event.model_dump())
+ if isinstance(event, ToolResultEndEvent):
+ snapshot = agent.checkpoint_snapshot(event)
+ assert snapshot is not None
+ snapshots.append(snapshot.context[0].model_dump())
+ self.assertEqual(
+ {
+ "events": published,
+ "snapshots": snapshots,
+ "checkpoints": checkpoints,
+ },
+ {
+ "events": [event.model_dump() for event in events],
+ "snapshots": [backend.state.context[0].model_dump()],
+ "checkpoints": [
+ (
+ "delegation_checkpoint",
+ {
+ "delegation_id": "task",
+ "reply_id": "backend",
+ "tool_call_id": "read",
+ },
+ ),
+ ],
+ },
+ )
+
+ async def test_client_delegation_resumes_after_confirmation(self) -> None:
+ """Realtime control routes approval back to the delegated agent."""
+ model = _model()
+ backend = _ConfirmingDelegationAgent()
+ model.send_delegation_result = ( # type: ignore[method-assign]
+ AsyncMock()
+ )
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ model,
+ delegation_agent_factory=lambda _state: backend,
+ )
+
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id="delegation_1",
+ target="client",
+ ),
+ )
+ await asyncio.gather(*list(agent._tasks))
+ self.assertDictEqual(
+ agent._delegated_tool_calls,
+ {"tool_1": "delegation_1"},
+ )
+ delegation = agent._delegations["delegation_1"]
+ self.assertListEqual(delegation.buffered_events, [])
+
+ sync_calls = 0
+ original_sync = agent._sync_delegation_state
+
+ def _count_sync(current: object) -> None:
+ nonlocal sync_calls
+ sync_calls += 1
+ original_sync(current) # type: ignore[arg-type]
+
+ agent._sync_delegation_state = ( # type: ignore[method-assign]
+ _count_sync
+ )
+
+ await agent.send(
+ UserConfirmResultEvent(
+ reply_id="backend_reply",
+ confirm_results=[
+ ConfirmResult(
+ confirmed=True,
+ tool_call=backend.tool_call,
+ ),
+ ],
+ ),
+ )
+ await asyncio.gather(*list(agent._tasks))
+
+ model.send_delegation_result.assert_awaited_once_with(
+ "delegation_1",
+ "Order 42 updated.",
+ )
+ self.assertDictEqual(agent._delegated_tool_calls, {})
+ self.assertEqual(sync_calls, 2)
+
+ async def test_delegations_preserve_middleware_state(self) -> None:
+ """Carry middleware data across tasks without sharing nested state."""
+ counts = []
+ backends = []
+
+ class Counter(MiddlewareBase):
+ """Record the number of replies in persistent middleware data."""
+
+ async def on_reply(
+ self,
+ agent: Agent,
+ input_kwargs: dict,
+ next_handler: Callable[..., AsyncGenerator],
+ ) -> AsyncGenerator:
+ """Increment state before running the actual Agent reply."""
+ values = agent.state.middle_context.setdefault(
+ "counter",
+ {"count": 0, "history": []},
+ )
+ values["count"] += 1
+ values["history"].append(values["count"])
+ counts.append(values["count"])
+ async for item in next_handler(**input_kwargs):
+ yield item
+
+ def create_backend(state: AgentState) -> Agent:
+ """Build a real Agent with deterministic model output."""
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[TextBlock(text="Done")],
+ is_last=True,
+ ),
+ ],
+ )
+ backend = Agent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ middlewares=[Counter()],
+ )
+ backends.append(backend)
+ return backend
+
+ model = _model()
+ model.send_delegation_result = AsyncMock()
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ model,
+ delegation_agent_factory=create_backend,
+ )
+ for task_id in ("first", "second"):
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id=task_id,
+ target="client",
+ ),
+ )
+ while agent._tasks:
+ await asyncio.gather(*list(agent._tasks))
+
+ backends[0].state.middle_context["counter"]["history"].append(99)
+ backends[1].state.middle_context["counter"]["history"].append(100)
+ self.assertEqual(
+ {
+ "counts": counts,
+ "saved": agent.state.middle_context,
+ "backend_states": [
+ backend.state.middle_context for backend in backends
+ ],
+ },
+ {
+ "counts": [1, 2],
+ "saved": {"counter": {"count": 2, "history": [1, 2]}},
+ "backend_states": [
+ {"counter": {"count": 1, "history": [1, 99]}},
+ {"counter": {"count": 2, "history": [1, 2, 100]}},
+ ],
+ },
+ )
+
+ async def test_client_delegations_use_latest_serial_state(self) -> None:
+ """Queued tasks are created only after the active task syncs state."""
+ model = _model()
+ model.send_delegation_result = ( # type: ignore[method-assign]
+ AsyncMock()
+ )
+ state = AgentState()
+ factory_snapshots: list[list[str]] = []
+
+ def _factory(snapshot: AgentState) -> _StateDelegationAgent:
+ factory_snapshots.append(
+ list(snapshot.tool_context.activated_groups),
+ )
+ marker = f"task-{len(factory_snapshots)}"
+ return _StateDelegationAgent(snapshot, marker)
+
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ delegation_agent_factory=_factory, # type: ignore[arg-type]
+ )
+
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id="delegation_1",
+ target="client",
+ ),
+ )
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id="delegation_2",
+ target="client",
+ ),
+ )
+ while agent._tasks:
+ await asyncio.gather(*list(agent._tasks))
+
+ self.assertDictEqual(
+ {
+ "factory_snapshots": factory_snapshots,
+ "shared_groups": state.tool_context.activated_groups,
+ "delegations": agent._delegations,
+ "queue": list(agent._delegation_queue),
+ "active": agent._active_delegation_id,
+ },
+ {
+ "factory_snapshots": [[], ["task-1"]],
+ "shared_groups": ["task-1", "task-2"],
+ "delegations": {},
+ "queue": [],
+ "active": None,
+ },
+ )
+
+ async def test_client_delegation_resumes_external_execution(self) -> None:
+ """External results route back to the parked delegated agent."""
+ model = _model()
+ backend = _ExternalDelegationAgent()
+ model.send_delegation_result = ( # type: ignore[method-assign]
+ AsyncMock()
+ )
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ model,
+ delegation_agent_factory=lambda _state: backend,
+ )
+
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id="delegation_1",
+ target="client",
+ ),
+ )
+ await asyncio.gather(*list(agent._tasks))
+ model.send_delegation_result.assert_not_awaited()
+ self.assertDictEqual(
+ agent._delegated_tool_calls,
+ {"external_1": "delegation_1"},
+ )
+ queued = []
+ while not agent._out.empty():
+ item = agent._out.get_nowait()
+ queued.append(
+ (type(item.event).__name__, item.checkpoint is not None),
+ )
+ self.assertIn(
+ ("RequireExternalExecutionEvent", True),
+ queued,
+ )
+
+ external_result = ExternalExecutionResultEvent(
+ reply_id="external_reply",
+ execution_results=[
+ ToolResultBlock(
+ id="external_1",
+ name="ask_user",
+ output=[TextBlock(text="Order 42")],
+ state=ToolResultState.SUCCESS,
+ ),
+ ],
+ )
+ await agent._on_control(
+ ControlFrame(
+ type=ControlFrameType.EXTERNAL_EXECUTION_RESULT,
+ data=external_result.model_dump(mode="json"),
+ ),
+ )
+ await asyncio.gather(*list(agent._tasks))
+
+ self.assertEqual(backend.received[-1], external_result)
+ model.send_delegation_result.assert_awaited_once_with(
+ "delegation_1",
+ "Order 42 selected.",
+ )
+ self.assertDictEqual(agent._delegated_tool_calls, {})
+
+ async def test_parked_delegation_intentionally_holds_fifo(self) -> None:
+ """A parked task holds FIFO until its input deadline expires."""
+ model = _model()
+ first = _ExternalDelegationAgent()
+ second = _FakeDelegationAgent()
+ created: list[object] = []
+
+ def _factory(state: AgentState) -> object:
+ del state
+ backend = first if not created else second
+ created.append(backend)
+ return backend
+
+ model.send_delegation_result = ( # type: ignore[method-assign]
+ AsyncMock()
+ )
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ model,
+ delegation_agent_factory=_factory, # type: ignore[arg-type]
+ )
+
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id="delegation_1",
+ target="client",
+ ),
+ )
+ await asyncio.gather(*list(agent._tasks))
+ await agent._on_model_event(
+ DelegationCreatedEvent(
+ delegation_id="delegation_2",
+ target="client",
+ ),
+ )
+
+ self.assertDictEqual(
+ {
+ "created": created,
+ "active": agent._active_delegation_id,
+ "queue": list(agent._delegation_queue),
+ },
+ {
+ "created": [first],
+ "active": "delegation_1",
+ "queue": ["delegation_2"],
+ },
+ )
+
+ delegation = agent._delegations["delegation_1"]
+ self.assertIsNotNone(delegation.timeout_task)
+ agent._cancel_delegation_timeout(delegation)
+ with patch(
+ "agentscope.agent._realtime._agent."
+ "_DELEGATION_INPUT_TIMEOUT_SECS",
+ 0,
+ ):
+ await agent._expire_client_delegation(delegation)
+ while agent._tasks:
+ await asyncio.gather(*list(agent._tasks))
+
+ self.assertDictEqual(
+ {
+ "first_received": [
+ type(item).__name__ for item in first.received
+ ],
+ "created": created,
+ "active": agent._active_delegation_id,
+ "queue": list(agent._delegation_queue),
+ "delegations": agent._delegations,
+ "expired": delegation.expiring,
+ },
+ {
+ "first_received": ["Msg", "UserInterruptEvent"],
+ "created": [first, second],
+ "active": None,
+ "queue": [],
+ "delegations": {},
+ "expired": True,
+ },
+ )
+ self.assertListEqual(
+ model.send_delegation_result.await_args_list,
+ [
+ unittest.mock.call(
+ "delegation_1",
+ "The delegated task was cancelled because the required "
+ "user input or external execution was not provided in "
+ "time.",
+ ),
+ unittest.mock.call(
+ "delegation_2",
+ "Order 42 shipped.",
+ ),
+ ],
+ )
+
+ async def test_unmatched_external_result_is_logged(self) -> None:
+ """An external result without a delegation is not silent."""
+ agent = RealtimeAgent("Friday", "Be brief.", _model())
+ event = ExternalExecutionResultEvent(
+ reply_id="external_reply",
+ execution_results=[
+ ToolResultBlock(
+ id="unknown_tool",
+ name="ask_user",
+ output=[TextBlock(text="Order 42")],
+ state=ToolResultState.SUCCESS,
+ ),
+ ],
+ )
+
+ with self.assertLogs("as", level="WARNING") as captured:
+ await agent.send(event)
+
+ self.assertListEqual(
+ captured.output,
+ [
+ "WARNING:as:RealtimeAgent: ignored external execution "
+ "results without an active delegation: ['unknown_tool']",
+ ],
+ )
+
+ async def test_live_transcript_samples_exact_snapshots(
+ self,
+ ) -> None:
+ """Only sampled deltas carry their exact persistence boundary."""
+ model = _model()
+ agent = RealtimeAgent("Friday", "Be brief.", model)
+ events = [
+ TimelineTranscriptDeltaEvent(
+ role="user",
+ delta=delta,
+ start_ms=start_ms,
+ end_ms=end_ms,
+ )
+ for delta, start_ms, end_ms in [
+ ("one", 0, 100),
+ ("two", 100, 200),
+ ("three", 2_000, 2_100),
+ ("four", 2_100, 2_200),
+ ]
+ ]
+
+ with patch(
+ "agentscope.agent._realtime._agent.time.monotonic",
+ return_value=1.0,
+ ):
+ for event in events:
+ await agent._on_model_event(event)
+
+ checkpoints = []
+ while not agent._out.empty():
+ queued = agent._out.get_nowait()
+ if isinstance(queued.event, TextBlockDeltaEvent):
+ checkpoints.append(
+ {
+ "delta": queued.event.delta,
+ "checkpoint": (
+ None
+ if queued.checkpoint is None
+ else queued.checkpoint.context[
+ 0
+ ].get_text_content()
+ ),
+ },
+ )
+
+ self.assertListEqual(
+ checkpoints,
+ [
+ {"delta": "one", "checkpoint": "one"},
+ {"delta": "two", "checkpoint": None},
+ {"delta": "three", "checkpoint": "onetwothree"},
+ {"delta": "four", "checkpoint": None},
+ ],
+ )
+
+ async def test_live_transcript_keeps_a_timestamped_bounded_window(
+ self,
+ ) -> None:
+ """Prompt history stays bounded while retaining timeline metadata."""
+ model = _model()
+ agent = RealtimeAgent("Friday", "Be brief.", model)
+ events = [
+ TimelineTranscriptDeltaEvent(
+ role="user",
+ delta="12345",
+ start_ms=10,
+ end_ms=20,
+ ),
+ TimelineTranscriptDeltaEvent(
+ role="assistant",
+ delta="abcde",
+ start_ms=21,
+ end_ms=30,
+ ),
+ TimelineTranscriptDeltaEvent(
+ role="assistant",
+ delta="fg",
+ start_ms=31,
+ end_ms=35,
+ ),
+ ]
+
+ with patch(
+ "agentscope.agent._realtime._agent._HISTORY_MAX_TEXT_CHARS",
+ 8,
+ ):
+ for event in events:
+ await agent._on_model_event(event)
+
+ self.assertDictEqual(
+ {
+ "transcript": list(agent._live_transcript),
+ "chars": agent._live_transcript_chars,
+ },
+ {
+ "transcript": [
+ ("assistant", "abcde", 21, 30),
+ ("assistant", "fg", 31, 35),
+ ],
+ "chars": 7,
+ },
+ )
+
+
+class RealtimeDelegationLifecycleTest(unittest.IsolatedAsyncioTestCase):
+ """Check delegation ownership across reconnects and compaction."""
+
+ async def _reconnect(
+ self,
+ agent: RealtimeAgent,
+ socket: _FakeWebSocket,
+ ) -> None:
+ """Wait for the provider disconnect before opening a new session."""
+ await socket.close()
+
+ async def wait_disconnected() -> None:
+ while agent._connected:
+ await asyncio.sleep(0)
+
+ await asyncio.wait_for(wait_disconnected(), timeout=1)
+ await agent.connect()
+
+ async def test_reconnect_retires_parked_and_queued_delegations(
+ self,
+ ) -> None:
+ """Keep completed work, interrupt HITL and discard old queue items."""
+ sockets = [_FakeWebSocket(), _FakeWebSocket()]
+ executed = []
+ backends = []
+
+ def read() -> str:
+ """Record a completed read before the permission request."""
+ executed.append("read")
+ return "contents"
+
+ def write() -> str:
+ """Record a write only if the user approves it."""
+ executed.append("write")
+ return "written"
+
+ def factory(state: AgentState) -> Agent:
+ """Park the first task and let the next session finish normally."""
+ model = MockModel(context_size=10000)
+ responses = [
+ ChatResponse(content=[TextBlock(text="Done")], is_last=True),
+ ]
+ if not backends:
+ responses = [
+ ChatResponse(
+ content=[
+ ToolCallBlock(
+ id=name,
+ name=name,
+ input="{}",
+ ),
+ ],
+ is_last=True,
+ )
+ for name in ("read", "write")
+ ]
+ model.set_responses(responses)
+ backend = Agent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ toolkit=Toolkit(
+ tools=[
+ FunctionTool(read, is_read_only=True),
+ FunctionTool(write),
+ ],
+ ),
+ )
+ backends.append(backend)
+ return backend
+
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ _model(),
+ delegation_agent_factory=factory,
+ )
+ self.addAsyncCleanup(agent.close)
+ with patch("websockets.connect", AsyncMock(side_effect=sockets)):
+ await agent.connect()
+ agent._start_client_delegation("old")
+ while agent._tasks:
+ await asyncio.gather(*list(agent._tasks))
+ old = agent._delegations["old"]
+ timeout = old.timeout_task
+ agent._start_client_delegation("queued")
+ await self._reconnect(agent, sockets[0])
+ self.assertEqual(
+ {
+ "active": agent._active_delegation_id,
+ "queue": list(agent._delegation_queue),
+ "delegations": agent._delegations,
+ "routes": agent._delegated_tool_calls,
+ "timeout_done": timeout is not None and timeout.done(),
+ "awaiting": backends[0].state.has_awaiting_tool_calls(
+ "Friday",
+ ),
+ "executed": executed,
+ "results": [
+ (block.id, block.state)
+ for message in agent.state.context
+ for block in message.get_content_blocks("tool_result")
+ ],
+ },
+ {
+ "active": None,
+ "queue": [],
+ "delegations": {},
+ "routes": {},
+ "timeout_done": True,
+ "awaiting": False,
+ "executed": ["read"],
+ "results": [
+ ("read", ToolResultState.SUCCESS),
+ ("write", ToolResultState.INTERRUPTED),
+ ],
+ },
+ )
+ # A late callback must not revive a task from the old connection.
+ await agent._drive_client_delegation(old, old.prompt)
+ agent._start_client_delegation("new")
+ while agent._tasks:
+ await asyncio.gather(*list(agent._tasks))
+ self.assertEqual(
+ {
+ "backends": len(backends),
+ "commentary": [
+ (item["delegation_id"], item["content"])
+ for item in sockets[1].sent
+ if item["type"] == "session.commentary.append"
+ ],
+ },
+ {"backends": 2, "commentary": [("new", "Done")]},
+ )
+
+ async def test_reconnect_cancels_running_backend_before_new_session(
+ self,
+ ) -> None:
+ """Do not send an old task's late result on the new connection."""
+ started = asyncio.Event()
+ cancelled = asyncio.Event()
+
+ class Backend(_FakeDelegationAgent):
+ """Keep a task running until reconnect cancels its execution."""
+
+ async def reply_stream(
+ self,
+ inputs: Msg,
+ yield_final_msg: bool,
+ ) -> AsyncIterator[Msg]:
+ """Return late output after receiving cancellation."""
+ self.state.middle_context["completed_operation"] = True
+ started.set()
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ cancelled.set()
+ async for item in super().reply_stream(
+ inputs,
+ yield_final_msg,
+ ):
+ yield item
+
+ backend = Backend()
+ sockets = [_FakeWebSocket(), _FakeWebSocket()]
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ _model(),
+ delegation_agent_factory=lambda _: backend,
+ )
+ self.addAsyncCleanup(agent.close)
+ with patch("websockets.connect", AsyncMock(side_effect=sockets)):
+ await agent.connect()
+ agent._start_client_delegation("old")
+ await asyncio.wait_for(started.wait(), timeout=1)
+ await self._reconnect(agent, sockets[0])
+ self.assertEqual(
+ {
+ "cancelled": cancelled.is_set(),
+ "tasks": len(agent._tasks),
+ "active": agent._active_delegation_id,
+ "state": agent.state.middle_context,
+ "commentary": [
+ item
+ for item in sockets[1].sent
+ if item["type"] == "session.commentary.append"
+ ],
+ },
+ {
+ "cancelled": True,
+ "tasks": 0,
+ "active": None,
+ "state": {"completed_operation": True},
+ "commentary": [],
+ },
+ )
+
+ async def test_compression_prunes_only_delegated_history(self) -> None:
+ """Sync real compression while retaining concurrent Live messages."""
+ recent = UserMsg("user", "3" * 40, id="recent")
+ parent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ _model(),
+ state=AgentState(
+ summary="old summary",
+ context=[
+ UserMsg("user", "1" * 120, id="old1"),
+ AssistantMsg("Friday", "2" * 40, id="old2"),
+ recent,
+ ],
+ ),
+ )
+ model = MockModel(context_size=100)
+ model.set_structured_response(
+ StructuredResponse(
+ content={
+ "task_overview": "Compressed",
+ "current_state": "Done",
+ "important_discoveries": "Facts",
+ "next_steps": "Continue",
+ "context_to_preserve": "Keep this",
+ },
+ ),
+ )
+ backend = Agent(
+ "Friday",
+ "0" * 80,
+ model,
+ toolkit=Toolkit(),
+ state=parent._copy_delegation_state(),
+ context_config=ContextConfig(trigger_ratio=0.7, reserve_ratio=0.4),
+ )
+ delegation = _ClientDelegation(
+ delegation_id="task",
+ prompt=UserMsg("user", "Task"),
+ generation=parent._connection_generation,
+ agent=backend,
+ baseline_ids={msg.id for msg in backend.state.context},
+ )
+ backend.state.context.insert(
+ 2,
+ AssistantMsg(
+ "Friday",
+ id="tool_reply",
+ content=[
+ ToolResultBlock(
+ id="read",
+ name="Read",
+ output="contents " * 100,
+ state=ToolResultState.SUCCESS,
+ ),
+ ],
+ ),
+ )
+ parent._sync_delegation_state(delegation)
+ self.assertIn("tool_reply", [msg.id for msg in parent.state.context])
+ await parent._on_model_event(
+ TimelineTranscriptDeltaEvent(
+ role="user",
+ delta="Concurrent speech",
+ start_ms=0,
+ end_ms=10,
+ ),
+ )
+ await backend.compress_context()
+ self.assertEqual([msg.id for msg in backend.state.context], ["recent"])
+ self.assertNotEqual(backend.state.summary, "old summary")
+ await parent._on_model_event(
+ TimelineTranscriptDeltaEvent(
+ role="user",
+ delta=" continues",
+ start_ms=10,
+ end_ms=20,
+ ),
+ )
+ live = parent.state.context[-1].model_dump()
+ parent._sync_delegation_state(delegation)
+ # Repeated checkpoints must not reintroduce compacted tool messages.
+ parent._sync_delegation_state(delegation)
+ self.assertEqual(
+ {
+ "summary": parent.state.summary,
+ "messages": [msg.model_dump() for msg in parent.state.context],
+ "next_messages": [
+ msg.model_dump()
+ for msg in parent._copy_delegation_state().context
+ ],
+ },
+ {
+ "summary": backend.state.summary,
+ "messages": [recent.model_dump(), live],
+ "next_messages": [recent.model_dump()],
+ },
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/realtime_openai_test.py b/tests/realtime_openai_test.py
index d589fc1a5..6af825114 100644
--- a/tests/realtime_openai_test.py
+++ b/tests/realtime_openai_test.py
@@ -85,6 +85,7 @@ class OpenAICardsTest(unittest.TestCase):
"gpt-realtime-2": "OpenAIRealtimeModel",
"gpt-realtime-2.1": "OpenAIRealtimeModel",
"gpt-realtime-2.1-mini": "OpenAIRealtimeModel",
+ "gpt-live-1": "OpenAILiveModel",
},
)
diff --git a/tests/realtime_service_test.py b/tests/realtime_service_test.py
index 2ddbd10a5..c12eac179 100644
--- a/tests/realtime_service_test.py
+++ b/tests/realtime_service_test.py
@@ -1,21 +1,35 @@
# -*- coding: utf-8 -*-
"""Tests for realtime service runtime assembly."""
+# pylint: disable=protected-access
import unittest
from types import SimpleNamespace
from typing import cast
from unittest.mock import AsyncMock, patch
+from utils import MockModel
+
+from agentscope.agent import Agent
from agentscope.app._service._realtime import RealtimeService
from agentscope.app.channel import ChatKind
+from agentscope.app.message_bus import MessageBusKeys
+from agentscope.app.middleware import StateChangeMiddleware
from agentscope.app.storage import (
AgentData,
AgentRecord,
+ ChatModelConfig,
ChannelOrigin,
SessionConfig,
+ SessionKnowledgeConfig,
SessionRecord,
)
-from agentscope.realtime import RealtimeModelBase
+from agentscope.middleware import RAGMiddleware
+from agentscope.message import AssistantMsg, TextBlock, ToolCallBlock, UserMsg
+from agentscope.model import ChatResponse
+from agentscope.permission import PermissionMode
+from agentscope.realtime import InteractionMode, RealtimeModelBase
+from agentscope.state import AgentState
+from agentscope.tool import FunctionTool, TaskCreate, Toolkit
class RealtimeServiceTest(unittest.IsolatedAsyncioTestCase):
@@ -75,7 +89,10 @@ class RealtimeServiceTest(unittest.IsolatedAsyncioTestCase):
extra_factory = AsyncMock()
templates = {"researcher": object()}
toolkit = object()
- model = object()
+ model = SimpleNamespace(
+ interaction_mode=InteractionMode.TURN_BASED,
+ card=SimpleNamespace(delegation_backend="none"),
+ )
service = RealtimeService(
storage=storage,
workspace_manager=workspace_manager,
@@ -220,7 +237,13 @@ class RealtimeServiceTest(unittest.IsolatedAsyncioTestCase):
),
)
toolkit = object()
- model = cast(RealtimeModelBase, object())
+ model = cast(
+ RealtimeModelBase,
+ SimpleNamespace(
+ interaction_mode=InteractionMode.TURN_BASED,
+ card=SimpleNamespace(delegation_backend="none"),
+ ),
+ )
with patch(
"agentscope.app._service._realtime.get_toolkit",
@@ -260,6 +283,418 @@ class RealtimeServiceTest(unittest.IsolatedAsyncioTestCase):
channel.chat_kind.assert_awaited_once_with("chat-1")
channel.chat_name.assert_not_awaited()
+ async def test_gpt_live_rejects_pending_chat_interactions(self) -> None:
+ """Check freshly loaded state before allocating runtime resources."""
+ record = AgentRecord(
+ id="agent",
+ user_id="user",
+ data=AgentData(name="Friday"),
+ )
+ session = SessionRecord(
+ id="session",
+ user_id="user",
+ agent_id="agent",
+ config=SessionConfig(workspace_id="workspace"),
+ )
+ workspace_manager = SimpleNamespace(get_workspace=AsyncMock())
+ service = RealtimeService(
+ storage=SimpleNamespace(
+ get_session=AsyncMock(return_value=session),
+ ),
+ workspace_manager=workspace_manager,
+ scheduler_manager=object(),
+ background_task_manager=object(),
+ message_bus=object(),
+ resource_access_service=SimpleNamespace(
+ resolve_agent=AsyncMock(return_value=record),
+ ),
+ )
+ model = cast(
+ RealtimeModelBase,
+ SimpleNamespace(
+ card=SimpleNamespace(delegation_backend="session_chat"),
+ ),
+ )
+ for state in ("asking", "submitted"):
+ with self.subTest(state=state):
+ session.state.context = [
+ AssistantMsg(
+ name="Friday",
+ id=session.state.reply_id,
+ content=[
+ ToolCallBlock(
+ id="old_tool",
+ name="Write",
+ input="{}",
+ state=state,
+ ),
+ ],
+ ),
+ UserMsg("user", "Later speech"),
+ AssistantMsg("Friday", "Voice response"),
+ ]
+ original = session.state.model_dump()
+ with self.assertRaises(ValueError) as raised:
+ await service.create_agent(
+ user_id="user",
+ agent_id="agent",
+ session_id="session",
+ model=model,
+ )
+ self.assertEqual(
+ {
+ "error": str(raised.exception),
+ "state": session.state.model_dump(),
+ "workspace_calls": (
+ workspace_manager.get_workspace.await_count
+ ),
+ },
+ {
+ "error": (
+ "Complete the pending Chat confirmation or "
+ "external execution before starting GPT Live."
+ ),
+ "state": original,
+ "workspace_calls": 0,
+ },
+ )
+
+ async def test_gpt_live_reuses_the_session_chat_agent(self) -> None:
+ """Client delegation follows the current session chat model."""
+ user_id = "user-1"
+ agent_id = "agent-1"
+ session_id = "session-1"
+ agent_record = AgentRecord(
+ id=agent_id,
+ user_id=user_id,
+ data=AgentData(name="Friday", system_prompt="Be concise."),
+ )
+ chat_config = ChatModelConfig(
+ type="openai_chat",
+ credential_id="credential-1",
+ model="chat-model",
+ parameters={},
+ )
+ session = SessionRecord(
+ id=session_id,
+ user_id=user_id,
+ agent_id=agent_id,
+ config=SessionConfig(
+ workspace_id="workspace-1",
+ chat_model_config=chat_config,
+ ),
+ )
+ session.state.context = [
+ UserMsg(
+ id="kept-message",
+ name="user",
+ content="Persisted chat context.",
+ ),
+ UserMsg(
+ id="live:user:current",
+ name="user",
+ content="Duplicated Live transcript.",
+ ),
+ ]
+ storage = SimpleNamespace(
+ get_session=AsyncMock(return_value=session),
+ )
+ workspace = SimpleNamespace(workdir="/workspace/session-1")
+ middleware = SimpleNamespace(
+ is_implemented=lambda name: name == "on_reply",
+ )
+ middleware_calls: list[tuple] = []
+
+ async def _extra_middlewares(*args: object) -> list[object]:
+ middleware_calls.append(args)
+ return [middleware]
+
+ service = RealtimeService(
+ storage=storage,
+ workspace_manager=SimpleNamespace(
+ get_workspace=AsyncMock(return_value=workspace),
+ ),
+ scheduler_manager=object(),
+ background_task_manager=object(),
+ message_bus=object(),
+ resource_access_service=SimpleNamespace(
+ resolve_agent=AsyncMock(return_value=agent_record),
+ ),
+ extra_agent_middlewares=(
+ _extra_middlewares # type: ignore[arg-type]
+ ),
+ )
+ realtime_model = cast(
+ RealtimeModelBase,
+ SimpleNamespace(
+ interaction_mode=InteractionMode.FULL_DUPLEX,
+ card=SimpleNamespace(
+ delegation_backend="session_chat",
+ ),
+ ),
+ )
+ toolkit = object()
+ backend_model = object()
+
+ with (
+ patch(
+ "agentscope.app._service._realtime.get_toolkit",
+ new=AsyncMock(return_value=toolkit),
+ ) as get_toolkit_mock,
+ patch(
+ "agentscope.app._service._realtime.get_model",
+ new=AsyncMock(return_value=backend_model),
+ ) as get_model_mock,
+ ):
+ realtime_agent = await service.create_agent(
+ user_id=user_id,
+ agent_id=agent_id,
+ session_id=session_id,
+ model=realtime_model,
+ )
+
+ factory = realtime_agent._delegation_agent_factory
+ self.assertIsNotNone(factory)
+ assert factory is not None
+ delegated_state = realtime_agent._copy_delegation_state()
+ delegated_agent = factory(delegated_state)
+ notifier = delegated_agent._reply_middlewares[0]
+ self.assertIsInstance(notifier, StateChangeMiddleware)
+ self.assertDictEqual(
+ {
+ "model": delegated_agent.model,
+ "toolkit": delegated_agent.toolkit,
+ "system_prompt": delegated_agent._system_prompt,
+ "state_texts": [
+ message.get_text_content()
+ for message in delegated_agent.state.context
+ ],
+ "state_isolated": delegated_agent.state is not session.state,
+ "message_isolated": (
+ delegated_agent.state.context[0]
+ is not session.state.context[0]
+ ),
+ "reply_middlewares": delegated_agent._reply_middlewares,
+ "middleware_calls": middleware_calls,
+ },
+ {
+ "model": backend_model,
+ "toolkit": toolkit,
+ "system_prompt": (
+ "Be concise.\n\n<system-notification>You're within a "
+ "session (id=session-1).</system-notification>"
+ ),
+ "state_texts": [
+ "Persisted chat context.",
+ "Duplicated Live transcript.",
+ ],
+ "state_isolated": True,
+ "message_isolated": True,
+ "reply_middlewares": [notifier, middleware],
+ "middleware_calls": [
+ (user_id, agent_id, session_id, workspace),
+ ],
+ },
+ )
+ self.assertListEqual(
+ get_toolkit_mock.await_args.kwargs["middlewares"],
+ [notifier, middleware],
+ )
+ get_model_mock.assert_awaited_once_with(
+ user_id,
+ chat_config,
+ service._access,
+ )
+
+ async def test_delegation_publishes_state_and_team_changes(self) -> None:
+ """Assembled middleware publishes tool changes to the session bus."""
+ bus = SimpleNamespace(
+ log_append=AsyncMock(return_value="1-0"),
+ publish=AsyncMock(),
+ )
+ service = RealtimeService(
+ storage=object(),
+ workspace_manager=object(),
+ scheduler_manager=object(),
+ background_task_manager=object(),
+ message_bus=bus,
+ resource_access_service=object(),
+ )
+ session = SessionRecord(
+ id="session",
+ user_id="user",
+ agent_id="agent",
+ config=SessionConfig(workspace_id="workspace"),
+ )
+ middlewares = await service._build_delegation_middlewares(
+ user_id="user",
+ agent_id="agent",
+ session_id=session.id,
+ session=session,
+ workspace=object(),
+ )
+ state = AgentState()
+ initial_permission = state.permission_context.model_dump(mode="json")
+
+ def update_permission() -> str:
+ """Simulate a tool changing session permission state."""
+ state.permission_context.mode = PermissionMode.ACCEPT_EDITS
+ return "Updated"
+
+ def create_team() -> str:
+ """Simulate a successful team tool without external storage."""
+ return "Created"
+
+ model = MockModel(context_size=10000)
+ model.set_responses(
+ [
+ ChatResponse(
+ content=[
+ ToolCallBlock(
+ id="task",
+ name="TaskCreate",
+ input=(
+ '{"subject":"Test",'
+ '"description":"Notify UI"}'
+ ),
+ ),
+ ],
+ is_last=True,
+ ),
+ ChatResponse(
+ content=[
+ ToolCallBlock(
+ id="permission",
+ name="update_permission",
+ input="{}",
+ ),
+ ],
+ is_last=True,
+ ),
+ ChatResponse(
+ content=[
+ ToolCallBlock(
+ id="team",
+ name="TeamCreate",
+ input="{}",
+ ),
+ ],
+ is_last=True,
+ ),
+ ChatResponse(content=[TextBlock(text="Done")], is_last=True),
+ ],
+ )
+ agent = Agent(
+ "Friday",
+ "Be brief.",
+ model,
+ state=state,
+ middlewares=middlewares,
+ toolkit=Toolkit(
+ tools=[
+ TaskCreate(),
+ FunctionTool(update_permission, is_read_only=True),
+ FunctionTool(
+ create_team,
+ name="TeamCreate",
+ is_read_only=True,
+ ),
+ ],
+ ),
+ )
+ async for _ in agent.reply_stream(
+ UserMsg(name="user", content="Create a task and team."),
+ ):
+ pass
+
+ final_state = {
+ "tasks_context": state.tasks_context.model_dump(mode="json"),
+ "permission_context": state.permission_context.model_dump(
+ mode="json",
+ ),
+ }
+ key = MessageBusKeys.session_events(session.id)
+ self.assertEqual(
+ {
+ "task_count": len(state.tasks_context.tasks),
+ "notifications": [
+ (call.args[0], call.args[1]["name"], call.args[1]["value"])
+ for call in bus.publish.call_args_list
+ ],
+ },
+ {
+ "task_count": 1,
+ "notifications": [
+ (
+ key,
+ "state_updated",
+ {
+ **final_state,
+ "permission_context": initial_permission,
+ },
+ ),
+ (key, "state_updated", final_state),
+ (key, "team_updated", {}),
+ (key, "state_updated", final_state),
+ ],
+ },
+ )
+
+ async def test_gpt_live_builds_knowledge_middleware(self) -> None:
+ """Attach configured session knowledge to delegated Chat agents."""
+ knowledge = object()
+ manager = SimpleNamespace(
+ get_knowledge=AsyncMock(return_value=knowledge),
+ )
+ access = SimpleNamespace(
+ resolve_knowledge_base=AsyncMock(
+ return_value=SimpleNamespace(user_id="owner-1"),
+ ),
+ )
+ service = RealtimeService(
+ storage=object(), # type: ignore[arg-type]
+ workspace_manager=object(), # type: ignore[arg-type]
+ scheduler_manager=object(), # type: ignore[arg-type]
+ background_task_manager=object(), # type: ignore[arg-type]
+ message_bus=object(), # type: ignore[arg-type]
+ resource_access_service=access, # type: ignore[arg-type]
+ knowledge_base_manager=manager, # type: ignore[arg-type]
+ )
+ session = SessionRecord(
+ id="session-1",
+ user_id="user-1",
+ agent_id="agent-1",
+ config=SessionConfig(
+ workspace_id="workspace-1",
+ knowledge_config=SessionKnowledgeConfig(
+ knowledge_base_ids=["knowledge-1"],
+ parameters={"top_k": 3},
+ ),
+ ),
+ )
+
+ middlewares = await service._build_delegation_middlewares(
+ user_id="user-1",
+ agent_id="agent-1",
+ session_id="session-1",
+ session=session,
+ workspace=object(),
+ )
+
+ self.assertEqual(len(middlewares), 2)
+ self.assertIsInstance(middlewares[0], StateChangeMiddleware)
+ self.assertIsInstance(middlewares[1], RAGMiddleware)
+ self.assertListEqual(middlewares[1]._knowledge_bases, [knowledge])
+ self.assertEqual(middlewares[1]._parameters.top_k, 3)
+ access.resolve_knowledge_base.assert_awaited_once_with(
+ "user-1",
+ "knowledge-1",
+ )
+ manager.get_knowledge.assert_awaited_once_with(
+ "owner-1",
+ "knowledge-1",
+ )
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/service_team_failure_report_test.py b/tests/service_team_failure_report_test.py
index 7c63d5446..571c584e9 100644
--- a/tests/service_team_failure_report_test.py
+++ b/tests/service_team_failure_report_test.py
@@ -456,6 +456,7 @@ class TeamFailureReportTest(IsolatedAsyncioTestCase):
self.state.context.append(
AssistantMsg(
name=self.name,
+ id=self.state.reply_id,
content=[
ToolCallBlock(
id="call-1",
diff --git a/tests/service_wakeup_dispatcher_test.py b/tests/service_wakeup_dispatcher_test.py
index 0c83bed76..a6cbcc6e6 100644
--- a/tests/service_wakeup_dispatcher_test.py
+++ b/tests/service_wakeup_dispatcher_test.py
@@ -90,7 +90,10 @@ class _FakeStorage:
],
),
]
- return SimpleNamespace(state=AgentState(context=context))
+ state = AgentState(context=context)
+ if context:
+ state.reply_id = context[0].id
+ return SimpleNamespace(state=state)
async def get_agent(self, _user_id: str, _agent_id: str) -> object:
"""Return the agent name used by canonical HITL detection."""
diff --git a/tests/webrtc_audio_transport_test.py b/tests/webrtc_audio_transport_test.py
index de57699da..62121a6c0 100644
--- a/tests/webrtc_audio_transport_test.py
+++ b/tests/webrtc_audio_transport_test.py
@@ -5,7 +5,7 @@
import asyncio
import json
import unittest
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
from contextlib import asynccontextmanager
from fractions import Fraction
from itertools import count
@@ -34,6 +34,7 @@ from agentscope.app._service._webrtc_session import WebRTCSession
from agentscope.app.message_bus import InMemoryMessageBus, MessageBusKeys
from agentscope.app._bus_ops import enqueue_chat_input
from agentscope.app.storage import CredentialRecord, RealtimeModelConfig
+from agentscope.credential import OpenAICredential
from agentscope.event import (
DataBlockDeltaEvent,
DataBlockEndEvent,
@@ -46,15 +47,23 @@ from agentscope.event import (
TextBlockStartEvent,
ToolResultEndEvent,
)
-from agentscope.message import ToolCallBlock, ToolResultState, UserMsg
+from agentscope.message import (
+ AssistantMsg,
+ ToolCallBlock,
+ ToolResultState,
+ UserMsg,
+)
from agentscope.realtime import (
AudioFrame,
DashScopeAudioRealtimeModel,
GeminiRealtimeModel,
+ OpenAILiveModel,
RealtimeModelBase,
RealtimeModelCard,
+ TimelineTranscriptDeltaEvent,
)
from agentscope.realtime import _events as me
+from agentscope.state import AgentState
from agentscope.tool import FunctionTool, Toolkit
@@ -154,6 +163,10 @@ class _FakeAgent:
for event in self.events:
yield event
+ def drain_events(self) -> Iterator[Any]:
+ """No additional cleanup events for the default fixture."""
+ return iter(())
+
def checkpoint_snapshot(self, event: Any) -> object | None:
"""Return state for the same boundaries as the real agent."""
if isinstance(
@@ -168,6 +181,44 @@ class _FakeAgent:
return None
+class _FailingClosingAgent(_FakeAgent):
+ """Fail after starting a reply and enqueue terminal events on exit."""
+
+ def __init__(self) -> None:
+ super().__init__({"id": "reply-1"}, [])
+ self.failure = RuntimeError("stream failed")
+ self.started = ReplyStartEvent(
+ session_id="session-1",
+ reply_id="reply-1",
+ name="Friday",
+ )
+ self.terminal = [
+ ToolResultEndEvent(
+ reply_id="reply-1",
+ tool_call_id="tool-1",
+ state=ToolResultState.INTERRUPTED,
+ ),
+ ReplyEndEvent(session_id="session-1", reply_id="reply-1"),
+ ]
+
+ async def __aexit__(self, *exc: object) -> None:
+ self.events.extend(self.terminal)
+
+ async def reply_stream(
+ self,
+ transport: object,
+ ) -> AsyncIterator[Any]:
+ """Start the reply before simulating a provider stream error."""
+ del transport
+ yield self.started
+ raise self.failure
+
+ def drain_events(self) -> Iterator[Any]:
+ """Consume the events produced by context-manager cleanup."""
+ events, self.events = self.events, []
+ return iter(events)
+
+
class _UnacknowledgedSetupSocket:
"""Keep a real Gemini connect waiting for its setup acknowledgement."""
@@ -922,6 +973,59 @@ class RealtimeOfferTest(unittest.IsolatedAsyncioTestCase):
},
)
+ async def test_gpt_live_offer_rejects_pending_chat(self) -> None:
+ """Reject parked Chat interactions before WebRTC negotiation."""
+ self.model.card = SimpleNamespace(delegation_backend="session_chat")
+ self.session.config.chat_model_config = object()
+ self.access.resolve_agent.return_value = SimpleNamespace(
+ data=SimpleNamespace(name="Friday"),
+ )
+ for state in ("asking", "submitted"):
+ with self.subTest(state=state):
+ self.session.state = AgentState(
+ context=[
+ AssistantMsg(
+ name="Friday",
+ content=[
+ ToolCallBlock(
+ id="old_tool",
+ name="Write",
+ input="{}",
+ state=state,
+ ),
+ ],
+ ),
+ ],
+ )
+ self.session.state.reply_id = self.session.state.context[0].id
+ self.session.state.context.extend(
+ [
+ UserMsg("user", "Later speech"),
+ AssistantMsg("Friday", "Voice response"),
+ ],
+ )
+ with self.assertRaises(HTTPException) as raised:
+ await self._offer()
+ self.assertEqual(
+ {
+ "status": raised.exception.status_code,
+ "detail": raised.exception.detail,
+ "negotiation_calls": (
+ self.peer.setRemoteDescription.await_count
+ ),
+ "connections": self.connections,
+ },
+ {
+ "status": 409,
+ "detail": (
+ "Complete the pending Chat confirmation or "
+ "external execution before starting GPT Live."
+ ),
+ "negotiation_calls": 0,
+ "connections": {},
+ },
+ )
+
async def test_offer_replaces_the_previous_connection(self) -> None:
"""A reconnect closes the previous local runner before reload."""
previous = SimpleNamespace(close=AsyncMock())
@@ -995,9 +1099,100 @@ class RealtimeOfferTest(unittest.IsolatedAsyncioTestCase):
self.peer.close.assert_awaited_once_with()
+# pylint: disable-next=too-many-public-methods
class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):
"""Verify that a WebRTC run keeps the normal session contract."""
+ async def test_sampled_live_checkpoint_persists_immediately(self) -> None:
+ """Persist a sampled transcript snapshot at its published cursor."""
+
+ class _LiveFakeAgent(_FakeAgent):
+ allows_open_user_reply_checkpoint = True
+
+ def checkpoint_snapshot(self, event: Any) -> object | None:
+ if isinstance(event, TextBlockDeltaEvent):
+ return self.state
+ return super().checkpoint_snapshot(event)
+
+ events = [
+ ReplyStartEvent(
+ session_id="session-1",
+ reply_id="user-1",
+ name="user",
+ role="user",
+ ),
+ TextBlockDeltaEvent(
+ reply_id="user-1",
+ block_id="text-1",
+ delta="hello",
+ ),
+ ]
+ agent = _LiveFakeAgent(
+ {"id": "user-1", "role": "user"},
+ events,
+ )
+ message_bus = _FakeMessageBus(sequence_entries=True)
+ storage = _FakeStorage()
+ closed = asyncio.Event()
+ session = _make_session(
+ agent_factory=AsyncMock(return_value=agent),
+ storage=storage,
+ message_bus=message_bus,
+ on_closed=lambda _: closed.set(),
+ )
+
+ session.start()
+ await asyncio.wait_for(closed.wait(), timeout=1)
+
+ self.assertDictEqual(
+ {
+ "storage_methods": [item["method"] for item in storage.calls],
+ "cursor_values": [
+ item["value"]
+ for item in message_bus.calls
+ if item["method"] == "registry_set"
+ ],
+ },
+ {
+ "storage_methods": [
+ "update_session_state",
+ "update_session_state",
+ ],
+ "cursor_values": ["2-0"],
+ },
+ )
+
+ async def test_older_checkpoint_cannot_regress_cursor(self) -> None:
+ """A delayed flush cannot overwrite a newer checkpoint cursor."""
+ message_bus = _FakeMessageBus()
+ session = _make_session(
+ agent_factory=AsyncMock(),
+ storage=_FakeStorage(),
+ message_bus=message_bus,
+ on_closed=lambda _: None,
+ )
+ session._checkpoint_entry_id = "2-0"
+ session._persist_state = AsyncMock() # type: ignore[method-assign]
+
+ await session._checkpoint_state("1-0", cast(Any, object()))
+
+ self.assertDictEqual(
+ {
+ "persist_calls": session._persist_state.await_args_list,
+ "checkpoint_entry_id": session._checkpoint_entry_id,
+ "registry_calls": [
+ item
+ for item in message_bus.calls
+ if item["method"] == "registry_set"
+ ],
+ },
+ {
+ "persist_calls": [],
+ "checkpoint_entry_id": "2-0",
+ "registry_calls": [],
+ },
+ )
+
async def test_transcript_between_responses_preserves_full_reply(
self,
) -> None:
@@ -1536,6 +1731,193 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):
},
)
+ async def test_pruned_live_segments_remain_in_message_history(
+ self,
+ ) -> None:
+ """Checkpoint pruning bounds state without erasing voice history."""
+ agent = RealtimeAgent(
+ "Friday",
+ "Be brief.",
+ OpenAILiveModel(
+ "gpt-live-1",
+ OpenAICredential(api_key="fake"),
+ ),
+ )
+ storage = _FakeStorage()
+ session = _make_session(
+ agent_factory=AsyncMock(),
+ storage=storage,
+ message_bus=_FakeMessageBus(),
+ on_closed=lambda _: None,
+ )
+ session.agent = agent
+ for index in range(50):
+ for role in ("user", "assistant"):
+ await agent._on_model_event(
+ TimelineTranscriptDeltaEvent(
+ role=role,
+ delta=role[0] * 1_000,
+ start_ms=index * 2_100,
+ end_ms=index * 2_100 + 100,
+ ),
+ )
+ for event in agent.drain_events():
+ snapshot = agent.checkpoint_snapshot(event)
+ if snapshot is not None:
+ await session._persist_state(snapshot)
+ await agent.close()
+ for event in agent.drain_events():
+ snapshot = agent.checkpoint_snapshot(event)
+ if snapshot is not None:
+ await session._persist_state(snapshot)
+ await session._persist_state()
+ self.assertEqual(
+ {
+ "history": {
+ role: "".join(
+ msg.get_text_content() or ""
+ for msg in storage.messages.values()
+ if msg.role == role
+ )
+ for role in ("user", "assistant")
+ },
+ "deletions": [
+ call
+ for call in storage.calls
+ if call["method"] == "delete_message"
+ ],
+ "cached_ids": set(session._persisted_messages),
+ },
+ {
+ "history": {
+ "user": "u" * 50_000,
+ "assistant": "a" * 50_000,
+ },
+ "deletions": [],
+ "cached_ids": {msg.id for msg in agent.state.context},
+ },
+ )
+ self.assertLessEqual(
+ sum(
+ len(msg.get_text_content() or "")
+ for msg in agent.state.context
+ ),
+ 32_000,
+ )
+ self.assertLess(len(agent.state.context), len(storage.messages))
+
+ async def test_close_events_are_published_before_final_persistence(
+ self,
+ ) -> None:
+ """Events emitted by Agent exit are not stranded in its queue."""
+ ended = ReplyEndEvent(
+ session_id="session-1",
+ reply_id="reply-1",
+ )
+
+ class ClosingAgent(_FakeAgent):
+ """Emit the final reply only after the runtime closes."""
+
+ def drain_events(self) -> Iterator[Any]:
+ return iter([ended])
+
+ agent = ClosingAgent({"id": "reply-1"}, [])
+ bus = _FakeMessageBus()
+ storage = _FakeStorage()
+ session = _make_session(
+ agent_factory=AsyncMock(return_value=agent),
+ storage=storage,
+ message_bus=bus,
+ on_closed=lambda _: None,
+ )
+ await session._run()
+ self.assertEqual(session._last_entry_id, "1-0")
+ self.assertEqual(session._checkpoint_entry_id, "1-0")
+ self.assertEqual(session.transport.errors, [])
+
+ async def test_failed_stream_publishes_close_events(self) -> None:
+ """Publish terminal events and their checkpoint before reporting."""
+ agent = _FailingClosingAgent()
+ bus = _FakeMessageBus(sequence_entries=True)
+ storage = _FakeStorage()
+ session = _make_session(
+ agent_factory=AsyncMock(return_value=agent),
+ storage=storage,
+ message_bus=bus,
+ on_closed=lambda _: None,
+ )
+ with self.assertLogs("as", level="ERROR"):
+ await session._run()
+ self.assertEqual(
+ {
+ "published": [
+ call["event"]
+ for call in bus.calls
+ if call["method"] == "publish"
+ ],
+ "cursor": session._checkpoint_entry_id,
+ "errors": session.transport.errors,
+ "transport_closed": session.transport.closed,
+ "peer_closed": session.peer_connection.closed,
+ "remaining": list(agent.drain_events()),
+ },
+ {
+ "published": [
+ {
+ **event.model_dump(mode="json"),
+ "_entry_id": f"{index}-0",
+ }
+ for index, event in enumerate(
+ [agent.started, *agent.terminal],
+ 1,
+ )
+ ],
+ "cursor": "3-0",
+ "errors": ["stream failed"],
+ "transport_closed": True,
+ "peer_closed": True,
+ "remaining": [],
+ },
+ )
+
+ async def test_failed_stream_preserves_original_exception(self) -> None:
+ """Drain closing events before propagating the same exception."""
+ agent = _FailingClosingAgent()
+ session = _make_session(
+ agent_factory=AsyncMock(return_value=agent),
+ storage=_FakeStorage(),
+ message_bus=_FakeMessageBus(),
+ on_closed=lambda _: None,
+ )
+ session.agent = agent
+ events = []
+ with self.assertRaises(RuntimeError) as raised:
+ async for event in session._agent_events():
+ events.append(event)
+ self.assertIs(raised.exception, agent.failure)
+ self.assertEqual(events, [agent.started, *agent.terminal])
+
+ async def test_event_generator_close_does_not_yield(self) -> None:
+ """Explicit generator close and cancellation must not yield."""
+ for cancel in (False, True):
+ with self.subTest(cancel=cancel):
+ agent = _FailingClosingAgent()
+ session = _make_session(
+ agent_factory=AsyncMock(return_value=agent),
+ storage=_FakeStorage(),
+ message_bus=_FakeMessageBus(),
+ on_closed=lambda _: None,
+ )
+ session.agent = agent
+ events = session._agent_events()
+ self.assertEqual(await anext(events), agent.started)
+ if cancel:
+ with self.assertRaises(asyncio.CancelledError):
+ await events.athrow(asyncio.CancelledError())
+ else:
+ await events.aclose()
+ self.assertEqual(list(agent.drain_events()), agent.terminal)
+
async def test_pcm_events_are_not_published_to_session_events(
self,
) -> None:
@@ -1994,6 +2376,7 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):
agent = AsyncMock()
agent.state = SimpleNamespace(context=[])
agent.__aenter__.side_effect = RuntimeError("Setup failed.")
+ agent.drain_events = MagicMock(return_value=iter(()))
storage = BlockingStorage(message_bus)
session = _make_session(
agent_factory=AsyncMock(return_value=agent),
@@ -2026,6 +2409,7 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(message_bus.held_locks, set())
self.assertEqual(closed_sessions, [session])
agent.close.assert_awaited_once_with()
+ self.assertEqual(session.transport.errors, ["Setup failed."])
finally:
release_write.set()
if session._task is not None:
@@ -2066,7 +2450,7 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):
transport = _FakeTransport()
peer_connection = _FakePeerConnection()
storage = _FakeStorage()
- message_bus = _FakeMessageBus()
+ message_bus = _FakeMessageBus(sequence_entries=True)
closed = asyncio.Event()
async def _create_agent() -> _FakeAgent:
@@ -2100,7 +2484,7 @@ class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):
{
"storage_methods": ["update_session_state"] * 3,
"published_event_types": [event.type for event in events],
- "cursor_values": ["1-0"] * 2,
+ "cursor_values": ["2-0", "3-0"],
},
)