Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(realtime): support Gemini 3.8 Live Extended Thinking (#3700)
qbc4704aaf9d174
diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.pyindex ee335dc40..1bc8d48b7 100644--- a/src/agentscope/agent/_realtime/_agent.py+++ b/src/agentscope/agent/_realtime/_agent.py@@ -6,6 +6,7 @@ import timeimport uuidfrom collections import dequefrom collections.abc import Callable, Iterator+from contextlib import nullcontextfrom copy import deepcopyfrom dataclasses import dataclass, fieldfrom typing import Any, AsyncIterator, TYPE_CHECKING@@ -107,6 +108,7 @@ class _UserTurn:item_id: str = ""open: bool = Truespeech_ended: bool = False+ partial_transcription: bool = False@dataclass@@ -117,6 +119,15 @@ class _QueuedEvent:checkpoint: AgentState | None = None+@dataclass+class _BackgroundTools:+ """Work that must settle before a logical reply can complete."""++ pending: int = 0+ model_done: bool = False+ lock: asyncio.Lock = field(default_factory=asyncio.Lock)++@dataclassclass _ClientDelegation:"""One application-managed task requested by a Live session."""@@ -253,6 +264,8 @@ class RealtimeAgent:self._metrics = TurnMetrics()self._pending_tools: dict[str, ToolCallBlock] = {}self._confirmations: dict[str, asyncio.Future[ConfirmResult]] = {}+ self._confirmation_replies: dict[str, str] = {}+ self._background_tools: dict[str, _BackgroundTools] = {}self._tasks: set[asyncio.Task] = set()self._barge_lock = asyncio.Lock()self._live_session_id = ""@@ -436,6 +449,7 @@ class RealtimeAgent:self._delegation_queue.clear()self._delegated_tool_calls.clear()self._active_delegation_id = None+ self._reset_user_turns()self._connected = Falseself._connected_event.clear()await self.model.close()@@ -640,8 +654,21 @@ class RealtimeAgent:returnfor result in inputs.confirm_results:future = self._confirmations.get(result.tool_call.id)- if future and not future.done():- future.set_result(result)+ if future is None or future.done():+ continue+ expected_reply = self._confirmation_replies.get(+ result.tool_call.id,+ )+ if expected_reply != inputs.reply_id:+ logger.warning(+ "RealtimeAgent: ignored confirmation for tool %s "+ "from reply %s; expected reply %s",+ result.tool_call.id,+ inputs.reply_id,+ expected_reply,+ )+ continue+ future.set_result(result)case ExternalExecutionResultEvent():tool_call_ids = [@@ -746,6 +773,7 @@ class RealtimeAgent:if speech is SpeechTransition.STARTED:self._start_user_turn()+ await self.model.start_turn()await self._barge_in()elif speech is SpeechTransition.ENDED:if self._user_turn is not None:@@ -818,7 +846,10 @@ class RealtimeAgent:await self._transport.clear_audio()self._audio_pending = False- if active_reply is not None and self._connected:+ if self._connected and (+ active_reply is not None+ or self._reply_id in self._background_tools+ ):await self.model.cancel_response()if self._reply_id:@@ -883,6 +914,9 @@ class RealtimeAgent:case me.InputTranscriptionEvent():self._on_transcription(event)+ case me.InputTranscriptionDeltaEvent():+ self._on_transcription_delta(event)+case me.InputTranscriptionFailedEvent():self._finish_transcription(self._find_user_turn(event.item_id),@@ -918,6 +952,11 @@ class RealtimeAgent:self._reply_id,event.tool_call,)+ if event.non_blocking:+ self._schedule_tools(+ self._reply_id,+ request_response=False,+ )case me.ResponseDoneEvent():async with self._barge_lock:@@ -940,6 +979,11 @@ class RealtimeAgent:reply_id = reply.reply_idself._finish_response()self._schedule_tools(reply_id)+ elif work := self._background_tools.get(reply.reply_id):+ self._finish_response()+ work.model_done = True+ if not work.pending:+ self._finish_reply(ReplyFinishedReason.COMPLETED)else:self._finish_reply(ReplyFinishedReason.COMPLETED)@@ -1928,19 +1972,32 @@ class RealtimeAgent:"""Retire only the turn whose transcript succeeded or failed."""if turn is None:return+ self._discard_partial_transcription(turn)self._end_user_turn(turn)self._pending_user_turns.pop(turn.reply_id, None)if self._user_turn is turn:self._user_turn = Nonedef _reset_user_turns(self) -> None:- """Discard pending identities when their model session ends."""- for turn in list(self._pending_user_turns.values()):+ """Discard unfinished input before closing its model session."""+ turns = list(self._pending_user_turns.values())+ if self._user_turn is not None and self._user_turn not in turns:+ turns.append(self._user_turn)+ for turn in turns:+ self._discard_partial_transcription(turn)+ for turn in turns:self._end_user_turn(turn)- self._end_user_turn()self._pending_user_turns.clear()self._user_turn = None+ def _discard_partial_transcription(self, turn: _UserTurn) -> None:+ """Remove a provisional message without touching accepted input."""+ if turn.partial_transcription:+ message = self._find_user_message(turn.reply_id)+ if message is not None:+ self.state.context.remove(message)+ turn.partial_transcription = False+def _retire_unbound_user_turns(self) -> None:"""End pending user turns when their transport goes, keeping onlythe ones a late transcript could still find.@@ -1957,6 +2014,9 @@ class RealtimeAgent:turns = list(self._pending_user_turns.values())if self._user_turn is not None and self._user_turn not in turns:turns.append(self._user_turn)+ for turn in turns:+ if not turn.item_id and not turn.speech_ended:+ self._discard_partial_transcription(turn)for turn in turns:self._end_user_turn(turn)for turn in turns:@@ -1965,13 +2025,60 @@ class RealtimeAgent:if self._user_turn is turn:self._user_turn = None+ def _on_transcription_delta(+ self,+ event: me.InputTranscriptionDeltaEvent,+ ) -> None:+ """Keep input fragments in one message before their response."""+ turn = self._find_user_turn(event.item_id)+ if turn is None:+ turn = self._new_user_turn(event.item_id)+ turn.partial_transcription = True+ message = self._find_user_message(turn.reply_id)+ if message is not None:+ text = message.get_text_content()+ message.content = [TextBlock(text=f"{text}{event.delta}")]+ return+ response_id = event.response_id+ if self._reply is not None and self._reply.item_id == response_id:+ response_id = self._reply.reply_id+ context = self.state.context+ position = next(+ (+ i+ for i, msg in enumerate(context)+ if msg.id == response_id and msg.role == "assistant"+ ),+ len(context),+ )+ context.insert(+ position,+ UserMsg(name="user", content=event.delta, id=turn.reply_id),+ )++ def _find_user_message(self, reply_id: str) -> Msg | None:+ """Find an input turn already reserved by transcription fragments."""+ return next(+ (+ msg+ for msg in self.state.context+ if msg.id == reply_id and msg.role == "user"+ ),+ None,+ )+def _on_transcription(self, event: me.InputTranscriptionEvent) -> None:"""Record a settled user turn, merging a split one back together,and report its text on the user's reply."""turn = self.aggregator.take(event.text)user_turn = self._find_user_turn(event.item_id)+ message = self._find_user_message(+ user_turn.reply_id if user_turn else event.item_id,+ )if turn is None:logger.debug("RealtimeAgent: dropping %r", event.text)+ if message is not None:+ self.state.context.remove(message)self._finish_transcription(user_turn)return@@ -1990,15 +2097,20 @@ class RealtimeAgent:)self._emit(TextBlockEndEvent(reply_id=reply_id, block_id=block_id))- if not (- self.aggregator.merges_with_previous() and self._merge_user(turn)- ):+ merged = self.aggregator.merges_with_previous() and self._merge_user(+ turn,+ message,+ )+ if not merged and message is not None:+ message.content = [TextBlock(id=block_id, text=turn)]+ elif not merged:self.state.context.append(UserMsg(name="user", content=turn, id=reply_id),)+ user_turn.partial_transcription = Falseself._finish_transcription(user_turn)- def _merge_user(self, text: str) -> bool:+ def _merge_user(self, text: str, reserved: Msg | None = None) -> bool:"""Append *text* to the previous user turn that endpointing split.The stub assistant message between the two halves is dropped: it@@ -2006,16 +2118,25 @@ class RealtimeAgent:answers half a question nobody finished asking."""context = self.state.context- if context and context[-1].role == "assistant":- tail = context[-1]+ position = (+ context.index(reserved) if reserved is not None else len(context)+ )+ if position and context[position - 1].role == "assistant":+ tail = context[position - 1]if not tail.get_text_content().strip() and all(isinstance(b, TextBlock) for b in tail.content):- context.pop()- if not context or context[-1].role != "user":+ context.pop(position - 1)+ position -= 1+ if not position or context[position - 1].role != "user":+ return False+ previous_turn = self._pending_user_turns.get(context[position - 1].id)+ if previous_turn is not None and previous_turn.partial_transcription:return False- previous = context[-1].get_text_content()- context[-1].content = [TextBlock(text=f"{previous}{text}")]+ previous = context[position - 1].get_text_content()+ context[position - 1].content = [TextBlock(text=f"{previous}{text}")]+ if reserved is not None:+ context.remove(reserved)return Truedef _start_reply(self, item_id: str) -> _Reply | None:@@ -2049,6 +2170,8 @@ class RealtimeAgent:),)self._reply = _Reply(item_id=item_id, reply_id=self._reply_id)+ if work := self._background_tools.get(self._reply_id):+ work.model_done = Falseself._emit(ModelCallStartEvent(reply_id=self._reply_id,@@ -2097,6 +2220,11 @@ class RealtimeAgent:self._pending_tools.clear()if not self._reply_id:return+ self._background_tools.pop(self._reply_id, None)+ if reason != ReplyFinishedReason.COMPLETED:+ for call_id, future in self._confirmations.items():+ if self._confirmation_replies.get(call_id) == self._reply_id:+ future.cancel()event = ReplyEndEvent(session_id=self.state.session_id,reply_id=self._reply_id,@@ -2142,6 +2270,22 @@ class RealtimeAgent:def _emit_text(self, reply: _Reply, delta: str) -> None:"""Emit a transcript delta, opening the block on first use."""+ message = self._find_reply_message(reply.reply_id)+ blocks = message.content if message is not None else None+ if (+ reply.text_started+ and blocks+ and not isinstance(blocks[-1], TextBlock)+ ):+ # Background tools can separate speech within one response.+ self._emit(+ TextBlockEndEvent(+ reply_id=reply.reply_id,+ block_id=reply.text_block_id,+ ),+ )+ reply.text_started = False+ reply.text_block_id = uuid.uuid4().hexif not reply.text_started:reply.text_started = Trueself._emit(@@ -2152,8 +2296,6 @@ class RealtimeAgent:)# Grow the tail text block rather than appending one per delta,# or the context ends up as dozens of one-word blocks.- message = self._find_reply_message(reply.reply_id)- blocks = message.content if message is not None else Noneif (isinstance(blocks, list)and blocks@@ -2212,6 +2354,17 @@ class RealtimeAgent:),):snapshot = self.state.model_copy(deep=True)+ partial_ids = {+ turn.reply_id+ for turn in self._pending_user_turns.values()+ if turn.partial_transcription+ }+ if partial_ids:+ snapshot.context = [+ msg+ for msg in snapshot.context+ if msg.role != "user" or msg.id not in partial_ids+ ]self._out.put_nowait(_QueuedEvent(event, snapshot))def checkpoint_snapshot(self, event: AgentEvent) -> AgentState | None:@@ -2225,14 +2378,25 @@ class RealtimeAgent:# Tools# ------------------------------------------------------------------- def _schedule_tools(self, reply_id: str) -> None:- """Run the tool calls of the finished reply, then ask for more."""+ def _schedule_tools(+ self,+ reply_id: str,+ *,+ request_response: bool = True,+ ) -> None:+ """Run pending tools without blocking the model event stream."""if not reply_id or not self._pending_tools or self.toolkit is None:returncalls = list(self._pending_tools.values())self._pending_tools.clear()+ if not request_response:+ work = self._background_tools.setdefault(+ reply_id,+ _BackgroundTools(),+ )+ work.pending += 1task = asyncio.create_task(- self._run_tools(reply_id, calls),+ self._run_tools(reply_id, calls, request_response),name="rt-tools",)self._tasks.add(task)@@ -2242,23 +2406,46 @@ class RealtimeAgent:self,reply_id: str,calls: list[ToolCallBlock],+ request_response: bool = True,) -> None:- """Execute *calls* in order, then trigger the follow-up response."""+ """Execute tools serially, optionally requesting a follow-up."""try:- for call in calls:- async with self._barge_lock:- if self._reply_id != reply_id:- return- await self._run_tool(reply_id, call)+ work = self._background_tools.get(reply_id)+ lock = work.lock if work is not None else nullcontext()+ async with lock:+ for call in calls:+ async with self._barge_lock:+ interrupted = self._reply_id != reply_id+ if interrupted:+ await self._report_tool(+ reply_id,+ call,+ "Reply interrupted before tool execution.",+ state=ToolResultState.INTERRUPTED,+ )+ continue+ await self._run_tool(reply_id, call)# Unless the user cut the reply short while the tools ran.async with self._barge_lock:- if self._reply_id == reply_id:+ if request_response and self._reply_id == reply_id:await self.model.request_response()except Exception: # noqa: BLE001logger.exception("RealtimeAgent: tool execution failed")async with self._barge_lock:if self._reply_id == reply_id:self._finish_reply(ReplyFinishedReason.ERROR)+ finally:+ if not request_response:+ async with self._barge_lock:+ work = self._background_tools.get(reply_id)+ if work is not None:+ work.pending -= 1+ if (+ not work.pending+ and work.model_done+ and self._reply_id == reply_id+ ):+ self._finish_reply(ReplyFinishedReason.COMPLETED)async def _run_tool(self, reply_id: str, call: ToolCallBlock) -> None:"""Check permission for one call, run it, and report the result."""@@ -2292,6 +2479,14 @@ class RealtimeAgent:returndecision = await self._engine.check_permission(tool, tool_input)+ if self._reply_id != reply_id:+ await self._report_tool(+ reply_id,+ call,+ "Reply interrupted before tool execution.",+ state=ToolResultState.DENIED,+ )+ returnif decision.behavior in (PermissionBehavior.ASK,PermissionBehavior.PASSTHROUGH,@@ -2384,13 +2579,17 @@ class RealtimeAgent:ConfirmResult] = asyncio.get_running_loop().create_future()self._confirmations[call.id] = future+ self._confirmation_replies[call.id] = reply_idtry:result = await asyncio.wait_for(future, timeout=300)except (asyncio.TimeoutError, asyncio.CancelledError):return Falsefinally:self._confirmations.pop(call.id, None)+ self._confirmation_replies.pop(call.id, None)+ if self._reply_id != reply_id:+ return Falsefor rule in result.rules or []:self._engine.add_rule(rule)call.state = (@@ -2441,4 +2640,12 @@ class RealtimeAgent:state=state,),)- await self.model.push_tool_result(block)+ # The result starts more model work. An IDLE received before this+ # send cannot complete the reply, and a completion received during+ # the send must not be overwritten after it returns.+ if work := self._background_tools.get(reply_id):+ work.model_done = False+ if self._reply_id == reply_id:+ await self.model.push_tool_result(block)+ else:+ await self.model.push_tool_result_silently(block)diff --git a/src/agentscope/realtime/_base.py b/src/agentscope/realtime/_base.pyindex 2c30bac0b..ec703d814 100644--- a/src/agentscope/realtime/_base.py+++ b/src/agentscope/realtime/_base.py@@ -236,11 +236,27 @@ class RealtimeModelBase(ABC):async def push_tool_result(self, block: ToolResultBlock) -> None:"""Feed the result of a tool the caller executed."""+ async def push_tool_result_silently(self, block: ToolResultBlock) -> None:+ """Feed a late result without requesting another response.++ The default delegates to :meth:`push_tool_result`, for adapters that+ require a separate :meth:`request_response`. Adapters with implicit+ generation should override this when silent delivery is supported.+ """+ await self.push_tool_result(block)+@abstractmethodasync def commit_turn(self) -> None:"""Declare the user turn finished. Only called when the caller ownsendpointing, i.e. the session was opened with turn detection off."""+ async def start_turn(self) -> None:+ """Notify the model that local VAD detected a new user turn.++ Adapters may settle pending input here even without an active reply.+ The default implementation does nothing.+ """+# ------------------------------------------------------------------# Response control# ------------------------------------------------------------------diff --git a/src/agentscope/realtime/_events.py b/src/agentscope/realtime/_events.pyindex f65950f35..ac9e1526a 100644--- a/src/agentscope/realtime/_events.py+++ b/src/agentscope/realtime/_events.py@@ -64,6 +64,14 @@ class InputTranscriptionEvent(ModelEvent):text: str+class InputTranscriptionDeltaEvent(ModelEvent):+ """Unsettled input text, anchored before the response it prompted."""++ item_id: str+ delta: str+ response_id: str = ""++class InputTranscriptionFailedEvent(ModelEvent):"""The provider could not transcribe one committed user turn."""@@ -130,6 +138,8 @@ class ToolCallEvent(ModelEvent):item_id: strtool_call: ToolCallBlock+ non_blocking: bool = False+ """Run immediately while the provider continues the current response."""class ResponseDoneEvent(ModelEvent):diff --git a/src/agentscope/realtime/_gemini/_model.py b/src/agentscope/realtime/_gemini/_model.pyindex 3ea563492..6e9e3c5f3 100644--- a/src/agentscope/realtime/_gemini/_model.py+++ b/src/agentscope/realtime/_gemini/_model.py@@ -14,7 +14,12 @@ from .._model_card import RealtimeModelCardfrom ..._logging import loggerfrom ..._utils._common import _flatten_json_schemafrom ...credential import GeminiCredential-from ...message import TextBlock, ToolCallBlock, ToolResultBlock+from ...message import (+ TextBlock,+ ToolCallBlock,+ ToolResultBlock,+ ToolResultState,+)from ...model._gemini._model import _sanitize_schema_for_gemini_LIVE_URL = (@@ -35,7 +40,7 @@ class GeminiRealtimeModel(RealtimeModelBase):starts the reply, so :meth:`request_response` sends nothing.- Interruption is the server's job — it reports``serverContent.interrupted`` and drops the rest of the reply- itself — so :meth:`cancel_response` is a no-op.+ itself — so :meth:`cancel_response` sends no wire frame.With caller-owned turns the caller silences the reply locallyinstead, and the server is told not to interrupt on its own."""@@ -44,6 +49,11 @@ class GeminiRealtimeModel(RealtimeModelBase):"""Tuneables surfaced to the UI."""voice: str = Field(default="Puck", title="Voice")+ thinking_level: Literal["low", "medium", "high"] | None = Field(+ default=None,+ title="Thinking Level",+ description="Background reasoning depth for Extended Thinking.",+ )turn_detection: Literal["automatic", "none"] = Field(default="automatic",title="Turn Detection",@@ -94,6 +104,11 @@ class GeminiRealtimeModel(RealtimeModelBase):"""super().__init__(model, credential, parameters, model_card)self.parameters: GeminiRealtimeModel.Parameters+ self._extended_thinking = self.card.supports_extended_thinking+ if self.parameters.thinking_level and not self._extended_thinking:+ raise ValueError(+ "thinking_level requires Gemini Live Extended Thinking.",+ )self.resumption_handle = """""The latest handle the server offered. Not used on reconnect:the agent carries the transcript in the instructions instead."""@@ -101,10 +116,14 @@ class GeminiRealtimeModel(RealtimeModelBase):self._reader: asyncio.Task | None = Noneself._queue: asyncio.Queue[me.ModelEvent | None] = asyncio.Queue()self._response_id = ""+ self._transcription_response_id = ""+ self._response_cancelled = Falseself._user_id = ""self._input_text = ""self._usage: dict = {}self._activity = False+ self._awaiting_tool_reply = False+ self._pending_tool_calls: set[str] = set()self._ready = asyncio.Event()# ------------------------------------------------------------------@@ -130,11 +149,19 @@ class GeminiRealtimeModel(RealtimeModelBase):update={"turn_detection": "none"},)self._response_id, self._user_id, self._input_text = "", "", ""+ self._response_cancelled = Falseself._usage, self._activity = {}, False+ self._awaiting_tool_reply = False+ self._pending_tool_calls.clear()credential: GeminiCredential = self.credential # type: ignore+ url = (+ _LIVE_URL.replace("v1beta", "v1alpha")+ if self._extended_thinking+ else _LIVE_URL+ )self._ws = await websockets.connect(- f"{_LIVE_URL}?key={credential.api_key.get_secret_value()}",+ f"{url}?key={credential.api_key.get_secret_value()}",)self._ready.clear()self._reader = asyncio.create_task(self._read(), name="gemini-rt")@@ -160,6 +187,7 @@ class GeminiRealtimeModel(RealtimeModelBase):self._reader.cancel()await asyncio.gather(self._reader, return_exceptions=True)self._reader = None+ self._transcription_response_id = ""async def events(self) -> AsyncIterator[me.ModelEvent]:"""Yield events until the session ends."""@@ -193,6 +221,11 @@ class GeminiRealtimeModel(RealtimeModelBase):async def push_text(self, text: str) -> None:"""Send a complete text turn, which the model answers at once."""+ if self._extended_thinking:+ for event in self._settle_input():+ self._queue.put_nowait(event)+ self._user_id = ""+ self._transcription_response_id = ""await self._send({"clientContent": {@@ -204,6 +237,23 @@ class GeminiRealtimeModel(RealtimeModelBase):async def push_tool_result(self, block: ToolResultBlock) -> None:"""Send one ``functionResponse``; the model resumes on its own."""+ await self._push_tool_result(block, silent=False)++ async def push_tool_result_silently(self, block: ToolResultBlock) -> None:+ """Keep a stale Thinking result without triggering generation.++ Scheduling is only supported for non-blocking tools. Ordinary Live+ retains its existing blocking-tool result behavior.+ """+ await self._push_tool_result(block, silent=True)++ async def _push_tool_result(+ self,+ block: ToolResultBlock,+ *,+ silent: bool,+ ) -> None:+ """Serialize a tool result with its generation and error semantics."""output = (block.outputif isinstance(block.output, str)@@ -211,20 +261,41 @@ class GeminiRealtimeModel(RealtimeModelBase):_.text for _ in block.output if isinstance(_, TextBlock)))+ if self._extended_thinking and block.id in self._pending_tool_calls:+ if not silent:+ self._awaiting_tool_reply = True+ self._pending_tool_calls.discard(block.id)+ failed = block.state in (+ ToolResultState.ERROR,+ ToolResultState.DENIED,+ ToolResultState.INTERRUPTED,+ )+ response = {+ "id": block.id,+ "name": block.name,+ "response": {"error" if failed else "output": output},+ }+ if self._extended_thinking and silent:+ response["scheduling"] = "SILENT"await self._send({"toolResponse": {- "functionResponses": [- {- "id": block.id,- "name": block.name,- "response": {"output": output},- },- ],+ "functionResponses": [response],},},)+ async def start_turn(self) -> None:+ """Settle prior input at a local speech boundary, even after cancel."""+ if (+ self._extended_thinking+ and self.parameters.turn_detection == "none"+ ):+ self._transcription_response_id = ""+ for event in self._settle_input():+ self._queue.put_nowait(event)+ self._user_id = ""+async def commit_turn(self) -> None:"""Close the activity window, which asks for the reply."""if self._activity:@@ -239,7 +310,20 @@ class GeminiRealtimeModel(RealtimeModelBase):"""No-op: a finished turn or a tool response is the request."""async def cancel_response(self) -> None:- """No-op: the protocol has no cancel frame."""+ """Settle locally interrupted input; there is no wire cancel."""+ if (+ self._extended_thinking+ and self.parameters.turn_detection == "none"+ and (self._response_id or self._transcription_response_id)+ and not self._response_cancelled+ ):+ self._response_cancelled = True+ for event in self._settle_input():+ self._queue.put_nowait(event)+ self._user_id = ""+ self._transcription_response_id = ""+ self._awaiting_tool_reply = False+ self._pending_tool_calls.clear()# ------------------------------------------------------------------# Wire@@ -296,12 +380,21 @@ class GeminiRealtimeModel(RealtimeModelBase):}if p.input_audio_transcription:setup["inputAudioTranscription"] = {}+ if self._extended_thinking and p.thinking_level:+ setup["generationConfig"]["thinkingConfig"] = {+ "thinkingLevel": p.thinking_level.upper(),+ }if tools and self.card.supports_tools:setup["tools"] = [{"functionDeclarations": [{**schema["function"],+ **(+ {"behavior": "NON_BLOCKING"}+ if self._extended_thinking+ else {}+ ),"parameters": _sanitize_schema_for_gemini(_flatten_json_schema(schema["function"].get("parameters", {}),@@ -349,6 +442,7 @@ class GeminiRealtimeModel(RealtimeModelBase):logger.error("GeminiRealtimeModel: connection lost: %s", exc)finally:self._ws = None+ self._transcription_response_id = ""self._ready.set() # release a connect() still waitingself._queue.put_nowait(me.SessionEndedEvent(reason="closed"))self._queue.put_nowait(None)@@ -358,10 +452,21 @@ class GeminiRealtimeModel(RealtimeModelBase):the id every event of this turn carries."""if self._response_id:return []+ events = [] if self._extended_thinking else self._settle_input()+ self._response_id = uuid.uuid4().hex+ if self._extended_thinking and not self._transcription_response_id:+ # All background continuations belong to the first response.+ self._transcription_response_id = self._response_id+ self._response_cancelled = False+ events.append(me.ResponseCreatedEvent(item_id=self._response_id))+ return events++ def _settle_input(self) -> list[me.ModelEvent]:+ """Flush the transcription fragments of the pending user turn."""events: list[me.ModelEvent] = []if self._input_text:# Input transcription arrives in fragments; the user turn is- # settled only once the model starts answering it.+ # settled when the model answers or finishes background work.events.append(me.InputTranscriptionEvent(item_id=self._user_id,@@ -369,25 +474,45 @@ class GeminiRealtimeModel(RealtimeModelBase):),)self._user_id, self._input_text = "", ""- self._response_id = uuid.uuid4().hex- events.append(me.ResponseCreatedEvent(item_id=self._response_id))return eventsdef _parse(self, data: dict) -> list[me.ModelEvent]:"""Translate one server message into model events."""+ if "setupComplete" in data:+ self._ready.set()if usage := data.get("usageMetadata"):self._usage = usage- if "serverContent" in data:- return self._parse_content(data["serverContent"])+ content = data.get("serverContent") or {}+ # The Thinking guide includes root status alongside toolCall.+ # Source: https://ai.google.dev/gemini-api/docs/live-api/thinking+ status = content.get("interactionStatus") or data.get(+ "interactionStatus",+ )+ events: list[me.ModelEvent] = []+ if (+ self._extended_thinking+ and self._awaiting_tool_reply+ and status == "IDLE"+ and not content.get("interrupted")+ ):+ # A result may receive a silent completion after an earlier+ # IDLE already closed its original model response.+ events += self._open_response()+ if (+ self._extended_thinking+ and status == "IN_PROGRESS"+ and not content.get("interrupted")+ ):+ events += self._open_response()+ events += self._parse_content(content)if tool_call := data.get("toolCall"):- # The server hands the floor over with a tool call, so the- # reply ends here and the follow-up is a turn of its own.- events = self._open_response()- events += [+ events += self._open_response()+ tool_events = [me.ToolCallEvent(item_id=self._response_id,+ non_blocking=self._extended_thinking,tool_call=ToolCallBlock(id=call.get("id") or uuid.uuid4().hex,name=call.get("name", ""),@@ -399,7 +524,20 @@ class GeminiRealtimeModel(RealtimeModelBase):)for call in tool_call.get("functionCalls") or []]- return events + self._close_response()+ events += tool_events+ if self._extended_thinking:+ self._pending_tool_calls.update(+ event.tool_call.id for event in tool_events+ )+ # Blocking tools hand over the floor; background tools do not.+ if not self._extended_thinking:+ events += self._close_response()++ if self._extended_thinking and status == "IDLE":+ self._awaiting_tool_reply = False+ events += self._close_response()+ if "serverContent" in data or tool_call or status:+ return eventsif go_away := data.get("goAway"):# Advance notice only; the session ends when the socket does.@@ -414,9 +552,7 @@ class GeminiRealtimeModel(RealtimeModelBase):self.resumption_handle = update.get("newHandle", "")return []- if "setupComplete" in data:- self._ready.set()- else:+ if "setupComplete" not in data:logger.debug("GeminiRealtimeModel: ignoring %s", list(data))return []@@ -424,22 +560,53 @@ class GeminiRealtimeModel(RealtimeModelBase):"""Translate one ``serverContent`` message into model events."""events: list[me.ModelEvent] = []+ if self._extended_thinking and content.get("interrupted"):+ self._awaiting_tool_reply = False+ self._pending_tool_calls.clear()+ self._transcription_response_id = ""+ events += self._settle_input()+ self._user_id, self._response_id, self._usage = "", "", {}++ parts = (content.get("modelTurn") or {}).get("parts") or []+ transcript = (content.get("outputTranscription") or {}).get("text", "")+ if (+ self._extended_thinking+ and not content.get("interrupted")+ and (parts or transcript)+ ):+ # Bind same-frame input to the response before emitting it.+ events += self._open_response()+if text := (content.get("inputTranscription") or {}).get("text"):self._user_id = self._user_id or uuid.uuid4().hexself._input_text += text+ if self._extended_thinking:+ events.append(+ me.InputTranscriptionDeltaEvent(+ item_id=self._user_id,+ delta=text,+ response_id=(+ ""+ if self._response_cancelled+ else self._transcription_response_id+ ),+ ),+ )if content.get("interrupted"):# The only notice that the user spoke over the reply; the# server has already dropped the rest of it.self._user_id = self._user_id or uuid.uuid4().hexself._response_id, self._usage = "", {}- return [me.SpeechStartedEvent(item_id=self._user_id)]+ return events + [me.SpeechStartedEvent(item_id=self._user_id)]- parts = (content.get("modelTurn") or {}).get("parts") or []- transcript = (content.get("outputTranscription") or {}).get("text", "")if parts or transcript:events += self._open_response()for part in parts:+ if part.get("thought"):+ # Provider thought summaries are not spoken transcript.+ # They can still open a response that finishes at IDLE.+ continueif data := (part.get("inlineData") or {}).get("data"):events.append(me.AudioDeltaEvent(@@ -463,18 +630,34 @@ class GeminiRealtimeModel(RealtimeModelBase):),)- if content.get("turnComplete"):+ if content.get("turnComplete") and not self._extended_thinking:events += self._close_response()return eventsdef _close_response(self) -> list[me.ModelEvent]:"""End the open model turn with the usage last reported."""+ # A thinking status can arrive before input transcription. Proactive+ # audio may also finish processing without producing any speech.+ # A cancelled response's late IDLE does not end the next user turn.+ # Tool handoffs keep fragments buffered until the interaction ends.+ interaction_done = not (+ self._pending_tool_calls or self._awaiting_tool_reply+ )+ events = (+ self._settle_input()+ if self._extended_thinking+ and interaction_done+ and not self._response_cancelled+ else []+ )+ if interaction_done:+ self._transcription_response_id = ""if not self._response_id:- return []+ return eventsevent = me.ResponseDoneEvent(item_id=self._response_id,input_tokens=self._usage.get("promptTokenCount", 0),output_tokens=self._usage.get("responseTokenCount", 0),)self._response_id, self._usage = "", {}- return [event]+ return events + [event]diff --git a/src/agentscope/realtime/_gemini/_models/gemini-2.5-flash-native-audio-preview-12-2025.yaml b/src/agentscope/realtime/_gemini/_models/gemini-2.5-flash-native-audio-preview-12-2025.yamlindex 2646f7c3f..a0e2c5734 100644--- a/src/agentscope/realtime/_gemini/_models/gemini-2.5-flash-native-audio-preview-12-2025.yaml+++ b/src/agentscope/realtime/_gemini/_models/gemini-2.5-flash-native-audio-preview-12-2025.yaml@@ -23,6 +23,8 @@ max_context_tokens: 131072max_session_duration_s: 900parameter_overrides:+ thinking_level:+ hidden: truevoice:enum: ["Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus","Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",diff --git a/src/agentscope/realtime/_gemini/_models/gemini-3.1-flash-live-preview.yaml b/src/agentscope/realtime/_gemini/_models/gemini-3.1-flash-live-preview.yamlindex e5a51b3f7..3ee43cb1f 100644--- a/src/agentscope/realtime/_gemini/_models/gemini-3.1-flash-live-preview.yaml+++ b/src/agentscope/realtime/_gemini/_models/gemini-3.1-flash-live-preview.yaml@@ -23,6 +23,8 @@ max_context_tokens: 131072max_session_duration_s: 900parameter_overrides:+ thinking_level:+ hidden: truevoice:enum: ["Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus","Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",diff --git a/src/agentscope/realtime/_gemini/_models/gemini-3.8-live-extended-thinking.yaml b/src/agentscope/realtime/_gemini/_models/gemini-3.8-live-extended-thinking.yamlnew file mode 100644index 000000000..93d525007--- /dev/null+++ b/src/agentscope/realtime/_gemini/_models/gemini-3.8-live-extended-thinking.yaml@@ -0,0 +1,33 @@+# Source: https://ai.google.dev/gemini-api/docs/models/gemini-3.8-live-extended-thinking+type: realtime_model+name: gemini-3.8-live-extended-thinking+label: Gemini 3.8 Live Extended Thinking+status: active+# Source: https://aistudio.google.com/docs/changelog#09-15-2026+release_date: 2026-09-15++# Image input is not implemented by the current realtime adapter.+input_types:+ - audio/pcm+ - text/plain+output_types:+ - audio/pcm+ - text/plain++input_sample_rate: 16000+output_sample_rate: 24000+supports_tools: true+supports_extended_thinking: true+max_context_tokens: 131072+# Audio-only duration without context-window compression.+# Source: https://aistudio.google.com/docs/live-api/session-management+max_session_duration_s: 900++parameter_overrides:+ voice:+ enum: ["Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus",+ "Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",+ "Umbriel", "Algieba", "Despina", "Erinome", "Algenib",+ "Rasalgethi", "Laomedeia", "Achernar", "Alnilam", "Schedar",+ "Gacrux", "Pulcherrima", "Achird", "Zubenelgenubi",+ "Vindemiatrix", "Sadachbia", "Sadaltager", "Sulafat"]diff --git a/src/agentscope/realtime/_gemini/_models/gemini-3.8-live.yaml b/src/agentscope/realtime/_gemini/_models/gemini-3.8-live.yamlindex c6f1a914d..5e8b836c8 100644--- a/src/agentscope/realtime/_gemini/_models/gemini-3.8-live.yaml+++ b/src/agentscope/realtime/_gemini/_models/gemini-3.8-live.yaml@@ -24,6 +24,8 @@ max_session_duration_s: 900# Source: https://ai.google.dev/gemini-api/docs/live-api/capabilitiesparameter_overrides:+ thinking_level:+ hidden: truevoice:enum: ["Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus","Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",diff --git a/src/agentscope/realtime/_model_card.py b/src/agentscope/realtime/_model_card.pyindex 73216c8e9..c8956ad4e 100644--- a/src/agentscope/realtime/_model_card.py+++ b/src/agentscope/realtime/_model_card.py@@ -64,6 +64,11 @@ class RealtimeModelCard(BaseModel):default=False,description="Whether this model accepts tool schemas.",)+ supports_extended_thinking: bool = Field(+ default=False,+ description="Whether the model uses background thinking with "+ "non-blocking tools and interaction-status completion.",+ )interaction_mode: Literal["turn_based", "full_duplex"] = Field(default="turn_based",description="Whether speech is turn-based or full duplex.",diff --git a/tests/realtime_agent_test.py b/tests/realtime_agent_test.pyindex 6e6ea8376..0b5a5dffc 100644--- a/tests/realtime_agent_test.py+++ b/tests/realtime_agent_test.py@@ -3,11 +3,11 @@transport — no network, no sound card."""# pylint: disable=protected-access, unused-argumentimport asyncio-from itertools import count+from itertools import count, productfrom types import SimpleNamespacefrom typing import Any, AsyncIterator, Sequencefrom unittest.async_case import IsolatedAsyncioTestCase-from unittest.mock import patch+from unittest.mock import AsyncMock, patchfrom utils import AnyString, MockModelfrom agentscope.agent import (@@ -16,13 +16,14 @@ from agentscope.agent import (RealtimeAgent,TurnAggregator,)-from agentscope.credential import DashScopeCredential+from agentscope.credential import DashScopeCredential, GeminiCredentialfrom agentscope.event import (DataBlockDeltaEvent,ReplyEndEvent,ReplyStartEvent,TextBlockDeltaEvent,TextBlockEndEvent,+ TextBlockStartEvent,)from agentscope.message import AssistantMsg, Msg, Usage, UserMsgfrom agentscope.model import ChatResponse@@ -30,6 +31,7 @@ from agentscope.realtime import (AudioFrame,ControlFrame,ControlFrameType,+ GeminiRealtimeModel,ModelDisconnectedError,PlayoutPosition,RealtimeModelBase,@@ -2038,10 +2040,12 @@ class BlockingTool(StreamTool):def __init__(self) -> None:super().__init__()+ self.started = asyncio.Event()self.release = asyncio.Event()async def __call__(self, q: str, **kwargs: Any) -> Any:"""Hold the tool result until a barge-in has completed."""+ self.started.set()await self.release.wait()yield ToolResponse(content=[TextBlock(text=f"{q}-final")])@@ -2096,10 +2100,15 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):self,tool: ToolBase,confirm: bool | None = None,+ non_blocking: bool = False,) -> tuple[list[tuple[Any, ...]], ScriptedModel]:"""Run one tool-calling reply; answer a permission prompt with*confirm* if one appears. Returns the tool-related events."""- model = ScriptedModel([_tool_script(tool.name)])+ script = _tool_script(tool.name)+ if non_blocking:+ script[1].non_blocking = True+ script.pop()+ model = ScriptedModel([script])agent = RealtimeAgent("Friday","be brief",@@ -2162,6 +2171,31 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):)return events, model+ async def test_non_blocking_permissions_and_errors(self) -> None:+ """Immediate scheduling preserves permission and error reporting."""+ for tool, confirm, output in (+ (AskTool(), True, "x-final"),+ (AskTool(), False, 'Tool "ask_tool" denied by user.'),+ (BrokenTool(), None, "boom"),+ ):+ with self.subTest(tool=tool.name, confirm=confirm):+ expected, _ = await self._run_tool_scenario(tool, confirm)+ actual, model = await self._run_tool_scenario(+ tool,+ confirm,+ non_blocking=True,+ )+ self.assertEqual(actual, expected)+ self.assertEqual(+ [call for call in model.calls if call != "push_audio"],+ [+ "connect(session=1,td_off=False)",+ f"tool_result(c1,{output!r})",+ "cancel",+ "close",+ ],+ )+async def test_streamed_tool_result_is_not_duplicated(self) -> None:"""Chunks are shown as they come; the provider gets only thecompleted result, once."""@@ -2296,6 +2330,1854 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):)+class GeminiThinkingAgentTest(IsolatedAsyncioTestCase):+ """Drive the shared agent with actual Gemini background frames."""++ async def test_partial_transcriptions_discarded_on_session_end(+ self,+ ) -> None:+ """Session loss removes all fragments before any end checkpoint."""+ for failure in (False, True):+ with self.subTest(failure=failure):+ model = QueueSessionModel()+ model.parameters.input_audio_transcription = True+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ aggregator=TurnAggregator(min_chars=4),+ )+ try:+ await agent.connect()+ await agent._on_model_event(+ me.InputTranscriptionEvent(+ item_id="kept",+ text="complete",+ ),+ )+ while not agent._out.empty():+ agent._out.get_nowait()+ for item_id in ("u1", "u2"):+ await agent._on_model_event(+ me.InputTranscriptionDeltaEvent(+ item_id=item_id,+ delta="a",+ ),+ )+ await agent._on_model_event(+ me.TranscriptDeltaEvent(item_id="r1", delta="Draft"),+ )+ if failure:+ model.event_queues[0].put_nowait(+ me.ModelErrorEvent(code="closed", message="lost"),+ )+ model.event_queues[0].put_nowait(None)+ await asyncio.wait_for(+ model.iterator_finished[0].wait(),+ timeout=1,+ )+ checkpoints = []+ while not agent._out.empty():+ queued = agent._out.get_nowait()+ if queued.checkpoint is not None:+ checkpoints.append(+ (+ queued.event.reply_id,+ [+ (msg.id, msg.get_text_content())+ for msg in queued.checkpoint.context+ ],+ ),+ )+ history = [("kept", "complete"), ("r1", "Draft")]+ self.assertEqual(+ {+ "context": [+ (msg.id, msg.get_text_content())+ for msg in agent.state.context+ ],+ "pending": list(agent._pending_user_turns),+ "checkpoints": checkpoints,+ },+ {+ "context": history,+ "pending": [],+ "checkpoints": [+ (item_id, history)+ for item_id in ("u1", "u2", "r1")+ ],+ },+ )+ await agent.connect()+ self.assertEqual(+ model.instructions,+ "\n\n## Conversation so far\nuser: complete",+ )+ finally:+ await agent.close()++ async def test_close_discards_partial_transcription_after_detach(+ self,+ ) -> None:+ """Permanent close retires input only after its reader has stopped."""+ for finalized in (False, True):+ with self.subTest(finalized=finalized):+ model = QueueSessionModel()+ model.parameters.input_audio_transcription = True+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ aggregator=TurnAggregator(min_chars=4),+ )+ try:+ await agent.connect()+ await asyncio.wait_for(+ model.iterator_started[0].wait(),+ timeout=1,+ )+ await agent._on_model_event(+ me.SpeechStartedEvent(item_id="u1"),+ )+ await agent._on_model_event(+ me.InputTranscriptionDeltaEvent(+ item_id="u1",+ delta="a",+ ),+ )+ _ = [+ event+ async for event in agent.reply_stream(+ FakeTransport(0),+ )+ ]+ self.assertEqual(+ (+ list(agent._pending_user_turns),+ model.iterator_finished[0].is_set(),+ ),+ (["u1"], False),+ )+ if finalized:+ await agent._on_model_event(+ me.InputTranscriptionEvent(+ item_id="u1",+ text="accepted",+ ),+ )+ await agent.close()+ await agent.close()+ self.assertEqual(+ {+ "context": [+ (msg.id, msg.get_text_content())+ for msg in agent.state.context+ ],+ "pending": agent._pending_user_turns,+ "current": agent._user_turn,+ "reader_stopped": (+ model.iterator_finished[0].is_set()+ ),+ },+ {+ "context": (+ [("u1", "accepted")] if finalized else []+ ),+ "pending": {},+ "current": None,+ "reader_stopped": True,+ },+ )+ finally:+ await agent.close()++ async def test_partial_transcription_settlement(self) -> None:+ """Only accepted final text survives failure, filtering or reset."""+ for outcome, detached in product(+ ("failed", "filtered", "accepted"),+ (False, True),+ ):+ with self.subTest(outcome=outcome, detached=detached):+ agent = RealtimeAgent(+ "Friday",+ "",+ ScriptedModel([], input_audio_transcription=True),+ aggregator=TurnAggregator(min_chars=4),+ )+ await agent._on_model_event(+ me.InputTranscriptionDeltaEvent(item_id="u1", delta="a"),+ )+ # Transport detachment preserves the turn for a late final.+ if detached:+ agent._retire_unbound_user_turns()+ early = []+ while not agent._out.empty():+ queued = agent._out.get_nowait()+ if queued.checkpoint is not None:+ early.append(queued.checkpoint.context)+ self.assertEqual(early, [[]] if detached else [])+ self.assertEqual(+ [+ (msg.id, msg.get_text_content())+ for msg in agent.state.context+ ],+ [("u1", "a")],+ )+ if outcome == "failed":+ event = me.InputTranscriptionFailedEvent(item_id="u1")+ else:+ event = me.InputTranscriptionEvent(+ item_id="u1",+ text="accepted" if outcome == "accepted" else "a",+ )+ await agent._on_model_event(event)+ agent._reset_user_turns()+ history = [("u1", "accepted")] if outcome == "accepted" else []+ snapshots = []+ while not agent._out.empty():+ queued = agent._out.get_nowait()+ if queued.checkpoint is not None:+ snapshots.append(+ [+ (msg.id, msg.get_text_content())+ for msg in queued.checkpoint.context+ ],+ )+ self.assertEqual(+ (+ [+ (msg.id, msg.get_text_content())+ for msg in agent.state.context+ ],+ list(agent._pending_user_turns),+ snapshots,+ ),+ (history, [], [] if detached else [history]),+ )++ async def test_partial_transcription_cannot_absorb_accepted_turn(+ self,+ ) -> None:+ """Merging into an unfinished predecessor would lose accepted text."""+ agent = RealtimeAgent(+ "Friday",+ "",+ ScriptedModel([], input_audio_transcription=True),+ )+ with patch(+ "agentscope.agent._realtime._aggregator.time.monotonic",+ return_value=100,+ ):+ for event in (+ me.InputTranscriptionEvent(item_id="kept", text="previous"),+ me.InputTranscriptionDeltaEvent(item_id="u1", delta="a"),+ me.InputTranscriptionDeltaEvent(item_id="u2", delta="b"),+ me.InputTranscriptionEvent(item_id="u2", text="accepted"),+ ):+ await agent._on_model_event(event)+ self.assertTrue(agent.aggregator.merges_with_previous())+ agent._reset_user_turns()+ self.assertEqual(+ [(msg.id, msg.get_text_content()) for msg in agent.state.context],+ [("kept", "previous"), ("u2", "accepted")],+ )++ async def test_unbound_partial_transcription_discarded_on_detach(+ self,+ ) -> None:+ """Retiring an unmatchable turn must also remove its placeholder."""+ agent = RealtimeAgent(+ "Friday",+ "",+ ScriptedModel([], input_audio_transcription=True),+ )+ await agent._on_model_event(+ me.InputTranscriptionDeltaEvent(item_id="", delta="a"),+ )+ agent._retire_unbound_user_turns()+ self.assertEqual(+ (agent.state.context, agent._pending_user_turns),+ ([], {}),+ )++ async def test_tool_result_precedes_followup(self) -> None:+ """Early IDLE cannot close the reply before a result's continuation."""+ for delivery in ("after_send", "during_send", "silent", "interrupted"):+ with self.subTest(delivery=delivery):+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ )+ tool = BlockingTool()+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ toolkit=Toolkit(tools=[tool]),+ )++ async def feed(+ content: dict,+ current_model: GeminiRealtimeModel = model,+ current_agent: RealtimeAgent = agent,+ ) -> None:+ """Deliver a server frame through the real adapter."""+ for event in current_model._parse(content):+ await current_agent._on_model_event(event)++ final_content: dict = {"interactionStatus": "IDLE"}+ if delivery not in ("silent", "interrupted"):+ final_content["outputTranscription"] = {"text": "Sunny."}+ final_frame = {"serverContent": final_content}++ async def send_result(+ _payload: dict,+ mode: str = delivery,+ deliver: Any = feed,+ ) -> None:+ """Allow provider completion before send returns."""+ if mode == "during_send":+ await deliver(+ {+ "serverContent": {+ "interactionStatus": "IDLE",+ "outputTranscription": {"text": "Sunny."},+ },+ },+ )++ model._send = AsyncMock(side_effect=send_result)+ await feed(+ {+ "serverContent": {+ "outputTranscription": {"text": "Checking."},+ },+ },+ )+ reply_id = agent._reply_id+ await feed(+ {+ "toolCall": {+ "functionCalls": [+ {+ "id": "c1",+ "name": tool.name,+ "args": {"q": "x"},+ },+ ],+ },+ },+ )+ tasks = list(agent._tasks)+ try:+ await asyncio.wait_for(tool.started.wait(), timeout=1)+ await feed(+ {"serverContent": {"interactionStatus": "IDLE"}},+ )+ if delivery == "interrupted":+ agent._connected = True+ await agent.interrupt()+ tool.release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)+ if delivery != "during_send":+ self.assertEqual(+ agent._reply_id,+ "" if delivery == "interrupted" else reply_id,+ )+ await feed(final_frame)+ lifecycle = []+ while not agent._out.empty():+ event = agent._out.get_nowait().event+ if isinstance(event, (ReplyStartEvent, ReplyEndEvent)):+ lifecycle.append((event.type, event.reply_id))+ self.assertEqual(+ lifecycle,+ [+ ("REPLY_START", reply_id),+ ("REPLY_END", reply_id),+ ],+ )+ self.assertEqual(+ [+ (+ msg.id,+ [+ block.text+ for block in msg.content+ if isinstance(block, TextBlock)+ ],+ )+ for msg in agent.state.context+ ],+ [+ (+ reply_id,+ ["Checking."]+ + (+ []+ if delivery in ("silent", "interrupted")+ else ["Sunny."]+ ),+ ),+ ],+ )+ self.assertEqual(agent._background_tools, {})+ self.assertEqual(+ (+ model._awaiting_tool_reply,+ model._pending_tool_calls,+ ),+ (False, set()),+ )+ finally:+ tool.release.set()+ await asyncio.gather(*tasks, return_exceptions=True)++ async def test_fragment_placeholders_preserve_pause_merging(self) -> None:+ """Reserved input must honor the same aggregation as final input."""+ for with_delta in (False, True):+ for separator in ("none", "empty", "spoken", "tool"):+ with self.subTest(delta=with_delta, separator=separator):+ agent = RealtimeAgent("Friday", "", ScriptedModel([]))+ with patch(+ "agentscope.agent._realtime."+ "_aggregator.time.monotonic",+ return_value=100.0,+ ):+ agent._on_transcription(+ me.InputTranscriptionEvent(+ item_id="u1",+ text="北京",+ ),+ )+ if separator != "none":+ content = (+ [+ ToolCallBlock(+ id="c1",+ name="tool",+ input="{}",+ ),+ ]+ if separator == "tool"+ else [+ TextBlock(+ text="好的"+ if separator == "spoken"+ else "",+ ),+ ]+ )+ agent.state.context.append(+ Msg(+ id="between",+ name="Friday",+ role="assistant",+ content=content,+ ),+ )+ if with_delta:+ agent._on_transcription_delta(+ me.InputTranscriptionDeltaEvent(+ item_id="u2",+ delta="天",+ ),+ )+ agent.state.context.append(+ Msg(+ id="answer",+ name="Friday",+ role="assistant",+ content=[TextBlock(text="晴天")],+ ),+ )+ agent._on_transcription_delta(+ me.InputTranscriptionDeltaEvent(+ item_id="u2",+ delta="气",+ response_id="answer",+ ),+ )+ agent._on_transcription(+ me.InputTranscriptionEvent(+ item_id="u2",+ text="天气",+ ),+ )+ self.assertTrue(agent.aggregator.merges_with_previous())+ expected = [("u1", "user", "北京天气")]+ if separator in ("spoken", "tool"):+ expected = [+ ("u1", "user", "北京"),+ (+ "between",+ "assistant",+ "好的" if separator == "spoken" else "",+ ),+ ("u2", "user", "天气"),+ ]+ if with_delta:+ expected.append(("answer", "assistant", "晴天"))+ self.assertEqual(+ [+ (msg.id, msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ expected,+ )++ async def test_idle_waits_for_running_and_queued_tools(self) -> None:+ """Model completion must not discard work or precede its results."""+ model = ScriptedModel([])+ tool = BlockingTool()+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ toolkit=Toolkit(tools=[tool]),+ )+ for call_id in ("c1", "c2"):+ await agent._on_model_event(+ me.ToolCallEvent(+ item_id="r1",+ non_blocking=True,+ tool_call=ToolCallBlock(+ id=call_id,+ name=tool.name,+ input='{"q":"x"}',+ ),+ ),+ )+ tasks = list(agent._tasks)+ try:+ await asyncio.wait_for(tool.started.wait(), timeout=1)+ await agent._on_model_event(me.ResponseDoneEvent(item_id="r1"))+ tool.release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)+ self.assertEqual(agent._reply_id, "r1")+ await agent._on_model_event(me.ResponseCreatedEvent(item_id="r2"))+ await agent._on_model_event(me.ResponseDoneEvent(item_id="r2"))+ events = []+ while not agent._out.empty():+ event = agent._out.get_nowait().event+ if isinstance(event, (ToolResultEndEvent, ReplyEndEvent)):+ events.append(+ (event.type, getattr(event, "tool_call_id", "")),+ )+ self.assertEqual(+ (model.calls, events),+ (+ ["tool_result(c1,'x-final')", "tool_result(c2,'x-final')"],+ [+ ("TOOL_RESULT_END", "c1"),+ ("TOOL_RESULT_END", "c2"),+ ("REPLY_END", ""),+ ],+ ),+ )+ finally:+ tool.release.set()+ await asyncio.gather(*tasks, return_exceptions=True)++ async def test_transcription_keeps_one_turn_before_answer(self) -> None:+ """Fragments around thinking remain one user message before output."""+ for split in (False, True):+ with self.subTest(split=split):+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ )+ agent = RealtimeAgent("Friday", "", model)+ frames: list[dict] = []+ if split:+ frames.append({"inputTranscription": {"text": "Check "}})+ frames.extend(+ [+ {"interactionStatus": "IN_PROGRESS"},+ {"outputTranscription": {"text": "Sunny."}},+ {+ "inputTranscription": {+ "text": "weather"+ if split+ else "Check weather",+ },+ },+ {"interactionStatus": "IDLE"},+ ],+ )+ for content in frames:+ for event in model._parse({"serverContent": content}):+ await agent._on_model_event(event)+ self.assertEqual(+ [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ [("user", "Check weather"), ("assistant", "Sunny.")],+ )+ user_replies = []+ snapshots = []+ while not agent._out.empty():+ queued = agent._out.get_nowait()+ if isinstance(queued.event, ReplyStartEvent):+ if queued.event.role == "user":+ user_replies.append(queued.event.reply_id)+ if queued.checkpoint is not None:+ snapshots.append(+ [+ (msg.role, msg.get_text_content())+ for msg in queued.checkpoint.context+ ],+ )+ self.assertEqual(user_replies, [agent.state.context[0].id])+ self.assertEqual(+ snapshots[-1],+ [+ ("user", "Check weather"),+ ("assistant", "Sunny."),+ ],+ )+ model.connect = AsyncMock()+ try:+ await agent.connect()+ self.assertEqual(+ model.connect.await_args.kwargs["instructions"],+ "\n\n## Conversation so far\n"+ "user: Check weather\nFriday: Sunny.",+ )+ finally:+ await agent.close()++ async def test_late_transcription_precedes_tool_followup(self) -> None:+ """Late input precedes the logical reply across model responses."""+ for delivery in (+ "separate",+ "same_frame",+ "content_status",+ "root_status",+ "before_tool_result",+ "after_tool_result",+ ):+ with self.subTest(delivery=delivery):+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ )+ model._send = AsyncMock()+ tool = BlockingTool()+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ toolkit=Toolkit(tools=[tool]),+ )++ async def feed(+ frame: dict,+ current_model: GeminiRealtimeModel = model,+ current_agent: RealtimeAgent = agent,+ ) -> None:+ """Deliver a frame through the real Gemini adapter."""+ for event in current_model._parse(frame):+ await current_agent._on_model_event(event)++ tasks = []+ try:+ await feed(+ {+ "serverContent": {+ "outputTranscription": {"text": "Checking."},+ },+ },+ )+ reply_id = agent._reply_id+ await feed(+ {+ "toolCall": {+ "functionCalls": [+ {+ "id": "c1",+ "name": tool.name,+ "args": {"q": "weather"},+ },+ ],+ },+ },+ )+ tasks = list(agent._tasks)+ await asyncio.wait_for(tool.started.wait(), timeout=1)+ await feed(+ {"serverContent": {"interactionStatus": "IDLE"}},+ )+ transcription = {+ "inputTranscription": {"text": "Weather?"},+ }+ if delivery == "before_tool_result":+ await feed({"serverContent": transcription})+ tool.release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)+ output = {"outputTranscription": {"text": "Sunny."}}+ if delivery == "same_frame":+ await feed(+ {"serverContent": {**output, **transcription}},+ )+ elif delivery in ("content_status", "root_status"):+ frame: dict = {"serverContent": transcription}+ status_target = (+ frame+ if delivery == "root_status"+ else frame["serverContent"]+ )+ status_target["interactionStatus"] = "IN_PROGRESS"+ await feed(frame)+ await feed({"serverContent": output})+ elif delivery in (+ "before_tool_result",+ "after_tool_result",+ ):+ if delivery == "after_tool_result":+ await feed({"serverContent": transcription})+ await feed({"serverContent": output})+ else:+ await feed({"serverContent": output})+ await feed({"serverContent": transcription})+ self.assertNotEqual(model._response_id, reply_id)+ self.assertEqual(agent._reply_id, reply_id)+ await feed(+ {"serverContent": {"interactionStatus": "IDLE"}},+ )+ expected = [+ ("user", "Weather?"),+ ("assistant", "Checking.\nSunny."),+ ]+ snapshots = [+ agent.checkpoint_snapshot(event)+ for event in agent.drain_events()+ ]+ checkpoint = next(+ state for state in reversed(snapshots) if state+ )+ self.assertEqual(+ {+ "context": [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ "checkpoint": [+ (msg.role, msg.get_text_content())+ for msg in checkpoint.context+ ],+ },+ {"context": expected, "checkpoint": expected},+ )+ model.connect = AsyncMock()+ await agent.connect()+ self.assertEqual(+ model.connect.await_args.kwargs["instructions"],+ "\n\n## Conversation so far\n"+ "user: Weather?\nFriday: Checking.\nSunny.",+ )+ finally:+ tool.release.set()+ await asyncio.gather(*tasks, return_exceptions=True)+ await agent.close()++ async def test_tool_idle_preserves_input_fragments(self) -> None:+ """Tool handoffs must not strip or filter individual fragments."""+ for min_chars, idle_count in product((1, 4), (1, 2)):+ with self.subTest(min_chars=min_chars, idle_count=idle_count):+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ )+ model._send = AsyncMock()+ tool = BlockingTool()+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ toolkit=Toolkit(tools=[tool]),+ aggregator=TurnAggregator(min_chars=min_chars),+ )++ async def feed(+ frame: dict,+ current_model: GeminiRealtimeModel = model,+ current_agent: RealtimeAgent = agent,+ ) -> None:+ """Deliver real adapter events without a network."""+ for event in current_model._parse(frame):+ await current_agent._on_model_event(event)++ tasks = []+ try:+ await feed(+ {+ "serverContent": {+ "inputTranscription": {"text": "Go "},+ "outputTranscription": {"text": "Checking."},+ },+ "toolCall": {+ "functionCalls": [+ {+ "id": "c1",+ "name": tool.name,+ "args": {"q": "x"},+ },+ ],+ },+ },+ )+ user_id = model._user_id+ tasks = list(agent._tasks)+ await asyncio.wait_for(tool.started.wait(), timeout=1)+ for _ in range(idle_count):+ await feed(+ {"serverContent": {"interactionStatus": "IDLE"}},+ )+ self.assertEqual(+ (model._user_id, model._input_text),+ (user_id, "Go "),+ )+ tool.release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)+ await feed(+ {+ "serverContent": {+ "inputTranscription": {"text": "there"},+ },+ },+ )+ self.assertEqual(+ (model._user_id, model._input_text),+ (user_id, "Go there"),+ )+ await feed(+ {+ "serverContent": {+ "outputTranscription": {"text": "Done."},+ "interactionStatus": "IDLE",+ },+ },+ )+ events = list(agent.drain_events())+ user_reply_id = agent.state.context[0].id+ self.assertEqual(+ {+ "context": [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ "user_text": [+ event.delta+ for event in events+ if isinstance(event, TextBlockDeltaEvent)+ and event.reply_id == user_reply_id+ ],+ "pending": agent._pending_user_turns,+ "input": (model._user_id, model._input_text),+ },+ {+ "context": [+ ("user", "Go there"),+ ("assistant", "Checking.\nDone."),+ ],+ "user_text": ["Go there"],+ "pending": {},+ "input": ("", ""),+ },+ )+ finally:+ tool.release.set()+ await asyncio.gather(*tasks, return_exceptions=True)+ await agent.close()++ async def test_interruption_keeps_transcription_identities(self) -> None:+ """A new utterance cannot inherit the interrupted input buffer."""+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ )+ agent = RealtimeAgent("Friday", "", model)+ agent._connected = True+ frames = [+ {"inputTranscription": {"text": "Check weather"}},+ {"interactionStatus": "IN_PROGRESS"},+ {"outputTranscription": {"text": "Checking."}},+ {"interrupted": True, "inputTranscription": {"text": "Change "}},+ {"interactionStatus": "IN_PROGRESS"},+ {"outputTranscription": {"text": "Okay."}},+ {"inputTranscription": {"text": "city"}},+ {"interactionStatus": "IDLE"},+ ]+ for content in frames:+ for event in model._parse({"serverContent": content}):+ await agent._on_model_event(event)+ self.assertEqual(+ [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ [+ ("user", "Check weather"),+ ("assistant", "Checking."),+ ("user", "Change city"),+ ("assistant", "Okay."),+ ],+ )+ self.assertNotEqual(+ agent.state.context[0].id,+ agent.state.context[2].id,+ )++ async def test_local_vad_interruption_preserves_input_history(+ self,+ ) -> None:+ """Local speech starts settle old input and reserve a new turn."""+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ parameters=GeminiRealtimeModel.Parameters(turn_detection="none"),+ )+ model._send = AsyncMock()+ agent = RealtimeAgent("Friday", "", model)+ agent._connected = True++ async def feed(content: dict) -> None:+ """Deliver parsed provider frames in model event order."""+ for event in model._parse({"serverContent": content}):+ await agent._on_model_event(event)++ await agent._forward_audio(b"\x00\x00", SpeechTransition.STARTED)+ await feed({"inputTranscription": {"text": "Check weather"}})+ await agent._forward_audio(b"\x00\x00", SpeechTransition.ENDED)+ await feed(+ {+ "interactionStatus": "IN_PROGRESS",+ "outputTranscription": {"text": "Checking."},+ },+ )+ await agent._forward_audio(b"\x00\x00", SpeechTransition.STARTED)+ while not model._queue.empty():+ await agent._on_model_event(model._queue.get_nowait())+ await feed({"inputTranscription": {"text": "Change "}})+ await agent._forward_audio(b"\x00\x00", SpeechTransition.ENDED)+ await feed(+ {+ "inputTranscription": {"text": "city"},+ "interactionStatus": "IDLE",+ },+ )+ await feed(+ {+ "interactionStatus": "IN_PROGRESS",+ "outputTranscription": {"text": "Okay."},+ },+ )+ await feed({"interactionStatus": "IDLE"})+ self.assertEqual(+ [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ [+ ("user", "Check weather"),+ ("assistant", "Checking."),+ ("user", "Change city"),+ ("assistant", "Okay."),+ ],+ )+ self.assertNotEqual(+ agent.state.context[0].id,+ agent.state.context[2].id,+ )+ self.assertEqual(agent._pending_user_turns, {})++ async def test_stale_idle_preserves_ongoing_local_input(self) -> None:+ """Cancelled completions cannot split speech or filter a fragment."""+ for backchannels in (frozenset(), frozenset({"New"})):+ with self.subTest(backchannels=backchannels):+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ parameters=GeminiRealtimeModel.Parameters(+ turn_detection="none",+ ),+ )+ model._send = AsyncMock()+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ aggregator=TurnAggregator(backchannels=backchannels),+ )+ agent._connected = True++ async def feed(+ content: dict,+ current_model: GeminiRealtimeModel = model,+ current_agent: RealtimeAgent = agent,+ ) -> None:+ """Deliver provider events without a network."""+ for event in current_model._parse(+ {"serverContent": content},+ ):+ await current_agent._on_model_event(event)++ try:+ with patch(+ "agentscope.agent._realtime._aggregator.time",+ ) as clock:+ clock.monotonic.return_value = 10.0+ await feed(+ {"outputTranscription": {"text": "Old answer"}},+ )+ await agent._forward_audio(+ b"\x00\x00",+ SpeechTransition.STARTED,+ )+ user_turn = agent._user_turn+ await feed({"inputTranscription": {"text": "New "}})+ provider_id = model._user_id+ for _ in range(2):+ await feed({"interactionStatus": "IDLE"})+ self.assertIs(agent._user_turn, user_turn)+ self.assertEqual(+ (+ user_turn.open,+ user_turn.speech_ended,+ model._user_id,+ model._input_text,+ ),+ (True, False, provider_id, "New "),+ )+ clock.monotonic.return_value = 12.0+ await feed(+ {"inputTranscription": {"text": "question"}},+ )+ self.assertEqual(model._user_id, provider_id)+ await agent._forward_audio(+ b"\x00\x00",+ SpeechTransition.ENDED,+ )+ await feed(+ {+ "interactionStatus": "IN_PROGRESS",+ "outputTranscription": {"text": "Okay."},+ },+ )+ await feed({"interactionStatus": "IDLE"})+ events = list(agent.drain_events())+ self.assertEqual(+ {+ "context": [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ "user_starts": [+ event.reply_id+ for event in events+ if isinstance(event, ReplyStartEvent)+ and event.role == "user"+ ],+ "user_ends": [+ event.reply_id+ for event in events+ if isinstance(event, ReplyEndEvent)+ and event.reply_id == user_turn.reply_id+ ],+ "pending": agent._pending_user_turns,+ "current": agent._user_turn,+ },+ {+ "context": [+ ("assistant", "Old answer"),+ ("user", "New question"),+ ("assistant", "Okay."),+ ],+ "user_starts": [user_turn.reply_id],+ "user_ends": [user_turn.reply_id],+ "pending": {},+ "current": None,+ },+ )+ finally:+ await agent.close()++ async def test_repeated_local_input_before_cancelled_response_idle(+ self,+ ) -> None:+ """Every local input start separates text, even with no live reply."""+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ parameters=GeminiRealtimeModel.Parameters(turn_detection="none"),+ )+ model._send = AsyncMock()+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ aggregator=TurnAggregator(merge_window_ms=-1),+ )+ agent._connected = True+ user_ids = []+ provider_ids = []+ transcripts = ["Check weather", "Change city", "Tomorrow instead"]+ for index, text in enumerate(transcripts):+ await agent._forward_audio(b"\x00\x00", SpeechTransition.STARTED)+ user_ids.append(agent._user_turn.reply_id)+ while not model._queue.empty():+ await agent._on_model_event(model._queue.get_nowait())+ for event in model._parse(+ {"serverContent": {"inputTranscription": {"text": text}}},+ ):+ provider_ids.append(event.item_id)+ await agent._on_model_event(event)+ await agent._forward_audio(b"\x00\x00", SpeechTransition.ENDED)+ if index == 0:+ for event in model._parse(+ {+ "serverContent": {+ "interactionStatus": "IN_PROGRESS",+ "outputTranscription": {"text": "Checking."},+ },+ },+ ):+ await agent._on_model_event(event)+ else:+ self.assertEqual(agent._reply_id, "")+ for event in model._parse({"interactionStatus": "IDLE"}):+ await agent._on_model_event(event)+ self.assertIsNotNone(agent._user_turn)+ for content in (+ {"interactionStatus": "IN_PROGRESS"},+ {"interactionStatus": "IDLE"},+ ):+ for event in model._parse({"serverContent": content}):+ await agent._on_model_event(event)+ starts = []+ ends = []+ while not agent._out.empty():+ event = agent._out.get_nowait().event+ if isinstance(event, ReplyStartEvent) and event.role == "user":+ starts.append(event.reply_id)+ elif (+ isinstance(event, ReplyEndEvent) and event.reply_id in user_ids+ ):+ ends.append(event.reply_id)+ self.assertEqual(+ {+ "history": [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ "user_ids": [+ msg.id for msg in agent.state.context if msg.role == "user"+ ],+ "provider_id_count": len(set(provider_ids)),+ "starts": starts,+ "ends": ends,+ "pending": agent._pending_user_turns,+ "current": agent._user_turn,+ },+ {+ "history": [+ ("user", transcripts[0]),+ ("assistant", "Checking."),+ ("user", transcripts[1]),+ ("user", transcripts[2]),+ ],+ "user_ids": user_ids,+ "provider_id_count": 3,+ "starts": user_ids,+ "ends": user_ids,+ "pending": {},+ "current": None,+ },+ )++ async def test_followup_keeps_reply_open_after_tools_finish(self) -> None:+ """New model output after early IDLE resets model completion."""+ model = ScriptedModel([])+ tool = BlockingTool()+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ toolkit=Toolkit(tools=[tool]),+ )+ await agent._on_model_event(+ me.ToolCallEvent(+ item_id="r1",+ non_blocking=True,+ tool_call=ToolCallBlock(+ id="c1",+ name=tool.name,+ input='{"q":"x"}',+ ),+ ),+ )+ tasks = list(agent._tasks)+ try:+ await asyncio.wait_for(tool.started.wait(), timeout=1)+ await agent._on_model_event(me.ResponseDoneEvent(item_id="r1"))+ await agent._on_model_event(me.ResponseCreatedEvent(item_id="r2"))+ tool.release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)+ self.assertEqual(agent._reply_id, "r1")+ await agent._on_model_event(me.ResponseDoneEvent(item_id="r2"))+ replies = []+ while not agent._out.empty():+ event = agent._out.get_nowait().event+ if isinstance(event, ReplyEndEvent):+ replies.append((event.reply_id, event.finished_reason))+ self.assertEqual(replies, [("r1", "completed")])+ self.assertEqual(agent._background_tools, {})+ finally:+ tool.release.set()+ await asyncio.gather(*tasks, return_exceptions=True)++ async def test_interrupted_slow_tool_does_not_block_new_reply(+ self,+ ) -> None:+ """Serialize within a reply while allowing the next reply to run."""+ model = ScriptedModel([])+ model.push_tool_result_silently = AsyncMock(+ wraps=model.push_tool_result_silently,+ )+ old_tool = BlockingTool()+ old_tool.name = "slow_tool"+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ toolkit=Toolkit(tools=[old_tool, StreamTool()]),+ )+ try:+ for call_id in ("running", "queued"):+ await agent._on_model_event(+ me.ToolCallEvent(+ item_id="r1",+ non_blocking=True,+ tool_call=ToolCallBlock(+ id=call_id,+ name=old_tool.name,+ input='{"q":"old"}',+ ),+ ),+ )+ old_tasks = set(agent._tasks)+ await asyncio.wait_for(old_tool.started.wait(), timeout=1)+ started_calls = []+ while not agent._out.empty():+ event = agent._out.get_nowait().event+ if isinstance(event, ToolCallStartEvent):+ started_calls.append(event.tool_call_id)+ self.assertEqual(started_calls, ["running"])+ await agent._barge_in()+ await agent._on_model_event(+ me.ToolCallEvent(+ item_id="r2",+ non_blocking=True,+ tool_call=ToolCallBlock(+ id="new",+ name="stream_tool",+ input='{"q":"new"}',+ ),+ ),+ )+ new_tasks = agent._tasks - old_tasks+ await asyncio.wait_for(asyncio.gather(*new_tasks), timeout=1)+ self.assertEqual(+ (old_tool.release.is_set(), model.calls, agent._reply_id),+ (False, ["tool_result(new,'new-final')"], "r2"),+ )+ old_tool.release.set()+ await asyncio.wait_for(asyncio.gather(*old_tasks), timeout=1)+ self.assertEqual(+ model.calls,+ [+ "tool_result(new,'new-final')",+ "tool_result(running,'old-final')",+ "tool_result(queued,"+ "'Reply interrupted before tool execution.')",+ ],+ )+ self.assertEqual(+ [+ (call.args[0].id, call.args[0].state)+ for call in model.push_tool_result_silently.await_args_list+ ],+ [("running", "success"), ("queued", "interrupted")],+ )+ self.assertEqual(+ [+ (+ msg.id,+ [+ (block.type, block.id, block.state)+ for block in msg.content+ if isinstance(+ block,+ (ToolCallBlock, ToolResultBlock),+ )+ ],+ )+ for msg in agent.state.context+ ],+ [+ (+ "r1",+ [+ ("tool_call", "running", "finished"),+ ("tool_call", "queued", "finished"),+ ("tool_result", "running", "success"),+ ("tool_result", "queued", "interrupted"),+ ],+ ),+ (+ "r2",+ [+ ("tool_call", "new", "finished"),+ ("tool_result", "new", "success"),+ ],+ ),+ ],+ )+ self.assertEqual(+ [+ (event.type, event.tool_call_id)+ for event in agent.drain_events()+ if isinstance(+ event,+ (ToolCallStartEvent, ToolResultEndEvent),+ )+ ],+ [+ ("TOOL_CALL_START", "new"),+ ("TOOL_RESULT_END", "new"),+ ("TOOL_RESULT_END", "running"),+ ("TOOL_RESULT_END", "queued"),+ ],+ )+ finally:+ old_tool.release.set()+ await agent.close()++ async def test_interruption_releases_confirmation_lock(self) -> None:+ """An unanswered old prompt cannot block the next reply's tool."""+ factories = [None]+ if eager_factory := getattr(asyncio, "eager_task_factory", None):+ factories.append(eager_factory)+ loop = asyncio.get_running_loop()+ original_factory = loop.get_task_factory()+ for factory in factories:+ with self.subTest(eager=factory is not None):+ try:+ loop.set_task_factory(factory)+ await self._check_interrupted_confirmation()+ finally:+ loop.set_task_factory(original_factory)++ async def test_interrupted_gemini_tool_results_are_silent(self) -> None:+ """Late results cannot restart a closed reply or disturb a new one."""+ for confirmation, new_reply in product((False, True), repeat=2):+ with self.subTest(confirm=confirmation, new_reply=new_reply):+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ )+ model._send = AsyncMock()+ tool = AskTool() if confirmation else BlockingTool()+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ toolkit=Toolkit(tools=[tool]),+ )+ agent._connected = True++ async def feed(+ frame: dict,+ current_model: GeminiRealtimeModel = model,+ current_agent: RealtimeAgent = agent,+ ) -> None:+ """Deliver real adapter events without a network."""+ for event in current_model._parse(frame):+ await current_agent._on_model_event(event)++ await feed(+ {+ "serverContent": {+ "outputTranscription": {"text": "Checking."},+ },+ "toolCall": {+ "functionCalls": [+ {+ "id": call_id,+ "name": tool.name,+ "args": {"q": "x"},+ }+ for call_id in ("c1", "c2")+ ],+ },+ },+ )+ tasks = list(agent._tasks)+ try:+ if confirmation:++ async def wait_for_confirmation(+ current_agent: RealtimeAgent = agent,+ ) -> None:+ """Wait until the prompt can be cancelled."""+ while "c1" not in current_agent._confirmations:+ await asyncio.sleep(0)++ await asyncio.wait_for(+ wait_for_confirmation(),+ timeout=1,+ )+ else:+ await asyncio.wait_for(tool.started.wait(), timeout=1)+ await agent.interrupt()+ await feed(+ {"serverContent": {"interactionStatus": "IDLE"}},+ )+ expected_context = [("assistant", "Checking.")]+ if new_reply:+ await feed(+ {+ "serverContent": {+ "inputTranscription": {"text": "Next"},+ "outputTranscription": {"text": "Ready."},+ },+ },+ )+ expected_context += [+ ("user", "Next"),+ ("assistant", "Ready."),+ ]+ current_reply = agent._reply_id+ list(agent.drain_events())+ if not confirmation:+ tool.release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)+ result = (+ {"error": 'Tool "ask_tool" denied by user.'}+ if confirmation+ else {"output": "x-final"}+ )+ self.assertEqual(+ {+ "frames": [+ call.args[0]+ for call in model._send.await_args_list+ ],+ "reply": agent._reply_id,+ "new_reply_events": [+ event.type+ for event in agent.drain_events()+ if isinstance(event, ReplyStartEvent)+ ],+ "awaiting_followup": model._awaiting_tool_reply,+ "context": [+ (msg.role, msg.get_text_content())+ for msg in agent.state.context+ ],+ },+ {+ "frames": [+ {+ "toolResponse": {+ "functionResponses": [+ {+ "id": "c1",+ "name": tool.name,+ "response": result,+ "scheduling": "SILENT",+ },+ ],+ },+ },+ {+ "toolResponse": {+ "functionResponses": [+ {+ "id": "c2",+ "name": tool.name,+ "response": {+ "error": (+ "Reply interrupted "+ "before tool "+ "execution."+ ),+ },+ "scheduling": "SILENT",+ },+ ],+ },+ },+ ],+ "reply": current_reply,+ "new_reply_events": [],+ "awaiting_followup": False,+ "context": expected_context,+ },+ )+ finally:+ if not confirmation:+ tool.release.set()+ await agent.close()+ await asyncio.gather(*tasks, return_exceptions=True)++ async def _check_interrupted_confirmation(self) -> None:+ """Validate cancellation and approval ownership under one scheduler."""+ model = ScriptedModel([])+ agent = RealtimeAgent(+ "Friday",+ "",+ model,+ toolkit=Toolkit(tools=[AskTool(), StreamTool()]),+ )+ old_call = ToolCallBlock(id="old", name="ask_tool", input='{"q":"x"}')+ await agent._on_model_event(+ me.ToolCallEvent(+ item_id="r1",+ non_blocking=True,+ tool_call=old_call,+ ),+ )+ tasks = list(agent._tasks)+ try:++ async def wait_for_confirmation() -> None:+ """Wait for the controlled prompt to be registered."""+ while "old" not in agent._confirmations:+ await asyncio.sleep(0)++ await asyncio.wait_for(wait_for_confirmation(), timeout=1)+ old_future = agent._confirmations["old"]+ await agent._barge_in()+ await agent._on_model_event(+ me.ToolCallEvent(+ item_id="r2",+ non_blocking=True,+ tool_call=ToolCallBlock(+ id="new",+ name="stream_tool",+ input='{"q":"y"}',+ ),+ ),+ )+ tasks = list(agent._tasks)+ await asyncio.wait_for(+ asyncio.shield(asyncio.gather(*tasks)),+ timeout=1,+ )+ self.assertTrue(old_future.cancelled())+ new_call = ToolCallBlock(+ id="old",+ name="ask_tool",+ input='{"q":"z"}',+ )+ await agent._on_model_event(+ me.ToolCallEvent(+ item_id="r2",+ non_blocking=True,+ tool_call=new_call,+ ),+ )+ tasks = list(agent._tasks)+ await asyncio.wait_for(wait_for_confirmation(), timeout=1)+ with patch(+ "agentscope.agent._realtime._agent.logger.warning",+ ) as warning:+ await agent.send(+ UserConfirmResultEvent(+ reply_id="r1",+ confirm_results=[+ ConfirmResult(tool_call=old_call, confirmed=True),+ ],+ ),+ )+ warning.assert_called_once_with(+ "RealtimeAgent: ignored confirmation for tool %s "+ "from reply %s; expected reply %s",+ "old",+ "r1",+ "r2",+ )+ self.assertFalse(agent._confirmations["old"].done())+ await agent.send(+ UserConfirmResultEvent(+ reply_id="r2",+ confirm_results=[+ ConfirmResult(tool_call=new_call, confirmed=True),+ ],+ ),+ )+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)+ self.assertEqual(+ {+ "independent_results": sorted(model.calls[:2]),+ "confirmed_result": model.calls[2:],+ },+ {+ "independent_results": [+ "tool_result(new,'y-final')",+ "tool_result(old,'Tool \"ask_tool\" denied by user.')",+ ],+ "confirmed_result": ["tool_result(old,'z-final')"],+ },+ )+ self.assertEqual(agent._confirmations, {})+ self.assertEqual(agent._confirmation_replies, {})+ finally:+ for future in list(agent._confirmations.values()):+ future.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)++ async def test_tools_and_speech_share_an_interaction(self) -> None:+ """Tools start before completion; audio continues during execution."""+ for interrupted in (False, True):+ with self.subTest(interrupted=interrupted):+ model = GeminiRealtimeModel(+ "gemini-3.8-live-extended-thinking",+ GeminiCredential(api_key="test"),+ )+ model._send = AsyncMock()+ tool = BlockingTool()+ agent = RealtimeAgent(+ "Friday",+ "be brief",+ model,+ toolkit=Toolkit(tools=[tool]),+ )+ transport = FakeTransport(frames=0)+ agent._transport = transport++ async def feed(+ frame: dict,+ current_model: GeminiRealtimeModel = model,+ current_agent: RealtimeAgent = agent,+ ) -> None:+ """Normalize a provider frame and deliver its events."""+ for event in current_model._parse(frame):+ await current_agent._on_model_event(event)++ await feed(+ {+ "serverContent": {+ "interactionStatus": "IN_PROGRESS",+ "outputTranscription": {"text": "Checking."},+ "turnComplete": True,+ },+ "toolCall": {+ "functionCalls": [+ {+ "id": "c1",+ "name": "stream_tool",+ "args": {"q": "x"},+ },+ ],+ },+ },+ )+ tasks = list(agent._tasks)+ try:+ await asyncio.wait_for(tool.started.wait(), timeout=1)+ reply_id = agent._reply_id+ await feed(+ {+ "toolCall": {+ "functionCalls": [+ {+ "id": "c2",+ "name": "stream_tool",+ "args": {"q": "y"},+ },+ ],+ },+ },+ )+ tasks = list(agent._tasks)+ await feed(+ {+ "serverContent": {+ "modelTurn": {+ "parts": [+ {+ "inlineData": {"data": "AQA="},+ },+ ],+ },+ "outputTranscription": {+ "text": "Still checking.",+ },+ "turnComplete": True,+ },+ },+ )+ self.assertEqual(+ {+ "reply": agent._reply_id,+ "audio_item": transport.item,+ "pending": agent._pending_tools,+ "done": [task.done() for task in tasks],+ "sent": model._send.await_args_list,+ },+ {+ "reply": reply_id,+ "audio_item": reply_id,+ "pending": {},+ "done": [False, False],+ "sent": [],+ },+ )+ if interrupted:+ await feed({"serverContent": {"interrupted": True}})+ tool.release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)+ if not interrupted:+ await feed(+ {+ "serverContent": {+ "outputTranscription": {"text": "Sunny."},+ },+ },+ )+ await feed(+ {+ "serverContent": {+ "interactionStatus": "IDLE",+ },+ },+ )+ await feed(+ {+ "serverContent": {+ "interactionStatus": "IDLE",+ },+ },+ )+ self.assertEqual(+ sorted(+ [+ call.args[0]+ for call in model._send.await_args_list+ ],+ key=lambda frame: frame["toolResponse"][+ "functionResponses"+ ][0]["id"],+ ),+ [+ {+ "toolResponse": {+ "functionResponses": [+ {+ "id": call_id,+ "name": "stream_tool",+ "response": result,+ **(+ {"scheduling": "SILENT"}+ if interrupted+ else {}+ ),+ },+ ],+ },+ }+ for call_id, result in [+ ("c1", {"output": "x-final"}),+ (+ "c2",+ {+ "error": "Reply interrupted before "+ "tool execution.",+ }+ if interrupted+ else {"output": "y-final"},+ ),+ ]+ ],+ )+ events = []+ while not agent._out.empty():+ events.append(agent._out.get_nowait().event)+ self.assertEqual(+ [+ (event.reply_id, event.finished_reason)+ for event in events+ if isinstance(event, ReplyEndEvent)+ ],+ [+ (+ reply_id,+ "interrupted" if interrupted else "completed",+ ),+ ],+ )+ text = agent.state.context[0].get_content_blocks("text")+ self.assertEqual(+ [block.text for block in text],+ ["Checking.", "Still checking."]+ + ([] if interrupted else ["Sunny."]),+ )+ self.assertEqual(+ len({block.id for block in text}),+ len(text),+ )+ self.assertEqual(+ [+ (event.type, event.block_id)+ for event in events+ if isinstance(+ event,+ (+ TextBlockStartEvent,+ TextBlockEndEvent,+ ),+ )+ ],+ [+ (kind, block.id)+ for block in text+ for kind in (+ "TEXT_BLOCK_START",+ "TEXT_BLOCK_END",+ )+ ],+ )+ self.assertEqual(+ sorted(+ [+ (event.type, event.tool_call_id)+ for event in events+ if isinstance(+ event,+ (+ ToolCallStartEvent,+ ToolResultEndEvent,+ ),+ )+ ],+ key=lambda event: (event[1], event[0]),+ ),+ [+ ("TOOL_CALL_START", "c1"),+ ("TOOL_RESULT_END", "c1"),+ ]+ + ([] if interrupted else [("TOOL_CALL_START", "c2")])+ + [("TOOL_RESULT_END", "c2")],+ )+ self.assertEqual(agent._reply_id, "")+ finally:+ tool.release.set()+ await asyncio.gather(*tasks, return_exceptions=True)++class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"""The complete event stream of a turn that calls a tool and thenanswers with speech, asserted as one structure."""diff --git a/tests/realtime_dashscope_test.py b/tests/realtime_dashscope_test.pyindex e6e7f8719..ad7883ea6 100644--- a/tests/realtime_dashscope_test.py+++ b/tests/realtime_dashscope_test.py@@ -643,6 +643,7 @@ class DashScopeParseTest(unittest.TestCase):"suggested_rules": [],"finished_at": None,},+ "non_blocking": False,},)self.assertEqual(self.model._tool_names, {})diff --git a/tests/realtime_gemini_test.py b/tests/realtime_gemini_test.pyindex 248443371..b776ea3d6 100644--- a/tests/realtime_gemini_test.py+++ b/tests/realtime_gemini_test.py@@ -2,14 +2,18 @@"""Unit tests for the Gemini Live realtime adapter: cards, the setupmessage and frame parsing. Nothing here opens a connection."""# pylint: disable=protected-access+import asyncioimport base64import unittest+from itertools import product+from types import SimpleNamespacefrom unittest.async_case import IsolatedAsyncioTestCase+from unittest.mock import AsyncMock, patchfrom utils import AnyStringfrom agentscope.credential import GeminiCredential-from agentscope.message import ToolResultBlock+from agentscope.message import ToolResultBlock, ToolResultStatefrom agentscope.realtime import (GeminiRealtimeModel,ModelDisconnectedError,@@ -17,6 +21,7 @@ from agentscope.realtime import (CRED = GeminiCredential(api_key="key-x")MODEL = "gemini-3.1-flash-live-preview"+THINKING_MODEL = "gemini-3.8-live-extended-thinking"TOOLS = [{"type": "function",@@ -180,6 +185,505 @@ class GeminiSetupTest(unittest.TestCase):)+class GeminiThinkingTest(unittest.TestCase):+ """Extended Thinking configuration and interaction boundaries."""++ def test_setup_and_model_discovery(self) -> None:+ """Only the new model exposes reasoning and non-blocking tools."""+ cards = CRED.list_realtime_models()+ self.assertEqual(+ {+ card.name: "thinking_level"+ in card.parameter_schema["properties"]+ for card in cards+ },+ {+ "gemini-2.5-flash-native-audio-preview-12-2025": False,+ MODEL: False,+ "gemini-3.8-live": False,+ THINKING_MODEL: True,+ },+ )+ for level in (None, "low", "medium", "high"):+ with self.subTest(level=level):+ expected = GeminiRealtimeModel(MODEL, CRED)._setup(+ "be nice",+ TOOLS,+ "handle",+ )+ setup = expected["setup"]+ setup["model"] = f"models/{THINKING_MODEL}"+ setup["tools"][0]["functionDeclarations"][0][+ "behavior"+ ] = "NON_BLOCKING"+ if level:+ setup["generationConfig"]["thinkingConfig"] = {+ "thinkingLevel": level.upper(),+ }+ model = GeminiRealtimeModel(+ THINKING_MODEL,+ CRED,+ parameters=GeminiRealtimeModel.Parameters(+ thinking_level=level,+ ),+ )+ self.assertEqual(+ model._setup("be nice", TOOLS, "handle"),+ expected,+ )++ def test_thinking_capability_comes_from_card(self) -> None:+ """Aliases and hidden UI parameters do not change protocol mode."""+ original = GeminiRealtimeModel(THINKING_MODEL, CRED).card+ self.assertEqual(+ {+ "enabled": original.supports_extended_thinking,+ "release": original.model_dump(mode="json")["release_date"],+ "session_limit": original.max_session_duration_s,+ },+ {"enabled": True, "release": "2026-09-15", "session_limit": 900},+ )+ for name in (THINKING_MODEL, "custom-thinking-alias"):+ for enabled in (False, True):+ with self.subTest(name=name, enabled=enabled):+ card = original.model_copy(+ update={+ "name": name,+ "supports_extended_thinking": enabled,+ "parameter_schema": {},+ },+ )+ model = GeminiRealtimeModel(name, CRED, model_card=card)+ declaration = model._setup("", TOOLS, "")["setup"][+ "tools"+ ][0]["functionDeclarations"][0]+ self.assertEqual(+ {+ "thinking": model._extended_thinking,+ "behavior": declaration.get("behavior"),+ "events": [+ type(event).__name__+ for event in model._parse(+ {+ "serverContent": {+ "interactionStatus": "IN_PROGRESS",+ },+ },+ )+ ],+ },+ {+ "thinking": enabled,+ "behavior": "NON_BLOCKING" if enabled else None,+ "events": ["ResponseCreatedEvent"]+ if enabled+ else [],+ },+ )++ def test_setup_acknowledgment_survives_mixed_frames(self) -> None:+ """A setup acknowledgment is processed before content early returns."""+ extras: list[dict] = [+ {},+ {"interactionStatus": "IN_PROGRESS"},+ {"serverContent": {"interactionStatus": "IN_PROGRESS"}},+ ]+ for extra in extras:+ with self.subTest(extra=extra):+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)+ events = model._parse({"setupComplete": {}, **extra})+ self.assertEqual(+ {"ready": model._ready.is_set(), "events": _dump(events)},+ {+ "ready": True,+ "events": [+ (+ "ResponseCreatedEvent",+ {+ "item_id": model._response_id,+ },+ ),+ ]+ if extra+ else [],+ },+ )++ def test_invalid_thinking_configuration(self) -> None:+ """Unsupported levels and older models fail before connecting."""+ with self.assertRaises(ValueError):+ GeminiRealtimeModel.Parameters(thinking_level="minimal")+ for card in GeminiRealtimeModel.list_models():+ if card.name == THINKING_MODEL:+ continue+ with self.subTest(model=card.name):+ with self.assertRaisesRegex(ValueError, "requires"):+ GeminiRealtimeModel(+ card.name,+ CRED,+ parameters=GeminiRealtimeModel.Parameters(+ thinking_level="low",+ ),+ )++ def test_background_interaction(self) -> None:+ """Fillers and tools share one response until final IDLE."""+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)+ messages = [+ {"serverContent": {"interactionStatus": "IN_PROGRESS"}},+ {+ "serverContent": {+ "modelTurn": {+ "parts": [{"thought": True, "text": "Plan"}],+ },+ },+ },+ {+ "serverContent": {+ "modelTurn": {+ "parts": [{"inlineData": {"data": "AQA="}}],+ },+ "outputTranscription": {"text": "Checking."},+ "turnComplete": True,+ "interactionStatus": "IN_PROGRESS",+ },+ },+ {+ "toolCall": {+ "functionCalls": [+ {"id": "c1", "name": "get_weather", "args": {}},+ ],+ },+ "interactionStatus": "IN_PROGRESS",+ },+ {"serverContent": {"turnComplete": True}},+ {+ "serverContent": {+ "outputTranscription": {"text": "Sunny."},+ "interactionStatus": "IDLE",+ "turnComplete": True,+ },+ "usageMetadata": {+ "promptTokenCount": 12,+ "responseTokenCount": 7,+ },+ },+ {"serverContent": {"interactionStatus": "IDLE"}},+ ]+ with patch(+ "agentscope.realtime._gemini._model.uuid.uuid4",+ return_value=SimpleNamespace(hex="r1"),+ ):+ actual = [_dump(model._parse(frame)) for frame in messages]+ self.assertEqual(+ actual,+ [+ [("ResponseCreatedEvent", {"item_id": "r1"})],+ [],+ [+ (+ "AudioDeltaEvent",+ {+ "item_id": "r1",+ "pcm": b"\x01\x00",+ "sample_rate": 24000,+ },+ ),+ (+ "TranscriptDeltaEvent",+ {"item_id": "r1", "delta": "Checking."},+ ),+ ],+ [+ (+ "ToolCallEvent",+ {+ "item_id": "r1",+ "non_blocking": True,+ "tool_call": {+ "type": "tool_call",+ "id": "c1",+ "name": "get_weather",+ "input": "{}",+ "state": "pending",+ "suggested_rules": [],+ "created_at": AnyString(),+ "finished_at": None,+ },+ },+ ),+ ],+ [],+ [+ (+ "TranscriptDeltaEvent",+ {"item_id": "r1", "delta": "Sunny."},+ ),+ (+ "ResponseDoneEvent",+ {+ "item_id": "r1",+ "input_tokens": 12,+ "output_tokens": 7,+ },+ ),+ ],+ [],+ ],+ )+ self.assertEqual((model._response_id, model._usage), ("", {}))++ def test_status_only_completion_and_interruption(self) -> None:+ """Root/content statuses finish silent work and reset on barge-in."""+ for at_root in (False, True):+ for interrupted in (False, True):+ with self.subTest(root=at_root, interrupted=interrupted):+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)+ content: dict = {"interactionStatus": "IN_PROGRESS"}+ frame: dict = (+ content if at_root else {"serverContent": content}+ )+ opened = model._parse(frame)+ reply_id = opened[0].item_id+ if interrupted:+ events = model._parse(+ {"serverContent": {"interrupted": True}},+ )+ self.assertEqual(+ _dump(events),+ [+ (+ "SpeechStartedEvent",+ {+ "item_id": AnyString(),+ "at_ms": 0,+ },+ ),+ ],+ )+ content = {"interactionStatus": "IDLE"}+ frame = content if at_root else {"serverContent": content}+ self.assertEqual(+ _dump(model._parse(frame)),+ []+ if interrupted+ else [+ (+ "ResponseDoneEvent",+ {+ "item_id": reply_id,+ "input_tokens": 0,+ "output_tokens": 0,+ },+ ),+ ],+ )+ self.assertEqual(model._parse(frame), [])+ next_turn = model._parse(+ {+ "serverContent": {+ "outputTranscription": {"text": "Next"},+ },+ },+ )+ self.assertEqual(+ _dump(next_turn),+ [+ (+ "ResponseCreatedEvent",+ {+ "item_id": AnyString(),+ },+ ),+ (+ "TranscriptDeltaEvent",+ {+ "item_id": next_turn[0].item_id,+ "delta": "Next",+ },+ ),+ ],+ )+ self.assertNotEqual(next_turn[0].item_id, reply_id)++ def test_same_frame_transcription_and_status(self) -> None:+ """Open status-only responses before input, except on interruption."""+ for at_root in (False, True):+ for interrupted in (False, True):+ with self.subTest(root=at_root, interrupted=interrupted):+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)+ content: dict = {+ "inputTranscription": {"text": "Weather?"},+ "interrupted": interrupted,+ }+ frame: dict = {"serverContent": content}+ status_target = frame if at_root else content+ status_target["interactionStatus"] = "IN_PROGRESS"+ events = model._parse(frame)+ response_id = model._response_id+ expected = [+ (+ "InputTranscriptionDeltaEvent",+ {+ "item_id": model._user_id,+ "delta": "Weather?",+ "response_id": response_id,+ },+ ),+ ]+ if interrupted:+ self.assertEqual(response_id, "")+ expected.append(+ (+ "SpeechStartedEvent",+ {"item_id": model._user_id, "at_ms": 0},+ ),+ )+ else:+ self.assertTrue(response_id)+ expected.insert(+ 0,+ ("ResponseCreatedEvent", {"item_id": response_id}),+ )+ self.assertEqual(_dump(events), expected)++ def test_same_frame_transcription_and_output(self) -> None:+ """Bind same-frame input to output unless the frame interrupts it."""+ outputs: list[dict] = [+ {"outputTranscription": {"text": "Sunny."}},+ {"modelTurn": {"parts": [{"text": "Sunny."}]}},+ {+ "modelTurn": {+ "parts": [+ {+ "inlineData": {+ "data": base64.b64encode(b"\x01\x00").decode(),+ },+ },+ ],+ },+ },+ ]+ for output in outputs:+ for interrupted in (False, True):+ with self.subTest(output=output, interrupted=interrupted):+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)+ events = model._parse(+ {+ "serverContent": {+ **output,+ "inputTranscription": {"text": "Weather?"},+ "interrupted": interrupted,+ },+ },+ )+ response_id = model._response_id+ expected = [+ (+ "InputTranscriptionDeltaEvent",+ {+ "item_id": model._user_id,+ "delta": "Weather?",+ "response_id": response_id,+ },+ ),+ ]+ if interrupted:+ expected.append(+ (+ "SpeechStartedEvent",+ {"item_id": model._user_id, "at_ms": 0},+ ),+ )+ self.assertEqual(response_id, "")+ else:+ self.assertTrue(response_id)+ expected.insert(+ 0,+ ("ResponseCreatedEvent", {"item_id": response_id}),+ )+ if output == outputs[-1]:+ expected.append(+ (+ "AudioDeltaEvent",+ {+ "item_id": response_id,+ "pcm": b"\x01\x00",+ "sample_rate": 24000,+ },+ ),+ )+ else:+ expected.append(+ (+ "TranscriptDeltaEvent",+ {+ "item_id": response_id,+ "delta": "Sunny.",+ },+ ),+ )+ self.assertEqual(_dump(events), expected)++ def test_late_transcription_and_silent_completion(self) -> None:+ """Thinking before transcription and silent replies retain input."""+ for starts_thinking in (False, True):+ with self.subTest(starts_thinking=starts_thinking):+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)+ if starts_thinking:+ model._parse({"interactionStatus": "IN_PROGRESS"})+ reply_id = model._response_id+ for text in ("Check ", "weather"):+ events = model._parse(+ {+ "serverContent": {+ "inputTranscription": {"text": text},+ },+ },+ )+ self.assertEqual(+ _dump(events),+ [+ (+ "InputTranscriptionDeltaEvent",+ {+ "item_id": model._user_id,+ "delta": text,+ "response_id": reply_id,+ },+ ),+ ],+ )+ expected = [+ (+ "InputTranscriptionEvent",+ {+ "item_id": model._user_id,+ "text": "Check weather",+ },+ ),+ ]+ if starts_thinking:+ expected.append(+ (+ "ResponseDoneEvent",+ {+ "item_id": reply_id,+ "input_tokens": 0,+ "output_tokens": 0,+ },+ ),+ )+ self.assertEqual(+ _dump(model._parse({"interactionStatus": "IDLE"})),+ expected,+ )+ self.assertEqual(+ (model._input_text, model._user_id, model._response_id),+ ("", "", ""),+ )++class GeminiParseTest(unittest.TestCase):"""Server messages -> model events."""@@ -286,6 +790,7 @@ class GeminiParseTest(unittest.TestCase):"ToolCallEvent",{"item_id": AnyString(),+ "non_blocking": False,"tool_call": {"type": "tool_call","id": "c1",@@ -394,6 +899,407 @@ class GeminiSendTest(IsolatedAsyncioTestCase):self.model._send = capture # type: ignore[method-assign]+ async def test_tool_result_state_and_scheduling(self) -> None:+ """All Gemini models encode errors; only Thinking schedules output."""+ names = (+ THINKING_MODEL,+ MODEL,+ "gemini-3.8-live",+ "gemini-2.5-flash-native-audio-preview-12-2025",+ )+ for name, state, silent in product(+ names,+ ToolResultState,+ (False, True),+ ):+ with self.subTest(model=name, state=state, silent=silent):+ model = GeminiRealtimeModel(name, CRED)+ thinking = name == THINKING_MODEL+ model._send = AsyncMock()+ if thinking:+ model._pending_tool_calls.add("c1")+ block = ToolResultBlock(+ id="c1",+ name="weather",+ output="result",+ state=state,+ )+ if silent:+ await model.push_tool_result_silently(block)+ else:+ await model.push_tool_result(block)+ failed = state in (+ ToolResultState.ERROR,+ ToolResultState.DENIED,+ ToolResultState.INTERRUPTED,+ )+ response: dict = {+ "id": "c1",+ "name": "weather",+ "response": {+ "error" if failed else "output": "result",+ },+ }+ if thinking and silent:+ response["scheduling"] = "SILENT"+ self.assertEqual(+ {+ "frames": [+ call.args[0]+ for call in model._send.await_args_list+ ],+ "awaiting": model._awaiting_tool_reply,+ "pending": model._pending_tool_calls,+ },+ {+ "frames": [+ {+ "toolResponse": {+ "functionResponses": [response],+ },+ },+ ],+ "awaiting": thinking and not silent,+ "pending": set(),+ },+ )++ async def test_silent_result_preserves_newer_followup(self) -> None:+ """An old result cannot clear another tool's pending response."""+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)+ model._send = AsyncMock()+ model._awaiting_tool_reply = True+ model._pending_tool_calls.add("new")+ await model.push_tool_result_silently(+ ToolResultBlock(id="old", name="weather", output="done"),+ )+ self.assertEqual(+ {+ "awaiting": model._awaiting_tool_reply,+ "pending": model._pending_tool_calls,+ "frame": model._send.await_args.args[0],+ },+ {+ "awaiting": True,+ "pending": {"new"},+ "frame": {+ "toolResponse": {+ "functionResponses": [+ {+ "id": "old",+ "name": "weather",+ "response": {"output": "done"},+ "scheduling": "SILENT",+ },+ ],+ },+ },+ },+ )++ async def test_transcription_tracks_first_background_response(+ self,+ ) -> None:+ """A gap after multiple responses still refers to the first one."""+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)+ model._send = AsyncMock()+ first_response = ""+ model._parse(+ {"serverContent": {"inputTranscription": {"text": "Go "}}},+ )+ user_id = model._user_id+ for call_id in ("c1", "c2"):+ model._parse(+ {+ "serverContent": {+ "outputTranscription": {"text": "Checking."},+ },+ "toolCall": {+ "functionCalls": [+ {+ "id": call_id,+ "name": "weather",+ "args": {},+ },+ ],+ },+ },+ )+ if not first_response:+ first_response = model._response_id+ else:+ self.assertNotEqual(model._response_id, first_response)+ closed = model._parse(+ {"serverContent": {"interactionStatus": "IDLE"}},+ )+ self.assertEqual(+ (+ [type(event).__name__ for event in closed],+ model._user_id,+ model._input_text,+ ),+ (["ResponseDoneEvent"], user_id, "Go "),+ )+ await model.push_tool_result(+ ToolResultBlock(id=call_id, name="weather", output="done"),+ )+ events = model._parse(+ {"serverContent": {"inputTranscription": {"text": "there"}}},+ )+ self.assertEqual(+ _dump(events),+ [+ (+ "InputTranscriptionDeltaEvent",+ {+ "item_id": user_id,+ "delta": "there",+ "response_id": first_response,+ },+ ),+ ],+ )+ completed = model._parse(+ {"serverContent": {"interactionStatus": "IDLE"}},+ )+ self.assertEqual(+ _dump(completed),+ [+ (+ "ResponseCreatedEvent",+ {"item_id": completed[0].item_id},+ ),+ (+ "InputTranscriptionEvent",+ {"item_id": user_id, "text": "Go there"},+ ),+ (+ "ResponseDoneEvent",+ {+ "item_id": completed[0].item_id,+ "input_tokens": 0,+ "output_tokens": 0,+ },+ ),+ ],+ )++ async def test_transcription_association_cleared_at_boundaries(+ self,+ ) -> None:+ """New input cannot inherit a completed or interrupted interaction."""+ for boundary in (+ "completed",+ "cancel",+ "provider_interrupt",+ "new_turn",+ "text",+ "close",+ ):+ with self.subTest(boundary=boundary):+ model = GeminiRealtimeModel(+ THINKING_MODEL,+ CRED,+ parameters=GeminiRealtimeModel.Parameters(+ turn_detection="none",+ ),+ )+ model._send = AsyncMock()+ model._parse(+ {+ "serverContent": {+ "outputTranscription": {"text": "Checking."},+ },+ "toolCall": {+ "functionCalls": [+ {+ "id": "c1",+ "name": "weather",+ "args": {},+ },+ ],+ },+ },+ )+ if boundary != "close":+ model._parse(+ {+ "serverContent": {+ "inputTranscription": {+ "text": "Old request",+ },+ },+ },+ )+ old_user_id = model._user_id+ model._parse({"serverContent": {"interactionStatus": "IDLE"}})+ self.assertTrue(model._transcription_response_id)+ settled = (+ "InputTranscriptionEvent",+ {"item_id": old_user_id, "text": "Old request"},+ )+ content: dict = {"inputTranscription": {"text": "Next"}}+ if boundary == "completed":+ await model.push_tool_result(+ ToolResultBlock(id="c1", name="weather", output="ok"),+ )+ model._parse(+ {+ "serverContent": {"interactionStatus": "IDLE"},+ },+ )+ elif boundary == "cancel":+ await model.cancel_response()+ elif boundary == "provider_interrupt":+ content["interrupted"] = True+ elif boundary == "new_turn":+ await model.start_turn()+ elif boundary == "text":+ await model.push_text("New request")+ else:+ await model.close()+ events = model._parse({"serverContent": content})+ self.assertNotEqual(model._user_id, old_user_id)+ expected = [+ (+ "InputTranscriptionDeltaEvent",+ {+ "item_id": model._user_id,+ "delta": "Next",+ "response_id": "",+ },+ ),+ ]+ if boundary == "provider_interrupt":+ expected.insert(0, settled)+ expected.append(+ (+ "SpeechStartedEvent",+ {+ "item_id": model._user_id,+ "at_ms": 0,+ },+ ),+ )+ self.assertEqual(_dump(events), expected)+ self.assertEqual(model._transcription_response_id, "")+ queued = []+ while not model._queue.empty():+ queued.append(model._queue.get_nowait())+ self.assertEqual(+ _dump(queued),+ [settled]+ if boundary in ("cancel", "new_turn", "text")+ else [],+ )++ async def test_local_cancel_settles_thinking_input(self) -> None:+ """Local VAD interruption must separate successive input turns."""+ model = GeminiRealtimeModel(+ THINKING_MODEL,+ CRED,+ parameters=GeminiRealtimeModel.Parameters(turn_detection="none"),+ )+ model._send = AsyncMock()+ setup = model._setup("", None, "")["setup"]+ self.assertEqual(+ setup["realtimeInputConfig"]["activityHandling"],+ "NO_INTERRUPTION",+ )+ first = model._parse(+ {+ "serverContent": {+ "inputTranscription": {"text": "Check weather"},+ "interactionStatus": "IN_PROGRESS",+ },+ },+ )+ user_id = first[1].item_id+ await model.cancel_response()+ await model.cancel_response()+ settled = []+ while not model._queue.empty():+ settled.append(model._queue.get_nowait())+ self.assertEqual(+ _dump(settled),+ [+ (+ "InputTranscriptionEvent",+ {"item_id": user_id, "text": "Check weather"},+ ),+ ],+ )+ second = model._parse(+ {"serverContent": {"inputTranscription": {"text": "Change "}}},+ )+ await model.cancel_response()+ self.assertTrue(model._queue.empty())+ stale = model._parse({"serverContent": {"interactionStatus": "IDLE"}})+ self.assertEqual(+ _dump(stale),+ [+ (+ "ResponseDoneEvent",+ {+ "item_id": first[0].item_id,+ "input_tokens": 0,+ "output_tokens": 0,+ },+ ),+ ],+ )+ self.assertEqual(+ model._parse({"serverContent": {"interactionStatus": "IDLE"}}),+ [],+ )+ self.assertEqual(+ (model._user_id, model._input_text),+ (second[0].item_id, "Change "),+ )+ second += model._parse(+ {"serverContent": {"inputTranscription": {"text": "city"}}},+ )+ self.assertNotEqual(second[0].item_id, user_id)+ self.assertEqual(+ _dump(second),+ [+ (+ "InputTranscriptionDeltaEvent",+ {+ "item_id": second[0].item_id,+ "delta": text,+ "response_id": "",+ },+ )+ for text in ("Change ", "city")+ ],+ )+ opened = model._parse(+ {"serverContent": {"interactionStatus": "IN_PROGRESS"}},+ )+ self.assertEqual(+ _dump(+ model._parse(+ {"serverContent": {"interactionStatus": "IDLE"}},+ ),+ ),+ [+ (+ "InputTranscriptionEvent",+ {"item_id": second[0].item_id, "text": "Change city"},+ ),+ (+ "ResponseDoneEvent",+ {+ "item_id": opened[0].item_id,+ "input_tokens": 0,+ "output_tokens": 0,+ },+ ),+ ],+ )+ model._send.assert_not_awaited()+async def test_text_turn_and_tool_result(self) -> None:"""A text turn is complete on arrival; a tool result resumes themodel without asking for a response."""@@ -483,6 +1389,57 @@ class GeminiSendTest(IsolatedAsyncioTestCase):class GeminiDisconnectTest(IsolatedAsyncioTestCase):"""A closed WebSocket surfaces as ModelDisconnectedError."""+ async def test_thinking_endpoint_and_reconnect(self) -> None:+ """Thinking uses alpha; ordinary Live retains its beta endpoint."""+ for name, version in (+ (THINKING_MODEL, "v1alpha"),+ ("custom-thinking-alias", "v1alpha"),+ (MODEL, "v1beta"),+ ):+ with self.subTest(model=name):+ original = THINKING_MODEL if version == "v1alpha" else MODEL+ card = GeminiRealtimeModel(original, CRED).card.model_copy(+ update={"name": name},+ )+ model = GeminiRealtimeModel(name, CRED, model_card=card)+ socket = AsyncMock()++ async def acknowledge(+ session: GeminiRealtimeModel = model,+ ) -> None:+ """Acknowledge setup and keep the reader alive."""+ session._ready.set()+ await asyncio.Event().wait()++ with patch(+ "websockets.connect",+ new_callable=AsyncMock,+ return_value=socket,+ ) as connect, patch.object(model, "_read", acknowledge):+ model._response_id = "old-response"+ model._transcription_response_id = "old-interaction"+ model._usage = {"responseTokenCount": 99}+ model._awaiting_tool_reply = True+ model._pending_tool_calls.add("old-call")+ await model.connect("hi")+ self.assertEqual(+ (+ model._response_id,+ model._transcription_response_id,+ model._usage,+ model._awaiting_tool_reply,+ model._pending_tool_calls,+ ),+ ("", "", {}, False, set()),+ )+ connect.assert_awaited_once_with(+ "wss://generativelanguage.googleapis.com/ws/"+ f"google.ai.generativelanguage.{version}."+ "GenerativeService.BidiGenerateContent?key=key-x",+ )+ await model.close()+ socket.close.assert_awaited_once_with()+async def test_send_on_closed_socket(self) -> None:"""websockets' ConnectionClosed becomes the realtime-level errorand the socket reference is dropped."""diff --git a/tests/realtime_openai_test.py b/tests/realtime_openai_test.pyindex 6af825114..04eabea2c 100644--- a/tests/realtime_openai_test.py+++ b/tests/realtime_openai_test.py@@ -444,6 +444,7 @@ class OpenAIParseTest(unittest.TestCase):"created_at": AnyString(),"finished_at": None,},+ "non_blocking": False,},)diff --git a/tests/realtime_xai_test.py b/tests/realtime_xai_test.pyindex 207163b45..211b9f4b1 100644--- a/tests/realtime_xai_test.py+++ b/tests/realtime_xai_test.py@@ -299,6 +299,7 @@ class XAIParseTest(unittest.TestCase):"created_at": AnyString(),"finished_at": None,},+ "non_blocking": False,},{"item_id": "resp_001",@@ -312,6 +313,7 @@ class XAIParseTest(unittest.TestCase):"created_at": AnyString(),"finished_at": None,},+ "non_blocking": False,},],)diff --git a/tests/webrtc_audio_transport_test.py b/tests/webrtc_audio_transport_test.pyindex 62121a6c0..1145af698 100644--- a/tests/webrtc_audio_transport_test.py+++ b/tests/webrtc_audio_transport_test.py@@ -19,7 +19,7 @@ from aiortc.mediastreams import MediaStreamErrorfrom av import AudioFrame as AVAudioFramefrom fastapi import HTTPException-from agentscope.agent import RealtimeAgent+from agentscope.agent import RealtimeAgent, TurnAggregatorfrom agentscope.credential import GeminiCredentialfrom agentscope.app._router._realtime import _create_realtime_offerfrom agentscope.app._router._schema import (@@ -1103,6 +1103,60 @@ class RealtimeOfferTest(unittest.IsolatedAsyncioTestCase):class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):"""Verify that a WebRTC run keeps the normal session contract."""+ async def test_final_persistence_excludes_unfinished_transcription(+ self,+ ) -> None:+ """Direct state persistence after close must not save fragments."""+ agent = RealtimeAgent(+ "Friday",+ "",+ MagicMock(+ spec=RealtimeModelBase,+ output_sample_rate=24000,+ input_transcription_enabled=True,+ ),+ aggregator=TurnAggregator(min_chars=4),+ )+ storage = _FakeStorage()+ session = _make_session(+ agent_factory=AsyncMock(),+ storage=storage,+ message_bus=_FakeMessageBus(),+ on_closed=lambda _: None,+ )+ session.agent = agent+ await agent._on_model_event(+ me.InputTranscriptionEvent(item_id="kept", text="accepted"),+ )+ accepted = agent.state.context[0].model_dump(mode="json")+ await agent._on_model_event(+ me.InputTranscriptionDeltaEvent(item_id="u1", delta="a"),+ )+ agent._retire_unbound_user_turns()+ self.assertEqual(+ [(msg.id, msg.get_text_content()) for msg in agent.state.context],+ [("kept", "accepted"), ("u1", "a")],+ )+ await agent.close()+ await session._persist_state()+ self.assertEqual(+ {+ "messages": {+ key: value.model_dump(mode="json")+ for key, value in storage.messages.items()+ },+ "states": [+ [+ msg.model_dump(mode="json")+ for msg in call["state"].context+ ]+ for call in storage.calls+ if call["method"] == "update_session_state"+ ],+ },+ {"messages": {"kept": accepted}, "states": [[accepted]]},+ )+async def test_sampled_live_checkpoint_persists_immediately(self) -> None:"""Persist a sampled transcript snapshot at its published cursor."""