Skip to content

Build and run agents you can see, understand and trust.

feat(realtime): support Gemini 3.8 Live Extended Thinking (#3700)

qbc4704aaf9d174
diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.py
index ee335dc40..1bc8d48b7 100644
--- a/src/agentscope/agent/_realtime/_agent.py
+++ b/src/agentscope/agent/_realtime/_agent.py
@@ -6,6 +6,7 @@ import time
import uuid
from collections import deque
from collections.abc import Callable, Iterator
+from contextlib import nullcontext
from copy import deepcopy
from dataclasses import dataclass, field
from typing import Any, AsyncIterator, TYPE_CHECKING
@@ -107,6 +108,7 @@ class _UserTurn:
item_id: str = ""
open: bool = True
speech_ended: bool = False
+ partial_transcription: bool = False
@dataclass
@@ -117,6 +119,15 @@ class _QueuedEvent:
checkpoint: AgentState | None = None
+@dataclass
+class _BackgroundTools:
+ """Work that must settle before a logical reply can complete."""
+
+ pending: int = 0
+ model_done: bool = False
+ lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+
+
@dataclass
class _ClientDelegation:
"""One application-managed task requested by a Live session."""
@@ -253,6 +264,8 @@ class RealtimeAgent:
self._metrics = TurnMetrics()
self._pending_tools: dict[str, ToolCallBlock] = {}
self._confirmations: dict[str, asyncio.Future[ConfirmResult]] = {}
+ self._confirmation_replies: dict[str, str] = {}
+ self._background_tools: dict[str, _BackgroundTools] = {}
self._tasks: set[asyncio.Task] = set()
self._barge_lock = asyncio.Lock()
self._live_session_id = ""
@@ -436,6 +449,7 @@ class RealtimeAgent:
self._delegation_queue.clear()
self._delegated_tool_calls.clear()
self._active_delegation_id = None
+ self._reset_user_turns()
self._connected = False
self._connected_event.clear()
await self.model.close()
@@ -640,8 +654,21 @@ class RealtimeAgent:
return
for result in inputs.confirm_results:
future = self._confirmations.get(result.tool_call.id)
- if future and not future.done():
- future.set_result(result)
+ if future is None or future.done():
+ continue
+ expected_reply = self._confirmation_replies.get(
+ result.tool_call.id,
+ )
+ if expected_reply != inputs.reply_id:
+ logger.warning(
+ "RealtimeAgent: ignored confirmation for tool %s "
+ "from reply %s; expected reply %s",
+ result.tool_call.id,
+ inputs.reply_id,
+ expected_reply,
+ )
+ continue
+ future.set_result(result)
case ExternalExecutionResultEvent():
tool_call_ids = [
@@ -746,6 +773,7 @@ class RealtimeAgent:
if speech is SpeechTransition.STARTED:
self._start_user_turn()
+ await self.model.start_turn()
await self._barge_in()
elif speech is SpeechTransition.ENDED:
if self._user_turn is not None:
@@ -818,7 +846,10 @@ class RealtimeAgent:
await self._transport.clear_audio()
self._audio_pending = False
- if active_reply is not None and self._connected:
+ if self._connected and (
+ active_reply is not None
+ or self._reply_id in self._background_tools
+ ):
await self.model.cancel_response()
if self._reply_id:
@@ -883,6 +914,9 @@ class RealtimeAgent:
case me.InputTranscriptionEvent():
self._on_transcription(event)
+ case me.InputTranscriptionDeltaEvent():
+ self._on_transcription_delta(event)
+
case me.InputTranscriptionFailedEvent():
self._finish_transcription(
self._find_user_turn(event.item_id),
@@ -918,6 +952,11 @@ class RealtimeAgent:
self._reply_id,
event.tool_call,
)
+ if event.non_blocking:
+ self._schedule_tools(
+ self._reply_id,
+ request_response=False,
+ )
case me.ResponseDoneEvent():
async with self._barge_lock:
@@ -940,6 +979,11 @@ class RealtimeAgent:
reply_id = reply.reply_id
self._finish_response()
self._schedule_tools(reply_id)
+ elif work := self._background_tools.get(reply.reply_id):
+ self._finish_response()
+ work.model_done = True
+ if not work.pending:
+ self._finish_reply(ReplyFinishedReason.COMPLETED)
else:
self._finish_reply(ReplyFinishedReason.COMPLETED)
@@ -1928,19 +1972,32 @@ class RealtimeAgent:
"""Retire only the turn whose transcript succeeded or failed."""
if turn is None:
return
+ self._discard_partial_transcription(turn)
self._end_user_turn(turn)
self._pending_user_turns.pop(turn.reply_id, None)
if self._user_turn is turn:
self._user_turn = None
def _reset_user_turns(self) -> None:
- """Discard pending identities when their model session ends."""
- for turn in list(self._pending_user_turns.values()):
+ """Discard unfinished input before closing its model session."""
+ turns = list(self._pending_user_turns.values())
+ if self._user_turn is not None and self._user_turn not in turns:
+ turns.append(self._user_turn)
+ for turn in turns:
+ self._discard_partial_transcription(turn)
+ for turn in turns:
self._end_user_turn(turn)
- self._end_user_turn()
self._pending_user_turns.clear()
self._user_turn = None
+ def _discard_partial_transcription(self, turn: _UserTurn) -> None:
+ """Remove a provisional message without touching accepted input."""
+ if turn.partial_transcription:
+ message = self._find_user_message(turn.reply_id)
+ if message is not None:
+ self.state.context.remove(message)
+ turn.partial_transcription = False
+
def _retire_unbound_user_turns(self) -> None:
"""End pending user turns when their transport goes, keeping only
the ones a late transcript could still find.
@@ -1957,6 +2014,9 @@ class RealtimeAgent:
turns = list(self._pending_user_turns.values())
if self._user_turn is not None and self._user_turn not in turns:
turns.append(self._user_turn)
+ for turn in turns:
+ if not turn.item_id and not turn.speech_ended:
+ self._discard_partial_transcription(turn)
for turn in turns:
self._end_user_turn(turn)
for turn in turns:
@@ -1965,13 +2025,60 @@ class RealtimeAgent:
if self._user_turn is turn:
self._user_turn = None
+ def _on_transcription_delta(
+ self,
+ event: me.InputTranscriptionDeltaEvent,
+ ) -> None:
+ """Keep input fragments in one message before their response."""
+ turn = self._find_user_turn(event.item_id)
+ if turn is None:
+ turn = self._new_user_turn(event.item_id)
+ turn.partial_transcription = True
+ message = self._find_user_message(turn.reply_id)
+ if message is not None:
+ text = message.get_text_content()
+ message.content = [TextBlock(text=f"{text}{event.delta}")]
+ return
+ response_id = event.response_id
+ if self._reply is not None and self._reply.item_id == response_id:
+ response_id = self._reply.reply_id
+ context = self.state.context
+ position = next(
+ (
+ i
+ for i, msg in enumerate(context)
+ if msg.id == response_id and msg.role == "assistant"
+ ),
+ len(context),
+ )
+ context.insert(
+ position,
+ UserMsg(name="user", content=event.delta, id=turn.reply_id),
+ )
+
+ def _find_user_message(self, reply_id: str) -> Msg | None:
+ """Find an input turn already reserved by transcription fragments."""
+ return next(
+ (
+ msg
+ for msg in self.state.context
+ if msg.id == reply_id and msg.role == "user"
+ ),
+ None,
+ )
+
def _on_transcription(self, event: me.InputTranscriptionEvent) -> None:
"""Record a settled user turn, merging a split one back together,
and report its text on the user's reply."""
turn = self.aggregator.take(event.text)
user_turn = self._find_user_turn(event.item_id)
+ message = self._find_user_message(
+ user_turn.reply_id if user_turn else event.item_id,
+ )
if turn is None:
logger.debug("RealtimeAgent: dropping %r", event.text)
+ if message is not None:
+ self.state.context.remove(message)
self._finish_transcription(user_turn)
return
@@ -1990,15 +2097,20 @@ class RealtimeAgent:
)
self._emit(TextBlockEndEvent(reply_id=reply_id, block_id=block_id))
- if not (
- self.aggregator.merges_with_previous() and self._merge_user(turn)
- ):
+ merged = self.aggregator.merges_with_previous() and self._merge_user(
+ turn,
+ message,
+ )
+ if not merged and message is not None:
+ message.content = [TextBlock(id=block_id, text=turn)]
+ elif not merged:
self.state.context.append(
UserMsg(name="user", content=turn, id=reply_id),
)
+ user_turn.partial_transcription = False
self._finish_transcription(user_turn)
- def _merge_user(self, text: str) -> bool:
+ def _merge_user(self, text: str, reserved: Msg | None = None) -> bool:
"""Append *text* to the previous user turn that endpointing split.
The stub assistant message between the two halves is dropped: it
@@ -2006,16 +2118,25 @@ class RealtimeAgent:
answers half a question nobody finished asking.
"""
context = self.state.context
- if context and context[-1].role == "assistant":
- tail = context[-1]
+ position = (
+ context.index(reserved) if reserved is not None else len(context)
+ )
+ if position and context[position - 1].role == "assistant":
+ tail = context[position - 1]
if not tail.get_text_content().strip() and all(
isinstance(b, TextBlock) for b in tail.content
):
- context.pop()
- if not context or context[-1].role != "user":
+ context.pop(position - 1)
+ position -= 1
+ if not position or context[position - 1].role != "user":
+ return False
+ previous_turn = self._pending_user_turns.get(context[position - 1].id)
+ if previous_turn is not None and previous_turn.partial_transcription:
return False
- previous = context[-1].get_text_content()
- context[-1].content = [TextBlock(text=f"{previous}{text}")]
+ previous = context[position - 1].get_text_content()
+ context[position - 1].content = [TextBlock(text=f"{previous}{text}")]
+ if reserved is not None:
+ context.remove(reserved)
return True
def _start_reply(self, item_id: str) -> _Reply | None:
@@ -2049,6 +2170,8 @@ class RealtimeAgent:
),
)
self._reply = _Reply(item_id=item_id, reply_id=self._reply_id)
+ if work := self._background_tools.get(self._reply_id):
+ work.model_done = False
self._emit(
ModelCallStartEvent(
reply_id=self._reply_id,
@@ -2097,6 +2220,11 @@ class RealtimeAgent:
self._pending_tools.clear()
if not self._reply_id:
return
+ self._background_tools.pop(self._reply_id, None)
+ if reason != ReplyFinishedReason.COMPLETED:
+ for call_id, future in self._confirmations.items():
+ if self._confirmation_replies.get(call_id) == self._reply_id:
+ future.cancel()
event = ReplyEndEvent(
session_id=self.state.session_id,
reply_id=self._reply_id,
@@ -2142,6 +2270,22 @@ class RealtimeAgent:
def _emit_text(self, reply: _Reply, delta: str) -> None:
"""Emit a transcript delta, opening the block on first use."""
+ message = self._find_reply_message(reply.reply_id)
+ blocks = message.content if message is not None else None
+ if (
+ reply.text_started
+ and blocks
+ and not isinstance(blocks[-1], TextBlock)
+ ):
+ # Background tools can separate speech within one response.
+ self._emit(
+ TextBlockEndEvent(
+ reply_id=reply.reply_id,
+ block_id=reply.text_block_id,
+ ),
+ )
+ reply.text_started = False
+ reply.text_block_id = uuid.uuid4().hex
if not reply.text_started:
reply.text_started = True
self._emit(
@@ -2152,8 +2296,6 @@ class RealtimeAgent:
)
# Grow the tail text block rather than appending one per delta,
# or the context ends up as dozens of one-word blocks.
- message = self._find_reply_message(reply.reply_id)
- blocks = message.content if message is not None else None
if (
isinstance(blocks, list)
and blocks
@@ -2212,6 +2354,17 @@ class RealtimeAgent:
),
):
snapshot = self.state.model_copy(deep=True)
+ partial_ids = {
+ turn.reply_id
+ for turn in self._pending_user_turns.values()
+ if turn.partial_transcription
+ }
+ if partial_ids:
+ snapshot.context = [
+ msg
+ for msg in snapshot.context
+ if msg.role != "user" or msg.id not in partial_ids
+ ]
self._out.put_nowait(_QueuedEvent(event, snapshot))
def checkpoint_snapshot(self, event: AgentEvent) -> AgentState | None:
@@ -2225,14 +2378,25 @@ class RealtimeAgent:
# Tools
# ------------------------------------------------------------------
- def _schedule_tools(self, reply_id: str) -> None:
- """Run the tool calls of the finished reply, then ask for more."""
+ def _schedule_tools(
+ self,
+ reply_id: str,
+ *,
+ request_response: bool = True,
+ ) -> None:
+ """Run pending tools without blocking the model event stream."""
if not reply_id or not self._pending_tools or self.toolkit is None:
return
calls = list(self._pending_tools.values())
self._pending_tools.clear()
+ if not request_response:
+ work = self._background_tools.setdefault(
+ reply_id,
+ _BackgroundTools(),
+ )
+ work.pending += 1
task = asyncio.create_task(
- self._run_tools(reply_id, calls),
+ self._run_tools(reply_id, calls, request_response),
name="rt-tools",
)
self._tasks.add(task)
@@ -2242,23 +2406,46 @@ class RealtimeAgent:
self,
reply_id: str,
calls: list[ToolCallBlock],
+ request_response: bool = True,
) -> None:
- """Execute *calls* in order, then trigger the follow-up response."""
+ """Execute tools serially, optionally requesting a follow-up."""
try:
- for call in calls:
- async with self._barge_lock:
- if self._reply_id != reply_id:
- return
- await self._run_tool(reply_id, call)
+ work = self._background_tools.get(reply_id)
+ lock = work.lock if work is not None else nullcontext()
+ async with lock:
+ for call in calls:
+ async with self._barge_lock:
+ interrupted = self._reply_id != reply_id
+ if interrupted:
+ await self._report_tool(
+ reply_id,
+ call,
+ "Reply interrupted before tool execution.",
+ state=ToolResultState.INTERRUPTED,
+ )
+ continue
+ await self._run_tool(reply_id, call)
# Unless the user cut the reply short while the tools ran.
async with self._barge_lock:
- if self._reply_id == reply_id:
+ if request_response and self._reply_id == reply_id:
await self.model.request_response()
except Exception: # noqa: BLE001
logger.exception("RealtimeAgent: tool execution failed")
async with self._barge_lock:
if self._reply_id == reply_id:
self._finish_reply(ReplyFinishedReason.ERROR)
+ finally:
+ if not request_response:
+ async with self._barge_lock:
+ work = self._background_tools.get(reply_id)
+ if work is not None:
+ work.pending -= 1
+ if (
+ not work.pending
+ and work.model_done
+ and self._reply_id == reply_id
+ ):
+ self._finish_reply(ReplyFinishedReason.COMPLETED)
async def _run_tool(self, reply_id: str, call: ToolCallBlock) -> None:
"""Check permission for one call, run it, and report the result."""
@@ -2292,6 +2479,14 @@ class RealtimeAgent:
return
decision = await self._engine.check_permission(tool, tool_input)
+ if self._reply_id != reply_id:
+ await self._report_tool(
+ reply_id,
+ call,
+ "Reply interrupted before tool execution.",
+ state=ToolResultState.DENIED,
+ )
+ return
if decision.behavior in (
PermissionBehavior.ASK,
PermissionBehavior.PASSTHROUGH,
@@ -2384,13 +2579,17 @@ class RealtimeAgent:
ConfirmResult
] = asyncio.get_running_loop().create_future()
self._confirmations[call.id] = future
+ self._confirmation_replies[call.id] = reply_id
try:
result = await asyncio.wait_for(future, timeout=300)
except (asyncio.TimeoutError, asyncio.CancelledError):
return False
finally:
self._confirmations.pop(call.id, None)
+ self._confirmation_replies.pop(call.id, None)
+ if self._reply_id != reply_id:
+ return False
for rule in result.rules or []:
self._engine.add_rule(rule)
call.state = (
@@ -2441,4 +2640,12 @@ class RealtimeAgent:
state=state,
),
)
- await self.model.push_tool_result(block)
+ # The result starts more model work. An IDLE received before this
+ # send cannot complete the reply, and a completion received during
+ # the send must not be overwritten after it returns.
+ if work := self._background_tools.get(reply_id):
+ work.model_done = False
+ if self._reply_id == reply_id:
+ await self.model.push_tool_result(block)
+ else:
+ await self.model.push_tool_result_silently(block)
diff --git a/src/agentscope/realtime/_base.py b/src/agentscope/realtime/_base.py
index 2c30bac0b..ec703d814 100644
--- a/src/agentscope/realtime/_base.py
+++ b/src/agentscope/realtime/_base.py
@@ -236,11 +236,27 @@ class RealtimeModelBase(ABC):
async def push_tool_result(self, block: ToolResultBlock) -> None:
"""Feed the result of a tool the caller executed."""
+ async def push_tool_result_silently(self, block: ToolResultBlock) -> None:
+ """Feed a late result without requesting another response.
+
+ The default delegates to :meth:`push_tool_result`, for adapters that
+ require a separate :meth:`request_response`. Adapters with implicit
+ generation should override this when silent delivery is supported.
+ """
+ await self.push_tool_result(block)
+
@abstractmethod
async def commit_turn(self) -> None:
"""Declare the user turn finished. Only called when the caller owns
endpointing, i.e. the session was opened with turn detection off."""
+ async def start_turn(self) -> None:
+ """Notify the model that local VAD detected a new user turn.
+
+ Adapters may settle pending input here even without an active reply.
+ The default implementation does nothing.
+ """
+
# ------------------------------------------------------------------
# Response control
# ------------------------------------------------------------------
diff --git a/src/agentscope/realtime/_events.py b/src/agentscope/realtime/_events.py
index f65950f35..ac9e1526a 100644
--- a/src/agentscope/realtime/_events.py
+++ b/src/agentscope/realtime/_events.py
@@ -64,6 +64,14 @@ class InputTranscriptionEvent(ModelEvent):
text: str
+class InputTranscriptionDeltaEvent(ModelEvent):
+ """Unsettled input text, anchored before the response it prompted."""
+
+ item_id: str
+ delta: str
+ response_id: str = ""
+
+
class InputTranscriptionFailedEvent(ModelEvent):
"""The provider could not transcribe one committed user turn."""
@@ -130,6 +138,8 @@ class ToolCallEvent(ModelEvent):
item_id: str
tool_call: ToolCallBlock
+ non_blocking: bool = False
+ """Run immediately while the provider continues the current response."""
class ResponseDoneEvent(ModelEvent):
diff --git a/src/agentscope/realtime/_gemini/_model.py b/src/agentscope/realtime/_gemini/_model.py
index 3ea563492..6e9e3c5f3 100644
--- a/src/agentscope/realtime/_gemini/_model.py
+++ b/src/agentscope/realtime/_gemini/_model.py
@@ -14,7 +14,12 @@ from .._model_card import RealtimeModelCard
from ..._logging import logger
from ..._utils._common import _flatten_json_schema
from ...credential import GeminiCredential
-from ...message import TextBlock, ToolCallBlock, ToolResultBlock
+from ...message import (
+ TextBlock,
+ ToolCallBlock,
+ ToolResultBlock,
+ ToolResultState,
+)
from ...model._gemini._model import _sanitize_schema_for_gemini
_LIVE_URL = (
@@ -35,7 +40,7 @@ class GeminiRealtimeModel(RealtimeModelBase):
starts the reply, so :meth:`request_response` sends nothing.
- Interruption is the server's job — it reports
``serverContent.interrupted`` and drops the rest of the reply
- itself — so :meth:`cancel_response` is a no-op.
+ itself — so :meth:`cancel_response` sends no wire frame.
With caller-owned turns the caller silences the reply locally
instead, and the server is told not to interrupt on its own.
"""
@@ -44,6 +49,11 @@ class GeminiRealtimeModel(RealtimeModelBase):
"""Tuneables surfaced to the UI."""
voice: str = Field(default="Puck", title="Voice")
+ thinking_level: Literal["low", "medium", "high"] | None = Field(
+ default=None,
+ title="Thinking Level",
+ description="Background reasoning depth for Extended Thinking.",
+ )
turn_detection: Literal["automatic", "none"] = Field(
default="automatic",
title="Turn Detection",
@@ -94,6 +104,11 @@ class GeminiRealtimeModel(RealtimeModelBase):
"""
super().__init__(model, credential, parameters, model_card)
self.parameters: GeminiRealtimeModel.Parameters
+ self._extended_thinking = self.card.supports_extended_thinking
+ if self.parameters.thinking_level and not self._extended_thinking:
+ raise ValueError(
+ "thinking_level requires Gemini Live Extended Thinking.",
+ )
self.resumption_handle = ""
"""The latest handle the server offered. Not used on reconnect:
the agent carries the transcript in the instructions instead."""
@@ -101,10 +116,14 @@ class GeminiRealtimeModel(RealtimeModelBase):
self._reader: asyncio.Task | None = None
self._queue: asyncio.Queue[me.ModelEvent | None] = asyncio.Queue()
self._response_id = ""
+ self._transcription_response_id = ""
+ self._response_cancelled = False
self._user_id = ""
self._input_text = ""
self._usage: dict = {}
self._activity = False
+ self._awaiting_tool_reply = False
+ self._pending_tool_calls: set[str] = set()
self._ready = asyncio.Event()
# ------------------------------------------------------------------
@@ -130,11 +149,19 @@ class GeminiRealtimeModel(RealtimeModelBase):
update={"turn_detection": "none"},
)
self._response_id, self._user_id, self._input_text = "", "", ""
+ self._response_cancelled = False
self._usage, self._activity = {}, False
+ self._awaiting_tool_reply = False
+ self._pending_tool_calls.clear()
credential: GeminiCredential = self.credential # type: ignore
+ url = (
+ _LIVE_URL.replace("v1beta", "v1alpha")
+ if self._extended_thinking
+ else _LIVE_URL
+ )
self._ws = await websockets.connect(
- f"{_LIVE_URL}?key={credential.api_key.get_secret_value()}",
+ f"{url}?key={credential.api_key.get_secret_value()}",
)
self._ready.clear()
self._reader = asyncio.create_task(self._read(), name="gemini-rt")
@@ -160,6 +187,7 @@ class GeminiRealtimeModel(RealtimeModelBase):
self._reader.cancel()
await asyncio.gather(self._reader, return_exceptions=True)
self._reader = None
+ self._transcription_response_id = ""
async def events(self) -> AsyncIterator[me.ModelEvent]:
"""Yield events until the session ends."""
@@ -193,6 +221,11 @@ class GeminiRealtimeModel(RealtimeModelBase):
async def push_text(self, text: str) -> None:
"""Send a complete text turn, which the model answers at once."""
+ if self._extended_thinking:
+ for event in self._settle_input():
+ self._queue.put_nowait(event)
+ self._user_id = ""
+ self._transcription_response_id = ""
await self._send(
{
"clientContent": {
@@ -204,6 +237,23 @@ class GeminiRealtimeModel(RealtimeModelBase):
async def push_tool_result(self, block: ToolResultBlock) -> None:
"""Send one ``functionResponse``; the model resumes on its own."""
+ await self._push_tool_result(block, silent=False)
+
+ async def push_tool_result_silently(self, block: ToolResultBlock) -> None:
+ """Keep a stale Thinking result without triggering generation.
+
+ Scheduling is only supported for non-blocking tools. Ordinary Live
+ retains its existing blocking-tool result behavior.
+ """
+ await self._push_tool_result(block, silent=True)
+
+ async def _push_tool_result(
+ self,
+ block: ToolResultBlock,
+ *,
+ silent: bool,
+ ) -> None:
+ """Serialize a tool result with its generation and error semantics."""
output = (
block.output
if isinstance(block.output, str)
@@ -211,20 +261,41 @@ class GeminiRealtimeModel(RealtimeModelBase):
_.text for _ in block.output if isinstance(_, TextBlock)
)
)
+ if self._extended_thinking and block.id in self._pending_tool_calls:
+ if not silent:
+ self._awaiting_tool_reply = True
+ self._pending_tool_calls.discard(block.id)
+ failed = block.state in (
+ ToolResultState.ERROR,
+ ToolResultState.DENIED,
+ ToolResultState.INTERRUPTED,
+ )
+ response = {
+ "id": block.id,
+ "name": block.name,
+ "response": {"error" if failed else "output": output},
+ }
+ if self._extended_thinking and silent:
+ response["scheduling"] = "SILENT"
await self._send(
{
"toolResponse": {
- "functionResponses": [
- {
- "id": block.id,
- "name": block.name,
- "response": {"output": output},
- },
- ],
+ "functionResponses": [response],
},
},
)
+ async def start_turn(self) -> None:
+ """Settle prior input at a local speech boundary, even after cancel."""
+ if (
+ self._extended_thinking
+ and self.parameters.turn_detection == "none"
+ ):
+ self._transcription_response_id = ""
+ for event in self._settle_input():
+ self._queue.put_nowait(event)
+ self._user_id = ""
+
async def commit_turn(self) -> None:
"""Close the activity window, which asks for the reply."""
if self._activity:
@@ -239,7 +310,20 @@ class GeminiRealtimeModel(RealtimeModelBase):
"""No-op: a finished turn or a tool response is the request."""
async def cancel_response(self) -> None:
- """No-op: the protocol has no cancel frame."""
+ """Settle locally interrupted input; there is no wire cancel."""
+ if (
+ self._extended_thinking
+ and self.parameters.turn_detection == "none"
+ and (self._response_id or self._transcription_response_id)
+ and not self._response_cancelled
+ ):
+ self._response_cancelled = True
+ for event in self._settle_input():
+ self._queue.put_nowait(event)
+ self._user_id = ""
+ self._transcription_response_id = ""
+ self._awaiting_tool_reply = False
+ self._pending_tool_calls.clear()
# ------------------------------------------------------------------
# Wire
@@ -296,12 +380,21 @@ class GeminiRealtimeModel(RealtimeModelBase):
}
if p.input_audio_transcription:
setup["inputAudioTranscription"] = {}
+ if self._extended_thinking and p.thinking_level:
+ setup["generationConfig"]["thinkingConfig"] = {
+ "thinkingLevel": p.thinking_level.upper(),
+ }
if tools and self.card.supports_tools:
setup["tools"] = [
{
"functionDeclarations": [
{
**schema["function"],
+ **(
+ {"behavior": "NON_BLOCKING"}
+ if self._extended_thinking
+ else {}
+ ),
"parameters": _sanitize_schema_for_gemini(
_flatten_json_schema(
schema["function"].get("parameters", {}),
@@ -349,6 +442,7 @@ class GeminiRealtimeModel(RealtimeModelBase):
logger.error("GeminiRealtimeModel: connection lost: %s", exc)
finally:
self._ws = None
+ self._transcription_response_id = ""
self._ready.set() # release a connect() still waiting
self._queue.put_nowait(me.SessionEndedEvent(reason="closed"))
self._queue.put_nowait(None)
@@ -358,10 +452,21 @@ class GeminiRealtimeModel(RealtimeModelBase):
the id every event of this turn carries."""
if self._response_id:
return []
+ events = [] if self._extended_thinking else self._settle_input()
+ self._response_id = uuid.uuid4().hex
+ if self._extended_thinking and not self._transcription_response_id:
+ # All background continuations belong to the first response.
+ self._transcription_response_id = self._response_id
+ self._response_cancelled = False
+ events.append(me.ResponseCreatedEvent(item_id=self._response_id))
+ return events
+
+ def _settle_input(self) -> list[me.ModelEvent]:
+ """Flush the transcription fragments of the pending user turn."""
events: list[me.ModelEvent] = []
if self._input_text:
# Input transcription arrives in fragments; the user turn is
- # settled only once the model starts answering it.
+ # settled when the model answers or finishes background work.
events.append(
me.InputTranscriptionEvent(
item_id=self._user_id,
@@ -369,25 +474,45 @@ class GeminiRealtimeModel(RealtimeModelBase):
),
)
self._user_id, self._input_text = "", ""
- self._response_id = uuid.uuid4().hex
- events.append(me.ResponseCreatedEvent(item_id=self._response_id))
return events
def _parse(self, data: dict) -> list[me.ModelEvent]:
"""Translate one server message into model events."""
+ if "setupComplete" in data:
+ self._ready.set()
if usage := data.get("usageMetadata"):
self._usage = usage
- if "serverContent" in data:
- return self._parse_content(data["serverContent"])
+ content = data.get("serverContent") or {}
+ # The Thinking guide includes root status alongside toolCall.
+ # Source: https://ai.google.dev/gemini-api/docs/live-api/thinking
+ status = content.get("interactionStatus") or data.get(
+ "interactionStatus",
+ )
+ events: list[me.ModelEvent] = []
+ if (
+ self._extended_thinking
+ and self._awaiting_tool_reply
+ and status == "IDLE"
+ and not content.get("interrupted")
+ ):
+ # A result may receive a silent completion after an earlier
+ # IDLE already closed its original model response.
+ events += self._open_response()
+ if (
+ self._extended_thinking
+ and status == "IN_PROGRESS"
+ and not content.get("interrupted")
+ ):
+ events += self._open_response()
+ events += self._parse_content(content)
if tool_call := data.get("toolCall"):
- # The server hands the floor over with a tool call, so the
- # reply ends here and the follow-up is a turn of its own.
- events = self._open_response()
- events += [
+ events += self._open_response()
+ tool_events = [
me.ToolCallEvent(
item_id=self._response_id,
+ non_blocking=self._extended_thinking,
tool_call=ToolCallBlock(
id=call.get("id") or uuid.uuid4().hex,
name=call.get("name", ""),
@@ -399,7 +524,20 @@ class GeminiRealtimeModel(RealtimeModelBase):
)
for call in tool_call.get("functionCalls") or []
]
- return events + self._close_response()
+ events += tool_events
+ if self._extended_thinking:
+ self._pending_tool_calls.update(
+ event.tool_call.id for event in tool_events
+ )
+ # Blocking tools hand over the floor; background tools do not.
+ if not self._extended_thinking:
+ events += self._close_response()
+
+ if self._extended_thinking and status == "IDLE":
+ self._awaiting_tool_reply = False
+ events += self._close_response()
+ if "serverContent" in data or tool_call or status:
+ return events
if go_away := data.get("goAway"):
# Advance notice only; the session ends when the socket does.
@@ -414,9 +552,7 @@ class GeminiRealtimeModel(RealtimeModelBase):
self.resumption_handle = update.get("newHandle", "")
return []
- if "setupComplete" in data:
- self._ready.set()
- else:
+ if "setupComplete" not in data:
logger.debug("GeminiRealtimeModel: ignoring %s", list(data))
return []
@@ -424,22 +560,53 @@ class GeminiRealtimeModel(RealtimeModelBase):
"""Translate one ``serverContent`` message into model events."""
events: list[me.ModelEvent] = []
+ if self._extended_thinking and content.get("interrupted"):
+ self._awaiting_tool_reply = False
+ self._pending_tool_calls.clear()
+ self._transcription_response_id = ""
+ events += self._settle_input()
+ self._user_id, self._response_id, self._usage = "", "", {}
+
+ parts = (content.get("modelTurn") or {}).get("parts") or []
+ transcript = (content.get("outputTranscription") or {}).get("text", "")
+ if (
+ self._extended_thinking
+ and not content.get("interrupted")
+ and (parts or transcript)
+ ):
+ # Bind same-frame input to the response before emitting it.
+ events += self._open_response()
+
if text := (content.get("inputTranscription") or {}).get("text"):
self._user_id = self._user_id or uuid.uuid4().hex
self._input_text += text
+ if self._extended_thinking:
+ events.append(
+ me.InputTranscriptionDeltaEvent(
+ item_id=self._user_id,
+ delta=text,
+ response_id=(
+ ""
+ if self._response_cancelled
+ else self._transcription_response_id
+ ),
+ ),
+ )
if content.get("interrupted"):
# The only notice that the user spoke over the reply; the
# server has already dropped the rest of it.
self._user_id = self._user_id or uuid.uuid4().hex
self._response_id, self._usage = "", {}
- return [me.SpeechStartedEvent(item_id=self._user_id)]
+ return events + [me.SpeechStartedEvent(item_id=self._user_id)]
- parts = (content.get("modelTurn") or {}).get("parts") or []
- transcript = (content.get("outputTranscription") or {}).get("text", "")
if parts or transcript:
events += self._open_response()
for part in parts:
+ if part.get("thought"):
+ # Provider thought summaries are not spoken transcript.
+ # They can still open a response that finishes at IDLE.
+ continue
if data := (part.get("inlineData") or {}).get("data"):
events.append(
me.AudioDeltaEvent(
@@ -463,18 +630,34 @@ class GeminiRealtimeModel(RealtimeModelBase):
),
)
- if content.get("turnComplete"):
+ if content.get("turnComplete") and not self._extended_thinking:
events += self._close_response()
return events
def _close_response(self) -> list[me.ModelEvent]:
"""End the open model turn with the usage last reported."""
+ # A thinking status can arrive before input transcription. Proactive
+ # audio may also finish processing without producing any speech.
+ # A cancelled response's late IDLE does not end the next user turn.
+ # Tool handoffs keep fragments buffered until the interaction ends.
+ interaction_done = not (
+ self._pending_tool_calls or self._awaiting_tool_reply
+ )
+ events = (
+ self._settle_input()
+ if self._extended_thinking
+ and interaction_done
+ and not self._response_cancelled
+ else []
+ )
+ if interaction_done:
+ self._transcription_response_id = ""
if not self._response_id:
- return []
+ return events
event = me.ResponseDoneEvent(
item_id=self._response_id,
input_tokens=self._usage.get("promptTokenCount", 0),
output_tokens=self._usage.get("responseTokenCount", 0),
)
self._response_id, self._usage = "", {}
- return [event]
+ return events + [event]
diff --git a/src/agentscope/realtime/_gemini/_models/gemini-2.5-flash-native-audio-preview-12-2025.yaml b/src/agentscope/realtime/_gemini/_models/gemini-2.5-flash-native-audio-preview-12-2025.yaml
index 2646f7c3f..a0e2c5734 100644
--- a/src/agentscope/realtime/_gemini/_models/gemini-2.5-flash-native-audio-preview-12-2025.yaml
+++ b/src/agentscope/realtime/_gemini/_models/gemini-2.5-flash-native-audio-preview-12-2025.yaml
@@ -23,6 +23,8 @@ max_context_tokens: 131072
max_session_duration_s: 900
parameter_overrides:
+ thinking_level:
+ hidden: true
voice:
enum: ["Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus",
"Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",
diff --git a/src/agentscope/realtime/_gemini/_models/gemini-3.1-flash-live-preview.yaml b/src/agentscope/realtime/_gemini/_models/gemini-3.1-flash-live-preview.yaml
index e5a51b3f7..3ee43cb1f 100644
--- a/src/agentscope/realtime/_gemini/_models/gemini-3.1-flash-live-preview.yaml
+++ b/src/agentscope/realtime/_gemini/_models/gemini-3.1-flash-live-preview.yaml
@@ -23,6 +23,8 @@ max_context_tokens: 131072
max_session_duration_s: 900
parameter_overrides:
+ thinking_level:
+ hidden: true
voice:
enum: ["Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus",
"Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",
diff --git a/src/agentscope/realtime/_gemini/_models/gemini-3.8-live-extended-thinking.yaml b/src/agentscope/realtime/_gemini/_models/gemini-3.8-live-extended-thinking.yaml
new file mode 100644
index 000000000..93d525007
--- /dev/null
+++ b/src/agentscope/realtime/_gemini/_models/gemini-3.8-live-extended-thinking.yaml
@@ -0,0 +1,33 @@
+# Source: https://ai.google.dev/gemini-api/docs/models/gemini-3.8-live-extended-thinking
+type: realtime_model
+name: gemini-3.8-live-extended-thinking
+label: Gemini 3.8 Live Extended Thinking
+status: active
+# Source: https://aistudio.google.com/docs/changelog#09-15-2026
+release_date: 2026-09-15
+
+# Image input is not implemented by the current realtime adapter.
+input_types:
+ - audio/pcm
+ - text/plain
+output_types:
+ - audio/pcm
+ - text/plain
+
+input_sample_rate: 16000
+output_sample_rate: 24000
+supports_tools: true
+supports_extended_thinking: true
+max_context_tokens: 131072
+# Audio-only duration without context-window compression.
+# Source: https://aistudio.google.com/docs/live-api/session-management
+max_session_duration_s: 900
+
+parameter_overrides:
+ voice:
+ enum: ["Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus",
+ "Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",
+ "Umbriel", "Algieba", "Despina", "Erinome", "Algenib",
+ "Rasalgethi", "Laomedeia", "Achernar", "Alnilam", "Schedar",
+ "Gacrux", "Pulcherrima", "Achird", "Zubenelgenubi",
+ "Vindemiatrix", "Sadachbia", "Sadaltager", "Sulafat"]
diff --git a/src/agentscope/realtime/_gemini/_models/gemini-3.8-live.yaml b/src/agentscope/realtime/_gemini/_models/gemini-3.8-live.yaml
index c6f1a914d..5e8b836c8 100644
--- a/src/agentscope/realtime/_gemini/_models/gemini-3.8-live.yaml
+++ b/src/agentscope/realtime/_gemini/_models/gemini-3.8-live.yaml
@@ -24,6 +24,8 @@ max_session_duration_s: 900
# Source: https://ai.google.dev/gemini-api/docs/live-api/capabilities
parameter_overrides:
+ thinking_level:
+ hidden: true
voice:
enum: ["Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus",
"Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",
diff --git a/src/agentscope/realtime/_model_card.py b/src/agentscope/realtime/_model_card.py
index 73216c8e9..c8956ad4e 100644
--- a/src/agentscope/realtime/_model_card.py
+++ b/src/agentscope/realtime/_model_card.py
@@ -64,6 +64,11 @@ class RealtimeModelCard(BaseModel):
default=False,
description="Whether this model accepts tool schemas.",
)
+ supports_extended_thinking: bool = Field(
+ default=False,
+ description="Whether the model uses background thinking with "
+ "non-blocking tools and interaction-status completion.",
+ )
interaction_mode: Literal["turn_based", "full_duplex"] = Field(
default="turn_based",
description="Whether speech is turn-based or full duplex.",
diff --git a/tests/realtime_agent_test.py b/tests/realtime_agent_test.py
index 6e6ea8376..0b5a5dffc 100644
--- a/tests/realtime_agent_test.py
+++ b/tests/realtime_agent_test.py
@@ -3,11 +3,11 @@
transport — no network, no sound card."""
# pylint: disable=protected-access, unused-argument
import asyncio
-from itertools import count
+from itertools import count, product
from types import SimpleNamespace
from typing import Any, AsyncIterator, Sequence
from unittest.async_case import IsolatedAsyncioTestCase
-from unittest.mock import patch
+from unittest.mock import AsyncMock, patch
from utils import AnyString, MockModel
from agentscope.agent import (
@@ -16,13 +16,14 @@ from agentscope.agent import (
RealtimeAgent,
TurnAggregator,
)
-from agentscope.credential import DashScopeCredential
+from agentscope.credential import DashScopeCredential, GeminiCredential
from agentscope.event import (
DataBlockDeltaEvent,
ReplyEndEvent,
ReplyStartEvent,
TextBlockDeltaEvent,
TextBlockEndEvent,
+ TextBlockStartEvent,
)
from agentscope.message import AssistantMsg, Msg, Usage, UserMsg
from agentscope.model import ChatResponse
@@ -30,6 +31,7 @@ from agentscope.realtime import (
AudioFrame,
ControlFrame,
ControlFrameType,
+ GeminiRealtimeModel,
ModelDisconnectedError,
PlayoutPosition,
RealtimeModelBase,
@@ -2038,10 +2040,12 @@ class BlockingTool(StreamTool):
def __init__(self) -> None:
super().__init__()
+ self.started = asyncio.Event()
self.release = asyncio.Event()
async def __call__(self, q: str, **kwargs: Any) -> Any:
"""Hold the tool result until a barge-in has completed."""
+ self.started.set()
await self.release.wait()
yield ToolResponse(content=[TextBlock(text=f"{q}-final")])
@@ -2096,10 +2100,15 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):
self,
tool: ToolBase,
confirm: bool | None = None,
+ non_blocking: bool = False,
) -> tuple[list[tuple[Any, ...]], ScriptedModel]:
"""Run one tool-calling reply; answer a permission prompt with
*confirm* if one appears. Returns the tool-related events."""
- model = ScriptedModel([_tool_script(tool.name)])
+ script = _tool_script(tool.name)
+ if non_blocking:
+ script[1].non_blocking = True
+ script.pop()
+ model = ScriptedModel([script])
agent = RealtimeAgent(
"Friday",
"be brief",
@@ -2162,6 +2171,31 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):
)
return events, model
+ async def test_non_blocking_permissions_and_errors(self) -> None:
+ """Immediate scheduling preserves permission and error reporting."""
+ for tool, confirm, output in (
+ (AskTool(), True, "x-final"),
+ (AskTool(), False, 'Tool "ask_tool" denied by user.'),
+ (BrokenTool(), None, "boom"),
+ ):
+ with self.subTest(tool=tool.name, confirm=confirm):
+ expected, _ = await self._run_tool_scenario(tool, confirm)
+ actual, model = await self._run_tool_scenario(
+ tool,
+ confirm,
+ non_blocking=True,
+ )
+ self.assertEqual(actual, expected)
+ self.assertEqual(
+ [call for call in model.calls if call != "push_audio"],
+ [
+ "connect(session=1,td_off=False)",
+ f"tool_result(c1,{output!r})",
+ "cancel",
+ "close",
+ ],
+ )
+
async def test_streamed_tool_result_is_not_duplicated(self) -> None:
"""Chunks are shown as they come; the provider gets only the
completed result, once."""
@@ -2296,6 +2330,1854 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):
)
+class GeminiThinkingAgentTest(IsolatedAsyncioTestCase):
+ """Drive the shared agent with actual Gemini background frames."""
+
+ async def test_partial_transcriptions_discarded_on_session_end(
+ self,
+ ) -> None:
+ """Session loss removes all fragments before any end checkpoint."""
+ for failure in (False, True):
+ with self.subTest(failure=failure):
+ model = QueueSessionModel()
+ model.parameters.input_audio_transcription = True
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ aggregator=TurnAggregator(min_chars=4),
+ )
+ try:
+ await agent.connect()
+ await agent._on_model_event(
+ me.InputTranscriptionEvent(
+ item_id="kept",
+ text="complete",
+ ),
+ )
+ while not agent._out.empty():
+ agent._out.get_nowait()
+ for item_id in ("u1", "u2"):
+ await agent._on_model_event(
+ me.InputTranscriptionDeltaEvent(
+ item_id=item_id,
+ delta="a",
+ ),
+ )
+ await agent._on_model_event(
+ me.TranscriptDeltaEvent(item_id="r1", delta="Draft"),
+ )
+ if failure:
+ model.event_queues[0].put_nowait(
+ me.ModelErrorEvent(code="closed", message="lost"),
+ )
+ model.event_queues[0].put_nowait(None)
+ await asyncio.wait_for(
+ model.iterator_finished[0].wait(),
+ timeout=1,
+ )
+ checkpoints = []
+ while not agent._out.empty():
+ queued = agent._out.get_nowait()
+ if queued.checkpoint is not None:
+ checkpoints.append(
+ (
+ queued.event.reply_id,
+ [
+ (msg.id, msg.get_text_content())
+ for msg in queued.checkpoint.context
+ ],
+ ),
+ )
+ history = [("kept", "complete"), ("r1", "Draft")]
+ self.assertEqual(
+ {
+ "context": [
+ (msg.id, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ "pending": list(agent._pending_user_turns),
+ "checkpoints": checkpoints,
+ },
+ {
+ "context": history,
+ "pending": [],
+ "checkpoints": [
+ (item_id, history)
+ for item_id in ("u1", "u2", "r1")
+ ],
+ },
+ )
+ await agent.connect()
+ self.assertEqual(
+ model.instructions,
+ "\n\n## Conversation so far\nuser: complete",
+ )
+ finally:
+ await agent.close()
+
+ async def test_close_discards_partial_transcription_after_detach(
+ self,
+ ) -> None:
+ """Permanent close retires input only after its reader has stopped."""
+ for finalized in (False, True):
+ with self.subTest(finalized=finalized):
+ model = QueueSessionModel()
+ model.parameters.input_audio_transcription = True
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ aggregator=TurnAggregator(min_chars=4),
+ )
+ try:
+ await agent.connect()
+ await asyncio.wait_for(
+ model.iterator_started[0].wait(),
+ timeout=1,
+ )
+ await agent._on_model_event(
+ me.SpeechStartedEvent(item_id="u1"),
+ )
+ await agent._on_model_event(
+ me.InputTranscriptionDeltaEvent(
+ item_id="u1",
+ delta="a",
+ ),
+ )
+ _ = [
+ event
+ async for event in agent.reply_stream(
+ FakeTransport(0),
+ )
+ ]
+ self.assertEqual(
+ (
+ list(agent._pending_user_turns),
+ model.iterator_finished[0].is_set(),
+ ),
+ (["u1"], False),
+ )
+ if finalized:
+ await agent._on_model_event(
+ me.InputTranscriptionEvent(
+ item_id="u1",
+ text="accepted",
+ ),
+ )
+ await agent.close()
+ await agent.close()
+ self.assertEqual(
+ {
+ "context": [
+ (msg.id, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ "pending": agent._pending_user_turns,
+ "current": agent._user_turn,
+ "reader_stopped": (
+ model.iterator_finished[0].is_set()
+ ),
+ },
+ {
+ "context": (
+ [("u1", "accepted")] if finalized else []
+ ),
+ "pending": {},
+ "current": None,
+ "reader_stopped": True,
+ },
+ )
+ finally:
+ await agent.close()
+
+ async def test_partial_transcription_settlement(self) -> None:
+ """Only accepted final text survives failure, filtering or reset."""
+ for outcome, detached in product(
+ ("failed", "filtered", "accepted"),
+ (False, True),
+ ):
+ with self.subTest(outcome=outcome, detached=detached):
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ ScriptedModel([], input_audio_transcription=True),
+ aggregator=TurnAggregator(min_chars=4),
+ )
+ await agent._on_model_event(
+ me.InputTranscriptionDeltaEvent(item_id="u1", delta="a"),
+ )
+ # Transport detachment preserves the turn for a late final.
+ if detached:
+ agent._retire_unbound_user_turns()
+ early = []
+ while not agent._out.empty():
+ queued = agent._out.get_nowait()
+ if queued.checkpoint is not None:
+ early.append(queued.checkpoint.context)
+ self.assertEqual(early, [[]] if detached else [])
+ self.assertEqual(
+ [
+ (msg.id, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ [("u1", "a")],
+ )
+ if outcome == "failed":
+ event = me.InputTranscriptionFailedEvent(item_id="u1")
+ else:
+ event = me.InputTranscriptionEvent(
+ item_id="u1",
+ text="accepted" if outcome == "accepted" else "a",
+ )
+ await agent._on_model_event(event)
+ agent._reset_user_turns()
+ history = [("u1", "accepted")] if outcome == "accepted" else []
+ snapshots = []
+ while not agent._out.empty():
+ queued = agent._out.get_nowait()
+ if queued.checkpoint is not None:
+ snapshots.append(
+ [
+ (msg.id, msg.get_text_content())
+ for msg in queued.checkpoint.context
+ ],
+ )
+ self.assertEqual(
+ (
+ [
+ (msg.id, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ list(agent._pending_user_turns),
+ snapshots,
+ ),
+ (history, [], [] if detached else [history]),
+ )
+
+ async def test_partial_transcription_cannot_absorb_accepted_turn(
+ self,
+ ) -> None:
+ """Merging into an unfinished predecessor would lose accepted text."""
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ ScriptedModel([], input_audio_transcription=True),
+ )
+ with patch(
+ "agentscope.agent._realtime._aggregator.time.monotonic",
+ return_value=100,
+ ):
+ for event in (
+ me.InputTranscriptionEvent(item_id="kept", text="previous"),
+ me.InputTranscriptionDeltaEvent(item_id="u1", delta="a"),
+ me.InputTranscriptionDeltaEvent(item_id="u2", delta="b"),
+ me.InputTranscriptionEvent(item_id="u2", text="accepted"),
+ ):
+ await agent._on_model_event(event)
+ self.assertTrue(agent.aggregator.merges_with_previous())
+ agent._reset_user_turns()
+ self.assertEqual(
+ [(msg.id, msg.get_text_content()) for msg in agent.state.context],
+ [("kept", "previous"), ("u2", "accepted")],
+ )
+
+ async def test_unbound_partial_transcription_discarded_on_detach(
+ self,
+ ) -> None:
+ """Retiring an unmatchable turn must also remove its placeholder."""
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ ScriptedModel([], input_audio_transcription=True),
+ )
+ await agent._on_model_event(
+ me.InputTranscriptionDeltaEvent(item_id="", delta="a"),
+ )
+ agent._retire_unbound_user_turns()
+ self.assertEqual(
+ (agent.state.context, agent._pending_user_turns),
+ ([], {}),
+ )
+
+ async def test_tool_result_precedes_followup(self) -> None:
+ """Early IDLE cannot close the reply before a result's continuation."""
+ for delivery in ("after_send", "during_send", "silent", "interrupted"):
+ with self.subTest(delivery=delivery):
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ )
+ tool = BlockingTool()
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ toolkit=Toolkit(tools=[tool]),
+ )
+
+ async def feed(
+ content: dict,
+ current_model: GeminiRealtimeModel = model,
+ current_agent: RealtimeAgent = agent,
+ ) -> None:
+ """Deliver a server frame through the real adapter."""
+ for event in current_model._parse(content):
+ await current_agent._on_model_event(event)
+
+ final_content: dict = {"interactionStatus": "IDLE"}
+ if delivery not in ("silent", "interrupted"):
+ final_content["outputTranscription"] = {"text": "Sunny."}
+ final_frame = {"serverContent": final_content}
+
+ async def send_result(
+ _payload: dict,
+ mode: str = delivery,
+ deliver: Any = feed,
+ ) -> None:
+ """Allow provider completion before send returns."""
+ if mode == "during_send":
+ await deliver(
+ {
+ "serverContent": {
+ "interactionStatus": "IDLE",
+ "outputTranscription": {"text": "Sunny."},
+ },
+ },
+ )
+
+ model._send = AsyncMock(side_effect=send_result)
+ await feed(
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Checking."},
+ },
+ },
+ )
+ reply_id = agent._reply_id
+ await feed(
+ {
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": "c1",
+ "name": tool.name,
+ "args": {"q": "x"},
+ },
+ ],
+ },
+ },
+ )
+ tasks = list(agent._tasks)
+ try:
+ await asyncio.wait_for(tool.started.wait(), timeout=1)
+ await feed(
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ )
+ if delivery == "interrupted":
+ agent._connected = True
+ await agent.interrupt()
+ tool.release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)
+ if delivery != "during_send":
+ self.assertEqual(
+ agent._reply_id,
+ "" if delivery == "interrupted" else reply_id,
+ )
+ await feed(final_frame)
+ lifecycle = []
+ while not agent._out.empty():
+ event = agent._out.get_nowait().event
+ if isinstance(event, (ReplyStartEvent, ReplyEndEvent)):
+ lifecycle.append((event.type, event.reply_id))
+ self.assertEqual(
+ lifecycle,
+ [
+ ("REPLY_START", reply_id),
+ ("REPLY_END", reply_id),
+ ],
+ )
+ self.assertEqual(
+ [
+ (
+ msg.id,
+ [
+ block.text
+ for block in msg.content
+ if isinstance(block, TextBlock)
+ ],
+ )
+ for msg in agent.state.context
+ ],
+ [
+ (
+ reply_id,
+ ["Checking."]
+ + (
+ []
+ if delivery in ("silent", "interrupted")
+ else ["Sunny."]
+ ),
+ ),
+ ],
+ )
+ self.assertEqual(agent._background_tools, {})
+ self.assertEqual(
+ (
+ model._awaiting_tool_reply,
+ model._pending_tool_calls,
+ ),
+ (False, set()),
+ )
+ finally:
+ tool.release.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ async def test_fragment_placeholders_preserve_pause_merging(self) -> None:
+ """Reserved input must honor the same aggregation as final input."""
+ for with_delta in (False, True):
+ for separator in ("none", "empty", "spoken", "tool"):
+ with self.subTest(delta=with_delta, separator=separator):
+ agent = RealtimeAgent("Friday", "", ScriptedModel([]))
+ with patch(
+ "agentscope.agent._realtime."
+ "_aggregator.time.monotonic",
+ return_value=100.0,
+ ):
+ agent._on_transcription(
+ me.InputTranscriptionEvent(
+ item_id="u1",
+ text="北京",
+ ),
+ )
+ if separator != "none":
+ content = (
+ [
+ ToolCallBlock(
+ id="c1",
+ name="tool",
+ input="{}",
+ ),
+ ]
+ if separator == "tool"
+ else [
+ TextBlock(
+ text="好的"
+ if separator == "spoken"
+ else "",
+ ),
+ ]
+ )
+ agent.state.context.append(
+ Msg(
+ id="between",
+ name="Friday",
+ role="assistant",
+ content=content,
+ ),
+ )
+ if with_delta:
+ agent._on_transcription_delta(
+ me.InputTranscriptionDeltaEvent(
+ item_id="u2",
+ delta="天",
+ ),
+ )
+ agent.state.context.append(
+ Msg(
+ id="answer",
+ name="Friday",
+ role="assistant",
+ content=[TextBlock(text="晴天")],
+ ),
+ )
+ agent._on_transcription_delta(
+ me.InputTranscriptionDeltaEvent(
+ item_id="u2",
+ delta="气",
+ response_id="answer",
+ ),
+ )
+ agent._on_transcription(
+ me.InputTranscriptionEvent(
+ item_id="u2",
+ text="天气",
+ ),
+ )
+ self.assertTrue(agent.aggregator.merges_with_previous())
+ expected = [("u1", "user", "北京天气")]
+ if separator in ("spoken", "tool"):
+ expected = [
+ ("u1", "user", "北京"),
+ (
+ "between",
+ "assistant",
+ "好的" if separator == "spoken" else "",
+ ),
+ ("u2", "user", "天气"),
+ ]
+ if with_delta:
+ expected.append(("answer", "assistant", "晴天"))
+ self.assertEqual(
+ [
+ (msg.id, msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ expected,
+ )
+
+ async def test_idle_waits_for_running_and_queued_tools(self) -> None:
+ """Model completion must not discard work or precede its results."""
+ model = ScriptedModel([])
+ tool = BlockingTool()
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ toolkit=Toolkit(tools=[tool]),
+ )
+ for call_id in ("c1", "c2"):
+ await agent._on_model_event(
+ me.ToolCallEvent(
+ item_id="r1",
+ non_blocking=True,
+ tool_call=ToolCallBlock(
+ id=call_id,
+ name=tool.name,
+ input='{"q":"x"}',
+ ),
+ ),
+ )
+ tasks = list(agent._tasks)
+ try:
+ await asyncio.wait_for(tool.started.wait(), timeout=1)
+ await agent._on_model_event(me.ResponseDoneEvent(item_id="r1"))
+ tool.release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)
+ self.assertEqual(agent._reply_id, "r1")
+ await agent._on_model_event(me.ResponseCreatedEvent(item_id="r2"))
+ await agent._on_model_event(me.ResponseDoneEvent(item_id="r2"))
+ events = []
+ while not agent._out.empty():
+ event = agent._out.get_nowait().event
+ if isinstance(event, (ToolResultEndEvent, ReplyEndEvent)):
+ events.append(
+ (event.type, getattr(event, "tool_call_id", "")),
+ )
+ self.assertEqual(
+ (model.calls, events),
+ (
+ ["tool_result(c1,'x-final')", "tool_result(c2,'x-final')"],
+ [
+ ("TOOL_RESULT_END", "c1"),
+ ("TOOL_RESULT_END", "c2"),
+ ("REPLY_END", ""),
+ ],
+ ),
+ )
+ finally:
+ tool.release.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ async def test_transcription_keeps_one_turn_before_answer(self) -> None:
+ """Fragments around thinking remain one user message before output."""
+ for split in (False, True):
+ with self.subTest(split=split):
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ )
+ agent = RealtimeAgent("Friday", "", model)
+ frames: list[dict] = []
+ if split:
+ frames.append({"inputTranscription": {"text": "Check "}})
+ frames.extend(
+ [
+ {"interactionStatus": "IN_PROGRESS"},
+ {"outputTranscription": {"text": "Sunny."}},
+ {
+ "inputTranscription": {
+ "text": "weather"
+ if split
+ else "Check weather",
+ },
+ },
+ {"interactionStatus": "IDLE"},
+ ],
+ )
+ for content in frames:
+ for event in model._parse({"serverContent": content}):
+ await agent._on_model_event(event)
+ self.assertEqual(
+ [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ [("user", "Check weather"), ("assistant", "Sunny.")],
+ )
+ user_replies = []
+ snapshots = []
+ while not agent._out.empty():
+ queued = agent._out.get_nowait()
+ if isinstance(queued.event, ReplyStartEvent):
+ if queued.event.role == "user":
+ user_replies.append(queued.event.reply_id)
+ if queued.checkpoint is not None:
+ snapshots.append(
+ [
+ (msg.role, msg.get_text_content())
+ for msg in queued.checkpoint.context
+ ],
+ )
+ self.assertEqual(user_replies, [agent.state.context[0].id])
+ self.assertEqual(
+ snapshots[-1],
+ [
+ ("user", "Check weather"),
+ ("assistant", "Sunny."),
+ ],
+ )
+ model.connect = AsyncMock()
+ try:
+ await agent.connect()
+ self.assertEqual(
+ model.connect.await_args.kwargs["instructions"],
+ "\n\n## Conversation so far\n"
+ "user: Check weather\nFriday: Sunny.",
+ )
+ finally:
+ await agent.close()
+
+ async def test_late_transcription_precedes_tool_followup(self) -> None:
+ """Late input precedes the logical reply across model responses."""
+ for delivery in (
+ "separate",
+ "same_frame",
+ "content_status",
+ "root_status",
+ "before_tool_result",
+ "after_tool_result",
+ ):
+ with self.subTest(delivery=delivery):
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ )
+ model._send = AsyncMock()
+ tool = BlockingTool()
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ toolkit=Toolkit(tools=[tool]),
+ )
+
+ async def feed(
+ frame: dict,
+ current_model: GeminiRealtimeModel = model,
+ current_agent: RealtimeAgent = agent,
+ ) -> None:
+ """Deliver a frame through the real Gemini adapter."""
+ for event in current_model._parse(frame):
+ await current_agent._on_model_event(event)
+
+ tasks = []
+ try:
+ await feed(
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Checking."},
+ },
+ },
+ )
+ reply_id = agent._reply_id
+ await feed(
+ {
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": "c1",
+ "name": tool.name,
+ "args": {"q": "weather"},
+ },
+ ],
+ },
+ },
+ )
+ tasks = list(agent._tasks)
+ await asyncio.wait_for(tool.started.wait(), timeout=1)
+ await feed(
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ )
+ transcription = {
+ "inputTranscription": {"text": "Weather?"},
+ }
+ if delivery == "before_tool_result":
+ await feed({"serverContent": transcription})
+ tool.release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)
+ output = {"outputTranscription": {"text": "Sunny."}}
+ if delivery == "same_frame":
+ await feed(
+ {"serverContent": {**output, **transcription}},
+ )
+ elif delivery in ("content_status", "root_status"):
+ frame: dict = {"serverContent": transcription}
+ status_target = (
+ frame
+ if delivery == "root_status"
+ else frame["serverContent"]
+ )
+ status_target["interactionStatus"] = "IN_PROGRESS"
+ await feed(frame)
+ await feed({"serverContent": output})
+ elif delivery in (
+ "before_tool_result",
+ "after_tool_result",
+ ):
+ if delivery == "after_tool_result":
+ await feed({"serverContent": transcription})
+ await feed({"serverContent": output})
+ else:
+ await feed({"serverContent": output})
+ await feed({"serverContent": transcription})
+ self.assertNotEqual(model._response_id, reply_id)
+ self.assertEqual(agent._reply_id, reply_id)
+ await feed(
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ )
+ expected = [
+ ("user", "Weather?"),
+ ("assistant", "Checking.\nSunny."),
+ ]
+ snapshots = [
+ agent.checkpoint_snapshot(event)
+ for event in agent.drain_events()
+ ]
+ checkpoint = next(
+ state for state in reversed(snapshots) if state
+ )
+ self.assertEqual(
+ {
+ "context": [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ "checkpoint": [
+ (msg.role, msg.get_text_content())
+ for msg in checkpoint.context
+ ],
+ },
+ {"context": expected, "checkpoint": expected},
+ )
+ model.connect = AsyncMock()
+ await agent.connect()
+ self.assertEqual(
+ model.connect.await_args.kwargs["instructions"],
+ "\n\n## Conversation so far\n"
+ "user: Weather?\nFriday: Checking.\nSunny.",
+ )
+ finally:
+ tool.release.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ await agent.close()
+
+ async def test_tool_idle_preserves_input_fragments(self) -> None:
+ """Tool handoffs must not strip or filter individual fragments."""
+ for min_chars, idle_count in product((1, 4), (1, 2)):
+ with self.subTest(min_chars=min_chars, idle_count=idle_count):
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ )
+ model._send = AsyncMock()
+ tool = BlockingTool()
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ toolkit=Toolkit(tools=[tool]),
+ aggregator=TurnAggregator(min_chars=min_chars),
+ )
+
+ async def feed(
+ frame: dict,
+ current_model: GeminiRealtimeModel = model,
+ current_agent: RealtimeAgent = agent,
+ ) -> None:
+ """Deliver real adapter events without a network."""
+ for event in current_model._parse(frame):
+ await current_agent._on_model_event(event)
+
+ tasks = []
+ try:
+ await feed(
+ {
+ "serverContent": {
+ "inputTranscription": {"text": "Go "},
+ "outputTranscription": {"text": "Checking."},
+ },
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": "c1",
+ "name": tool.name,
+ "args": {"q": "x"},
+ },
+ ],
+ },
+ },
+ )
+ user_id = model._user_id
+ tasks = list(agent._tasks)
+ await asyncio.wait_for(tool.started.wait(), timeout=1)
+ for _ in range(idle_count):
+ await feed(
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ )
+ self.assertEqual(
+ (model._user_id, model._input_text),
+ (user_id, "Go "),
+ )
+ tool.release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)
+ await feed(
+ {
+ "serverContent": {
+ "inputTranscription": {"text": "there"},
+ },
+ },
+ )
+ self.assertEqual(
+ (model._user_id, model._input_text),
+ (user_id, "Go there"),
+ )
+ await feed(
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Done."},
+ "interactionStatus": "IDLE",
+ },
+ },
+ )
+ events = list(agent.drain_events())
+ user_reply_id = agent.state.context[0].id
+ self.assertEqual(
+ {
+ "context": [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ "user_text": [
+ event.delta
+ for event in events
+ if isinstance(event, TextBlockDeltaEvent)
+ and event.reply_id == user_reply_id
+ ],
+ "pending": agent._pending_user_turns,
+ "input": (model._user_id, model._input_text),
+ },
+ {
+ "context": [
+ ("user", "Go there"),
+ ("assistant", "Checking.\nDone."),
+ ],
+ "user_text": ["Go there"],
+ "pending": {},
+ "input": ("", ""),
+ },
+ )
+ finally:
+ tool.release.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ await agent.close()
+
+ async def test_interruption_keeps_transcription_identities(self) -> None:
+ """A new utterance cannot inherit the interrupted input buffer."""
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ )
+ agent = RealtimeAgent("Friday", "", model)
+ agent._connected = True
+ frames = [
+ {"inputTranscription": {"text": "Check weather"}},
+ {"interactionStatus": "IN_PROGRESS"},
+ {"outputTranscription": {"text": "Checking."}},
+ {"interrupted": True, "inputTranscription": {"text": "Change "}},
+ {"interactionStatus": "IN_PROGRESS"},
+ {"outputTranscription": {"text": "Okay."}},
+ {"inputTranscription": {"text": "city"}},
+ {"interactionStatus": "IDLE"},
+ ]
+ for content in frames:
+ for event in model._parse({"serverContent": content}):
+ await agent._on_model_event(event)
+ self.assertEqual(
+ [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ [
+ ("user", "Check weather"),
+ ("assistant", "Checking."),
+ ("user", "Change city"),
+ ("assistant", "Okay."),
+ ],
+ )
+ self.assertNotEqual(
+ agent.state.context[0].id,
+ agent.state.context[2].id,
+ )
+
+ async def test_local_vad_interruption_preserves_input_history(
+ self,
+ ) -> None:
+ """Local speech starts settle old input and reserve a new turn."""
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ parameters=GeminiRealtimeModel.Parameters(turn_detection="none"),
+ )
+ model._send = AsyncMock()
+ agent = RealtimeAgent("Friday", "", model)
+ agent._connected = True
+
+ async def feed(content: dict) -> None:
+ """Deliver parsed provider frames in model event order."""
+ for event in model._parse({"serverContent": content}):
+ await agent._on_model_event(event)
+
+ await agent._forward_audio(b"\x00\x00", SpeechTransition.STARTED)
+ await feed({"inputTranscription": {"text": "Check weather"}})
+ await agent._forward_audio(b"\x00\x00", SpeechTransition.ENDED)
+ await feed(
+ {
+ "interactionStatus": "IN_PROGRESS",
+ "outputTranscription": {"text": "Checking."},
+ },
+ )
+ await agent._forward_audio(b"\x00\x00", SpeechTransition.STARTED)
+ while not model._queue.empty():
+ await agent._on_model_event(model._queue.get_nowait())
+ await feed({"inputTranscription": {"text": "Change "}})
+ await agent._forward_audio(b"\x00\x00", SpeechTransition.ENDED)
+ await feed(
+ {
+ "inputTranscription": {"text": "city"},
+ "interactionStatus": "IDLE",
+ },
+ )
+ await feed(
+ {
+ "interactionStatus": "IN_PROGRESS",
+ "outputTranscription": {"text": "Okay."},
+ },
+ )
+ await feed({"interactionStatus": "IDLE"})
+ self.assertEqual(
+ [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ [
+ ("user", "Check weather"),
+ ("assistant", "Checking."),
+ ("user", "Change city"),
+ ("assistant", "Okay."),
+ ],
+ )
+ self.assertNotEqual(
+ agent.state.context[0].id,
+ agent.state.context[2].id,
+ )
+ self.assertEqual(agent._pending_user_turns, {})
+
+ async def test_stale_idle_preserves_ongoing_local_input(self) -> None:
+ """Cancelled completions cannot split speech or filter a fragment."""
+ for backchannels in (frozenset(), frozenset({"New"})):
+ with self.subTest(backchannels=backchannels):
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ parameters=GeminiRealtimeModel.Parameters(
+ turn_detection="none",
+ ),
+ )
+ model._send = AsyncMock()
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ aggregator=TurnAggregator(backchannels=backchannels),
+ )
+ agent._connected = True
+
+ async def feed(
+ content: dict,
+ current_model: GeminiRealtimeModel = model,
+ current_agent: RealtimeAgent = agent,
+ ) -> None:
+ """Deliver provider events without a network."""
+ for event in current_model._parse(
+ {"serverContent": content},
+ ):
+ await current_agent._on_model_event(event)
+
+ try:
+ with patch(
+ "agentscope.agent._realtime._aggregator.time",
+ ) as clock:
+ clock.monotonic.return_value = 10.0
+ await feed(
+ {"outputTranscription": {"text": "Old answer"}},
+ )
+ await agent._forward_audio(
+ b"\x00\x00",
+ SpeechTransition.STARTED,
+ )
+ user_turn = agent._user_turn
+ await feed({"inputTranscription": {"text": "New "}})
+ provider_id = model._user_id
+ for _ in range(2):
+ await feed({"interactionStatus": "IDLE"})
+ self.assertIs(agent._user_turn, user_turn)
+ self.assertEqual(
+ (
+ user_turn.open,
+ user_turn.speech_ended,
+ model._user_id,
+ model._input_text,
+ ),
+ (True, False, provider_id, "New "),
+ )
+ clock.monotonic.return_value = 12.0
+ await feed(
+ {"inputTranscription": {"text": "question"}},
+ )
+ self.assertEqual(model._user_id, provider_id)
+ await agent._forward_audio(
+ b"\x00\x00",
+ SpeechTransition.ENDED,
+ )
+ await feed(
+ {
+ "interactionStatus": "IN_PROGRESS",
+ "outputTranscription": {"text": "Okay."},
+ },
+ )
+ await feed({"interactionStatus": "IDLE"})
+ events = list(agent.drain_events())
+ self.assertEqual(
+ {
+ "context": [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ "user_starts": [
+ event.reply_id
+ for event in events
+ if isinstance(event, ReplyStartEvent)
+ and event.role == "user"
+ ],
+ "user_ends": [
+ event.reply_id
+ for event in events
+ if isinstance(event, ReplyEndEvent)
+ and event.reply_id == user_turn.reply_id
+ ],
+ "pending": agent._pending_user_turns,
+ "current": agent._user_turn,
+ },
+ {
+ "context": [
+ ("assistant", "Old answer"),
+ ("user", "New question"),
+ ("assistant", "Okay."),
+ ],
+ "user_starts": [user_turn.reply_id],
+ "user_ends": [user_turn.reply_id],
+ "pending": {},
+ "current": None,
+ },
+ )
+ finally:
+ await agent.close()
+
+ async def test_repeated_local_input_before_cancelled_response_idle(
+ self,
+ ) -> None:
+ """Every local input start separates text, even with no live reply."""
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ parameters=GeminiRealtimeModel.Parameters(turn_detection="none"),
+ )
+ model._send = AsyncMock()
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ aggregator=TurnAggregator(merge_window_ms=-1),
+ )
+ agent._connected = True
+ user_ids = []
+ provider_ids = []
+ transcripts = ["Check weather", "Change city", "Tomorrow instead"]
+ for index, text in enumerate(transcripts):
+ await agent._forward_audio(b"\x00\x00", SpeechTransition.STARTED)
+ user_ids.append(agent._user_turn.reply_id)
+ while not model._queue.empty():
+ await agent._on_model_event(model._queue.get_nowait())
+ for event in model._parse(
+ {"serverContent": {"inputTranscription": {"text": text}}},
+ ):
+ provider_ids.append(event.item_id)
+ await agent._on_model_event(event)
+ await agent._forward_audio(b"\x00\x00", SpeechTransition.ENDED)
+ if index == 0:
+ for event in model._parse(
+ {
+ "serverContent": {
+ "interactionStatus": "IN_PROGRESS",
+ "outputTranscription": {"text": "Checking."},
+ },
+ },
+ ):
+ await agent._on_model_event(event)
+ else:
+ self.assertEqual(agent._reply_id, "")
+ for event in model._parse({"interactionStatus": "IDLE"}):
+ await agent._on_model_event(event)
+ self.assertIsNotNone(agent._user_turn)
+ for content in (
+ {"interactionStatus": "IN_PROGRESS"},
+ {"interactionStatus": "IDLE"},
+ ):
+ for event in model._parse({"serverContent": content}):
+ await agent._on_model_event(event)
+ starts = []
+ ends = []
+ while not agent._out.empty():
+ event = agent._out.get_nowait().event
+ if isinstance(event, ReplyStartEvent) and event.role == "user":
+ starts.append(event.reply_id)
+ elif (
+ isinstance(event, ReplyEndEvent) and event.reply_id in user_ids
+ ):
+ ends.append(event.reply_id)
+ self.assertEqual(
+ {
+ "history": [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ "user_ids": [
+ msg.id for msg in agent.state.context if msg.role == "user"
+ ],
+ "provider_id_count": len(set(provider_ids)),
+ "starts": starts,
+ "ends": ends,
+ "pending": agent._pending_user_turns,
+ "current": agent._user_turn,
+ },
+ {
+ "history": [
+ ("user", transcripts[0]),
+ ("assistant", "Checking."),
+ ("user", transcripts[1]),
+ ("user", transcripts[2]),
+ ],
+ "user_ids": user_ids,
+ "provider_id_count": 3,
+ "starts": user_ids,
+ "ends": user_ids,
+ "pending": {},
+ "current": None,
+ },
+ )
+
+ async def test_followup_keeps_reply_open_after_tools_finish(self) -> None:
+ """New model output after early IDLE resets model completion."""
+ model = ScriptedModel([])
+ tool = BlockingTool()
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ toolkit=Toolkit(tools=[tool]),
+ )
+ await agent._on_model_event(
+ me.ToolCallEvent(
+ item_id="r1",
+ non_blocking=True,
+ tool_call=ToolCallBlock(
+ id="c1",
+ name=tool.name,
+ input='{"q":"x"}',
+ ),
+ ),
+ )
+ tasks = list(agent._tasks)
+ try:
+ await asyncio.wait_for(tool.started.wait(), timeout=1)
+ await agent._on_model_event(me.ResponseDoneEvent(item_id="r1"))
+ await agent._on_model_event(me.ResponseCreatedEvent(item_id="r2"))
+ tool.release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)
+ self.assertEqual(agent._reply_id, "r1")
+ await agent._on_model_event(me.ResponseDoneEvent(item_id="r2"))
+ replies = []
+ while not agent._out.empty():
+ event = agent._out.get_nowait().event
+ if isinstance(event, ReplyEndEvent):
+ replies.append((event.reply_id, event.finished_reason))
+ self.assertEqual(replies, [("r1", "completed")])
+ self.assertEqual(agent._background_tools, {})
+ finally:
+ tool.release.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ async def test_interrupted_slow_tool_does_not_block_new_reply(
+ self,
+ ) -> None:
+ """Serialize within a reply while allowing the next reply to run."""
+ model = ScriptedModel([])
+ model.push_tool_result_silently = AsyncMock(
+ wraps=model.push_tool_result_silently,
+ )
+ old_tool = BlockingTool()
+ old_tool.name = "slow_tool"
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ toolkit=Toolkit(tools=[old_tool, StreamTool()]),
+ )
+ try:
+ for call_id in ("running", "queued"):
+ await agent._on_model_event(
+ me.ToolCallEvent(
+ item_id="r1",
+ non_blocking=True,
+ tool_call=ToolCallBlock(
+ id=call_id,
+ name=old_tool.name,
+ input='{"q":"old"}',
+ ),
+ ),
+ )
+ old_tasks = set(agent._tasks)
+ await asyncio.wait_for(old_tool.started.wait(), timeout=1)
+ started_calls = []
+ while not agent._out.empty():
+ event = agent._out.get_nowait().event
+ if isinstance(event, ToolCallStartEvent):
+ started_calls.append(event.tool_call_id)
+ self.assertEqual(started_calls, ["running"])
+ await agent._barge_in()
+ await agent._on_model_event(
+ me.ToolCallEvent(
+ item_id="r2",
+ non_blocking=True,
+ tool_call=ToolCallBlock(
+ id="new",
+ name="stream_tool",
+ input='{"q":"new"}',
+ ),
+ ),
+ )
+ new_tasks = agent._tasks - old_tasks
+ await asyncio.wait_for(asyncio.gather(*new_tasks), timeout=1)
+ self.assertEqual(
+ (old_tool.release.is_set(), model.calls, agent._reply_id),
+ (False, ["tool_result(new,'new-final')"], "r2"),
+ )
+ old_tool.release.set()
+ await asyncio.wait_for(asyncio.gather(*old_tasks), timeout=1)
+ self.assertEqual(
+ model.calls,
+ [
+ "tool_result(new,'new-final')",
+ "tool_result(running,'old-final')",
+ "tool_result(queued,"
+ "'Reply interrupted before tool execution.')",
+ ],
+ )
+ self.assertEqual(
+ [
+ (call.args[0].id, call.args[0].state)
+ for call in model.push_tool_result_silently.await_args_list
+ ],
+ [("running", "success"), ("queued", "interrupted")],
+ )
+ self.assertEqual(
+ [
+ (
+ msg.id,
+ [
+ (block.type, block.id, block.state)
+ for block in msg.content
+ if isinstance(
+ block,
+ (ToolCallBlock, ToolResultBlock),
+ )
+ ],
+ )
+ for msg in agent.state.context
+ ],
+ [
+ (
+ "r1",
+ [
+ ("tool_call", "running", "finished"),
+ ("tool_call", "queued", "finished"),
+ ("tool_result", "running", "success"),
+ ("tool_result", "queued", "interrupted"),
+ ],
+ ),
+ (
+ "r2",
+ [
+ ("tool_call", "new", "finished"),
+ ("tool_result", "new", "success"),
+ ],
+ ),
+ ],
+ )
+ self.assertEqual(
+ [
+ (event.type, event.tool_call_id)
+ for event in agent.drain_events()
+ if isinstance(
+ event,
+ (ToolCallStartEvent, ToolResultEndEvent),
+ )
+ ],
+ [
+ ("TOOL_CALL_START", "new"),
+ ("TOOL_RESULT_END", "new"),
+ ("TOOL_RESULT_END", "running"),
+ ("TOOL_RESULT_END", "queued"),
+ ],
+ )
+ finally:
+ old_tool.release.set()
+ await agent.close()
+
+ async def test_interruption_releases_confirmation_lock(self) -> None:
+ """An unanswered old prompt cannot block the next reply's tool."""
+ factories = [None]
+ if eager_factory := getattr(asyncio, "eager_task_factory", None):
+ factories.append(eager_factory)
+ loop = asyncio.get_running_loop()
+ original_factory = loop.get_task_factory()
+ for factory in factories:
+ with self.subTest(eager=factory is not None):
+ try:
+ loop.set_task_factory(factory)
+ await self._check_interrupted_confirmation()
+ finally:
+ loop.set_task_factory(original_factory)
+
+ async def test_interrupted_gemini_tool_results_are_silent(self) -> None:
+ """Late results cannot restart a closed reply or disturb a new one."""
+ for confirmation, new_reply in product((False, True), repeat=2):
+ with self.subTest(confirm=confirmation, new_reply=new_reply):
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ )
+ model._send = AsyncMock()
+ tool = AskTool() if confirmation else BlockingTool()
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ toolkit=Toolkit(tools=[tool]),
+ )
+ agent._connected = True
+
+ async def feed(
+ frame: dict,
+ current_model: GeminiRealtimeModel = model,
+ current_agent: RealtimeAgent = agent,
+ ) -> None:
+ """Deliver real adapter events without a network."""
+ for event in current_model._parse(frame):
+ await current_agent._on_model_event(event)
+
+ await feed(
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Checking."},
+ },
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": call_id,
+ "name": tool.name,
+ "args": {"q": "x"},
+ }
+ for call_id in ("c1", "c2")
+ ],
+ },
+ },
+ )
+ tasks = list(agent._tasks)
+ try:
+ if confirmation:
+
+ async def wait_for_confirmation(
+ current_agent: RealtimeAgent = agent,
+ ) -> None:
+ """Wait until the prompt can be cancelled."""
+ while "c1" not in current_agent._confirmations:
+ await asyncio.sleep(0)
+
+ await asyncio.wait_for(
+ wait_for_confirmation(),
+ timeout=1,
+ )
+ else:
+ await asyncio.wait_for(tool.started.wait(), timeout=1)
+ await agent.interrupt()
+ await feed(
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ )
+ expected_context = [("assistant", "Checking.")]
+ if new_reply:
+ await feed(
+ {
+ "serverContent": {
+ "inputTranscription": {"text": "Next"},
+ "outputTranscription": {"text": "Ready."},
+ },
+ },
+ )
+ expected_context += [
+ ("user", "Next"),
+ ("assistant", "Ready."),
+ ]
+ current_reply = agent._reply_id
+ list(agent.drain_events())
+ if not confirmation:
+ tool.release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)
+ result = (
+ {"error": 'Tool "ask_tool" denied by user.'}
+ if confirmation
+ else {"output": "x-final"}
+ )
+ self.assertEqual(
+ {
+ "frames": [
+ call.args[0]
+ for call in model._send.await_args_list
+ ],
+ "reply": agent._reply_id,
+ "new_reply_events": [
+ event.type
+ for event in agent.drain_events()
+ if isinstance(event, ReplyStartEvent)
+ ],
+ "awaiting_followup": model._awaiting_tool_reply,
+ "context": [
+ (msg.role, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ },
+ {
+ "frames": [
+ {
+ "toolResponse": {
+ "functionResponses": [
+ {
+ "id": "c1",
+ "name": tool.name,
+ "response": result,
+ "scheduling": "SILENT",
+ },
+ ],
+ },
+ },
+ {
+ "toolResponse": {
+ "functionResponses": [
+ {
+ "id": "c2",
+ "name": tool.name,
+ "response": {
+ "error": (
+ "Reply interrupted "
+ "before tool "
+ "execution."
+ ),
+ },
+ "scheduling": "SILENT",
+ },
+ ],
+ },
+ },
+ ],
+ "reply": current_reply,
+ "new_reply_events": [],
+ "awaiting_followup": False,
+ "context": expected_context,
+ },
+ )
+ finally:
+ if not confirmation:
+ tool.release.set()
+ await agent.close()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ async def _check_interrupted_confirmation(self) -> None:
+ """Validate cancellation and approval ownership under one scheduler."""
+ model = ScriptedModel([])
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ model,
+ toolkit=Toolkit(tools=[AskTool(), StreamTool()]),
+ )
+ old_call = ToolCallBlock(id="old", name="ask_tool", input='{"q":"x"}')
+ await agent._on_model_event(
+ me.ToolCallEvent(
+ item_id="r1",
+ non_blocking=True,
+ tool_call=old_call,
+ ),
+ )
+ tasks = list(agent._tasks)
+ try:
+
+ async def wait_for_confirmation() -> None:
+ """Wait for the controlled prompt to be registered."""
+ while "old" not in agent._confirmations:
+ await asyncio.sleep(0)
+
+ await asyncio.wait_for(wait_for_confirmation(), timeout=1)
+ old_future = agent._confirmations["old"]
+ await agent._barge_in()
+ await agent._on_model_event(
+ me.ToolCallEvent(
+ item_id="r2",
+ non_blocking=True,
+ tool_call=ToolCallBlock(
+ id="new",
+ name="stream_tool",
+ input='{"q":"y"}',
+ ),
+ ),
+ )
+ tasks = list(agent._tasks)
+ await asyncio.wait_for(
+ asyncio.shield(asyncio.gather(*tasks)),
+ timeout=1,
+ )
+ self.assertTrue(old_future.cancelled())
+ new_call = ToolCallBlock(
+ id="old",
+ name="ask_tool",
+ input='{"q":"z"}',
+ )
+ await agent._on_model_event(
+ me.ToolCallEvent(
+ item_id="r2",
+ non_blocking=True,
+ tool_call=new_call,
+ ),
+ )
+ tasks = list(agent._tasks)
+ await asyncio.wait_for(wait_for_confirmation(), timeout=1)
+ with patch(
+ "agentscope.agent._realtime._agent.logger.warning",
+ ) as warning:
+ await agent.send(
+ UserConfirmResultEvent(
+ reply_id="r1",
+ confirm_results=[
+ ConfirmResult(tool_call=old_call, confirmed=True),
+ ],
+ ),
+ )
+ warning.assert_called_once_with(
+ "RealtimeAgent: ignored confirmation for tool %s "
+ "from reply %s; expected reply %s",
+ "old",
+ "r1",
+ "r2",
+ )
+ self.assertFalse(agent._confirmations["old"].done())
+ await agent.send(
+ UserConfirmResultEvent(
+ reply_id="r2",
+ confirm_results=[
+ ConfirmResult(tool_call=new_call, confirmed=True),
+ ],
+ ),
+ )
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)
+ self.assertEqual(
+ {
+ "independent_results": sorted(model.calls[:2]),
+ "confirmed_result": model.calls[2:],
+ },
+ {
+ "independent_results": [
+ "tool_result(new,'y-final')",
+ "tool_result(old,'Tool \"ask_tool\" denied by user.')",
+ ],
+ "confirmed_result": ["tool_result(old,'z-final')"],
+ },
+ )
+ self.assertEqual(agent._confirmations, {})
+ self.assertEqual(agent._confirmation_replies, {})
+ finally:
+ for future in list(agent._confirmations.values()):
+ future.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ async def test_tools_and_speech_share_an_interaction(self) -> None:
+ """Tools start before completion; audio continues during execution."""
+ for interrupted in (False, True):
+ with self.subTest(interrupted=interrupted):
+ model = GeminiRealtimeModel(
+ "gemini-3.8-live-extended-thinking",
+ GeminiCredential(api_key="test"),
+ )
+ model._send = AsyncMock()
+ tool = BlockingTool()
+ agent = RealtimeAgent(
+ "Friday",
+ "be brief",
+ model,
+ toolkit=Toolkit(tools=[tool]),
+ )
+ transport = FakeTransport(frames=0)
+ agent._transport = transport
+
+ async def feed(
+ frame: dict,
+ current_model: GeminiRealtimeModel = model,
+ current_agent: RealtimeAgent = agent,
+ ) -> None:
+ """Normalize a provider frame and deliver its events."""
+ for event in current_model._parse(frame):
+ await current_agent._on_model_event(event)
+
+ await feed(
+ {
+ "serverContent": {
+ "interactionStatus": "IN_PROGRESS",
+ "outputTranscription": {"text": "Checking."},
+ "turnComplete": True,
+ },
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": "c1",
+ "name": "stream_tool",
+ "args": {"q": "x"},
+ },
+ ],
+ },
+ },
+ )
+ tasks = list(agent._tasks)
+ try:
+ await asyncio.wait_for(tool.started.wait(), timeout=1)
+ reply_id = agent._reply_id
+ await feed(
+ {
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": "c2",
+ "name": "stream_tool",
+ "args": {"q": "y"},
+ },
+ ],
+ },
+ },
+ )
+ tasks = list(agent._tasks)
+ await feed(
+ {
+ "serverContent": {
+ "modelTurn": {
+ "parts": [
+ {
+ "inlineData": {"data": "AQA="},
+ },
+ ],
+ },
+ "outputTranscription": {
+ "text": "Still checking.",
+ },
+ "turnComplete": True,
+ },
+ },
+ )
+ self.assertEqual(
+ {
+ "reply": agent._reply_id,
+ "audio_item": transport.item,
+ "pending": agent._pending_tools,
+ "done": [task.done() for task in tasks],
+ "sent": model._send.await_args_list,
+ },
+ {
+ "reply": reply_id,
+ "audio_item": reply_id,
+ "pending": {},
+ "done": [False, False],
+ "sent": [],
+ },
+ )
+ if interrupted:
+ await feed({"serverContent": {"interrupted": True}})
+ tool.release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=1)
+ if not interrupted:
+ await feed(
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Sunny."},
+ },
+ },
+ )
+ await feed(
+ {
+ "serverContent": {
+ "interactionStatus": "IDLE",
+ },
+ },
+ )
+ await feed(
+ {
+ "serverContent": {
+ "interactionStatus": "IDLE",
+ },
+ },
+ )
+ self.assertEqual(
+ sorted(
+ [
+ call.args[0]
+ for call in model._send.await_args_list
+ ],
+ key=lambda frame: frame["toolResponse"][
+ "functionResponses"
+ ][0]["id"],
+ ),
+ [
+ {
+ "toolResponse": {
+ "functionResponses": [
+ {
+ "id": call_id,
+ "name": "stream_tool",
+ "response": result,
+ **(
+ {"scheduling": "SILENT"}
+ if interrupted
+ else {}
+ ),
+ },
+ ],
+ },
+ }
+ for call_id, result in [
+ ("c1", {"output": "x-final"}),
+ (
+ "c2",
+ {
+ "error": "Reply interrupted before "
+ "tool execution.",
+ }
+ if interrupted
+ else {"output": "y-final"},
+ ),
+ ]
+ ],
+ )
+ events = []
+ while not agent._out.empty():
+ events.append(agent._out.get_nowait().event)
+ self.assertEqual(
+ [
+ (event.reply_id, event.finished_reason)
+ for event in events
+ if isinstance(event, ReplyEndEvent)
+ ],
+ [
+ (
+ reply_id,
+ "interrupted" if interrupted else "completed",
+ ),
+ ],
+ )
+ text = agent.state.context[0].get_content_blocks("text")
+ self.assertEqual(
+ [block.text for block in text],
+ ["Checking.", "Still checking."]
+ + ([] if interrupted else ["Sunny."]),
+ )
+ self.assertEqual(
+ len({block.id for block in text}),
+ len(text),
+ )
+ self.assertEqual(
+ [
+ (event.type, event.block_id)
+ for event in events
+ if isinstance(
+ event,
+ (
+ TextBlockStartEvent,
+ TextBlockEndEvent,
+ ),
+ )
+ ],
+ [
+ (kind, block.id)
+ for block in text
+ for kind in (
+ "TEXT_BLOCK_START",
+ "TEXT_BLOCK_END",
+ )
+ ],
+ )
+ self.assertEqual(
+ sorted(
+ [
+ (event.type, event.tool_call_id)
+ for event in events
+ if isinstance(
+ event,
+ (
+ ToolCallStartEvent,
+ ToolResultEndEvent,
+ ),
+ )
+ ],
+ key=lambda event: (event[1], event[0]),
+ ),
+ [
+ ("TOOL_CALL_START", "c1"),
+ ("TOOL_RESULT_END", "c1"),
+ ]
+ + ([] if interrupted else [("TOOL_CALL_START", "c2")])
+ + [("TOOL_RESULT_END", "c2")],
+ )
+ self.assertEqual(agent._reply_id, "")
+ finally:
+ tool.release.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+
class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"""The complete event stream of a turn that calls a tool and then
answers with speech, asserted as one structure."""
diff --git a/tests/realtime_dashscope_test.py b/tests/realtime_dashscope_test.py
index e6e7f8719..ad7883ea6 100644
--- a/tests/realtime_dashscope_test.py
+++ b/tests/realtime_dashscope_test.py
@@ -643,6 +643,7 @@ class DashScopeParseTest(unittest.TestCase):
"suggested_rules": [],
"finished_at": None,
},
+ "non_blocking": False,
},
)
self.assertEqual(self.model._tool_names, {})
diff --git a/tests/realtime_gemini_test.py b/tests/realtime_gemini_test.py
index 248443371..b776ea3d6 100644
--- a/tests/realtime_gemini_test.py
+++ b/tests/realtime_gemini_test.py
@@ -2,14 +2,18 @@
"""Unit tests for the Gemini Live realtime adapter: cards, the setup
message and frame parsing. Nothing here opens a connection."""
# pylint: disable=protected-access
+import asyncio
import base64
import unittest
+from itertools import product
+from types import SimpleNamespace
from unittest.async_case import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, patch
from utils import AnyString
from agentscope.credential import GeminiCredential
-from agentscope.message import ToolResultBlock
+from agentscope.message import ToolResultBlock, ToolResultState
from agentscope.realtime import (
GeminiRealtimeModel,
ModelDisconnectedError,
@@ -17,6 +21,7 @@ from agentscope.realtime import (
CRED = GeminiCredential(api_key="key-x")
MODEL = "gemini-3.1-flash-live-preview"
+THINKING_MODEL = "gemini-3.8-live-extended-thinking"
TOOLS = [
{
"type": "function",
@@ -180,6 +185,505 @@ class GeminiSetupTest(unittest.TestCase):
)
+class GeminiThinkingTest(unittest.TestCase):
+ """Extended Thinking configuration and interaction boundaries."""
+
+ def test_setup_and_model_discovery(self) -> None:
+ """Only the new model exposes reasoning and non-blocking tools."""
+ cards = CRED.list_realtime_models()
+ self.assertEqual(
+ {
+ card.name: "thinking_level"
+ in card.parameter_schema["properties"]
+ for card in cards
+ },
+ {
+ "gemini-2.5-flash-native-audio-preview-12-2025": False,
+ MODEL: False,
+ "gemini-3.8-live": False,
+ THINKING_MODEL: True,
+ },
+ )
+ for level in (None, "low", "medium", "high"):
+ with self.subTest(level=level):
+ expected = GeminiRealtimeModel(MODEL, CRED)._setup(
+ "be nice",
+ TOOLS,
+ "handle",
+ )
+ setup = expected["setup"]
+ setup["model"] = f"models/{THINKING_MODEL}"
+ setup["tools"][0]["functionDeclarations"][0][
+ "behavior"
+ ] = "NON_BLOCKING"
+ if level:
+ setup["generationConfig"]["thinkingConfig"] = {
+ "thinkingLevel": level.upper(),
+ }
+ model = GeminiRealtimeModel(
+ THINKING_MODEL,
+ CRED,
+ parameters=GeminiRealtimeModel.Parameters(
+ thinking_level=level,
+ ),
+ )
+ self.assertEqual(
+ model._setup("be nice", TOOLS, "handle"),
+ expected,
+ )
+
+ def test_thinking_capability_comes_from_card(self) -> None:
+ """Aliases and hidden UI parameters do not change protocol mode."""
+ original = GeminiRealtimeModel(THINKING_MODEL, CRED).card
+ self.assertEqual(
+ {
+ "enabled": original.supports_extended_thinking,
+ "release": original.model_dump(mode="json")["release_date"],
+ "session_limit": original.max_session_duration_s,
+ },
+ {"enabled": True, "release": "2026-09-15", "session_limit": 900},
+ )
+ for name in (THINKING_MODEL, "custom-thinking-alias"):
+ for enabled in (False, True):
+ with self.subTest(name=name, enabled=enabled):
+ card = original.model_copy(
+ update={
+ "name": name,
+ "supports_extended_thinking": enabled,
+ "parameter_schema": {},
+ },
+ )
+ model = GeminiRealtimeModel(name, CRED, model_card=card)
+ declaration = model._setup("", TOOLS, "")["setup"][
+ "tools"
+ ][0]["functionDeclarations"][0]
+ self.assertEqual(
+ {
+ "thinking": model._extended_thinking,
+ "behavior": declaration.get("behavior"),
+ "events": [
+ type(event).__name__
+ for event in model._parse(
+ {
+ "serverContent": {
+ "interactionStatus": "IN_PROGRESS",
+ },
+ },
+ )
+ ],
+ },
+ {
+ "thinking": enabled,
+ "behavior": "NON_BLOCKING" if enabled else None,
+ "events": ["ResponseCreatedEvent"]
+ if enabled
+ else [],
+ },
+ )
+
+ def test_setup_acknowledgment_survives_mixed_frames(self) -> None:
+ """A setup acknowledgment is processed before content early returns."""
+ extras: list[dict] = [
+ {},
+ {"interactionStatus": "IN_PROGRESS"},
+ {"serverContent": {"interactionStatus": "IN_PROGRESS"}},
+ ]
+ for extra in extras:
+ with self.subTest(extra=extra):
+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)
+ events = model._parse({"setupComplete": {}, **extra})
+ self.assertEqual(
+ {"ready": model._ready.is_set(), "events": _dump(events)},
+ {
+ "ready": True,
+ "events": [
+ (
+ "ResponseCreatedEvent",
+ {
+ "item_id": model._response_id,
+ },
+ ),
+ ]
+ if extra
+ else [],
+ },
+ )
+
+ def test_invalid_thinking_configuration(self) -> None:
+ """Unsupported levels and older models fail before connecting."""
+ with self.assertRaises(ValueError):
+ GeminiRealtimeModel.Parameters(thinking_level="minimal")
+ for card in GeminiRealtimeModel.list_models():
+ if card.name == THINKING_MODEL:
+ continue
+ with self.subTest(model=card.name):
+ with self.assertRaisesRegex(ValueError, "requires"):
+ GeminiRealtimeModel(
+ card.name,
+ CRED,
+ parameters=GeminiRealtimeModel.Parameters(
+ thinking_level="low",
+ ),
+ )
+
+ def test_background_interaction(self) -> None:
+ """Fillers and tools share one response until final IDLE."""
+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)
+ messages = [
+ {"serverContent": {"interactionStatus": "IN_PROGRESS"}},
+ {
+ "serverContent": {
+ "modelTurn": {
+ "parts": [{"thought": True, "text": "Plan"}],
+ },
+ },
+ },
+ {
+ "serverContent": {
+ "modelTurn": {
+ "parts": [{"inlineData": {"data": "AQA="}}],
+ },
+ "outputTranscription": {"text": "Checking."},
+ "turnComplete": True,
+ "interactionStatus": "IN_PROGRESS",
+ },
+ },
+ {
+ "toolCall": {
+ "functionCalls": [
+ {"id": "c1", "name": "get_weather", "args": {}},
+ ],
+ },
+ "interactionStatus": "IN_PROGRESS",
+ },
+ {"serverContent": {"turnComplete": True}},
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Sunny."},
+ "interactionStatus": "IDLE",
+ "turnComplete": True,
+ },
+ "usageMetadata": {
+ "promptTokenCount": 12,
+ "responseTokenCount": 7,
+ },
+ },
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ ]
+ with patch(
+ "agentscope.realtime._gemini._model.uuid.uuid4",
+ return_value=SimpleNamespace(hex="r1"),
+ ):
+ actual = [_dump(model._parse(frame)) for frame in messages]
+ self.assertEqual(
+ actual,
+ [
+ [("ResponseCreatedEvent", {"item_id": "r1"})],
+ [],
+ [
+ (
+ "AudioDeltaEvent",
+ {
+ "item_id": "r1",
+ "pcm": b"\x01\x00",
+ "sample_rate": 24000,
+ },
+ ),
+ (
+ "TranscriptDeltaEvent",
+ {"item_id": "r1", "delta": "Checking."},
+ ),
+ ],
+ [
+ (
+ "ToolCallEvent",
+ {
+ "item_id": "r1",
+ "non_blocking": True,
+ "tool_call": {
+ "type": "tool_call",
+ "id": "c1",
+ "name": "get_weather",
+ "input": "{}",
+ "state": "pending",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ },
+ ),
+ ],
+ [],
+ [
+ (
+ "TranscriptDeltaEvent",
+ {"item_id": "r1", "delta": "Sunny."},
+ ),
+ (
+ "ResponseDoneEvent",
+ {
+ "item_id": "r1",
+ "input_tokens": 12,
+ "output_tokens": 7,
+ },
+ ),
+ ],
+ [],
+ ],
+ )
+ self.assertEqual((model._response_id, model._usage), ("", {}))
+
+ def test_status_only_completion_and_interruption(self) -> None:
+ """Root/content statuses finish silent work and reset on barge-in."""
+ for at_root in (False, True):
+ for interrupted in (False, True):
+ with self.subTest(root=at_root, interrupted=interrupted):
+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)
+ content: dict = {"interactionStatus": "IN_PROGRESS"}
+ frame: dict = (
+ content if at_root else {"serverContent": content}
+ )
+ opened = model._parse(frame)
+ reply_id = opened[0].item_id
+ if interrupted:
+ events = model._parse(
+ {"serverContent": {"interrupted": True}},
+ )
+ self.assertEqual(
+ _dump(events),
+ [
+ (
+ "SpeechStartedEvent",
+ {
+ "item_id": AnyString(),
+ "at_ms": 0,
+ },
+ ),
+ ],
+ )
+ content = {"interactionStatus": "IDLE"}
+ frame = content if at_root else {"serverContent": content}
+ self.assertEqual(
+ _dump(model._parse(frame)),
+ []
+ if interrupted
+ else [
+ (
+ "ResponseDoneEvent",
+ {
+ "item_id": reply_id,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ },
+ ),
+ ],
+ )
+ self.assertEqual(model._parse(frame), [])
+ next_turn = model._parse(
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Next"},
+ },
+ },
+ )
+ self.assertEqual(
+ _dump(next_turn),
+ [
+ (
+ "ResponseCreatedEvent",
+ {
+ "item_id": AnyString(),
+ },
+ ),
+ (
+ "TranscriptDeltaEvent",
+ {
+ "item_id": next_turn[0].item_id,
+ "delta": "Next",
+ },
+ ),
+ ],
+ )
+ self.assertNotEqual(next_turn[0].item_id, reply_id)
+
+ def test_same_frame_transcription_and_status(self) -> None:
+ """Open status-only responses before input, except on interruption."""
+ for at_root in (False, True):
+ for interrupted in (False, True):
+ with self.subTest(root=at_root, interrupted=interrupted):
+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)
+ content: dict = {
+ "inputTranscription": {"text": "Weather?"},
+ "interrupted": interrupted,
+ }
+ frame: dict = {"serverContent": content}
+ status_target = frame if at_root else content
+ status_target["interactionStatus"] = "IN_PROGRESS"
+ events = model._parse(frame)
+ response_id = model._response_id
+ expected = [
+ (
+ "InputTranscriptionDeltaEvent",
+ {
+ "item_id": model._user_id,
+ "delta": "Weather?",
+ "response_id": response_id,
+ },
+ ),
+ ]
+ if interrupted:
+ self.assertEqual(response_id, "")
+ expected.append(
+ (
+ "SpeechStartedEvent",
+ {"item_id": model._user_id, "at_ms": 0},
+ ),
+ )
+ else:
+ self.assertTrue(response_id)
+ expected.insert(
+ 0,
+ ("ResponseCreatedEvent", {"item_id": response_id}),
+ )
+ self.assertEqual(_dump(events), expected)
+
+ def test_same_frame_transcription_and_output(self) -> None:
+ """Bind same-frame input to output unless the frame interrupts it."""
+ outputs: list[dict] = [
+ {"outputTranscription": {"text": "Sunny."}},
+ {"modelTurn": {"parts": [{"text": "Sunny."}]}},
+ {
+ "modelTurn": {
+ "parts": [
+ {
+ "inlineData": {
+ "data": base64.b64encode(b"\x01\x00").decode(),
+ },
+ },
+ ],
+ },
+ },
+ ]
+ for output in outputs:
+ for interrupted in (False, True):
+ with self.subTest(output=output, interrupted=interrupted):
+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)
+ events = model._parse(
+ {
+ "serverContent": {
+ **output,
+ "inputTranscription": {"text": "Weather?"},
+ "interrupted": interrupted,
+ },
+ },
+ )
+ response_id = model._response_id
+ expected = [
+ (
+ "InputTranscriptionDeltaEvent",
+ {
+ "item_id": model._user_id,
+ "delta": "Weather?",
+ "response_id": response_id,
+ },
+ ),
+ ]
+ if interrupted:
+ expected.append(
+ (
+ "SpeechStartedEvent",
+ {"item_id": model._user_id, "at_ms": 0},
+ ),
+ )
+ self.assertEqual(response_id, "")
+ else:
+ self.assertTrue(response_id)
+ expected.insert(
+ 0,
+ ("ResponseCreatedEvent", {"item_id": response_id}),
+ )
+ if output == outputs[-1]:
+ expected.append(
+ (
+ "AudioDeltaEvent",
+ {
+ "item_id": response_id,
+ "pcm": b"\x01\x00",
+ "sample_rate": 24000,
+ },
+ ),
+ )
+ else:
+ expected.append(
+ (
+ "TranscriptDeltaEvent",
+ {
+ "item_id": response_id,
+ "delta": "Sunny.",
+ },
+ ),
+ )
+ self.assertEqual(_dump(events), expected)
+
+ def test_late_transcription_and_silent_completion(self) -> None:
+ """Thinking before transcription and silent replies retain input."""
+ for starts_thinking in (False, True):
+ with self.subTest(starts_thinking=starts_thinking):
+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)
+ if starts_thinking:
+ model._parse({"interactionStatus": "IN_PROGRESS"})
+ reply_id = model._response_id
+ for text in ("Check ", "weather"):
+ events = model._parse(
+ {
+ "serverContent": {
+ "inputTranscription": {"text": text},
+ },
+ },
+ )
+ self.assertEqual(
+ _dump(events),
+ [
+ (
+ "InputTranscriptionDeltaEvent",
+ {
+ "item_id": model._user_id,
+ "delta": text,
+ "response_id": reply_id,
+ },
+ ),
+ ],
+ )
+ expected = [
+ (
+ "InputTranscriptionEvent",
+ {
+ "item_id": model._user_id,
+ "text": "Check weather",
+ },
+ ),
+ ]
+ if starts_thinking:
+ expected.append(
+ (
+ "ResponseDoneEvent",
+ {
+ "item_id": reply_id,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ },
+ ),
+ )
+ self.assertEqual(
+ _dump(model._parse({"interactionStatus": "IDLE"})),
+ expected,
+ )
+ self.assertEqual(
+ (model._input_text, model._user_id, model._response_id),
+ ("", "", ""),
+ )
+
+
class GeminiParseTest(unittest.TestCase):
"""Server messages -> model events."""
@@ -286,6 +790,7 @@ class GeminiParseTest(unittest.TestCase):
"ToolCallEvent",
{
"item_id": AnyString(),
+ "non_blocking": False,
"tool_call": {
"type": "tool_call",
"id": "c1",
@@ -394,6 +899,407 @@ class GeminiSendTest(IsolatedAsyncioTestCase):
self.model._send = capture # type: ignore[method-assign]
+ async def test_tool_result_state_and_scheduling(self) -> None:
+ """All Gemini models encode errors; only Thinking schedules output."""
+ names = (
+ THINKING_MODEL,
+ MODEL,
+ "gemini-3.8-live",
+ "gemini-2.5-flash-native-audio-preview-12-2025",
+ )
+ for name, state, silent in product(
+ names,
+ ToolResultState,
+ (False, True),
+ ):
+ with self.subTest(model=name, state=state, silent=silent):
+ model = GeminiRealtimeModel(name, CRED)
+ thinking = name == THINKING_MODEL
+ model._send = AsyncMock()
+ if thinking:
+ model._pending_tool_calls.add("c1")
+ block = ToolResultBlock(
+ id="c1",
+ name="weather",
+ output="result",
+ state=state,
+ )
+ if silent:
+ await model.push_tool_result_silently(block)
+ else:
+ await model.push_tool_result(block)
+ failed = state in (
+ ToolResultState.ERROR,
+ ToolResultState.DENIED,
+ ToolResultState.INTERRUPTED,
+ )
+ response: dict = {
+ "id": "c1",
+ "name": "weather",
+ "response": {
+ "error" if failed else "output": "result",
+ },
+ }
+ if thinking and silent:
+ response["scheduling"] = "SILENT"
+ self.assertEqual(
+ {
+ "frames": [
+ call.args[0]
+ for call in model._send.await_args_list
+ ],
+ "awaiting": model._awaiting_tool_reply,
+ "pending": model._pending_tool_calls,
+ },
+ {
+ "frames": [
+ {
+ "toolResponse": {
+ "functionResponses": [response],
+ },
+ },
+ ],
+ "awaiting": thinking and not silent,
+ "pending": set(),
+ },
+ )
+
+ async def test_silent_result_preserves_newer_followup(self) -> None:
+ """An old result cannot clear another tool's pending response."""
+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)
+ model._send = AsyncMock()
+ model._awaiting_tool_reply = True
+ model._pending_tool_calls.add("new")
+ await model.push_tool_result_silently(
+ ToolResultBlock(id="old", name="weather", output="done"),
+ )
+ self.assertEqual(
+ {
+ "awaiting": model._awaiting_tool_reply,
+ "pending": model._pending_tool_calls,
+ "frame": model._send.await_args.args[0],
+ },
+ {
+ "awaiting": True,
+ "pending": {"new"},
+ "frame": {
+ "toolResponse": {
+ "functionResponses": [
+ {
+ "id": "old",
+ "name": "weather",
+ "response": {"output": "done"},
+ "scheduling": "SILENT",
+ },
+ ],
+ },
+ },
+ },
+ )
+
+ async def test_transcription_tracks_first_background_response(
+ self,
+ ) -> None:
+ """A gap after multiple responses still refers to the first one."""
+ model = GeminiRealtimeModel(THINKING_MODEL, CRED)
+ model._send = AsyncMock()
+ first_response = ""
+ model._parse(
+ {"serverContent": {"inputTranscription": {"text": "Go "}}},
+ )
+ user_id = model._user_id
+ for call_id in ("c1", "c2"):
+ model._parse(
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Checking."},
+ },
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": call_id,
+ "name": "weather",
+ "args": {},
+ },
+ ],
+ },
+ },
+ )
+ if not first_response:
+ first_response = model._response_id
+ else:
+ self.assertNotEqual(model._response_id, first_response)
+ closed = model._parse(
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ )
+ self.assertEqual(
+ (
+ [type(event).__name__ for event in closed],
+ model._user_id,
+ model._input_text,
+ ),
+ (["ResponseDoneEvent"], user_id, "Go "),
+ )
+ await model.push_tool_result(
+ ToolResultBlock(id=call_id, name="weather", output="done"),
+ )
+ events = model._parse(
+ {"serverContent": {"inputTranscription": {"text": "there"}}},
+ )
+ self.assertEqual(
+ _dump(events),
+ [
+ (
+ "InputTranscriptionDeltaEvent",
+ {
+ "item_id": user_id,
+ "delta": "there",
+ "response_id": first_response,
+ },
+ ),
+ ],
+ )
+ completed = model._parse(
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ )
+ self.assertEqual(
+ _dump(completed),
+ [
+ (
+ "ResponseCreatedEvent",
+ {"item_id": completed[0].item_id},
+ ),
+ (
+ "InputTranscriptionEvent",
+ {"item_id": user_id, "text": "Go there"},
+ ),
+ (
+ "ResponseDoneEvent",
+ {
+ "item_id": completed[0].item_id,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ },
+ ),
+ ],
+ )
+
+ async def test_transcription_association_cleared_at_boundaries(
+ self,
+ ) -> None:
+ """New input cannot inherit a completed or interrupted interaction."""
+ for boundary in (
+ "completed",
+ "cancel",
+ "provider_interrupt",
+ "new_turn",
+ "text",
+ "close",
+ ):
+ with self.subTest(boundary=boundary):
+ model = GeminiRealtimeModel(
+ THINKING_MODEL,
+ CRED,
+ parameters=GeminiRealtimeModel.Parameters(
+ turn_detection="none",
+ ),
+ )
+ model._send = AsyncMock()
+ model._parse(
+ {
+ "serverContent": {
+ "outputTranscription": {"text": "Checking."},
+ },
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": "c1",
+ "name": "weather",
+ "args": {},
+ },
+ ],
+ },
+ },
+ )
+ if boundary != "close":
+ model._parse(
+ {
+ "serverContent": {
+ "inputTranscription": {
+ "text": "Old request",
+ },
+ },
+ },
+ )
+ old_user_id = model._user_id
+ model._parse({"serverContent": {"interactionStatus": "IDLE"}})
+ self.assertTrue(model._transcription_response_id)
+ settled = (
+ "InputTranscriptionEvent",
+ {"item_id": old_user_id, "text": "Old request"},
+ )
+ content: dict = {"inputTranscription": {"text": "Next"}}
+ if boundary == "completed":
+ await model.push_tool_result(
+ ToolResultBlock(id="c1", name="weather", output="ok"),
+ )
+ model._parse(
+ {
+ "serverContent": {"interactionStatus": "IDLE"},
+ },
+ )
+ elif boundary == "cancel":
+ await model.cancel_response()
+ elif boundary == "provider_interrupt":
+ content["interrupted"] = True
+ elif boundary == "new_turn":
+ await model.start_turn()
+ elif boundary == "text":
+ await model.push_text("New request")
+ else:
+ await model.close()
+ events = model._parse({"serverContent": content})
+ self.assertNotEqual(model._user_id, old_user_id)
+ expected = [
+ (
+ "InputTranscriptionDeltaEvent",
+ {
+ "item_id": model._user_id,
+ "delta": "Next",
+ "response_id": "",
+ },
+ ),
+ ]
+ if boundary == "provider_interrupt":
+ expected.insert(0, settled)
+ expected.append(
+ (
+ "SpeechStartedEvent",
+ {
+ "item_id": model._user_id,
+ "at_ms": 0,
+ },
+ ),
+ )
+ self.assertEqual(_dump(events), expected)
+ self.assertEqual(model._transcription_response_id, "")
+ queued = []
+ while not model._queue.empty():
+ queued.append(model._queue.get_nowait())
+ self.assertEqual(
+ _dump(queued),
+ [settled]
+ if boundary in ("cancel", "new_turn", "text")
+ else [],
+ )
+
+ async def test_local_cancel_settles_thinking_input(self) -> None:
+ """Local VAD interruption must separate successive input turns."""
+ model = GeminiRealtimeModel(
+ THINKING_MODEL,
+ CRED,
+ parameters=GeminiRealtimeModel.Parameters(turn_detection="none"),
+ )
+ model._send = AsyncMock()
+ setup = model._setup("", None, "")["setup"]
+ self.assertEqual(
+ setup["realtimeInputConfig"]["activityHandling"],
+ "NO_INTERRUPTION",
+ )
+ first = model._parse(
+ {
+ "serverContent": {
+ "inputTranscription": {"text": "Check weather"},
+ "interactionStatus": "IN_PROGRESS",
+ },
+ },
+ )
+ user_id = first[1].item_id
+ await model.cancel_response()
+ await model.cancel_response()
+ settled = []
+ while not model._queue.empty():
+ settled.append(model._queue.get_nowait())
+ self.assertEqual(
+ _dump(settled),
+ [
+ (
+ "InputTranscriptionEvent",
+ {"item_id": user_id, "text": "Check weather"},
+ ),
+ ],
+ )
+ second = model._parse(
+ {"serverContent": {"inputTranscription": {"text": "Change "}}},
+ )
+ await model.cancel_response()
+ self.assertTrue(model._queue.empty())
+ stale = model._parse({"serverContent": {"interactionStatus": "IDLE"}})
+ self.assertEqual(
+ _dump(stale),
+ [
+ (
+ "ResponseDoneEvent",
+ {
+ "item_id": first[0].item_id,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ },
+ ),
+ ],
+ )
+ self.assertEqual(
+ model._parse({"serverContent": {"interactionStatus": "IDLE"}}),
+ [],
+ )
+ self.assertEqual(
+ (model._user_id, model._input_text),
+ (second[0].item_id, "Change "),
+ )
+ second += model._parse(
+ {"serverContent": {"inputTranscription": {"text": "city"}}},
+ )
+ self.assertNotEqual(second[0].item_id, user_id)
+ self.assertEqual(
+ _dump(second),
+ [
+ (
+ "InputTranscriptionDeltaEvent",
+ {
+ "item_id": second[0].item_id,
+ "delta": text,
+ "response_id": "",
+ },
+ )
+ for text in ("Change ", "city")
+ ],
+ )
+ opened = model._parse(
+ {"serverContent": {"interactionStatus": "IN_PROGRESS"}},
+ )
+ self.assertEqual(
+ _dump(
+ model._parse(
+ {"serverContent": {"interactionStatus": "IDLE"}},
+ ),
+ ),
+ [
+ (
+ "InputTranscriptionEvent",
+ {"item_id": second[0].item_id, "text": "Change city"},
+ ),
+ (
+ "ResponseDoneEvent",
+ {
+ "item_id": opened[0].item_id,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ },
+ ),
+ ],
+ )
+ model._send.assert_not_awaited()
+
async def test_text_turn_and_tool_result(self) -> None:
"""A text turn is complete on arrival; a tool result resumes the
model without asking for a response."""
@@ -483,6 +1389,57 @@ class GeminiSendTest(IsolatedAsyncioTestCase):
class GeminiDisconnectTest(IsolatedAsyncioTestCase):
"""A closed WebSocket surfaces as ModelDisconnectedError."""
+ async def test_thinking_endpoint_and_reconnect(self) -> None:
+ """Thinking uses alpha; ordinary Live retains its beta endpoint."""
+ for name, version in (
+ (THINKING_MODEL, "v1alpha"),
+ ("custom-thinking-alias", "v1alpha"),
+ (MODEL, "v1beta"),
+ ):
+ with self.subTest(model=name):
+ original = THINKING_MODEL if version == "v1alpha" else MODEL
+ card = GeminiRealtimeModel(original, CRED).card.model_copy(
+ update={"name": name},
+ )
+ model = GeminiRealtimeModel(name, CRED, model_card=card)
+ socket = AsyncMock()
+
+ async def acknowledge(
+ session: GeminiRealtimeModel = model,
+ ) -> None:
+ """Acknowledge setup and keep the reader alive."""
+ session._ready.set()
+ await asyncio.Event().wait()
+
+ with patch(
+ "websockets.connect",
+ new_callable=AsyncMock,
+ return_value=socket,
+ ) as connect, patch.object(model, "_read", acknowledge):
+ model._response_id = "old-response"
+ model._transcription_response_id = "old-interaction"
+ model._usage = {"responseTokenCount": 99}
+ model._awaiting_tool_reply = True
+ model._pending_tool_calls.add("old-call")
+ await model.connect("hi")
+ self.assertEqual(
+ (
+ model._response_id,
+ model._transcription_response_id,
+ model._usage,
+ model._awaiting_tool_reply,
+ model._pending_tool_calls,
+ ),
+ ("", "", {}, False, set()),
+ )
+ connect.assert_awaited_once_with(
+ "wss://generativelanguage.googleapis.com/ws/"
+ f"google.ai.generativelanguage.{version}."
+ "GenerativeService.BidiGenerateContent?key=key-x",
+ )
+ await model.close()
+ socket.close.assert_awaited_once_with()
+
async def test_send_on_closed_socket(self) -> None:
"""websockets' ConnectionClosed becomes the realtime-level error
and the socket reference is dropped."""
diff --git a/tests/realtime_openai_test.py b/tests/realtime_openai_test.py
index 6af825114..04eabea2c 100644
--- a/tests/realtime_openai_test.py
+++ b/tests/realtime_openai_test.py
@@ -444,6 +444,7 @@ class OpenAIParseTest(unittest.TestCase):
"created_at": AnyString(),
"finished_at": None,
},
+ "non_blocking": False,
},
)
diff --git a/tests/realtime_xai_test.py b/tests/realtime_xai_test.py
index 207163b45..211b9f4b1 100644
--- a/tests/realtime_xai_test.py
+++ b/tests/realtime_xai_test.py
@@ -299,6 +299,7 @@ class XAIParseTest(unittest.TestCase):
"created_at": AnyString(),
"finished_at": None,
},
+ "non_blocking": False,
},
{
"item_id": "resp_001",
@@ -312,6 +313,7 @@ class XAIParseTest(unittest.TestCase):
"created_at": AnyString(),
"finished_at": None,
},
+ "non_blocking": False,
},
],
)
diff --git a/tests/webrtc_audio_transport_test.py b/tests/webrtc_audio_transport_test.py
index 62121a6c0..1145af698 100644
--- a/tests/webrtc_audio_transport_test.py
+++ b/tests/webrtc_audio_transport_test.py
@@ -19,7 +19,7 @@ from aiortc.mediastreams import MediaStreamError
from av import AudioFrame as AVAudioFrame
from fastapi import HTTPException
-from agentscope.agent import RealtimeAgent
+from agentscope.agent import RealtimeAgent, TurnAggregator
from agentscope.credential import GeminiCredential
from agentscope.app._router._realtime import _create_realtime_offer
from agentscope.app._router._schema import (
@@ -1103,6 +1103,60 @@ class RealtimeOfferTest(unittest.IsolatedAsyncioTestCase):
class WebRTCSessionTest(unittest.IsolatedAsyncioTestCase):
"""Verify that a WebRTC run keeps the normal session contract."""
+ async def test_final_persistence_excludes_unfinished_transcription(
+ self,
+ ) -> None:
+ """Direct state persistence after close must not save fragments."""
+ agent = RealtimeAgent(
+ "Friday",
+ "",
+ MagicMock(
+ spec=RealtimeModelBase,
+ output_sample_rate=24000,
+ input_transcription_enabled=True,
+ ),
+ aggregator=TurnAggregator(min_chars=4),
+ )
+ storage = _FakeStorage()
+ session = _make_session(
+ agent_factory=AsyncMock(),
+ storage=storage,
+ message_bus=_FakeMessageBus(),
+ on_closed=lambda _: None,
+ )
+ session.agent = agent
+ await agent._on_model_event(
+ me.InputTranscriptionEvent(item_id="kept", text="accepted"),
+ )
+ accepted = agent.state.context[0].model_dump(mode="json")
+ await agent._on_model_event(
+ me.InputTranscriptionDeltaEvent(item_id="u1", delta="a"),
+ )
+ agent._retire_unbound_user_turns()
+ self.assertEqual(
+ [(msg.id, msg.get_text_content()) for msg in agent.state.context],
+ [("kept", "accepted"), ("u1", "a")],
+ )
+ await agent.close()
+ await session._persist_state()
+ self.assertEqual(
+ {
+ "messages": {
+ key: value.model_dump(mode="json")
+ for key, value in storage.messages.items()
+ },
+ "states": [
+ [
+ msg.model_dump(mode="json")
+ for msg in call["state"].context
+ ]
+ for call in storage.calls
+ if call["method"] == "update_session_state"
+ ],
+ },
+ {"messages": {"kept": accepted}, "states": [[accepted]]},
+ )
+
async def test_sampled_live_checkpoint_persists_immediately(self) -> None:
"""Persist a sampled transcript snapshot at its published cursor."""