Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
refactor(realtime): align RealtimeAgent naming with Agent (#2550)
DavdGao1aa0417f2c3b
diff --git a/examples/realtime/README.md b/examples/realtime/README.mdindex 33cf864ce..e72932adf 100644--- a/examples/realtime/README.md+++ b/examples/realtime/README.md@@ -11,7 +11,7 @@ Talk to a DashScope speech-to-speech model through your microphone.tool calls with permission prompts, per-turn latency metrics.The agent owns the model session, you own the transport, and one-`agent.run(transport)` borrows both until the transport ends.+`agent.reply_stream(transport)` borrows both until the transport ends.## Prerequisites@@ -82,9 +82,9 @@ agent gives up on a prompt after five minutes.- **Silence is fine.** DashScope closes a session after about three minuteswithout a response. The example logs that it happened and keeps the- microphone open; the next thing you say reconnects the model. The- conversation history stays in `agent.state`, but the provider starts fresh —- it cannot be seeded with prior turns.+ microphone open; the next thing you say reconnects the model, with the+ transcript so far appended to the system prompt so the model keeps the+ thread.- **Model limits are per turn, not per token.** `qwen3-omni-flash-realtime`remembers eight turns, `qwen-audio-3.0-realtime-*` fifty; older turns aredropped silently on the provider side. The cards underdiff --git a/examples/realtime/local_mic.py b/examples/realtime/local_mic.pyindex d4d91f7d3..a25115110 100644--- a/examples/realtime/local_mic.py+++ b/examples/realtime/local_mic.py@@ -32,8 +32,6 @@ from agentscope.event import (ToolResultEndEvent,ToolResultStartEvent,UserConfirmResultEvent,- UserInputAudioStartEvent,- UserInputTranscriptionEvent,)from agentscope.realtime import LocalAudioTransportfrom agentscope.tool import Bash, Edit, Read, Toolkit, Write@@ -59,7 +57,7 @@ async def main() -> None:agent = RealtimeAgent(name="Friday",- sys_prompt="你是一个中文语音助手,回答尽量简短。",+ system_prompt="你是一个中文语音助手,回答尽量简短。",model=model,toolkit=Toolkit(tools=[Bash(), Edit(), Write(), Read()]),)@@ -72,16 +70,20 @@ async def main() -> None:print(f"[{name}] listening... (Ctrl-C to quit)")# The agent owns the model session, we own the transport, and one- # run() borrows both until the transport ends.+ # reply_stream() borrows both until the transport ends.+ # The user's speech is reported as a reply too, with role "user":+ # its transcript arrives as text block events once it has settled.+ user_turns: set[str] = set()async with agent, transport:- async for event in agent.run(transport):+ async for event in agent.reply_stream(transport):match event:- case UserInputAudioStartEvent():+ case ReplyStartEvent(role="user"):+ user_turns.add(event.reply_id)print("\n[you] ...", end="", flush=True)- case UserInputTranscriptionEvent():- print(f"\r[you] {event.transcript}")case ReplyStartEvent():- print(f"[{agent.name}] ", end="", flush=True)+ print(f"\n[{agent.name}] ", end="", flush=True)+ case TextBlockDeltaEvent() if event.reply_id in user_turns:+ print(f"\r[you] {event.delta}")case TextBlockDeltaEvent():print(event.delta, end="", flush=True)case RequireUserConfirmEvent():@@ -90,7 +92,7 @@ async def main() -> None:print(f"\n [tool] {event.tool_call_name} running...")case ToolResultEndEvent():print(f" [tool] {event.state}")- case ReplyEndEvent():+ case ReplyEndEvent() if event.reply_id not in user_turns:m = agent.last_turn_metricsprint(f"\n ({event.finished_reason}"diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.pyindex ba3cb228b..6d8ddc54d 100644--- a/src/agentscope/agent/_realtime/_agent.py+++ b/src/agentscope/agent/_realtime/_agent.py@@ -26,7 +26,6 @@ from ...event import (DataBlockDeltaEvent,DataBlockEndEvent,DataBlockStartEvent,- ExternalExecutionResultEvent,ModelCallEndEvent,ModelCallStartEvent,ReplyEndEvent,@@ -41,8 +40,6 @@ from ...event import (ToolResultStartEvent,ToolResultTextDeltaEvent,UserConfirmResultEvent,- UserInputAudioStartEvent,- UserInputTranscriptionEvent,UserInterruptEvent,)from ...message import (@@ -51,6 +48,7 @@ from ...message import (ToolCallBlock,ToolResultBlock,ToolResultState,+ Usage,UserMsg,)from ...permission import PermissionBehavior, PermissionEngine@@ -68,9 +66,16 @@ class _Reply:to work out what the user actually heard."""item_id: str+ """The provider's response item; what playout and truncation refer to."""+ reply_id: str+ """The agent's reply this response belongs to, as seen in events and+ context. A reply spans every response up to the next user turn, so+ one that calls tools has several responses."""text_block_id: str = field(default_factory=lambda: uuid.uuid4().hex)audio_block_id: str = field(default_factory=lambda: uuid.uuid4().hex)text: str = ""+ final_text: str | None = None+ """Set on a barge-in: the text block is truncated to this."""audio_ms: float = 0.0text_started: bool = Falseaudio_started: bool = False@@ -105,14 +110,14 @@ class RealtimeAgent:Unlike :class:`~agentscope.agent.Agent` it is bidirectional and has norequest/reply boundary — audio flows in continuously while events flow- out of :meth:`run`.+ out of :meth:`reply_stream`.Three lifetimes are kept apart on purpose. The agent owns the modelsession and the state; a transport is owned by whoever created it; a- :meth:`run` borrows both for as long as both are alive. So a client- can drop and reconnect without losing the model session, and the model- session can time out during a long silence and be re-established on- the next word without touching the transport.+ :meth:`reply_stream` borrows both for as long as both are alive. So a+ client can drop and reconnect without losing the model session, and+ the model session can time out during a long silence and be+ re-established on the next word without touching the transport.The public methods are the only entry point for discrete input. Atransport carrying a browser's ``ControlFrame`` calls exactly those@@ -126,14 +131,14 @@ class RealtimeAgent:agent = RealtimeAgent("Friday", "Be brief.", model)async with agent: # model sessionasync with LocalAudioTransport() as t: # sound card- async for event in agent.run(t): # this call+ async for event in agent.reply_stream(t): # this callprint(event)"""def __init__(self,name: str,- sys_prompt: str,+ system_prompt: str,model: RealtimeModelBase,toolkit: Toolkit | None = None,state: AgentState | None = None,@@ -145,7 +150,7 @@ class RealtimeAgent:Args:name (`str`):Display name stamped on assistant messages and events.- sys_prompt (`str`):+ system_prompt (`str`):System instructions sent to the model on connect, with thetoolkit's skill instructions appended.model (`RealtimeModelBase`):@@ -175,7 +180,7 @@ class RealtimeAgent:omitted."""self.name = name- self.sys_prompt = sys_prompt+ self.system_prompt = system_promptself.model = modelself.toolkit = toolkitself.state = state or AgentState()@@ -187,6 +192,13 @@ class RealtimeAgent:self._out: asyncio.Queue = asyncio.Queue()self._reply: _Reply | None = Noneself._finished_item = ""+ # The agent's open reply, and whether the next response continues+ # it (after tool results) rather than starting a new one.+ self._reply_id = ""+ self._continuing = False+ # The user's turn in flight, reported as a reply of its own.+ self._user_turn = ""+ self._user_turn_open = Falseself._metrics = TurnMetrics()self._pending_tools: dict[str, ToolCallBlock] = {}self._confirmations: dict[str, asyncio.Future[ConfirmResult]] = {}@@ -221,7 +233,7 @@ class RealtimeAgent:if self._connected:return- instructions = self.sys_prompt+ instructions = self.system_prompttools = Noneif self.toolkit is not None:groups = self.state.tool_context.activated_groups@@ -244,8 +256,19 @@ class RealtimeAgent:# context. No provider documents whether the update applies# retroactively, so treat it as affecting future turns only. Do# not let it change `voice`: OpenAI locks it after first audio.+ # Providers differ in whether prior turns can be seeded, so the+ # transcript so far rides along in the instructions, which every+ # provider takes. Only matters on reconnect; first connect is empty.+ history = "\n".join(+ f"{m.name}: {text}"+ for m in self.state.context+ if (text := m.get_text_content())+ )+ if history:+ instructions = (+ f"{instructions}\n\n## Conversation so far\n{history}"+ )await self.model.connect(- context=self.state.context,instructions=instructions,tools=tools,turn_detection_disabled=self.vad is not None,@@ -313,7 +336,10 @@ class RealtimeAgent:# Lifecycle: one transport# ------------------------------------------------------------------- async def run(self, transport: TransportBase) -> AsyncIterator[AgentEvent]:+ async def reply_stream(+ self,+ transport: TransportBase,+ ) -> AsyncIterator[AgentEvent]:"""Pump *transport* and yield agent events until it ends.The transport is borrowed, not owned: it must already be started@@ -327,7 +353,7 @@ class RealtimeAgent:matters."""if self._transport is not None:- raise RuntimeError("RealtimeAgent.run is already active.")+ raise RuntimeError("RealtimeAgent.reply_stream is already active.")# Audio is forwarded as-is in both directions, so the rates must# already agree; resampling belongs to the transport, not here.pairs = {@@ -387,24 +413,17 @@ class RealtimeAgent:async def send(self,- inputs: (- str- | Msg- | UserConfirmResultEvent- | UserInterruptEvent- | ExternalExecutionResultEvent- ),+ inputs: (str | Msg | UserConfirmResultEvent | UserInterruptEvent),) -> None:"""Feed the agent anything that is not audio.Mirrors :meth:`Agent.reply`'s ``inputs``: a text turn (which cutsoff any reply in progress first), the outcome of a permission- prompt, an interrupt, or the result of a tool run elsewhere.+ prompt, or an interrupt.Raises:`NotImplementedError`: For a text turn when the provider takes- no text input, or for external execution results, which- are not supported yet.+ no text input."""match inputs:case UserInterruptEvent():@@ -416,11 +435,6 @@ class RealtimeAgent:if future and not future.done():future.set_result(result)- case ExternalExecutionResultEvent():- raise NotImplementedError(- "External tool execution is not supported yet.",- )-case str() | Msg():if not self.model.supports_text_input:raise NotImplementedError(@@ -497,8 +511,10 @@ class RealtimeAgent:pushed = Trueif speech is SpeechTransition.STARTED:+ self._start_user_turn()await self._barge_in()elif speech is SpeechTransition.ENDED:+ self._end_user_turn()now = time.monotonic()self._metrics.user_speech_end_at = nowawait self.model.commit_turn()@@ -516,8 +532,6 @@ class RealtimeAgent:await self.send(UserConfirmResultEvent(**frame.data))case ControlFrameType.INTERRUPT:await self.send(UserInterruptEvent())- case ControlFrameType.EXTERNAL_EXECUTION_RESULT:- await self.send(ExternalExecutionResultEvent(**frame.data))case _:logger.debug("RealtimeAgent: ignoring %s frame", frame.type)@@ -533,8 +547,8 @@ class RealtimeAgent:detection, by the VAD's own debounce otherwise.Reached from the uplink pump, the downlink pump, :meth:`send` and- :meth:`run`'s exit, so it is serialised; the losers find the reply- already closed and return.+ the exit of :meth:`reply_stream`, so it is serialised; the losers+ find the reply already closed and return."""async with self._barge_lock:await self._barge_in_locked()@@ -543,6 +557,8 @@ class RealtimeAgent:"""Body of :meth:`_barge_in`, run under the lock."""reply = self._replyif reply is None:+ # Nothing playing, but a reply may be waiting on its tools.+ self._finish_reply(ReplyFinishedReason.INTERRUPTED)returnspoken, played_ms = (@@ -563,6 +579,7 @@ class RealtimeAgent:spoken = reply.spoken_prefix(played_ms)self._truncate_reply(spoken)+ reply.final_text = spokenif self._connected:await self.model.truncate(reply.item_id, played_ms, spoken)await self.model.cancel_response()@@ -619,15 +636,11 @@ class RealtimeAgent:rate = self.model.output_sample_ratematch event:case me.SpeechStartedEvent():- self._emit(- UserInputAudioStartEvent(- session_id=self.state.session_id,- item_id=event.item_id,- ),- )+ self._start_user_turn(event.item_id)await self._barge_in()case me.SpeechEndedEvent():+ self._end_user_turn()# With provider turn detection this is also its commit.now = time.monotonic()self._metrics.user_speech_end_at = now@@ -664,12 +677,27 @@ class RealtimeAgent:case me.ToolCallEvent():if self._start_reply(event.item_id) is not None:self._pending_tools[event.tool_call.id] = event.tool_call+ self.state.append_context(self.name, [event.tool_call])case me.ResponseDoneEvent():self._metrics.input_tokens = event.input_tokensself._metrics.output_tokens = event.output_tokens- self._finish_reply(ReplyFinishedReason.COMPLETED)- self._schedule_tools()+ tail = self.state.context[-1] if self.state.context else None+ if tail is not None and tail.id == self._reply_id:+ tail.append_usage(+ Usage(+ input_tokens=event.input_tokens,+ output_tokens=event.output_tokens,+ ),+ )+ if self._pending_tools and self.toolkit is not None:+ # The reply goes on: tools run, then the next response+ # answers with their results.+ self._finish_response()+ self._continuing = True+ self._schedule_tools()+ else:+ self._finish_reply(ReplyFinishedReason.COMPLETED)case me.ModelErrorEvent():logger.error(@@ -680,28 +708,72 @@ class RealtimeAgent:self._finish_reply(ReplyFinishedReason.ERROR)case me.SessionEndedEvent():- pass # the events() iterator ends right after this+ # The events() iterator ends right after this.+ logger.info(+ "RealtimeAgent: model closed the session (%s)",+ event.reason,+ )++ def _start_user_turn(self, item_id: str = "") -> None:+ """Open the user's turn as a reply of its own. A local VAD sees it+ before the provider creates an item, hence the generated id."""+ if self._user_turn_open:+ return # providers repeat speech_started; one turn, one reply+ self._user_turn = item_id or uuid.uuid4().hex+ self._user_turn_open = True+ self._emit(+ ReplyStartEvent(+ session_id=self.state.session_id,+ reply_id=self._user_turn,+ name="user",+ role="user",+ ),+ )++ def _end_user_turn(self) -> None:+ """Close the user's turn; the transcript may still be on its way."""+ if not self._user_turn_open:+ return+ self._user_turn_open = False+ self._emit(+ ReplyEndEvent(+ session_id=self.state.session_id,+ reply_id=self._user_turn,+ finished_reason=ReplyFinishedReason.COMPLETED,+ ),+ )def _on_transcription(self, event: me.InputTranscriptionEvent) -> None:- """Record a settled user turn, merging a split one back together."""+ """Record a settled user turn, merging a split one back together,+ and report its text on the user's reply."""turn = self.aggregator.take(event.text)if turn is None:logger.debug("RealtimeAgent: dropping %r", event.text)return- if self.aggregator.merges_with_previous() and self._merge_user(turn):- transcript = self.state.context[-1].get_text_content() or turn- else:- self.state.context.append(UserMsg(name="user", content=turn))- transcript = turn-+ # A transcript with no detected speech, e.g. after a reconnect,+ # gets a turn of its own.+ if not self._user_turn:+ self._start_user_turn(event.item_id)+ reply_id, block_id = self._user_turn, uuid.uuid4().hex+ self._emit(TextBlockStartEvent(reply_id=reply_id, block_id=block_id))self._emit(- UserInputTranscriptionEvent(- session_id=self.state.session_id,- item_id=event.item_id,- transcript=transcript,+ TextBlockDeltaEvent(+ reply_id=reply_id,+ block_id=block_id,+ delta=turn,),)+ self._emit(TextBlockEndEvent(reply_id=reply_id, block_id=block_id))+ self._end_user_turn()+ self._user_turn = ""++ if not (+ self.aggregator.merges_with_previous() and self._merge_user(turn)+ ):+ self.state.context.append(+ UserMsg(name="user", content=turn, id=reply_id),+ )def _merge_user(self, text: str) -> bool:"""Append *text* to the previous user turn that endpointing split.@@ -712,7 +784,10 @@ class RealtimeAgent:"""context = self.state.contextif context and context[-1].role == "assistant":- if not (context[-1].get_text_content() or "").strip():+ tail = context[-1]+ if not (tail.get_text_content() or "").strip() and all(+ isinstance(b, TextBlock) for b in tail.content+ ):context.pop()if not context or context[-1].role != "user":return False@@ -721,7 +796,8 @@ class RealtimeAgent:return Truedef _start_reply(self, item_id: str) -> _Reply | None:- """Open a reply for *item_id*, emitting its start events once.+ """Open the response *item_id*, and the reply it belongs to unless+ one is waiting for it after tool results.Returns ``None`` for an item already closed — deltas still inflight after a barge-in must not reopen it.@@ -730,30 +806,37 @@ class RealtimeAgent:return self._replyif item_id == self._finished_item:return None-- self._reply = _Reply(item_id=item_id)- self.state.reply_id = item_id- self._metrics = TurnMetrics(- user_speech_end_at=self._metrics.user_speech_end_at,- turn_committed_at=self._metrics.turn_committed_at,- )- self._emit(- ReplyStartEvent(- session_id=self.state.session_id,- reply_id=item_id,- name=self.name,- ),- )+ if self._reply is not None:+ self._finish_response()++ if not self._continuing:+ # The agent takes the turn; the reply is named by its first+ # response so that every event of the turn shares one id.+ self._reply_id = item_id+ self.state.reply_id = item_id+ self._metrics = TurnMetrics(+ user_speech_end_at=self._metrics.user_speech_end_at,+ turn_committed_at=self._metrics.turn_committed_at,+ )+ self._emit(+ ReplyStartEvent(+ session_id=self.state.session_id,+ reply_id=item_id,+ name=self.name,+ ),+ )+ self._continuing = False+ self._reply = _Reply(item_id=item_id, reply_id=self._reply_id)self._emit(ModelCallStartEvent(- reply_id=item_id,+ reply_id=self._reply_id,model_name=self.model.model_name,),)return self._reply- def _finish_reply(self, reason: ReplyFinishedReason) -> None:- """Close the open reply, if any."""+ def _finish_response(self) -> None:+ """Close the open response: its blocks and the model call."""reply = self._replyif reply is None:return@@ -764,33 +847,42 @@ class RealtimeAgent:if reply.text_started:self._emit(TextBlockEndEvent(- reply_id=reply.item_id,+ reply_id=reply.reply_id,block_id=reply.text_block_id,+ text=reply.final_text,),)if reply.audio_started:self._emit(DataBlockEndEvent(- reply_id=reply.item_id,+ reply_id=reply.reply_id,block_id=reply.audio_block_id,),)self._emit(ModelCallEndEvent(- reply_id=reply.item_id,+ reply_id=reply.reply_id,input_tokens=self._metrics.input_tokens,output_tokens=self._metrics.output_tokens,),)+ self._finished_item = reply.item_id+ self._reply = None++ def _finish_reply(self, reason: ReplyFinishedReason) -> None:+ """Close the open reply, if any, response included."""+ self._finish_response()+ if not self._reply_id:+ returnself._emit(ReplyEndEvent(session_id=self.state.session_id,- reply_id=reply.item_id,+ reply_id=self._reply_id,finished_reason=reason,),)- self._finished_item = reply.item_id- self._reply = None+ self._reply_id = ""+ self._continuing = Falsedef _emit_text(self, reply: _Reply, delta: str) -> None:"""Emit a transcript delta, opening the block on first use."""@@ -798,7 +890,7 @@ class RealtimeAgent:reply.text_started = Trueself._emit(TextBlockStartEvent(- reply_id=reply.item_id,+ reply_id=reply.reply_id,block_id=reply.text_block_id,),)@@ -807,7 +899,7 @@ class RealtimeAgent:tail = self.state.context[-1] if self.state.context else Noneblocks = (tail.content- if tail is not None and tail.id == reply.item_id+ if tail is not None and tail.id == reply.reply_idelse None)if (@@ -820,7 +912,7 @@ class RealtimeAgent:self.state.append_context(self.name, [TextBlock(text=delta)])self._emit(TextBlockDeltaEvent(- reply_id=reply.item_id,+ reply_id=reply.reply_id,block_id=reply.text_block_id,delta=delta,),@@ -833,14 +925,14 @@ class RealtimeAgent:reply.audio_started = Trueself._emit(DataBlockStartEvent(- reply_id=reply.item_id,+ reply_id=reply.reply_id,block_id=reply.audio_block_id,media_type=media_type,),)self._emit(DataBlockDeltaEvent(- reply_id=reply.item_id,+ reply_id=reply.reply_id,block_id=reply.audio_block_id,data=base64.b64encode(pcm).decode("ascii"),media_type=media_type,@@ -848,7 +940,7 @@ class RealtimeAgent:)def _emit(self, event: AgentEvent) -> None:- """Queue one event for :meth:`run`."""+ """Queue one event for :meth:`reply_stream`."""self._out.put_nowait(event)# ------------------------------------------------------------------@@ -867,13 +959,17 @@ class RealtimeAgent:async def _run_tools(self, calls: list[ToolCallBlock]) -> None:"""Execute *calls* in order, then trigger the follow-up response."""- reply_id = self.state.reply_id+ reply_id = self._reply_idtry:for call in calls:await self._run_tool(reply_id, call)- await self.model.request_response()+ # Unless the user cut the reply short while the tools ran.+ if self._reply_id == reply_id:+ await self.model.request_response()except Exception: # noqa: BLE001logger.exception("RealtimeAgent: tool execution failed")+ if self._reply_id == reply_id:+ self._finish_reply(ReplyFinishedReason.ERROR)async def _run_tool(self, reply_id: str, call: ToolCallBlock) -> None:"""Check permission for one call, run it, and report the result."""@@ -1029,6 +1125,18 @@ class RealtimeAgent:state=state,),)- await self.model.push_tool_result(- ToolResultBlock(id=call.id, name=call.name, output=output),+ block = ToolResultBlock(+ id=call.id,+ name=call.name,+ output=output,+ state=state,)+ # The result belongs to the reply that made the call, which may no+ # longer be the current one if the user interrupted a slow tool.+ for msg in reversed(self.state.context):+ if msg.id == reply_id:+ msg.content.append(block)+ break+ else:+ self.state.append_context(self.name, [block])+ await self.model.push_tool_result(block)diff --git a/src/agentscope/event/__init__.py b/src/agentscope/event/__init__.pyindex dd9e40ac8..51632a731 100644--- a/src/agentscope/event/__init__.py+++ b/src/agentscope/event/__init__.py@@ -35,8 +35,6 @@ from ._event import (ExternalExecutionResultEvent,CustomEvent,AgentEvent,- UserInputAudioStartEvent,- UserInputTranscriptionEvent,ConfirmResult,)@@ -75,7 +73,5 @@ __all__ = ["ExternalExecutionResultEvent","CustomEvent","AgentEvent",- "UserInputAudioStartEvent",- "UserInputTranscriptionEvent","ConfirmResult",]diff --git a/src/agentscope/event/_event.py b/src/agentscope/event/_event.pyindex 8bce51cbd..40db34c7d 100644--- a/src/agentscope/event/_event.py+++ b/src/agentscope/event/_event.py@@ -64,9 +64,6 @@ class EventType(StrEnum):USER_INTERRUPT = "USER_INTERRUPT"EXTERNAL_EXECUTION_RESULT = "EXTERNAL_EXECUTION_RESULT"- USER_INPUT_AUDIO_START = "USER_INPUT_AUDIO_START"- USER_INPUT_TRANSCRIPTION = "USER_INPUT_TRANSCRIPTION"-CUSTOM = "CUSTOM"@@ -193,6 +190,9 @@ class TextBlockEndEvent(EventBase):"""ID of the reply message this block belongs to."""block_id: str"""Unique identifier of the text block."""+ text: str | None = None+ """The block's final text, when it is not the concatenation of the+ deltas: a voice reply cut short is truncated to what the user heard."""class DataBlockStartEvent(EventBase):@@ -565,34 +565,6 @@ class CustomEvent(EventBase):"""Arbitrary payload."""-class UserInputAudioStartEvent(EventBase):- """The user started speaking, as detected by VAD."""-- type: Literal[- EventType.USER_INPUT_AUDIO_START- ] = EventType.USER_INPUT_AUDIO_START- """Event type."""- session_id: str- """ID of the session this input belongs to."""- item_id: str- """ID of the user input item the provider assigned."""---class UserInputTranscriptionEvent(EventBase):- """The settled transcript of one spoken user turn."""-- type: Literal[- EventType.USER_INPUT_TRANSCRIPTION- ] = EventType.USER_INPUT_TRANSCRIPTION- """Event type."""- session_id: str- """ID of the session this input belongs to."""- item_id: str- """ID of the user input item the provider assigned."""- transcript: str- """What the user said."""--AgentEvent: TypeAlias = (ReplyStartEvent| ReplyEndEvent@@ -621,7 +593,5 @@ AgentEvent: TypeAlias = (| UserConfirmResultEvent| UserInterruptEvent| ExternalExecutionResultEvent- | UserInputAudioStartEvent- | UserInputTranscriptionEvent| CustomEvent)diff --git a/src/agentscope/message/_base.py b/src/agentscope/message/_base.pyindex 930c304bf..149f34d06 100644--- a/src/agentscope/message/_base.py+++ b/src/agentscope/message/_base.py@@ -307,6 +307,8 @@ class Msg(BaseModel):elif event.type == EventType.TEXT_BLOCK_DELTA:block.text += event.deltaelse:+ if event.text is not None:+ block.text = event.textblock.finished_at = event.created_atcase EventType.DATA_BLOCK_START:diff --git a/src/agentscope/realtime/DECISIONS.md b/src/agentscope/realtime/DECISIONS.mddeleted file mode 100644index d8b734a2b..000000000--- a/src/agentscope/realtime/DECISIONS.md+++ /dev/null@@ -1,57 +0,0 @@-# Realtime 设计决策记录--已定的事项不因单个提问重开;改变需要说明新事实。--## 已定--| 决策 | 依据 |-|---|---|-| 三层事实划分:类属性 = adapter 实现;card = 模型是什么;property = 各家归属不同的(采样率、turn detection) | 四家 API 调研:xAI 采样率可调、DashScope 固定 |-| `TruncationSupport` 三态 NONE / SERVER / EXPLICIT | OpenAI(WS)=EXPLICIT、Gemini=SERVER、DashScope/xAI=NONE |-| card 四个上下文限制字段并存,全部可选 | DashScope 按轮数+秒数、OpenAI/Gemini 按 token、同时生效 |-| 公开方法是离散输入的唯一入口,`ControlFrame` 是其线上形态 | livekit / openai SDK / pipecat / vocode 四家一致 |-| `_barge_in` 串行化,三个调用点共用一把锁 | pipecat CHANGELOG:4712 记录的双队列插队 bug |-| 复用 `AgentState`,不另建 `VoiceContext` | `append_context` 已覆盖进行中回合的追加语义 |-| 打断截断留在 `RealtimeAgent._truncate_reply`,不进 `AgentState` | 用户决定,不扩散到 `state/` |-| `SessionConfig` 二分:`chat`(Agent 驱动)/ `voice`(RealtimeAgent 驱动),内层再按 backend 判别 | 分类依据是驱动的运行时类,不是字段形状 |-| `TurnAggregator` 在 M1 做上下文合并那一半 | SERVER 模式无提交权,但脏历史会进持久化/重连/压缩 |-| M1 只做 DashScope,不做 OpenAI | 打断闭环除 provider RPC 外整条链 DashScope 都会跑到 |-| 不复用 `Agent` 实现级联 | 用户决定 |-| VAD 归 agent(`RealtimeAgent(vad=)`),不归 transport;无客户端上报的 speech 字段 | livekit 放 session 层、pipecat 主动从 transport 搬出、无一家采信浏览器 speech |-| 给了 `vad=` 就关 provider 的 turn detection,一个来源判回合;`TurnMode` 枚举删除,由 `vad is None` 推导 | livekit `_resolve_rt_turn_detection_enabled`;pipecat 警告双源发重复帧 |-| transport 由创建者拥有,`run(transport)` 只借用,不 start/close | livekit `start(room=)` 不关 room;三者平级:agent / transport / run |-| 下行泵归 agent(`connect()` 起),上行泵归 `run()`;两次 run 之间模型队列照常消费 | 否则空档里工具结果、模型帧无人处理 |-| 模型会话被 provider 关掉后**不主动重连**,下一帧用户音频触发 `connect()`(可重入,带退避,攒帧) | 静默时挂着 API 连接没必要 |-| 离散输入统一为 `send(inputs)`,镜像 `Agent.reply(inputs=)`;`interrupt()` 为快捷方式 | `send_text` / `send_confirm` 合并;`ExternalExecutionResultEvent` 入口留着,M1 抛 NotImplementedError |-| 流式接口叫 `run`,async generator,起动即迭代 | 与 `reply_stream` 结束条件不同(连接断才结束),不复用其名 |-| `TurnAggregator` 作为实体注入(`aggregator=`),不经 config;pydantic `TurnConfig` 只存在于 app 层 | 它不依赖 agent 内部件,可子类化(pipecat 的 turn strategy 家族) |-| `fade_ms` 归 transport 构造参数 | 淡出由 transport 执行 |-| provider 连接异常由 adapter 翻译成 `ModelDisconnectedError`,agent 只认它、不 import websockets;发送失败 = 断开,攒帧等下一帧重连,上行泵不死 | 真机复现:DashScope 180s 空闲关闭后 `push_audio` 撞上已关 socket,比下行泵察觉更早 |-| `run()` 入口校验 transport / VAD 与 model 的采样率,不一致直接拒绝;agent 不做重采样 | Copilot review;重采样归 transport |-| 工具结果只取 `ToolResponse.content`,`ToolChunk` 仅用于展示 | `ToolResponse` 是完整结果(与 `Agent` 一致),否则流式工具的文本会重复 |-| `RealtimeAgent` / `TurnAggregator` / `TurnMetrics` 放 `agent/_realtime/`;`realtime/` 只放模型侧(model、card、事件、transport、VAD) | 与 `Agent` 同目录;`agent/` 单向依赖 `realtime/` |-| 模型事件统一 `Event` 后缀(`AudioDeltaEvent` 等),基类 `ModelEvent` | 对齐 `agentscope.event` 的命名 |-| 命名保留 `realtime`,不改 `live`;agent 层将来挂级联后端时再评估 `VoiceAgent` | 六家里五家叫 Realtime,M1 的 provider 产品名就是 Qwen-Omni-Realtime |-| DashScope 的 Omni 和 Audio-3.0 两个 S2S API 用两个类,不在一个类里按模型名路由 | `supports_text_input` 等是 per-API 事实;`tts/_dashscope/` 一个 provider 三个类的先例 |-| 反向查找靠 card 上的 `model_type`(产出它的 class 的 `type` tag),服务层建 `{cls.type: cls}` dict,不扫名字 | 同名模型可出现在两个 class 的 card 里(OpenAI chat/responses)|-| `TurnMetrics` 是 agent 的**产出**(四个时刻),不是配置,不进构造函数;`last_turn_metrics` 读取 | 每回合的指标事件(livekit `metrics_collected`)待前端需要时再加 |-| `PlayoutPosition.first_played_at` 由 transport 在音频线程记录,agent 在回合结束时读取 | e2e_latency 的唯一真实来源;随 `LocalAudioTransport` 落地 |-| 用户打字先打断当前回复 | livekit / pipecat 默认;不打断需要额外的待处理状态 |--## 已推迟(保留概念,届时形态如下)--| 事项 | 阶段 | 预期形态 | 为什么现在不做 |-|---|---|---|---|-| filler(工具执行期间的填充语) | M2 | 本地 TTS 播放,不走 provider | DashScope 不收文本输入 |-| 独立的打断判定(叠在 VAD 之上) | M2 | 词数门槛(pipecat)或独立模型(livekit) | SERVER 模式下 provider 已决定;M1 无本地 VAD |-| `update_session(instructions, tools)` | M2 | OpenAI/DashScope/xAI 发 `session.update`;Gemini 重连+resumption handle;不改 `voice` | 见 `_agent.py` connect() 处 TODO |-| `ModelEvent.SessionResumption` | Gemini 接入时 | 随 Gemini adapter 一起回来 | Gemini 唯一的改 instructions 路径 |-| 语义端点检测 | M3 之后 | 先接 pipecat smart-turn ONNX | OpenAI/DashScope 已有 `semantic_vad` |-| 上下文压缩 | 重连功能时 | `summarize(messages) -> str` 灌进新 session | S2S 下压缩本地历史不影响 provider |-| WebSocket transport 兜底 | Safari 支持面确认后 | 同 9 字节头 + 二进制 Opus | 优先 WebTransport |--## 待定--- `SessionConfig` 二分改造单独 PR 的时机-- 模型重连后 UI 提示事件(M1 只记日志)-- `AudioDelta` 事件的 base64 惰性编码(local 场景全部白做)diff --git a/src/agentscope/realtime/_base.py b/src/agentscope/realtime/_base.pyindex f3d82b2db..dd7030d8f 100644--- a/src/agentscope/realtime/_base.py+++ b/src/agentscope/realtime/_base.py@@ -11,7 +11,7 @@ from pydantic import BaseModelfrom ._events import ModelEventfrom ._model_card import RealtimeModelCardfrom ..credential import CredentialBase-from ..message import Msg, ToolResultBlock+from ..message import ToolResultBlockclass ModelDisconnectedError(ConnectionError):@@ -138,12 +138,6 @@ class RealtimeModelBase(ABC):"""The PCM rate of the audio in :class:`AudioDelta`, in Hz."""return self.card.output_sample_rate- @property- @abstractmethod- def turn_detection_enabled(self) -> bool:- """Whether the provider decides turn boundaries. When ``False``- the caller owns endpointing and must call :meth:`commit_turn`."""-# ------------------------------------------------------------------# Lifecycle# ------------------------------------------------------------------@@ -151,16 +145,13 @@ class RealtimeModelBase(ABC):@abstractmethodasync def connect(self,- context: list[Msg],instructions: str,tools: list[dict] | None = None,**kwargs: Any,) -> None:- """Open the session, seeding it with prior *context*.+ """Open the session.Args:- context (`list[Msg]`):- Prior conversation, e.g. from ``AgentState.context``.instructions (`str`):System instructions.tools (`list[dict] | None`, optional):@@ -205,10 +196,8 @@ class RealtimeModelBase(ABC):@abstractmethodasync def commit_turn(self) -> None:- """Declare the user turn finished.-- Only called when :attr:`turn_detection_enabled` is ``False``.- """+ """Declare the user turn finished. Only called when the caller owns+ endpointing, i.e. the session was opened with turn detection off."""# ------------------------------------------------------------------# Response controldiff --git a/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-flash.yaml b/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-flash.yamlindex 5582971fd..cabece70f 100644--- a/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-flash.yaml+++ b/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-flash.yaml@@ -14,7 +14,6 @@ input_sample_rate: 16000output_sample_rate: 24000supports_tools: true-supports_session_resumption: false# Turns and cumulative audio seconds both apply; the provider discards# the oldest turn once either is exceeded.diff --git a/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-plus.yaml b/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-plus.yamlindex c02378a69..ef6df29fb 100644--- a/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-plus.yaml+++ b/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-plus.yaml@@ -14,7 +14,6 @@ input_sample_rate: 16000output_sample_rate: 24000supports_tools: true-supports_session_resumption: false# Turns and cumulative audio seconds both apply; the provider discards# the oldest turn once either is exceeded.diff --git a/src/agentscope/realtime/_dashscope/_model.py b/src/agentscope/realtime/_dashscope/_model.pyindex 9afd6461f..f350510c6 100644--- a/src/agentscope/realtime/_dashscope/_model.py+++ b/src/agentscope/realtime/_dashscope/_model.py@@ -16,7 +16,7 @@ from .._base import (from .._model_card import RealtimeModelCardfrom ..._logging import loggerfrom ...credential import DashScopeCredential-from ...message import Msg, TextBlock, ToolCallBlock, ToolResultBlock+from ...message import TextBlock, ToolCallBlock, ToolResultBlock_REALTIME_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime"@@ -79,36 +79,19 @@ class DashScopeRealtimeModel(RealtimeModelBase):self._tool_args: dict[str, str] = {}self._tool_names: dict[str, str] = {}- @property- def turn_detection_enabled(self) -> bool:- """Whether the provider decides turn boundaries."""- return self.parameters.turn_detection != "none"-# ------------------------------------------------------------------# Lifecycle# ------------------------------------------------------------------async def connect(self,- context: list[Msg],instructions: str,tools: list[dict] | None = None,**kwargs: Any,) -> None:- """Open the WebSocket and send the session config.-- DashScope offers no way to replay *context* into a new session, so- a non-empty one is logged and dropped rather than silently lost.- """+ """Open the WebSocket and send the session config."""import websockets- if context:- logger.warning(- "DashScopeRealtimeModel: %d prior messages cannot be "- "seeded into the session; the model starts fresh.",- len(context),- )-if kwargs.get("turn_detection_disabled"):self.parameters = self.parameters.model_copy(update={"turn_detection": "none"},diff --git a/src/agentscope/realtime/_dashscope/_models/qwen-omni-turbo-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen-omni-turbo-realtime.yamlindex 3aad9e615..f77c8be5b 100644--- a/src/agentscope/realtime/_dashscope/_models/qwen-omni-turbo-realtime.yaml+++ b/src/agentscope/realtime/_dashscope/_models/qwen-omni-turbo-realtime.yaml@@ -15,7 +15,6 @@ output_sample_rate: 24000# The model page lists "Function Calling: 不支持".supports_tools: false-supports_session_resumption: false# The model page states the context in tokens; the realtime guide's# turn/duration table does not cover this model.diff --git a/src/agentscope/realtime/_dashscope/_models/qwen3-omni-flash-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen3-omni-flash-realtime.yamlindex 8e3296584..0ad882b73 100644--- a/src/agentscope/realtime/_dashscope/_models/qwen3-omni-flash-realtime.yaml+++ b/src/agentscope/realtime/_dashscope/_models/qwen3-omni-flash-realtime.yaml@@ -15,7 +15,6 @@ output_sample_rate: 24000# Function calling is only offered on the qwen3.5 series.supports_tools: false-supports_session_resumption: false# History is bounded by dialog turns, not tokens; the provider discards# the oldest turn once exceeded.diff --git a/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-flash-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-flash-realtime.yamlindex 617b97472..df48a83c5 100644--- a/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-flash-realtime.yaml+++ b/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-flash-realtime.yaml@@ -14,7 +14,6 @@ input_sample_rate: 16000output_sample_rate: 24000supports_tools: true-supports_session_resumption: false# Turns and cumulative audio seconds both apply; the provider discards# the oldest turn once either is exceeded.diff --git a/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-plus-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-plus-realtime.yamlindex 8235b8999..e54af5d75 100644--- a/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-plus-realtime.yaml+++ b/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-plus-realtime.yaml@@ -14,7 +14,6 @@ input_sample_rate: 16000output_sample_rate: 24000supports_tools: true-supports_session_resumption: false# Turns and cumulative audio seconds both apply; the provider discards# the oldest turn once either is exceeded.diff --git a/src/agentscope/realtime/_model_card.py b/src/agentscope/realtime/_model_card.pyindex 77eeed6fa..575356658 100644--- a/src/agentscope/realtime/_model_card.py+++ b/src/agentscope/realtime/_model_card.py@@ -60,10 +60,6 @@ class RealtimeModelCard(BaseModel):default=False,description="Whether this model accepts tool schemas.",)- supports_session_resumption: bool = Field(- default=False,- description="Whether a previous session can be resumed by handle.",- )# Context limits. Providers use different units and several can apply# at once, so all are optional; ``None`` means unlimited or unknown.diff --git a/src/agentscope/realtime/_transport/_base.py b/src/agentscope/realtime/_transport/_base.pyindex a430daa69..8aba5f698 100644--- a/src/agentscope/realtime/_transport/_base.py+++ b/src/agentscope/realtime/_transport/_base.py@@ -34,9 +34,6 @@ class ControlFrameType(StrEnum):INTERRUPT = "interrupt""""The user pressed stop, as opposed to speaking over the reply."""- EXTERNAL_EXECUTION_RESULT = "external_execution_result"- """The result of a tool the client ran itself."""-class ControlFrame(BaseModel):"""One upstream control event."""@@ -99,10 +96,6 @@ class TransportBase(ABC):position can be attributed to it."""- @abstractmethod- async def send_event(self, event: dict) -> None:- """Send one control event downstream."""-@abstractmethodasync def clear_audio(self) -> PlayoutPosition:"""Cut playback short and report how much was actually heard.diff --git a/src/agentscope/realtime/_transport/_local.py b/src/agentscope/realtime/_transport/_local.pyindex 52339232a..e4f2003cd 100644--- a/src/agentscope/realtime/_transport/_local.py+++ b/src/agentscope/realtime/_transport/_local.py@@ -151,9 +151,6 @@ class LocalAudioTransport(TransportBase):self._first_played_at = Noneself._pending.extend(pcm)- async def send_event(self, event: dict) -> None:- """No peer to send to; events are consumed via the agent."""-def _on_output(self,outdata: Any,diff --git a/tests/a2a_agent_test.py b/tests/a2a_agent_test.pyindex e87c80a9f..2198e6c41 100644--- a/tests/a2a_agent_test.py+++ b/tests/a2a_agent_test.py@@ -209,6 +209,7 @@ class A2AAgentReplyTest(IsolatedAsyncioTestCase):"type": "TEXT_BLOCK_END","reply_id": AnyString(),"block_id": AnyString(),+ "text": None,},{"id": AnyString(),@@ -405,6 +406,7 @@ class A2AAgentReplyTest(IsolatedAsyncioTestCase):"type": "TEXT_BLOCK_END","reply_id": AnyString(),"block_id": AnyString(),+ "text": None,},{"id": AnyString(),@@ -430,6 +432,7 @@ class A2AAgentReplyTest(IsolatedAsyncioTestCase):"type": "TEXT_BLOCK_END","reply_id": AnyString(),"block_id": AnyString(),+ "text": None,},{"id": AnyString(),diff --git a/tests/agent_basic_test.py b/tests/agent_basic_test.pyindex 31870f724..b00ed6fe3 100644--- a/tests/agent_basic_test.py+++ b/tests/agent_basic_test.py@@ -293,6 +293,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):{"type": "TEXT_BLOCK_END","block_id": AnyString(),+ "text": None,},{"type": "MODEL_CALL_END",@@ -507,6 +508,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):{"type": "TEXT_BLOCK_END","block_id": AnyString(),+ "text": None,},{"type": "MODEL_CALL_END",@@ -1232,7 +1234,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"tool_call_id": tool_call_id_1,"delta": '{"input": ',},- {"type": "TEXT_BLOCK_END", "block_id": AnyString()},+ {"type": "TEXT_BLOCK_END", "block_id": AnyString(), "text": None},{"type": "TOOL_CALL_DELTA","tool_call_id": tool_call_id_1,@@ -1295,7 +1297,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"block_id": AnyString(),"delta": "ended",},- {"type": "TEXT_BLOCK_END", "block_id": AnyString()},+ {"type": "TEXT_BLOCK_END", "block_id": AnyString(), "text": None},{"type": "MODEL_CALL_END","input_tokens": 0,@@ -1553,7 +1555,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"block_id": AnyString(),"delta": "All done",},- {"type": "TEXT_BLOCK_END", "block_id": AnyString()},+ {"type": "TEXT_BLOCK_END", "block_id": AnyString(), "text": None},{"type": "MODEL_CALL_END","input_tokens": 0,@@ -1850,7 +1852,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"block_id": AnyString(),"delta": "All done",},- {"type": "TEXT_BLOCK_END", "block_id": AnyString()},+ {"type": "TEXT_BLOCK_END", "block_id": AnyString(), "text": None},{"type": "MODEL_CALL_END","input_tokens": 0,diff --git a/tests/hitl_external_execution_test.py b/tests/hitl_external_execution_test.pyindex d28c0048b..3fd98d8fe 100644--- a/tests/hitl_external_execution_test.py+++ b/tests/hitl_external_execution_test.py@@ -145,6 +145,7 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):{"type": "TEXT_BLOCK_END","block_id": AnyString(),+ "text": None,},{"type": "MODEL_CALL_END",diff --git a/tests/hitl_mixed_test.py b/tests/hitl_mixed_test.pyindex f4a2a8c80..f72164281 100644--- a/tests/hitl_mixed_test.py+++ b/tests/hitl_mixed_test.py@@ -149,6 +149,7 @@ class AgentMixTest(IsolatedAsyncioTestCase):{"type": "TEXT_BLOCK_END","block_id": AnyString(),+ "text": None,},{"type": "MODEL_CALL_END",diff --git a/tests/hitl_user_confirmation_test.py b/tests/hitl_user_confirmation_test.pyindex c7d2bce93..060d388c0 100644--- a/tests/hitl_user_confirmation_test.py+++ b/tests/hitl_user_confirmation_test.py@@ -239,6 +239,7 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):{"type": "TEXT_BLOCK_END","block_id": AnyString(),+ "text": None,},{"type": "MODEL_CALL_END",diff --git a/tests/realtime_agent_test.py b/tests/realtime_agent_test.pyindex b01b6806f..a92fd5d33 100644--- a/tests/realtime_agent_test.py+++ b/tests/realtime_agent_test.py@@ -11,8 +11,10 @@ from agentscope.agent import RealtimeAgent, TurnAggregatorfrom agentscope.credential import DashScopeCredentialfrom agentscope.event import (ReplyEndEvent,- UserInputTranscriptionEvent,+ ReplyStartEvent,+ TextBlockDeltaEvent,)+from agentscope.message import Msgfrom agentscope.realtime import (AudioFrame,ModelDisconnectedError,@@ -34,7 +36,7 @@ from agentscope.event import (ToolResultTextDeltaEvent,UserConfirmResultEvent,)-from agentscope.message import Msg, TextBlock, ToolCallBlock, ToolResultBlock+from agentscope.message import TextBlock, ToolCallBlock, ToolResultBlockfrom agentscope.permission import (PermissionBehavior,PermissionContext,@@ -86,26 +88,23 @@ class ScriptedModel(RealtimeModelBase):self.scripts = scriptsself.calls: list[str] = []self.sessions = 0+ self.instructions = ""self._open = asyncio.Event()self._requested = asyncio.Event()- @property- def turn_detection_enabled(self) -> bool:- """The provider owns turn boundaries unless a VAD is given."""- return True-async def connect(self,- context: list[Msg],instructions: str,tools: list[dict] | None = None,**kwargs: Any,) -> None:- """Record a session open and the turn-detection request."""+ """Record a session open, its instructions and the turn-detection+ request."""self.sessions += 1self._open.clear()+ self.instructions = instructionsself.calls.append(- f"connect(session={self.sessions},ctx={len(context)},"+ f"connect(session={self.sessions},"f"td_off={kwargs.get('turn_detection_disabled')})",)@@ -193,9 +192,6 @@ class FakeTransport(TransportBase):"""Remember which item is playing."""self.item = item_id- async def send_event(self, event: dict) -> None:- """No peer to send to."""-async def clear_audio(self) -> PlayoutPosition:"""Count cuts and report the fixed playout position."""self.cleared += 1@@ -211,7 +207,8 @@ class FakeTransport(TransportBase):class EndOnSecondFrameVAD(VADBase):- """Reports the user stopping on the second chunk."""+ """Reports the user starting on the first chunk and stopping on the+ second."""sample_rate = 16000@@ -219,8 +216,10 @@ class EndOnSecondFrameVAD(VADBase):self.seen = 0def push(self, pcm: bytes) -> SpeechTransition | None:- """ENDED on the second chunk, otherwise nothing."""+ """STARTED on the first chunk, ENDED on the second, else nothing."""self.seen += 1+ if self.seen == 1:+ return SpeechTransition.STARTEDreturn SpeechTransition.ENDED if self.seen == 2 else Nonedef reset(self) -> None:@@ -236,14 +235,20 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):agent: RealtimeAgent,transport: FakeTransport,) -> list[tuple[str, Any]]:- """Run the agent over *transport* and summarise the events."""+ """Run the agent over *transport* and summarise the events: the+ user's transcripts and how each of the agent's replies ended."""summary: list[tuple[str, Any]] = []+ user_turns: set[str] = set()async with transport:- async for event in agent.run(transport):- if isinstance(event, ReplyEndEvent):- summary.append(("reply_end", event.finished_reason))- elif isinstance(event, UserInputTranscriptionEvent):- summary.append(("user", event.transcript))+ async for event in agent.reply_stream(transport):+ if isinstance(event, ReplyStartEvent) and event.role == "user":+ user_turns.add(event.reply_id)+ elif isinstance(event, TextBlockDeltaEvent):+ if event.reply_id in user_turns:+ summary.append(("user", event.delta))+ elif isinstance(event, ReplyEndEvent):+ if event.reply_id not in user_turns:+ summary.append(("reply_end", event.finished_reason))return summaryasync def test_barge_in_truncates_to_what_was_heard(self) -> None:@@ -257,20 +262,47 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):agent = RealtimeAgent("Friday", "be brief", model)transport = FakeTransport(frames=3)- async with agent:- summary = await self._collect(agent, transport)+ # Rebuild the agent's message from its events the way a client+ # does; the text deltas ran ahead of what was heard.+ summary = []+ rebuilt = Msg(id="r1", role="assistant", name="Friday", content=[])+ async with agent, transport:+ async for event in agent.reply_stream(transport):+ if getattr(event, "reply_id", None) == "r1":+ rebuilt.append_event(event)+ if isinstance(event, ReplyStartEvent) and event.role == "user":+ summary.append(("user_start", event.reply_id))+ elif isinstance(event, TextBlockDeltaEvent):+ summary.append(event.delta)+ elif isinstance(event, ReplyEndEvent):+ summary.append(("reply_end", event.finished_reason))+ # The duplicate speech_started opens the user's turn only once.self.assertListEqual(summary,- [("user", "讲个故事"), ("reply_end", "interrupted")],+ [+ ("user_start", "u1"),+ "讲个故事",+ ("reply_end", "completed"), # the user's turn+ "从前",+ "有座山",+ "山里",+ "有座庙",+ "庙里",+ "有个",+ "老和尚",+ ("user_start", "u2"),+ ("reply_end", "interrupted"),+ ],)+ self.assertEqual(rebuilt.get_text_content(), "从前有座山山里有座庙")self.assertEqual(transport.cleared, 1)# Audio frames interleave with model events on the transport's# clock, so they are counted rather than positioned.self.assertListEqual([c for c in model.calls if c != "push_audio"],[- "connect(session=1,ctx=0,td_off=False)",+ "connect(session=1,td_off=False)","truncate(r1,320ms,'从前有座山山里有座庙')","cancel","close",@@ -309,8 +341,13 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):summary = await self._collect(agent, FakeTransport(frames=2))self.assertEqual(model.sessions, 2)- # The orphaned reply was dropped, so one message carried over.- self.assertIn("connect(session=2,ctx=1,td_off=False)", model.calls)+ # The orphaned reply was dropped, so one message carried over,+ # riding along in the instructions of the new session.+ self.assertIn("connect(session=2,td_off=False)", model.calls)+ self.assertEqual(+ model.instructions,+ "be brief\n\n## Conversation so far\nuser: 讲个故事",+ )# Events produced while no run was active are delivered first;# a reply streamed with nobody listening is cut off exactly once.self.assertListEqual(@@ -328,8 +365,9 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):)async def test_local_vad_owns_turns(self) -> None:- """Passing a VAD disables provider turn detection and commits the- turn when the VAD reports the user stopped."""+ """Passing a VAD disables provider turn detection, reports the+ user's speech as events and commits the turn when the VAD reports+ the user stopped."""model = ScriptedModel([[]])agent = RealtimeAgent("Friday",@@ -337,13 +375,36 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):model,vad=EndOnSecondFrameVAD(),)+ speech = []async with agent:- await self._collect(agent, FakeTransport(frames=3))-+ transport = FakeTransport(frames=3)+ async with transport:+ async for event in agent.reply_stream(transport):+ if isinstance(event, ReplyStartEvent):+ speech.append((event.type, event.role, event.reply_id))+ elif isinstance(event, ReplyEndEvent):+ speech.append(+ (+ event.type,+ event.finished_reason,+ event.reply_id,+ ),+ )++ # The user's turn is a reply of its own, with a locally generated id+ # since no provider item exists yet.+ self.assertListEqual(+ speech,+ [+ ("REPLY_START", "user", AnyString()),+ ("REPLY_END", "completed", AnyString()),+ ],+ )+ self.assertEqual(speech[0][2], speech[1][2])self.assertListEqual(model.calls,[- "connect(session=1,ctx=0,td_off=True)",+ "connect(session=1,td_off=True)","push_audio","commit_turn","push_audio",@@ -488,7 +549,7 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):async with agent:transport = FakeTransport(frames=4)async with transport:- async for event in agent.run(transport):+ async for event in agent.reply_stream(transport):match event:case ToolCallStartEvent():events.append(("call_start", event.tool_call_name))@@ -538,7 +599,7 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):self.assertListEqual([c for c in model.calls if not c.startswith("push_audio")],[- "connect(session=1,ctx=0,td_off=False)",+ "connect(session=1,td_off=False)","tool_result(c1,'x-final')","request_response","close",@@ -659,7 +720,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):async with agent:transport = FakeTransport(frames=6)async with transport:- async for event in agent.run(transport):+ async for event in agent.reply_stream(transport):events.append(event.model_dump(mode="json"))self.assertListEqual(@@ -669,10 +730,47 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"id": AnyString(),"created_at": AnyString(),"metadata": {},- "type": "USER_INPUT_TRANSCRIPTION",+ "type": "REPLY_START","session_id": AnyString(),- "item_id": "u1",- "transcript": "查天气",+ "reply_id": "u1",+ "name": "user",+ "role": "user",+ },+ {+ "id": AnyString(),+ "created_at": AnyString(),+ "metadata": {},+ "type": "TEXT_BLOCK_START",+ "reply_id": "u1",+ "block_id": AnyString(),+ },+ {+ "id": AnyString(),+ "created_at": AnyString(),+ "metadata": {},+ "type": "TEXT_BLOCK_DELTA",+ "reply_id": "u1",+ "block_id": AnyString(),+ "delta": "查天气",+ },+ {+ "id": AnyString(),+ "created_at": AnyString(),+ "metadata": {},+ "type": "TEXT_BLOCK_END",+ "reply_id": "u1",+ "block_id": AnyString(),+ "text": None,+ },+ {+ "id": AnyString(),+ "created_at": AnyString(),+ "metadata": {},+ "type": "REPLY_END",+ "session_id": AnyString(),+ "reply_id": "u1",+ "finished_reason": "completed",+ "error": None,},{"id": AnyString(),@@ -737,6 +835,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"type": "TEXT_BLOCK_END","reply_id": "r1","block_id": AnyString(),+ "text": None,},{"id": AnyString(),@@ -758,16 +857,6 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"cache_creation_input_tokens": 0,"finished_reason": "completed",},- {- "id": AnyString(),- "created_at": AnyString(),- "metadata": {},- "type": "REPLY_END",- "session_id": AnyString(),- "reply_id": "r1",- "finished_reason": "completed",- "error": None,- },{"id": AnyString(),"created_at": AnyString(),@@ -821,22 +910,12 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"tool_call_id": "c1","state": "success",},- {- "id": AnyString(),- "created_at": AnyString(),- "metadata": {},- "type": "REPLY_START",- "session_id": AnyString(),- "reply_id": "r2",- "name": "Friday",- "role": "assistant",- },{"id": AnyString(),"created_at": AnyString(),"metadata": {},"type": "MODEL_CALL_START",- "reply_id": "r2",+ "reply_id": "r1","model_name": "scripted",},{@@ -844,7 +923,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"created_at": AnyString(),"metadata": {},"type": "TEXT_BLOCK_START",- "reply_id": "r2",+ "reply_id": "r1","block_id": AnyString(),},{@@ -852,7 +931,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"created_at": AnyString(),"metadata": {},"type": "TEXT_BLOCK_DELTA",- "reply_id": "r2",+ "reply_id": "r1","block_id": AnyString(),"delta": "今天晴",},@@ -861,7 +940,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"created_at": AnyString(),"metadata": {},"type": "DATA_BLOCK_START",- "reply_id": "r2",+ "reply_id": "r1","block_id": AnyString(),"media_type": "audio/pcm;rate=24000","name": None,@@ -871,7 +950,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"created_at": AnyString(),"metadata": {},"type": "DATA_BLOCK_DELTA",- "reply_id": "r2",+ "reply_id": "r1","block_id": AnyString(),"media_type": "audio/pcm;rate=24000","data": "AQA=",@@ -882,15 +961,16 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"created_at": AnyString(),"metadata": {},"type": "TEXT_BLOCK_END",- "reply_id": "r2",+ "reply_id": "r1","block_id": AnyString(),+ "text": None,},{"id": AnyString(),"created_at": AnyString(),"metadata": {},"type": "DATA_BLOCK_END",- "reply_id": "r2",+ "reply_id": "r1","block_id": AnyString(),},{@@ -898,7 +978,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"created_at": AnyString(),"metadata": {},"type": "MODEL_CALL_END",- "reply_id": "r2",+ "reply_id": "r1","input_tokens": 9,"output_tokens": 3,"cache_input_tokens": 0,@@ -911,24 +991,98 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):"metadata": {},"type": "REPLY_END","session_id": AnyString(),- "reply_id": "r2",+ "reply_id": "r1","finished_reason": "completed","error": None,},],)+ # The context records the whole turn as one assistant message: the+ # first words, the tool call and its result, then the spoken answer.self.assertListEqual(- [(m.role, m.get_text_content()) for m in agent.state.context],+ [m.model_dump() for m in agent.state.context],[- ("user", "查天气"),- ("assistant", "我查一下"),- ("assistant", "今天晴"),+ {+ "name": "user",+ "role": "user",+ "id": "u1",+ "content": [+ {+ "type": "text",+ "text": "查天气",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": AnyString(),+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ {+ "name": "Friday",+ "role": "assistant",+ "id": "r1",+ "content": [+ {+ "type": "text",+ "text": "我查一下",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "tool_call",+ "id": "c1",+ "name": "stream_tool",+ "input": '{"q": "x"}',+ "state": "pending",+ "suggested_rules": [],+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "tool_result",+ "id": "c1",+ "name": "stream_tool",+ "output": "x-final",+ "state": "success",+ "metadata": {},+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": "今天晴",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "metadata": {},+ "created_at": AnyString(),+ # Both model calls of the reply, summed.+ "usage": {+ "input_tokens": 14,+ "output_tokens": 5,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ },+ "finished_at": None,+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },],)self.assertListEqual([c for c in model.calls if c != "push_audio"],[- "connect(session=1,ctx=0,td_off=False)",+ "connect(session=1,td_off=False)","tool_result(c1,'x-final')","request_response","close",@@ -966,15 +1120,15 @@ class RealtimeAgentDisconnectTest(IsolatedAsyncioTestCase):transport = FakeTransport(frames=3)with self.assertLogs("as", level="INFO") as logs:async with transport:- async for _ in agent.run(transport):+ async for _ in agent.reply_stream(transport):passself.assertEqual(model.sessions, 2)self.assertListEqual([c for c in model.calls if c != "push_audio"],[- "connect(session=1,ctx=0,td_off=False)",- "connect(session=2,ctx=0,td_off=False)",+ "connect(session=1,td_off=False)",+ "connect(session=2,td_off=False)","close",],)diff --git a/tests/realtime_dashscope_test.py b/tests/realtime_dashscope_test.pyindex 33c1eb012..4f7bf6d03 100644--- a/tests/realtime_dashscope_test.py+++ b/tests/realtime_dashscope_test.py@@ -186,7 +186,6 @@ class DashScopeSessionUpdateTest(unittest.TestCase):)session = model._session_update("x", None)["session"]self.assertIsNone(session["turn_detection"])- self.assertFalse(model.turn_detection_enabled)def test_adapter_facts(self) -> None:"""Protocol facts differ per adapter, not per model."""