Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(realtime): retire unbound local user turns when the transport dies (#3491)
Wilson_Jiang178fd3168309
diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.pyindex d8c8ed642..25653d128 100644--- a/src/agentscope/agent/_realtime/_agent.py+++ b/src/agentscope/agent/_realtime/_agent.py@@ -435,6 +435,7 @@ class RealtimeAgent:self._pump_uplink(transport),name="rt-up",)+ getter: asyncio.Future[_QueuedEvent] | None = Nonetry:while not uplink.done():getter = asyncio.ensure_future(self._out.get())@@ -460,9 +461,7 @@ class RealtimeAgent:# events that produced (the interrupted ReplyEnd) before the# stream ends rather than leaking them into the next run.await self._barge_in()- for turn in list(self._pending_user_turns.values()):- self._end_user_turn(turn)- self._end_user_turn()+ self._retire_unbound_user_turns()while not self._out.empty():queued = self._out.get_nowait()self._yielded_checkpoint = (@@ -478,10 +477,17 @@ class RealtimeAgent:if not uplink.cancelled() and uplink.exception() is not None:raise uplink.exception() # type: ignore[misc]finally:- if not uplink.done():- uplink.cancel()- await asyncio.gather(uplink, return_exceptions=True)- self._transport = None+ try:+ tasks = [task for task in (uplink, getter) if task is not None]+ for task in tasks:+ if not task.done():+ task.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)+ finally:+ # Early close and cancellation skip the normal event drain.+ # Stop readers before queuing the pending turns' end events.+ self._retire_unbound_user_turns()+ self._transport = None# ------------------------------------------------------------------# Discrete input@@ -872,6 +878,30 @@ class RealtimeAgent:self._pending_user_turns.clear()self._user_turn = None+ def _retire_unbound_user_turns(self) -> None:+ """End pending user turns when their transport goes, keeping only+ the ones a late transcript could still find.++ A turn bound to a provider item id stays in+ ``_pending_user_turns`` so a transcript that settles after the+ transport died still lands in the reply the caller already saw+ end. A local turn whose speech ended also stays: it can be+ committed without a provider item id, and its late transcript+ still needs ``_find_user_turn``'s ordered fallback. Drop only+ unfinished, unbound turns so they cannot capture the next run's+ first transcript.+ """+ turns = list(self._pending_user_turns.values())+ if self._user_turn is not None and self._user_turn not in turns:+ turns.append(self._user_turn)+ for turn in turns:+ self._end_user_turn(turn)+ for turn in turns:+ if not turn.item_id and not turn.speech_ended:+ self._pending_user_turns.pop(turn.reply_id, None)+ if self._user_turn is turn:+ self._user_turn = None+def _on_transcription(self, event: me.InputTranscriptionEvent) -> None:"""Record a settled user turn, merging a split one back together,and report its text on the user's reply."""diff --git a/tests/realtime_agent_test.py b/tests/realtime_agent_test.pyindex 241c7d97c..6e6ea8376 100644--- a/tests/realtime_agent_test.py+++ b/tests/realtime_agent_test.py@@ -1565,6 +1565,251 @@ class RealtimeAgentTranscriptionTest(IsolatedAsyncioTestCase):[("u2", "new")],)+ async def test_transport_end_retires_unbound_local_turns(self) -> None:+ """A transcript after reconnect lands on the new turn, not the+ dead one ended by the transport teardown."""+ model = ScriptedModel([[]], input_audio_transcription=True)+ agent = RealtimeAgent(+ "Friday",+ "be brief",+ model,+ vad=EndOnSecondFrameVAD(),+ )++ async with agent:+ # Run 1: the only audio frame starts a local (unbound) user+ # turn; the transport then dies before any transcript for it+ # can settle.+ transport = FakeTransport(frames=1)+ run1: list[tuple[Any, ...]] = []+ async with transport:+ async for event in agent.reply_stream(transport):+ if isinstance(event, ReplyStartEvent):+ run1.append(("start", event.reply_id))+ elif isinstance(event, ReplyEndEvent):+ run1.append(("end", event.reply_id))+ self.assertEqual(+ [kind for kind, *_ in run1],+ ["start", "end"],+ "the dead transport must hand back the turn's end",+ )+ dead_id = run1[0][1]++ # Run 2: a fresh transport on the same model session. The+ # user starts speaking again — a new local turn opens.+ await agent._forward_audio(b"", SpeechTransition.STARTED)+ run2_start = self._drain(agent)+ self.assertEqual(run2_start, [("start", run2_start[0][1])])+ new_id = run2_start[0][1]+ self.assertNotEqual(new_id, dead_id)++ # The provider settles the new turn's transcript. It must+ # land on the new turn — not be captured by the dead one.+ await agent._on_model_event(+ me.InputTranscriptionEvent(item_id="u9", text="hello"),+ )+ settled = self._drain(agent)++ self.assertEqual(+ settled,+ [("text", new_id, "hello"), ("end", new_id)],+ "the transcript must attach to the new turn",+ )+ self.assertEqual(+ [+ (message.id, message.get_text_content())+ for message in agent.state.context+ if message.role == "user"+ ],+ [(new_id, "hello")],+ "context must hold the transcript under the new turn's id",+ )++ async def test_early_stream_exit_cleans_up_local_turns(self) -> None:+ """Closing or cancelling retires only unfinished local turns."""+ for exit_mode in ["close", "cancel"]:+ for committed in [False, True]:+ with self.subTest(exit_mode=exit_mode, committed=committed):+ agent = self._make_agent()+ async with agent:+ await agent._forward_audio(+ b"",+ SpeechTransition.STARTED,+ )+ old_turn = agent._user_turn+ old_id = old_turn.reply_id+ if committed:+ await agent._forward_audio(+ b"",+ SpeechTransition.ENDED,+ )+ tasks_before = asyncio.all_tasks()+ stream = agent.reply_stream(GatedTransport([]))+ first = await anext(stream)+ self.assertEqual(+ (first.type, first.reply_id),+ ("REPLY_START", old_id),+ )+ if exit_mode == "close":+ await stream.aclose()+ else:+ consumer = asyncio.create_task(anext(stream))+ await asyncio.sleep(0)+ consumer.cancel()+ with self.assertRaises(asyncio.CancelledError):+ await consumer+ await asyncio.sleep(0)+ self.assertEqual(+ {+ "transport": agent._transport,+ "old_open": old_turn.open,+ "pending": list(agent._pending_user_turns),+ "events": self._drain(agent),+ "tasks": asyncio.all_tasks() - tasks_before,+ },+ {+ "transport": None,+ "old_open": False,+ "pending": [old_id] if committed else [],+ "events": [("end", old_id)],+ "tasks": set(),+ },+ )+ await agent._forward_audio(+ b"",+ SpeechTransition.STARTED,+ )+ new_id = self._drain(agent)[0][1]+ self.assertNotEqual(new_id, old_id)+ resumed = agent.reply_stream(GatedTransport([]))+ if committed:+ await agent._on_model_event(+ me.InputTranscriptionEvent(+ item_id="u1",+ text="first",+ ),+ )+ self.assertEqual(+ self._drain(agent),+ [("text", old_id, "first")],+ )+ self.assertTrue(agent._user_turn.open)+ await agent._on_model_event(+ me.InputTranscriptionEvent(+ item_id="u2",+ text="second",+ ),+ )+ try:+ events = []+ for _ in range(4):+ event = await asyncio.wait_for(+ anext(resumed),+ timeout=1,+ )+ events.append((event.type, event.reply_id))+ finally:+ await resumed.aclose()+ self.assertEqual(+ {+ "events": events,+ "context": [+ (msg.id, msg.get_text_content())+ for msg in agent.state.context+ ],+ },+ {+ "events": [+ ("TEXT_BLOCK_START", new_id),+ ("TEXT_BLOCK_DELTA", new_id),+ ("TEXT_BLOCK_END", new_id),+ ("REPLY_END", new_id),+ ],+ "context": (+ [(old_id, "first")] if committed else []+ )+ + [(new_id, "second")],+ },+ )++ async def test_transport_end_keeps_committed_local_turns(self) -> None:+ """A late local transcription must not settle the resumed turn."""+ for old_event, old_context in [+ (me.InputTranscriptionEvent(item_id="u1", text="first"), True),+ (me.InputTranscriptionFailedEvent(item_id="u1"), False),+ (me.InputTranscriptionEvent(item_id="u1", text=""), False),+ ]:+ with self.subTest(event=old_event):+ agent = self._make_agent()+ async with agent:+ await agent._forward_audio(b"", SpeechTransition.STARTED)+ old_id = self._drain(agent)[0][1]+ await agent._forward_audio(b"", SpeechTransition.ENDED)+ self.assertIn("commit_turn", agent.model.calls)++ ended = [+ (event.type, event.reply_id)+ async for event in agent.reply_stream(FakeTransport(0))+ ]+ self.assertEqual(ended, [("REPLY_END", old_id)])++ await agent._forward_audio(b"", SpeechTransition.STARTED)+ new_id = self._drain(agent)[0][1]+ await agent._on_model_event(old_event)+ self.assertEqual(+ {+ "events": self._drain(agent),+ "pending": [+ (turn.reply_id, turn.item_id, turn.open)+ for turn in agent._pending_user_turns.values()+ ],+ "context": [+ (msg.id, msg.get_text_content())+ for msg in agent.state.context+ ],+ },+ {+ "events": (+ [("text", old_id, "first")]+ if old_context+ else []+ ),+ "pending": [(new_id, "", True)],+ "context": (+ [(old_id, "first")] if old_context else []+ ),+ },+ )+ await agent._on_model_event(+ me.InputTranscriptionEvent(+ item_id="u2",+ text="second",+ ),+ )+ self.assertEqual(+ {+ "events": self._drain(agent),+ "pending": list(agent._pending_user_turns),+ "current": agent._user_turn,+ "context": [+ (msg.id, msg.get_text_content())+ for msg in agent.state.context+ ],+ },+ {+ "events": [+ ("text", new_id, "second"),+ ("end", new_id),+ ],+ "pending": [],+ "current": None,+ "context": (+ [(old_id, "first")] if old_context else []+ )+ + [(new_id, "second")],+ },+ )+async def test_transcription_after_transport_end_keeps_turn(self) -> None:"""A late transcript stays in the user reply already closed."""model = ScriptedModel(