Skip to content

Build and run agents you can see, understand and trust.

fix(realtime): retire unbound local user turns when the transport dies (#3491)

Wilson_Jiang178fd3168309
diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.py
index d8c8ed642..25653d128 100644
--- a/src/agentscope/agent/_realtime/_agent.py
+++ b/src/agentscope/agent/_realtime/_agent.py
@@ -435,6 +435,7 @@ class RealtimeAgent:
self._pump_uplink(transport),
name="rt-up",
)
+ getter: asyncio.Future[_QueuedEvent] | None = None
try:
while not uplink.done():
getter = asyncio.ensure_future(self._out.get())
@@ -460,9 +461,7 @@ class RealtimeAgent:
# events that produced (the interrupted ReplyEnd) before the
# stream ends rather than leaking them into the next run.
await self._barge_in()
- for turn in list(self._pending_user_turns.values()):
- self._end_user_turn(turn)
- self._end_user_turn()
+ self._retire_unbound_user_turns()
while not self._out.empty():
queued = self._out.get_nowait()
self._yielded_checkpoint = (
@@ -478,10 +477,17 @@ class RealtimeAgent:
if not uplink.cancelled() and uplink.exception() is not None:
raise uplink.exception() # type: ignore[misc]
finally:
- if not uplink.done():
- uplink.cancel()
- await asyncio.gather(uplink, return_exceptions=True)
- self._transport = None
+ try:
+ tasks = [task for task in (uplink, getter) if task is not None]
+ for task in tasks:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ finally:
+ # Early close and cancellation skip the normal event drain.
+ # Stop readers before queuing the pending turns' end events.
+ self._retire_unbound_user_turns()
+ self._transport = None
# ------------------------------------------------------------------
# Discrete input
@@ -872,6 +878,30 @@ class RealtimeAgent:
self._pending_user_turns.clear()
self._user_turn = None
+ def _retire_unbound_user_turns(self) -> None:
+ """End pending user turns when their transport goes, keeping only
+ the ones a late transcript could still find.
+
+ A turn bound to a provider item id stays in
+ ``_pending_user_turns`` so a transcript that settles after the
+ transport died still lands in the reply the caller already saw
+ end. A local turn whose speech ended also stays: it can be
+ committed without a provider item id, and its late transcript
+ still needs ``_find_user_turn``'s ordered fallback. Drop only
+ unfinished, unbound turns so they cannot capture the next run's
+ first transcript.
+ """
+ turns = list(self._pending_user_turns.values())
+ if self._user_turn is not None and self._user_turn not in turns:
+ turns.append(self._user_turn)
+ for turn in turns:
+ self._end_user_turn(turn)
+ for turn in turns:
+ if not turn.item_id and not turn.speech_ended:
+ self._pending_user_turns.pop(turn.reply_id, None)
+ if self._user_turn is turn:
+ self._user_turn = None
+
def _on_transcription(self, event: me.InputTranscriptionEvent) -> None:
"""Record a settled user turn, merging a split one back together,
and report its text on the user's reply."""
diff --git a/tests/realtime_agent_test.py b/tests/realtime_agent_test.py
index 241c7d97c..6e6ea8376 100644
--- a/tests/realtime_agent_test.py
+++ b/tests/realtime_agent_test.py
@@ -1565,6 +1565,251 @@ class RealtimeAgentTranscriptionTest(IsolatedAsyncioTestCase):
[("u2", "new")],
)
+ async def test_transport_end_retires_unbound_local_turns(self) -> None:
+ """A transcript after reconnect lands on the new turn, not the
+ dead one ended by the transport teardown."""
+ model = ScriptedModel([[]], input_audio_transcription=True)
+ agent = RealtimeAgent(
+ "Friday",
+ "be brief",
+ model,
+ vad=EndOnSecondFrameVAD(),
+ )
+
+ async with agent:
+ # Run 1: the only audio frame starts a local (unbound) user
+ # turn; the transport then dies before any transcript for it
+ # can settle.
+ transport = FakeTransport(frames=1)
+ run1: list[tuple[Any, ...]] = []
+ async with transport:
+ async for event in agent.reply_stream(transport):
+ if isinstance(event, ReplyStartEvent):
+ run1.append(("start", event.reply_id))
+ elif isinstance(event, ReplyEndEvent):
+ run1.append(("end", event.reply_id))
+ self.assertEqual(
+ [kind for kind, *_ in run1],
+ ["start", "end"],
+ "the dead transport must hand back the turn's end",
+ )
+ dead_id = run1[0][1]
+
+ # Run 2: a fresh transport on the same model session. The
+ # user starts speaking again — a new local turn opens.
+ await agent._forward_audio(b"", SpeechTransition.STARTED)
+ run2_start = self._drain(agent)
+ self.assertEqual(run2_start, [("start", run2_start[0][1])])
+ new_id = run2_start[0][1]
+ self.assertNotEqual(new_id, dead_id)
+
+ # The provider settles the new turn's transcript. It must
+ # land on the new turn — not be captured by the dead one.
+ await agent._on_model_event(
+ me.InputTranscriptionEvent(item_id="u9", text="hello"),
+ )
+ settled = self._drain(agent)
+
+ self.assertEqual(
+ settled,
+ [("text", new_id, "hello"), ("end", new_id)],
+ "the transcript must attach to the new turn",
+ )
+ self.assertEqual(
+ [
+ (message.id, message.get_text_content())
+ for message in agent.state.context
+ if message.role == "user"
+ ],
+ [(new_id, "hello")],
+ "context must hold the transcript under the new turn's id",
+ )
+
+ async def test_early_stream_exit_cleans_up_local_turns(self) -> None:
+ """Closing or cancelling retires only unfinished local turns."""
+ for exit_mode in ["close", "cancel"]:
+ for committed in [False, True]:
+ with self.subTest(exit_mode=exit_mode, committed=committed):
+ agent = self._make_agent()
+ async with agent:
+ await agent._forward_audio(
+ b"",
+ SpeechTransition.STARTED,
+ )
+ old_turn = agent._user_turn
+ old_id = old_turn.reply_id
+ if committed:
+ await agent._forward_audio(
+ b"",
+ SpeechTransition.ENDED,
+ )
+ tasks_before = asyncio.all_tasks()
+ stream = agent.reply_stream(GatedTransport([]))
+ first = await anext(stream)
+ self.assertEqual(
+ (first.type, first.reply_id),
+ ("REPLY_START", old_id),
+ )
+ if exit_mode == "close":
+ await stream.aclose()
+ else:
+ consumer = asyncio.create_task(anext(stream))
+ await asyncio.sleep(0)
+ consumer.cancel()
+ with self.assertRaises(asyncio.CancelledError):
+ await consumer
+ await asyncio.sleep(0)
+ self.assertEqual(
+ {
+ "transport": agent._transport,
+ "old_open": old_turn.open,
+ "pending": list(agent._pending_user_turns),
+ "events": self._drain(agent),
+ "tasks": asyncio.all_tasks() - tasks_before,
+ },
+ {
+ "transport": None,
+ "old_open": False,
+ "pending": [old_id] if committed else [],
+ "events": [("end", old_id)],
+ "tasks": set(),
+ },
+ )
+ await agent._forward_audio(
+ b"",
+ SpeechTransition.STARTED,
+ )
+ new_id = self._drain(agent)[0][1]
+ self.assertNotEqual(new_id, old_id)
+ resumed = agent.reply_stream(GatedTransport([]))
+ if committed:
+ await agent._on_model_event(
+ me.InputTranscriptionEvent(
+ item_id="u1",
+ text="first",
+ ),
+ )
+ self.assertEqual(
+ self._drain(agent),
+ [("text", old_id, "first")],
+ )
+ self.assertTrue(agent._user_turn.open)
+ await agent._on_model_event(
+ me.InputTranscriptionEvent(
+ item_id="u2",
+ text="second",
+ ),
+ )
+ try:
+ events = []
+ for _ in range(4):
+ event = await asyncio.wait_for(
+ anext(resumed),
+ timeout=1,
+ )
+ events.append((event.type, event.reply_id))
+ finally:
+ await resumed.aclose()
+ self.assertEqual(
+ {
+ "events": events,
+ "context": [
+ (msg.id, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ },
+ {
+ "events": [
+ ("TEXT_BLOCK_START", new_id),
+ ("TEXT_BLOCK_DELTA", new_id),
+ ("TEXT_BLOCK_END", new_id),
+ ("REPLY_END", new_id),
+ ],
+ "context": (
+ [(old_id, "first")] if committed else []
+ )
+ + [(new_id, "second")],
+ },
+ )
+
+ async def test_transport_end_keeps_committed_local_turns(self) -> None:
+ """A late local transcription must not settle the resumed turn."""
+ for old_event, old_context in [
+ (me.InputTranscriptionEvent(item_id="u1", text="first"), True),
+ (me.InputTranscriptionFailedEvent(item_id="u1"), False),
+ (me.InputTranscriptionEvent(item_id="u1", text=""), False),
+ ]:
+ with self.subTest(event=old_event):
+ agent = self._make_agent()
+ async with agent:
+ await agent._forward_audio(b"", SpeechTransition.STARTED)
+ old_id = self._drain(agent)[0][1]
+ await agent._forward_audio(b"", SpeechTransition.ENDED)
+ self.assertIn("commit_turn", agent.model.calls)
+
+ ended = [
+ (event.type, event.reply_id)
+ async for event in agent.reply_stream(FakeTransport(0))
+ ]
+ self.assertEqual(ended, [("REPLY_END", old_id)])
+
+ await agent._forward_audio(b"", SpeechTransition.STARTED)
+ new_id = self._drain(agent)[0][1]
+ await agent._on_model_event(old_event)
+ self.assertEqual(
+ {
+ "events": self._drain(agent),
+ "pending": [
+ (turn.reply_id, turn.item_id, turn.open)
+ for turn in agent._pending_user_turns.values()
+ ],
+ "context": [
+ (msg.id, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ },
+ {
+ "events": (
+ [("text", old_id, "first")]
+ if old_context
+ else []
+ ),
+ "pending": [(new_id, "", True)],
+ "context": (
+ [(old_id, "first")] if old_context else []
+ ),
+ },
+ )
+ await agent._on_model_event(
+ me.InputTranscriptionEvent(
+ item_id="u2",
+ text="second",
+ ),
+ )
+ self.assertEqual(
+ {
+ "events": self._drain(agent),
+ "pending": list(agent._pending_user_turns),
+ "current": agent._user_turn,
+ "context": [
+ (msg.id, msg.get_text_content())
+ for msg in agent.state.context
+ ],
+ },
+ {
+ "events": [
+ ("text", new_id, "second"),
+ ("end", new_id),
+ ],
+ "pending": [],
+ "current": None,
+ "context": (
+ [(old_id, "first")] if old_context else []
+ )
+ + [(new_id, "second")],
+ },
+ )
+
async def test_transcription_after_transport_end_keeps_turn(self) -> None:
"""A late transcript stays in the user reply already closed."""
model = ScriptedModel(