Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(realtime): reconnect before sending text input (#2591)
WuWenboced7a2a7cc70
diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.pyindex fe109cf0d..58dec6712 100644--- a/src/agentscope/agent/_realtime/_agent.py+++ b/src/agentscope/agent/_realtime/_agent.py@@ -158,7 +158,7 @@ class RealtimeAgent:The realtime model. Its session is opened by:meth:`connect` and lives as long as this agent — not aslong as any one transport — and is re-established on the- next user audio if the provider closes it.+ next user input if the provider closes it.toolkit (`Toolkit | None`, optional):Tools the model may call. Executed here, with permissionchecks against ``state.permission_context``.@@ -333,6 +333,21 @@ class RealtimeAgent:return Falsereturn True+ async def _ensure_connected(self) -> bool:+ """Re-establish the session and hand it the audio kept while it+ was down; ``False`` while the provider is still unreachable.++ The one way back to a live session, whichever input asks for it.+ """+ if self._connected:+ return True+ if not await self._try_connect():+ return False+ for buffered in self._backlog:+ await self.model.push_audio(buffered)+ self._backlog.clear()+ return True+# ------------------------------------------------------------------# Lifecycle: one transport# ------------------------------------------------------------------@@ -425,6 +440,8 @@ class RealtimeAgent:Raises:`NotImplementedError`: For a text turn when the provider takesno text input.+ `ModelDisconnectedError`: For a text turn when the provider+ cannot be reached, or disconnects before accepting it."""match inputs:case UserInterruptEvent():@@ -448,8 +465,18 @@ class RealtimeAgent:)text = msg.get_text_content() or ""await self._barge_in()+ # A typed turn reconnects the session the way audio does,+ # and takes the audio kept meanwhile along with it.+ if not await self._ensure_connected():+ raise ModelDisconnectedError(+ "Provider unreachable; the text turn was not sent.",+ )+ try:+ await self.model.push_text(text)+ except ModelDisconnectedError:+ self._mark_disconnected()+ raiseself.state.context.append(msg)- await self.model.push_text(text)async def interrupt(self) -> None:"""Stop the active reply, as when the user presses stop."""@@ -490,7 +517,7 @@ class RealtimeAgent:)def _mark_disconnected(self) -> None:- """Forget the model session so the next audio reconnects."""+ """Forget the model session so the next input reconnects."""self._connected = Falseself._connected_event.clear()@@ -504,11 +531,8 @@ class RealtimeAgent:if not self._connected:self._backlog.append(pcm)del self._backlog[:-_BACKLOG_FRAMES]- if not await self._try_connect():+ if not await self._ensure_connected():return- for buffered in self._backlog:- await self.model.push_audio(buffered)- self._backlog.clear()pushed = Trueif speech is SpeechTransition.STARTED:diff --git a/tests/realtime_agent_test.py b/tests/realtime_agent_test.pyindex e3c6612af..c87173212 100644--- a/tests/realtime_agent_test.py+++ b/tests/realtime_agent_test.py@@ -17,6 +17,8 @@ from agentscope.event import (from agentscope.message import Msgfrom agentscope.realtime import (AudioFrame,+ ControlFrame,+ ControlFrameType,ModelDisconnectedError,PlayoutPosition,RealtimeModelBase,@@ -89,6 +91,8 @@ class ScriptedModel(RealtimeModelBase):self.calls: list[str] = []self.sessions = 0self.instructions = ""+ self.connect_error: Exception | None = None+ self.push_text_error: Exception | None = Noneself._open = asyncio.Event()self._requested = asyncio.Event()@@ -100,6 +104,8 @@ class ScriptedModel(RealtimeModelBase):) -> None:"""Record a session open, its instructions and the turn-detectionrequest."""+ if self.connect_error is not None:+ raise self.connect_errorself.sessions += 1self._open.clear()self.instructions = instructions@@ -132,6 +138,8 @@ class ScriptedModel(RealtimeModelBase):async def push_text(self, text: str) -> None:"""Record a text turn; the agent gates on ``supports_text_input``before ever calling this."""+ if self.push_text_error is not None:+ raise self.push_text_errorself.calls.append(f"push_text({text!r})")async def push_tool_result(self, block: ToolResultBlock) -> None:@@ -364,6 +372,80 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):[("user", "讲个故事"), ("user", "你好"), ("assistant", "你好呀")],)+ async def test_provider_timeout_reconnects_on_text(self) -> None:+ """Text reconnects a timed-out provider without duplicating the+ new turn in the reconnect instructions."""+ model = ScriptedModel(+ [+ [me.SessionEndedEvent(reason="idle")],+ [],+ ],+ )+ model.supports_text_input = True+ agent = RealtimeAgent("Friday", "be brief", model)++ async with agent:+ await asyncio.sleep(0.1)+ self.assertFalse(agent._connected) # pylint: disable=W0212+ await agent.send("hello")++ self.assertEqual(model.instructions, "be brief")+ self.assertListEqual(+ model.calls,+ [+ "connect(session=1,td_off=False)",+ "connect(session=2,td_off=False)",+ "push_text('hello')",+ "close",+ ],+ )+ self.assertListEqual(+ [(m.role, m.get_text_content()) for m in agent.state.context],+ [("user", "hello")],+ )++ async def test_failed_text_delivery_does_not_change_context(self) -> None:+ """A disconnect while sending text leaves no undelivered turn."""+ model = ScriptedModel([[]])+ model.supports_text_input = True+ agent = RealtimeAgent("Friday", "be brief", model)+ error = ModelDisconnectedError("Not connected.")++ async with agent:+ model.push_text_error = error+ with self.assertRaisesRegex(+ ModelDisconnectedError,+ "Not connected",+ ):+ await agent._on_control( # pylint: disable=W0212+ ControlFrame(+ type=ControlFrameType.TEXT,+ data={"text": "hello"},+ ),+ )+ self.assertFalse(agent._connected) # pylint: disable=W0212++ self.assertListEqual(agent.state.context, [])++ async def test_failed_text_reconnect_does_not_change_context(self) -> None:+ """A failed reconnect leaves no undelivered text turn."""+ model = ScriptedModel(+ [[me.SessionEndedEvent(reason="idle")]],+ )+ model.supports_text_input = True+ agent = RealtimeAgent("Friday", "be brief", model)++ async with agent:+ await asyncio.sleep(0.1)+ model.connect_error = RuntimeError("reconnect failed")+ with self.assertRaisesRegex(+ ModelDisconnectedError,+ "Provider unreachable",+ ):+ await agent.send("hello")++ self.assertListEqual(agent.state.context, [])+async def test_local_vad_owns_turns(self) -> None:"""Passing a VAD disables provider turn detection, reports theuser's speech as events and commits the turn when the VAD reports@@ -1201,3 +1283,27 @@ class RealtimeAgentDisconnectTest(IsolatedAsyncioTestCase):any("keep talking" in line for line in logs.output),logs.output,)++ async def test_text_reconnect_delivers_the_kept_audio(self) -> None:+ """A typed turn reconnects, and the frame kept by the failed push+ rides along with that reconnect instead of being stranded."""+ model = DropsSocketModel()+ model.supports_text_input = True+ agent = RealtimeAgent("Friday", "be brief", model)++ async with agent:+ await agent._on_audio( # pylint: disable=W0212+ AudioFrame(pcm=b"\x00" * 3200),+ )+ await agent.send("hello")++ self.assertListEqual(+ model.calls,+ [+ "connect(session=1,td_off=False)",+ "connect(session=2,td_off=False)",+ "push_audio",+ "push_text('hello')",+ "close",+ ],+ )