Skip to content

Build and run agents you can see, understand and trust.

fix(realtime): reconnect before sending text input (#2591)

WuWenboced7a2a7cc70
diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.py
index fe109cf0d..58dec6712 100644
--- a/src/agentscope/agent/_realtime/_agent.py
+++ b/src/agentscope/agent/_realtime/_agent.py
@@ -158,7 +158,7 @@ class RealtimeAgent:
The realtime model. Its session is opened by
:meth:`connect` and lives as long as this agent — not as
long as any one transport — and is re-established on the
- next user audio if the provider closes it.
+ next user input if the provider closes it.
toolkit (`Toolkit | None`, optional):
Tools the model may call. Executed here, with permission
checks against ``state.permission_context``.
@@ -333,6 +333,21 @@ class RealtimeAgent:
return False
return True
+ async def _ensure_connected(self) -> bool:
+ """Re-establish the session and hand it the audio kept while it
+ was down; ``False`` while the provider is still unreachable.
+
+ The one way back to a live session, whichever input asks for it.
+ """
+ if self._connected:
+ return True
+ if not await self._try_connect():
+ return False
+ for buffered in self._backlog:
+ await self.model.push_audio(buffered)
+ self._backlog.clear()
+ return True
+
# ------------------------------------------------------------------
# Lifecycle: one transport
# ------------------------------------------------------------------
@@ -425,6 +440,8 @@ class RealtimeAgent:
Raises:
`NotImplementedError`: For a text turn when the provider takes
no text input.
+ `ModelDisconnectedError`: For a text turn when the provider
+ cannot be reached, or disconnects before accepting it.
"""
match inputs:
case UserInterruptEvent():
@@ -448,8 +465,18 @@ class RealtimeAgent:
)
text = msg.get_text_content() or ""
await self._barge_in()
+ # A typed turn reconnects the session the way audio does,
+ # and takes the audio kept meanwhile along with it.
+ if not await self._ensure_connected():
+ raise ModelDisconnectedError(
+ "Provider unreachable; the text turn was not sent.",
+ )
+ try:
+ await self.model.push_text(text)
+ except ModelDisconnectedError:
+ self._mark_disconnected()
+ raise
self.state.context.append(msg)
- await self.model.push_text(text)
async def interrupt(self) -> None:
"""Stop the active reply, as when the user presses stop."""
@@ -490,7 +517,7 @@ class RealtimeAgent:
)
def _mark_disconnected(self) -> None:
- """Forget the model session so the next audio reconnects."""
+ """Forget the model session so the next input reconnects."""
self._connected = False
self._connected_event.clear()
@@ -504,11 +531,8 @@ class RealtimeAgent:
if not self._connected:
self._backlog.append(pcm)
del self._backlog[:-_BACKLOG_FRAMES]
- if not await self._try_connect():
+ if not await self._ensure_connected():
return
- for buffered in self._backlog:
- await self.model.push_audio(buffered)
- self._backlog.clear()
pushed = True
if speech is SpeechTransition.STARTED:
diff --git a/tests/realtime_agent_test.py b/tests/realtime_agent_test.py
index e3c6612af..c87173212 100644
--- a/tests/realtime_agent_test.py
+++ b/tests/realtime_agent_test.py
@@ -17,6 +17,8 @@ from agentscope.event import (
from agentscope.message import Msg
from agentscope.realtime import (
AudioFrame,
+ ControlFrame,
+ ControlFrameType,
ModelDisconnectedError,
PlayoutPosition,
RealtimeModelBase,
@@ -89,6 +91,8 @@ class ScriptedModel(RealtimeModelBase):
self.calls: list[str] = []
self.sessions = 0
self.instructions = ""
+ self.connect_error: Exception | None = None
+ self.push_text_error: Exception | None = None
self._open = asyncio.Event()
self._requested = asyncio.Event()
@@ -100,6 +104,8 @@ class ScriptedModel(RealtimeModelBase):
) -> None:
"""Record a session open, its instructions and the turn-detection
request."""
+ if self.connect_error is not None:
+ raise self.connect_error
self.sessions += 1
self._open.clear()
self.instructions = instructions
@@ -132,6 +138,8 @@ class ScriptedModel(RealtimeModelBase):
async def push_text(self, text: str) -> None:
"""Record a text turn; the agent gates on ``supports_text_input``
before ever calling this."""
+ if self.push_text_error is not None:
+ raise self.push_text_error
self.calls.append(f"push_text({text!r})")
async def push_tool_result(self, block: ToolResultBlock) -> None:
@@ -364,6 +372,80 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):
[("user", "讲个故事"), ("user", "你好"), ("assistant", "你好呀")],
)
+ async def test_provider_timeout_reconnects_on_text(self) -> None:
+ """Text reconnects a timed-out provider without duplicating the
+ new turn in the reconnect instructions."""
+ model = ScriptedModel(
+ [
+ [me.SessionEndedEvent(reason="idle")],
+ [],
+ ],
+ )
+ model.supports_text_input = True
+ agent = RealtimeAgent("Friday", "be brief", model)
+
+ async with agent:
+ await asyncio.sleep(0.1)
+ self.assertFalse(agent._connected) # pylint: disable=W0212
+ await agent.send("hello")
+
+ self.assertEqual(model.instructions, "be brief")
+ self.assertListEqual(
+ model.calls,
+ [
+ "connect(session=1,td_off=False)",
+ "connect(session=2,td_off=False)",
+ "push_text('hello')",
+ "close",
+ ],
+ )
+ self.assertListEqual(
+ [(m.role, m.get_text_content()) for m in agent.state.context],
+ [("user", "hello")],
+ )
+
+ async def test_failed_text_delivery_does_not_change_context(self) -> None:
+ """A disconnect while sending text leaves no undelivered turn."""
+ model = ScriptedModel([[]])
+ model.supports_text_input = True
+ agent = RealtimeAgent("Friday", "be brief", model)
+ error = ModelDisconnectedError("Not connected.")
+
+ async with agent:
+ model.push_text_error = error
+ with self.assertRaisesRegex(
+ ModelDisconnectedError,
+ "Not connected",
+ ):
+ await agent._on_control( # pylint: disable=W0212
+ ControlFrame(
+ type=ControlFrameType.TEXT,
+ data={"text": "hello"},
+ ),
+ )
+ self.assertFalse(agent._connected) # pylint: disable=W0212
+
+ self.assertListEqual(agent.state.context, [])
+
+ async def test_failed_text_reconnect_does_not_change_context(self) -> None:
+ """A failed reconnect leaves no undelivered text turn."""
+ model = ScriptedModel(
+ [[me.SessionEndedEvent(reason="idle")]],
+ )
+ model.supports_text_input = True
+ agent = RealtimeAgent("Friday", "be brief", model)
+
+ async with agent:
+ await asyncio.sleep(0.1)
+ model.connect_error = RuntimeError("reconnect failed")
+ with self.assertRaisesRegex(
+ ModelDisconnectedError,
+ "Provider unreachable",
+ ):
+ await agent.send("hello")
+
+ self.assertListEqual(agent.state.context, [])
+
async def test_local_vad_owns_turns(self) -> None:
"""Passing a VAD disables provider turn detection, reports the
user's speech as events and commits the turn when the VAD reports
@@ -1201,3 +1283,27 @@ class RealtimeAgentDisconnectTest(IsolatedAsyncioTestCase):
any("keep talking" in line for line in logs.output),
logs.output,
)
+
+ async def test_text_reconnect_delivers_the_kept_audio(self) -> None:
+ """A typed turn reconnects, and the frame kept by the failed push
+ rides along with that reconnect instead of being stranded."""
+ model = DropsSocketModel()
+ model.supports_text_input = True
+ agent = RealtimeAgent("Friday", "be brief", model)
+
+ async with agent:
+ await agent._on_audio( # pylint: disable=W0212
+ AudioFrame(pcm=b"\x00" * 3200),
+ )
+ await agent.send("hello")
+
+ self.assertListEqual(
+ model.calls,
+ [
+ "connect(session=1,td_off=False)",
+ "connect(session=2,td_off=False)",
+ "push_audio",
+ "push_text('hello')",
+ "close",
+ ],
+ )