Skip to content

Build and run agents you can see, understand and trust.

fix(channel): propagate reply stream subscription failures (#3374)

WuWenbo9668b4c90c70
diff --git a/src/agentscope/app/channel/_stream.py b/src/agentscope/app/channel/_stream.py
index dbd9f2e05..4e951115b 100644
--- a/src/agentscope/app/channel/_stream.py
+++ b/src/agentscope/app/channel/_stream.py
@@ -19,6 +19,7 @@ from ..message_bus import MessageBus, MessageBusKeys
# How long to wait for the bus subscription to come up.
_SUBSCRIBE_TIMEOUT_SECS = 5.0
+_closing_feeders: set[asyncio.Task[None]] = set()
_TERMINAL_EVENTS = frozenset(
{
EventType.REPLY_END,
@@ -28,6 +29,13 @@ _TERMINAL_EVENTS = frozenset(
)
+def _feeder_closed(task: asyncio.Task[None]) -> None:
+ """Release a failed setup's feeder and observe any cleanup error."""
+ _closing_feeders.discard(task)
+ if not task.cancelled():
+ task.exception()
+
+
async def open_reply_stream(
bus: MessageBus,
session_id: str,
@@ -48,33 +56,57 @@ async def open_reply_stream(
"""
event_key = MessageBusKeys.session_events(session_id)
ready = asyncio.Event()
- queue: asyncio.Queue[dict] = asyncio.Queue()
+ queue: asyncio.Queue[dict | Exception] = asyncio.Queue()
async def feeder() -> None:
"""Buffer live subscription events into the local queue."""
try:
async for evt in bus.subscribe(event_key, on_ready=ready.set):
await queue.put(evt)
- except asyncio.CancelledError:
- pass
+ except Exception as error:
+ await queue.put(error)
+ raise
+ subscription_error = RuntimeError(
+ "Reply stream subscription ended unexpectedly",
+ )
+ await queue.put(subscription_error)
+ raise subscription_error
feeder_task = asyncio.create_task(feeder())
+ ready_task = asyncio.create_task(ready.wait())
try:
- await asyncio.wait_for(ready.wait(), timeout=_SUBSCRIBE_TIMEOUT_SECS)
+ done, _ = await asyncio.wait(
+ {ready_task, feeder_task},
+ timeout=_SUBSCRIBE_TIMEOUT_SECS,
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ if not ready.is_set():
+ if feeder_task in done:
+ await feeder_task
+ raise TimeoutError("Timed out waiting for the bus subscription")
cursor = await bus.registry_get(
MessageBusKeys.session_event_checkpoint(session_id),
MessageBusKeys.SESSION_EVENT_CURSOR_FIELD,
)
except BaseException:
+ # A bus may block while closing. Let it finish off the caller's path.
+ _closing_feeders.add(feeder_task)
+ feeder_task.add_done_callback(_feeder_closed)
feeder_task.cancel()
raise
+ finally:
+ ready_task.cancel()
+ try:
+ await ready_task
+ except asyncio.CancelledError:
+ pass
return _read(bus, event_key, queue, feeder_task, cursor)
async def _read(
bus: MessageBus,
event_key: str,
- queue: "asyncio.Queue[dict]",
+ queue: "asyncio.Queue[dict | Exception]",
feeder_task: "asyncio.Task",
after: str | None,
) -> AsyncGenerator[dict, None]:
@@ -86,7 +118,8 @@ async def _read(
Args:
bus (`MessageBus`): The application message bus.
event_key (`str`): The session's event log / channel key.
- queue (`asyncio.Queue[dict]`): Live events buffered so far.
+ queue (`asyncio.Queue[dict | Exception]`): Live events or a
+ subscription failure buffered so far.
feeder_task (`asyncio.Task`): The subscription, cancelled on close.
after (`str | None`): Persisted checkpoint cursor to replay after.
@@ -106,6 +139,8 @@ async def _read(
return
while True:
evt = await queue.get()
+ if isinstance(evt, Exception):
+ raise evt
eid = evt.get("_entry_id")
if eid is not None:
if str(eid) in seen:
diff --git a/tests/channel_stream_test.py b/tests/channel_stream_test.py
index 16d5ebd38..a72bf8ea1 100644
--- a/tests/channel_stream_test.py
+++ b/tests/channel_stream_test.py
@@ -10,7 +10,9 @@ these cover.
"""
import asyncio
from contextlib import aclosing
+from typing import AsyncGenerator, Callable
from unittest import IsolatedAsyncioTestCase
+from unittest.mock import patch
from agentscope.app._bus_ops import publish_session_event
from agentscope.app.channel._stream import open_reply_stream
@@ -102,6 +104,32 @@ class _SeamBus(InMemoryMessageBus):
return await super().log_read(key, **kwargs)
+class _BrokenSubscriptionBus(InMemoryMessageBus):
+ """A bus whose live subscription fails on demand."""
+
+ def __init__(self, fail_before_ready: bool = False) -> None:
+ super().__init__()
+ self.fail_before_ready = fail_before_ready
+ self.fail = asyncio.Event()
+ self.events: list[dict] = []
+
+ async def subscribe(
+ self,
+ key: str,
+ *,
+ on_ready: Callable[[], None] | None = None,
+ ) -> AsyncGenerator[dict, None]:
+ """Raise before or after reporting that the subscription is live."""
+ if self.fail_before_ready:
+ raise RuntimeError("subscription failed")
+ if callable(on_ready):
+ on_ready()
+ for event in self.events:
+ yield event
+ await self.fail.wait()
+ raise RuntimeError("subscription failed")
+
+
class EventStreamTest(IsolatedAsyncioTestCase):
"""The stream is gap-free and always terminates."""
@@ -151,6 +179,71 @@ class EventStreamTest(IsolatedAsyncioTestCase):
drained, _ = await asyncio.gather(_drain(bus, "s-1"), _run())
self.assertListEqual(drained, ["REPLY_START", "REPLY_END"])
+ async def test_subscription_failure_before_ready_is_propagated(
+ self,
+ ) -> None:
+ """Opening fails immediately if the subscription never comes up."""
+ bus = _BrokenSubscriptionBus(fail_before_ready=True)
+
+ with self.assertRaisesRegex(RuntimeError, "subscription failed"):
+ await open_reply_stream(bus, "s-1")
+
+ async def test_subscription_failure_while_reading_is_propagated(
+ self,
+ ) -> None:
+ """A lost live subscription does not leave the reader waiting."""
+ bus = _BrokenSubscriptionBus()
+ stream = await open_reply_stream(bus, "s-1")
+
+ async with aclosing(stream):
+ next_event = asyncio.create_task(anext(stream))
+ bus.fail.set()
+ with self.assertRaisesRegex(RuntimeError, "subscription failed"):
+ await asyncio.wait_for(next_event, timeout=0.5)
+
+ async def test_buffered_reply_survives_immediate_disconnect(self) -> None:
+ """Deliver the complete reply buffered before a disconnect."""
+ bus = _BrokenSubscriptionBus()
+ bus.events = [
+ _start().model_dump(mode="json"),
+ _end().model_dump(mode="json"),
+ ]
+ bus.fail.set()
+
+ stream = await open_reply_stream(bus, "s-1")
+ async with aclosing(stream):
+ self.assertEqual([event async for event in stream], bus.events)
+
+ async def test_setup_timeout_does_not_wait_for_bus_cleanup(self) -> None:
+ """Opening times out while subscription cleanup is pending."""
+ bus = InMemoryMessageBus()
+ release = asyncio.Event()
+ closed = asyncio.Event()
+
+ async def subscription() -> AsyncGenerator[dict, None]:
+ try:
+ await asyncio.Event().wait()
+ yield {}
+ finally:
+ await release.wait()
+ closed.set()
+
+ with (
+ patch.object(bus, "subscribe", return_value=subscription()),
+ patch("agentscope.app.channel._stream._SUBSCRIBE_TIMEOUT_SECS", 0),
+ ):
+ opening = asyncio.create_task(open_reply_stream(bus, "s-1"))
+ try:
+ done, _ = await asyncio.wait({opening}, timeout=1)
+ self.assertIn(opening, done)
+ with self.assertRaisesRegex(TimeoutError, "bus subscription"):
+ await opening
+ self.assertFalse(closed.is_set())
+ finally:
+ release.set()
+ await asyncio.gather(opening, return_exceptions=True)
+ await asyncio.wait_for(closed.wait(), timeout=1)
+
async def test_seam_events_are_not_delivered_twice(self) -> None:
"""An event landing between subscribe and replay reaches the
stream both ways; it must be yielded once.