Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(app): clean up wakeup startup failures after readiness (#3634)
Dan1fd57aef8e93
diff --git a/src/agentscope/app/_manager/_chat_input_queue.py b/src/agentscope/app/_manager/_chat_input_queue.pyindex 89f71f465..85f4f8f39 100644--- a/src/agentscope/app/_manager/_chat_input_queue.py+++ b/src/agentscope/app/_manager/_chat_input_queue.py@@ -65,8 +65,14 @@ class ChatInputQueue:self._retry_tasks: set[asyncio.Task] = set()self._closed = False- async def close(self) -> None:- """Cancel retained timers and release recovery leadership."""+ async def close(self, *, release_leadership: bool = True) -> None:+ """Cancel retained timers and optionally release recovery leadership.++ Args:+ release_leadership (`bool`):+ Whether to contact the bus to release the recovery lease.+ Failed startup skips this and lets any lease expire.+ """self._closed = Truetasks = list(self._retry_tasks)for task in tasks:@@ -74,7 +80,8 @@ class ChatInputQueue:await asyncio.gather(*tasks, return_exceptions=True)self._retry_tasks.clear()self._recovery_task = None- await self._release_recovery_leadership()+ if release_leadership:+ await self._release_recovery_leadership()def schedule_recovery(self) -> None:"""Retain at most one non-blocking recovery sweep per process."""diff --git a/src/agentscope/app/_manager/_wakeup_dispatcher.py b/src/agentscope/app/_manager/_wakeup_dispatcher.pyindex be23bf4f1..11a0d8320 100644--- a/src/agentscope/app/_manager/_wakeup_dispatcher.py+++ b/src/agentscope/app/_manager/_wakeup_dispatcher.py@@ -142,12 +142,39 @@ class WakeupDispatcher:name="wakeup-dispatcher",)await ready.wait()- self._fallback_task = asyncio.create_task(- self._fallback_loop(),- name="wakeup-dispatcher:fallback",- )- await self._bus.publish(MessageBusKeys.wakeup_signal(), {})- self._chat_queue.schedule_recovery()+ try:+ if self._task.done():+ raise RuntimeError("Wakeup subscription ended during startup")+ self._fallback_task = asyncio.create_task(+ self._fallback_loop(),+ name="wakeup-dispatcher:fallback",+ )+ await self._bus.publish(MessageBusKeys.wakeup_signal(), {})+ if self._task.done():+ raise RuntimeError("Wakeup subscription ended during startup")+ self._chat_queue.schedule_recovery()+ except BaseException:+ # Failed entry does not invoke __aexit__. The listener and+ # fallback may already have spawned retries or recovery work.+ tasks = [+ task+ for task in (+ self._task,+ self._fallback_task,+ *self._retry_tasks,+ )+ if task is not None+ ]+ for task in tasks:+ task.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)+ self._task = None+ self._fallback_task = None+ self._retry_tasks.clear()+ # Stop queue helpers and late run callbacks without contacting+ # the failed bus to release recovery leadership.+ await self._chat_queue.close(release_leadership=False)+ raisereturn selfasync def __aexit__(self, *exc: object) -> None:diff --git a/tests/service_wakeup_dispatcher_test.py b/tests/service_wakeup_dispatcher_test.pyindex 3e9a08926..781bcee4e 100644--- a/tests/service_wakeup_dispatcher_test.py+++ b/tests/service_wakeup_dispatcher_test.py@@ -1438,6 +1438,293 @@ class TestChatInputClaimReliability(IsolatedAsyncioTestCase):class TestWakeupDispatcherLifecycle(IsolatedAsyncioTestCase):"""Tests covering the ``__aenter__`` / ``__aexit__`` ACM behaviour."""+ async def test_startup_rejects_listener_ending_after_ready(self) -> None:+ """Readiness alone is insufficient if the listener already exited."""++ async def subscribe(+ _key: str,+ *,+ on_ready: Callable[[], None] | None = None,+ ) -> AsyncGenerator[dict, None]:+ if on_ready is not None:+ on_ready()+ return+ yield {} # pylint: disable=unreachable++ bus = _FakeBus()+ dispatcher = WakeupDispatcher(+ bus,+ _FakeStorage(),+ _FakeChatService(),+ ChatRunRegistry(),+ )+ with patch.object(bus, "subscribe", subscribe), patch.object(+ bus,+ "publish",+ wraps=bus.publish,+ ) as publish:+ with self.assertRaisesRegex(RuntimeError, "subscription"):+ # pylint: disable=unnecessary-dunder-call+ await asyncio.wait_for(dispatcher.__aenter__(), timeout=1)+ publish.assert_not_awaited()+ self.assertIsNone(dispatcher._task)+ self.assertIsNone(dispatcher._fallback_task)++ async def test_initial_publish_failure_cleans_up_started_tasks(+ self,+ ) -> None:+ """Failed entry must cancel both tasks without using the failed bus."""+ bus = _FakeBus()+ dispatcher = WakeupDispatcher(+ bus,+ _FakeStorage(),+ _FakeChatService(),+ ChatRunRegistry(),+ )+ error = ConnectionError("initial publish failed")+ started: list[asyncio.Task] = []++ async def publish(_key: str, _payload: dict) -> None:+ started.extend([dispatcher._task, dispatcher._fallback_task])+ raise error++ with patch.object(bus, "publish", publish), patch.object(+ bus,+ "acquire_lock",+ ) as acquire_lock:+ with self.assertRaises(ConnectionError) as raised:+ # pylint: disable=unnecessary-dunder-call+ await dispatcher.__aenter__()+ acquire_lock.assert_not_called()+ self.assertIs(raised.exception, error)+ self.assertEqual(len(started), 2)+ self.assertTrue(all(task.done() for task in started))+ self.assertIsNone(dispatcher._task)+ self.assertIsNone(dispatcher._fallback_task)++ async def test_cancelling_initial_publish_cleans_up_started_tasks(+ self,+ ) -> None:+ """Cancellation after readiness also cleans up the started tasks."""+ bus = _FakeBus()+ dispatcher = WakeupDispatcher(+ bus,+ _FakeStorage(),+ _FakeChatService(),+ ChatRunRegistry(),+ )+ publishing = asyncio.Event()++ async def publish(_key: str, _payload: dict) -> None:+ publishing.set()+ await asyncio.Event().wait()++ with patch.object(bus, "publish", publish):+ # pylint: disable=unnecessary-dunder-call+ startup = asyncio.create_task(dispatcher.__aenter__())+ await asyncio.wait_for(publishing.wait(), timeout=1)+ started = [dispatcher._task, dispatcher._fallback_task]+ startup.cancel()+ with self.assertRaises(asyncio.CancelledError):+ await startup+ self.assertTrue(all(task.done() for task in started))+ self.assertIsNone(dispatcher._task)+ self.assertIsNone(dispatcher._fallback_task)++ async def test_startup_rechecks_listener_after_initial_publish(+ self,+ ) -> None:+ """The listener may end while the initial publish is in flight."""+ bus = _FakeBus()+ dispatcher = WakeupDispatcher(+ bus,+ _FakeStorage(),+ _FakeChatService(),+ ChatRunRegistry(),+ )+ end_subscription = asyncio.Event()++ async def subscribe(+ _key: str,+ *,+ on_ready: Callable[[], None] | None = None,+ ) -> AsyncGenerator[dict, None]:+ if on_ready is not None:+ on_ready()+ await end_subscription.wait()+ return+ yield {} # pylint: disable=unreachable++ async def publish(_key: str, _payload: dict) -> None:+ end_subscription.set()+ await dispatcher._task++ with patch.object(bus, "subscribe", subscribe), patch.object(+ bus,+ "publish",+ publish,+ ):+ with self.assertRaisesRegex(RuntimeError, "subscription"):+ # pylint: disable=unnecessary-dunder-call+ await asyncio.wait_for(dispatcher.__aenter__(), timeout=1)+ self.assertIsNone(dispatcher._task)+ self.assertIsNone(dispatcher._fallback_task)++ async def test_failed_startup_cancels_dispatched_retries(self) -> None:+ """A signal can schedule retries before initial publish fails."""+ bus = _FakeBus()+ dispatcher = WakeupDispatcher(+ bus,+ _FakeStorage(),+ _FakeChatService(),+ ChatRunRegistry(),+ )+ bus._locks.add(MessageBusKeys.session_lock("s"))+ await enqueue_run_trigger(+ bus,+ user_id="u",+ session_id="s",+ agent_id="a",+ signal=False,+ )+ scheduled = asyncio.Event()+ schedule_retry = dispatcher._schedule_retry+ publish_signal = bus.publish+ started: list[asyncio.Task] = []+ error = ConnectionError("publish response lost")++ def schedule(*args: Any) -> None:+ schedule_retry(*args)+ started.extend(dispatcher._retry_tasks)+ scheduled.set()++ async def publish(key: str, payload: dict) -> None:+ await publish_signal(key, payload)+ await scheduled.wait()+ raise error++ with patch.object(dispatcher, "_schedule_retry", schedule), patch(+ "agentscope.app._manager._wakeup_dispatcher."+ "_RESUME_RETRY_BACKOFF_SECS",+ 60,+ ), patch.object(bus, "publish", publish):+ with self.assertRaises(ConnectionError) as raised:+ # pylint: disable=unnecessary-dunder-call+ await asyncio.wait_for(dispatcher.__aenter__(), timeout=1)++ self.assertIs(raised.exception, error)+ self.assertEqual(+ ([task.done() for task in started], dispatcher._retry_tasks),+ ([True], set()),+ )++ async def test_failed_startup_cancels_fallback_recovery(self) -> None:+ """A slow initial publish lets fallback start a recovery sweep."""+ bus = _FakeBus()+ dispatcher = WakeupDispatcher(+ bus,+ _FakeStorage(),+ _FakeChatService(),+ ChatRunRegistry(),+ )+ recovering = asyncio.Event()+ started: list[asyncio.Task] = []+ error = ConnectionError("initial publish timed out")+ publish_calls = 0++ async def registry_getall(_namespace: str) -> dict[str, str]:+ task = asyncio.current_task()+ assert task is not None+ started.append(task)+ recovering.set()+ await asyncio.Event().wait()+ return {}++ async def publish(_key: str, _payload: dict) -> None:+ nonlocal publish_calls+ publish_calls += 1+ if publish_calls == 1:+ await recovering.wait()+ raise error++ with patch(+ "agentscope.app._manager._wakeup_dispatcher."+ "_MESSAGE_FALLBACK_TICK_SECS",+ 0.001,+ ), patch.object(bus, "publish", publish), patch.object(+ bus,+ "registry_getall",+ registry_getall,+ ), patch.object(+ dispatcher._chat_queue,+ "_release_recovery_leadership",+ ) as release:+ with self.assertRaises(ConnectionError) as raised:+ # pylint: disable=unnecessary-dunder-call+ await asyncio.wait_for(dispatcher.__aenter__(), timeout=1)+ release.assert_not_awaited()++ self.assertIs(raised.exception, error)+ self.assertEqual(+ (+ [task.done() for task in started],+ dispatcher._chat_queue._retry_tasks,+ dispatcher._chat_queue._recovery_task,+ dispatcher._chat_queue._closed,+ ),+ ([True], set(), None, True),+ )++ async def test_failed_startup_blocks_late_run_nudges(self) -> None:+ """Runs finishing after failed startup must not create bus tasks."""+ bus = _FakeBus()+ registry = ChatRunRegistry()+ dispatcher = WakeupDispatcher(+ bus,+ _FakeStorage(),+ _FakeChatService(),+ registry,+ )+ running = asyncio.Event()+ finish_run = asyncio.Event()+ publish_signal = bus.publish+ await enqueue_run_trigger(+ bus,+ user_id="u",+ session_id="s",+ agent_id="a",+ signal=False,+ )++ async def run(**_kwargs: Any) -> None:+ running.set()+ await finish_run.wait()++ async def publish(key: str, payload: dict) -> None:+ await publish_signal(key, payload)+ await running.wait()+ raise ConnectionError("publish response lost")++ with patch.object(dispatcher._chat_service, "run", run), patch.object(+ bus,+ "publish",+ publish,+ ), patch.object(+ bus,+ "registry_get",+ return_value=None,+ ) as registry_get:+ async with registry:+ with self.assertRaises(ConnectionError):+ # pylint: disable=unnecessary-dunder-call+ await asyncio.wait_for(dispatcher.__aenter__(), timeout=1)+ task = registry.get("s")+ finish_run.set()+ await task+ await asyncio.sleep(0)+ registry_get.assert_not_awaited()+ self.assertEqual(dispatcher._chat_queue._retry_tasks, set())+async def test_recovery_sweep_has_one_cluster_leader(self) -> None:"""Two processes sharing a bus do not both run the O(N) sweep."""