Skip to content

Build and run agents you can see, understand and trust.

fix(app): clean up index worker tasks on cancellation (#2220)

Toddb22704db9a42
diff --git a/src/agentscope/app/_service/_index_worker.py b/src/agentscope/app/_service/_index_worker.py
index 14bcb8518..0625e7553 100644
--- a/src/agentscope/app/_service/_index_worker.py
+++ b/src/agentscope/app/_service/_index_worker.py
@@ -298,6 +298,15 @@ class IndexWorker:
with contextlib.suppress(asyncio.CancelledError):
await heartbeat_task
await pipeline_task
+ except asyncio.CancelledError:
+ pipeline_task.cancel()
+ heartbeat_task.cancel()
+ await asyncio.gather(
+ pipeline_task,
+ heartbeat_task,
+ return_exceptions=True,
+ )
+ raise
except Exception as exc: # noqa: BLE001 — terminal error sink
await self._mark_error(
user_id,
diff --git a/tests/index_worker_lease_test.py b/tests/index_worker_lease_test.py
index f5399b6b3..062299884 100644
--- a/tests/index_worker_lease_test.py
+++ b/tests/index_worker_lease_test.py
@@ -97,6 +97,8 @@ class _SlowPipelineWorker(IndexWorker):
self.pipeline_started = asyncio.Event()
self.pipeline_cancelled = False
self.pipeline_completed = False
+ self.heartbeat_started = asyncio.Event()
+ self.heartbeat_stopped = False
async def _run_pipeline(
self,
@@ -114,6 +116,23 @@ class _SlowPipelineWorker(IndexWorker):
self.pipeline_cancelled = True
raise
+ async def _heartbeat(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ ) -> None:
+ """Record heartbeat lifecycle while preserving worker behavior."""
+ self.heartbeat_started.set()
+ try:
+ await super()._heartbeat(
+ user_id,
+ knowledge_base_id,
+ document_id,
+ )
+ finally:
+ self.heartbeat_stopped = True
+
class IndexWorkerLeaseTest(IsolatedAsyncioTestCase):
"""Pipeline-vs-heartbeat race coverage."""
@@ -180,6 +199,46 @@ class IndexWorkerLeaseTest(IsolatedAsyncioTestCase):
)
self.assertEqual(len(storage.released), 1)
+ async def test_external_cancel_stops_child_tasks_and_releases(
+ self,
+ ) -> None:
+ """Caller cancellation must tear down pipeline and heartbeat."""
+ storage = _LeaseStorage()
+
+ worker = _SlowPipelineWorker(storage, pipeline_seconds=5.0)
+ process_task = asyncio.create_task(
+ worker.process("u", "kb", "doc-cancelled"),
+ )
+ await asyncio.wait_for(
+ asyncio.gather(
+ worker.pipeline_started.wait(),
+ worker.heartbeat_started.wait(),
+ ),
+ timeout=1.0,
+ )
+
+ process_task.cancel()
+ with self.assertRaises(asyncio.CancelledError):
+ await asyncio.wait_for(process_task, timeout=1.0)
+
+ self.assertTrue(
+ worker.pipeline_cancelled,
+ "Pipeline kept running after worker.process was cancelled.",
+ )
+ self.assertFalse(
+ worker.pipeline_completed,
+ "Pipeline completed after worker.process was cancelled.",
+ )
+ self.assertTrue(
+ worker.heartbeat_stopped,
+ "Heartbeat kept running after worker.process was cancelled.",
+ )
+ self.assertEqual(
+ [u for u in storage.status_updates if u["status"] == "error"],
+ [],
+ )
+ self.assertEqual(len(storage.released), 1)
+
async def test_not_acquired_short_circuits(self) -> None:
"""When the lease is already held by another worker, do nothing."""
storage = _LeaseStorage()