Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(schedule): serialize reconciliation to drop stale job snapshots (#3608)
ordinary being96e4f1a535f4
diff --git a/src/agentscope/app/_manager/_scheduler/_scheduler_manager.py b/src/agentscope/app/_manager/_scheduler/_scheduler_manager.pyindex c8903cdad..35380d77d 100644--- a/src/agentscope/app/_manager/_scheduler/_scheduler_manager.py+++ b/src/agentscope/app/_manager/_scheduler/_scheduler_manager.py@@ -91,6 +91,10 @@ class SchedulerManager:# ``updated_at`` of each registered job, so a reconcile can tell# an edited schedule from an unchanged one.self._versions: dict[str, datetime] = {}+ # Serialises reconcile. The periodic loop and the lifecycle listener+ # both call it, and a slow read must not apply its stale snapshot+ # after a newer reconcile has already changed the job set.+ self._reconcile_lock = asyncio.Lock()self._tasks: list[asyncio.Task] = []# ------------------------------------------------------------------@@ -466,7 +470,19 @@ class SchedulerManager:no longer enabled, and re-registers those whose ``updated_at``moved. Safe to call repeatedly — that is how a droppednotification heals.++ Both the periodic loop and the lifecycle listener call this+ method, so concurrent runs are serialised across the storage read+ and the resulting job updates. Without that ordering an older read+ can finish last and restore a job a newer reconcile had just+ deleted or disabled, or overwrite an edited job with its previous+ record."""+ async with self._reconcile_lock:+ await self._reconcile_locked()++ async def _reconcile_locked(self) -> None:+ """Apply one storage snapshot; caller must hold the reconcile lock."""try:records = await self._storage.list_all_schedules()except Exception: # pylint: disable=broad-exceptdiff --git a/tests/service_scheduler_test.py b/tests/service_scheduler_test.pyindex f48ec407a..c55b23759 100644--- a/tests/service_scheduler_test.py+++ b/tests/service_scheduler_test.py@@ -398,6 +398,99 @@ class TestSchedulerReconcile(_SchedulerOwnershipTestBase):self.assertListEqual(self.job_ids(), [good.id])+ async def _race_stale_and_fresh_reconcile(+ self,+ change: str,+ ) -> ScheduleRecord:+ """Pause the first read, apply *change*, then start a second run.++ The first reconcile captures the old records and waits; the second+ one sees the new storage state. The returned record is the one+ storage now holds.+ """+ record = _make_record()+ await self.storage.upsert_schedule(record.user_id, record)+ # ``_add_job`` needs a started scheduler to compute next_run_time.+ self.manager._scheduler.start(paused=True)+ self.addCleanup(+ self.manager._scheduler.shutdown,+ wait=False,+ )+ self.manager._add_job(record)++ original_list = self.storage.list_all_schedules+ first_read_captured = asyncio.Event()+ release_first_read = asyncio.Event()+ second_read_returned = asyncio.Event()+ calls = 0++ async def controlled_list() -> list[ScheduleRecord]:+ nonlocal calls+ calls += 1+ snapshot = await original_list()+ if calls == 1:+ first_read_captured.set()+ await release_first_read.wait()+ else:+ second_read_returned.set()+ return snapshot++ with patch.object(+ self.storage,+ "list_all_schedules",+ controlled_list,+ ):+ stale = asyncio.create_task(self.manager.reconcile())+ await first_read_captured.wait()++ if change == "disable":+ record.data.enabled = False+ else:+ record.data.cron_expression = "30 3 * * *"+ record.updated_at = datetime(2025, 6, 1)+ await self.storage.upsert_schedule(record.user_id, record)++ fresh = asyncio.create_task(self.manager.reconcile())+ try:+ # A serialised reconcile blocks the second read behind the+ # first one, so this wait times out. Without that ordering+ # the second read returns and applies immediately.+ await asyncio.wait_for(+ second_read_returned.wait(),+ timeout=0.2,+ )+ except asyncio.TimeoutError:+ pass+ else:+ # Let the fresh reconcile apply the new snapshot before the+ # stale one resumes.+ await asyncio.wait({fresh}, timeout=0.2)+ finally:+ release_first_read.set()+ await asyncio.gather(stale, fresh)++ return record++ async def test_a_stale_reconcile_cannot_restore_a_disabled_job(+ self,+ ) -> None:+ """A slow read that captured an enabled record must not re-add it."""+ await self._race_stale_and_fresh_reconcile("disable")++ self.assertListEqual(self.job_ids(), [])++ async def test_a_stale_reconcile_cannot_rewrite_an_edited_job(+ self,+ ) -> None:+ """A slow read must not restore the previous cron configuration."""+ record = await self._race_stale_and_fresh_reconcile("edit")++ self.assertListEqual(self.job_ids(), [record.id])+ self.assertEqual(+ self.manager._versions[record.id],+ record.updated_at,+ )+class TestSchedulerOwnership(_SchedulerOwnershipTestBase):"""Only the enabled node holds timers; every node can notify."""