AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): stop unpickling paused/fired schedules on every get_execution_schedules read (#14439)
Bentlyb042c69ae256
diff --git a/autogpt_platform/backend/backend/api/features/orgs/regression_test.py b/autogpt_platform/backend/backend/api/features/orgs/regression_test.pyindex 61ab7aaaf9..ac3a402447 100644--- a/autogpt_platform/backend/backend/api/features/orgs/regression_test.py+++ b/autogpt_platform/backend/backend/api/features/orgs/regression_test.py@@ -1374,6 +1374,13 @@ class TestRegressionSchedules:other_job.name = "other-job"scheduler.scheduler.get_jobs = MagicMock(return_value=[owned_job, other_job])+ # get_graph_execution_schedules (include_paused=False) reads the+ # active-jobs path, which queries the jobstore directly rather than+ # going through scheduler.get_jobs — see Scheduler._get_active_jobs_cached.+ scheduler._execution_jobstore = MagicMock()+ scheduler._execution_jobstore._get_jobs = MagicMock(+ return_value=[owned_job, other_job]+ )results = scheduler.get_graph_execution_schedules(user_id=USER_ID)diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex 1ca737b648..96cecff59a 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -1796,6 +1796,21 @@ class Scheduler(AppService):# Configure executors to limit concurrency without skipping jobsfrom apscheduler.executors.pool import ThreadPoolExecutor+ # Kept as a named reference (rather than only living inside the+ # ``jobstores=`` dict below) so ``_get_active_jobs_cached`` can query+ # its table directly with a server-side filter — see that method for+ # why the stock ``get_all_jobs()`` isn't enough.+ self._execution_jobstore = SQLAlchemyJobStore(+ engine=create_engine(+ url=db_url,+ pool_size=self.db_pool_size(),+ max_overflow=0,+ ),+ metadata=MetaData(schema=db_schema),+ # this one is pre-existing so it keeps the default table name.+ tablename="apscheduler_jobs",+ )+self.scheduler = BackgroundScheduler(executors={"default": ThreadPoolExecutor(@@ -1808,17 +1823,7 @@ class Scheduler(AppService):"misfire_grace_time": None, # No time limit for missed jobs},jobstores={- Jobstores.EXECUTION.value: SQLAlchemyJobStore(- engine=create_engine(- url=db_url,- pool_size=self.db_pool_size(),- max_overflow=0,- ),- metadata=MetaData(schema=db_schema),- # this one is pre-existing so it keeps the- # default table name.- tablename="apscheduler_jobs",- ),+ Jobstores.EXECUTION.value: self._execution_jobstore,Jobstores.BATCHED_NOTIFICATIONS.value: SQLAlchemyJobStore(engine=create_engine(url=db_url,@@ -2281,14 +2286,15 @@ class Scheduler(AppService):if isinstance(info, GraphExecutionJobInfo)]- # Process-wide cache for ``scheduler.get_jobs(EXECUTION)``. APScheduler- # has no SQL-level user_id / kind filter — it loads every row and- # unpickles each ``job.kwargs`` in Python. The /library page now- # fires THREE separate calls into this method on cold load (existing- # graph schedules + new copilot followups + briefing-pill counts),- # so we memoise the unfiltered list for a few seconds. Mutations- # (`add_*_schedule`, `delete_schedule`) clear the cache so user-visible- # latency on writes is unchanged.+ # Process-wide cache for ``scheduler.get_jobs(EXECUTION)`` — the fully+ # unfiltered row set, including paused schedules and already-fired+ # one-shot jobs. APScheduler has no SQL-level user_id / kind filter+ # either way — it loads every row and unpickles each ``job.kwargs`` in+ # Python — so this is now only worth paying for on the rare+ # ``include_paused=True`` lifecycle lookups; see the sibling+ # ``_get_active_jobs_cached`` below for the path everything else takes.+ # Mutations (`add_*_schedule`, `delete_schedule`) clear both caches so+ # user-visible latency on writes is unchanged._JOBS_CACHE_TTL_S = 5.0_jobs_cache: list[JobObj] | None = None_jobs_cache_expires_at: float = 0.0@@ -2324,6 +2330,40 @@ class Scheduler(AppService):if self._jobs_cache_version == version_at_start:self._jobs_cache = jobsself._jobs_cache_expires_at = time.monotonic() + self._JOBS_CACHE_TTL_S+ return jobs++ # Second cache, keyed off the same lock/version, for the ``next_run_time+ # IS NOT NULL`` (non-paused) rows only. This is what every caller except+ # the pause/resume lifecycle lookups (``include_paused=True``) actually+ # wants, and unlike ``_get_jobs_cached`` it pushes that filter down to+ # SQL instead of unpickling every paused/already-fired row in Python+ # only to throw it away — ``apscheduler_jobs`` accumulates those forever+ # (nothing deletes a paused or fired-once job), so on a table with a+ # meaningful history the unfiltered scan is what Sentry was flagging as+ # a slow, unbounded query. ``next_run_time`` already carries a btree+ # index from APScheduler's own table definition, so this needs no+ # schema change.+ _active_jobs_cache: list[JobObj] | None = None+ _active_jobs_cache_expires_at: float = 0.0++ def _get_active_jobs_cached(self) -> list[JobObj]:+ with self._jobs_cache_lock:+ now = time.monotonic()+ if (+ self._active_jobs_cache is not None+ and now < self._active_jobs_cache_expires_at+ ):+ return self._active_jobs_cache+ version_at_start = self._jobs_cache_version+ jobs = self._execution_jobstore._get_jobs(+ self._execution_jobstore.jobs_t.c.next_run_time.isnot(None)+ )+ with self._jobs_cache_lock:+ if self._jobs_cache_version == version_at_start:+ self._active_jobs_cache = jobs+ self._active_jobs_cache_expires_at = (+ time.monotonic() + self._JOBS_CACHE_TTL_S+ )# The one scheduler metric with an alert on it was never set.# Only an accepted read may publish it: a read that was# invalidated mid-query is stale by definition and must not@@ -2337,6 +2377,8 @@ class Scheduler(AppService):with self._jobs_cache_lock:self._jobs_cache = Noneself._jobs_cache_expires_at = 0.0+ self._active_jobs_cache = None+ self._active_jobs_cache_expires_at = 0.0self._jobs_cache_version += 1@expose@@ -2363,18 +2405,22 @@ class Scheduler(AppService):schedules remain owner-only for scoped calls. Trusted global callersthat provide neither *user_id* nor *organization_id* receive all jobs.- Paused jobs (``next_run_time is None``) are hidden by default, which- is what keeps a fired expert's suspended schedules out of every user- -facing listing. *include_paused* is for the lifecycle callers that- have to find them again to resume them. Fired one-shot jobs share- the same null marker, so they resurface too — filter by kind/id if- that matters to the caller.+ Paused jobs (``next_run_time is None``) are hidden by default —+ excluded at the SQL level via ``_get_active_jobs_cached`` rather than+ filtered out afterwards — which is what keeps a fired expert's+ suspended schedules out of every user-facing listing. *include_paused*+ is for the lifecycle callers that have to find them again to resume+ them; that path reads the unfiltered ``_get_jobs_cached`` instead.+ Fired one-shot jobs share the same null marker, so they resurface too+ — filter by kind/id if that matters to the caller."""- jobs: list[JobObj] = self._get_jobs_cached()+ jobs: list[JobObj] = (+ self._get_jobs_cached()+ if include_paused+ else self._get_active_jobs_cached()+ )results: list[Union[GraphExecutionJobInfo, CopilotTurnJobInfo]] = []for job in jobs:- if job.next_run_time is None and not include_paused:- continueinfo = _job_to_info(job)if info is None:continuediff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.pyindex 5c7c1e0b0f..bf146d8f3e 100644--- a/autogpt_platform/backend/backend/executor/scheduler_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py@@ -53,6 +53,50 @@ async def test_agent_schedule(server: SpinTestServer):assert len(schedules) == 0[email protected](loop_scope="session")+async def test_paused_schedule_excluded_unless_include_paused(server: SpinTestServer):+ """AUTOGPT-SERVER-9NB regression: get_execution_schedules must exclude a+ paused schedule via the SQL-level ``next_run_time IS NOT NULL`` filter+ (Scheduler._get_active_jobs_cached), not just via the old in-Python+ check — and ``include_paused=True`` must still find it through the+ unfiltered ``_get_jobs_cached`` path used by pause/resume lifecycle+ callers. Exercised against the real apscheduler_jobs table."""+ await db.connect()+ test_user = await create_test_user()+ test_graph = await server.agent_server.test_create_graph(+ create_graph=CreateGraph(graph=create_test_graph()),+ user_id=test_user.id,+ )++ scheduler = get_scheduler_client()+ schedule = await scheduler.add_execution_schedule(+ graph_id=test_graph.id,+ user_id=test_user.id,+ graph_version=1,+ cron="0 0 * * *",+ input_data={"input": "data"},+ input_credentials={},+ )++ assert await scheduler.pause_schedule(schedule.id, user_id=test_user.id) is True++ active_only = await scheduler.get_execution_schedules(+ graph_id=test_graph.id, user_id=test_user.id+ )+ assert len(active_only) == 0++ with_paused = await scheduler.get_execution_schedules(+ graph_id=test_graph.id, user_id=test_user.id, include_paused=True+ )+ assert [s.id for s in with_paused] == [schedule.id]++ assert await scheduler.resume_schedule(schedule.id, user_id=test_user.id) is True+ active_after_resume = await scheduler.get_execution_schedules(+ graph_id=test_graph.id, user_id=test_user.id+ )+ assert [s.id for s in active_after_resume] == [schedule.id]++# ---------------------------------------------------------------------------# Community rebuild @expose methods — lightweight unit tests#@@ -855,20 +899,20 @@ def _counter(name: str, **labels) -> float:return REGISTRY.get_sample_value(name, labels) or 0.0-def test_get_jobs_cached_sets_the_scheduler_jobs_gauge():+def test_get_active_jobs_cached_sets_the_scheduler_jobs_gauge():"""autogpt_scheduler_jobs has an alert on it and was never set."""from unittest.mock import MagicMockfrom backend.executor.scheduler import Schedulers = Scheduler.__new__(Scheduler)- s._jobs_cache = None- s._jobs_cache_expires_at = 0.0+ s._active_jobs_cache = None+ s._active_jobs_cache_expires_at = 0.0s._jobs_cache_version = 0- s.scheduler = MagicMock()- s.scheduler.get_jobs.return_value = [object(), object(), object()]+ s._execution_jobstore = MagicMock()+ s._execution_jobstore._get_jobs.return_value = [object(), object(), object()]- assert len(s._get_jobs_cached()) == 3+ assert len(s._get_active_jobs_cached()) == 3assert (_counter("autogpt_scheduler_jobs", job_type="execution", status="scheduled")== 3@@ -876,7 +920,7 @@ def test_get_jobs_cached_sets_the_scheduler_jobs_gauge():def test_stale_read_invalidated_mid_query_does_not_overwrite_the_gauge():- """An invalidation that lands while get_jobs() is in flight rejects the+ """An invalidation that lands while the DB query is in flight rejects thecache write; the gauge must be rejected with it, or a slow stale read canoverwrite a newer count that another reader already published."""from unittest.mock import MagicMock@@ -884,14 +928,14 @@ def test_stale_read_invalidated_mid_query_does_not_overwrite_the_gauge():from backend.executor.scheduler import Schedulers = Scheduler.__new__(Scheduler)- s._jobs_cache = None- s._jobs_cache_expires_at = 0.0+ s._active_jobs_cache = None+ s._active_jobs_cache_expires_at = 0.0s._jobs_cache_version = 0- s.scheduler = MagicMock()+ s._execution_jobstore = MagicMock()# A fresh, accepted read publishes 5.- s.scheduler.get_jobs.return_value = [object()] * 5- s._get_jobs_cached()+ s._execution_jobstore._get_jobs.return_value = [object()] * 5+ s._get_active_jobs_cached()def gauge() -> float:return _counter(@@ -902,14 +946,14 @@ def test_stale_read_invalidated_mid_query_does_not_overwrite_the_gauge():# Now a read whose query is interrupted by an invalidation: it returns a# different count, but the version moved, so the cache write is skipped.- s._jobs_cache = None- s._jobs_cache_expires_at = 0.0+ s._active_jobs_cache = None+ s._active_jobs_cache_expires_at = 0.0- def _slow_query_then_invalidated(**_):+ def _slow_query_then_invalidated(*_args, **_kwargs):s._invalidate_jobs_cache()return [object()] * 2- s.scheduler.get_jobs.side_effect = _slow_query_then_invalidated- assert len(s._get_jobs_cached()) == 2 # caller still gets the list- assert s._jobs_cache is None # write-back was rejected+ s._execution_jobstore._get_jobs.side_effect = _slow_query_then_invalidated+ assert len(s._get_active_jobs_cached()) == 2 # caller still gets the list+ assert s._active_jobs_cache is None # write-back was rejectedassert gauge() == 5 # and so was the gauge updatediff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.pyindex 61790fa13b..0cced5e5d5 100644--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py@@ -1647,6 +1647,10 @@ class TestScheduleOrgVisibility:sched, jobs, fake_job_to_info = self._scheduler_with_jobs(infos)with (patch.object(Scheduler, "_get_jobs_cached", lambda self: jobs),+ # None of these fixture jobs are paused, so the active-only path+ # (what get_execution_schedules actually calls unless+ # include_paused=True) can return the same list.+ patch.object(Scheduler, "_get_active_jobs_cached", lambda self: jobs),patch("backend.executor.scheduler._job_to_info",side_effect=fake_job_to_info,