AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): index schedule reads by user, and run one jobstore query per cache miss (#14148)
Nicholas Tindled142863d62ec
diff --git a/autogpt_platform/backend/backend/executor/schedule_index.py b/autogpt_platform/backend/backend/executor/schedule_index.pynew file mode 100644index 0000000000..8d3e11aa9b--- /dev/null+++ b/autogpt_platform/backend/backend/executor/schedule_index.py@@ -0,0 +1,182 @@+"""Queryable sidecar index for the scheduler's EXECUTION jobstore.++APScheduler's ``SQLAlchemyJobStore`` persists each job as a pickled blob, so+the only SQL-level filters it supports are ``id`` and ``next_run_time`` —+answering "which schedules does user X have?" means loading and unpickling+*every* row. On a busy deployment that full scan takes tens of seconds and+made ``get_graph_execution_schedules`` the dominant cost of the library+page (30s+ proxy timeouts surfacing as 504s — BUILDER-3A7).++This module maintains a companion table holding just the filterable columns+(user, kind, graph, session, org/team/expert) keyed by job id. It is a+*candidate* index, never an authority:++- Reads use it only to narrow the job-id set; every candidate is then+ loaded from the real jobstore and re-checked against the same in-Python+ predicate as the full-scan path. A wrong or stale index row can therefore+ never produce a wrong result — only a missing row can hide a schedule,+ and those are healed by the startup backfill / periodic reconcile in+ ``scheduler.py``.+- Rows whose job has vanished (e.g. fired one-shots that APScheduler+ auto-removes) are dropped lazily when a read trips over them.++The table is owned and auto-created by the scheduler service over the same+SQLAlchemy engine as the jobstore itself — mirroring how APScheduler manages+``apscheduler_jobs`` — rather than via Prisma, which the scheduler process+never connects to.++Writes are plain delete+insert inside one transaction (portable across+Postgres and the SQLite used in unit tests); schedule mutations are rare+enough that dialect-native upserts aren't worth the coupling.+"""++import logging+from collections.abc import Sequence+from typing import Optional++from pydantic import BaseModel+from sqlalchemy import Column, MetaData, String, Table, delete, insert, or_, select+from sqlalchemy.engine import Engine+from sqlalchemy.exc import IntegrityError++logger = logging.getLogger(__name__)++# Matches APScheduler's default id column length so every job id that fits+# the jobstore fits the index.+_JOB_ID_LENGTH = 191+_WRITE_BATCH_SIZE = 500+++class ScheduleIndexEntry(BaseModel):+ """The filterable identity of one schedule, keyed by its jobstore id."""++ job_id: str+ user_id: str+ kind: str+ graph_id: Optional[str] = None+ session_id: Optional[str] = None+ # Normalized: legacy rows carry ``organization_id=""`` which must never+ # match an org-scoped query, so empty strings are stored as NULL.+ organization_id: Optional[str] = None+ team_id: Optional[str] = None+ expert_id: Optional[str] = None+++class ScheduleIndex:+ def __init__(self, engine: Engine, schema: str | None = None):+ self._engine = engine+ self._metadata = MetaData(schema=schema)+ self._table = Table(+ "apscheduler_jobs_index",+ self._metadata,+ Column("job_id", String(_JOB_ID_LENGTH), primary_key=True),+ Column("user_id", String, nullable=False, index=True),+ Column("kind", String, nullable=False),+ Column("graph_id", String, nullable=True, index=True),+ Column("session_id", String, nullable=True, index=True),+ Column("organization_id", String, nullable=True, index=True),+ Column("team_id", String, nullable=True),+ Column("expert_id", String, nullable=True),+ )++ def ensure_table(self) -> None:+ """Create the table if missing (same auto-DDL model as APScheduler)."""+ self._metadata.create_all(self._engine, checkfirst=True)++ def upsert(self, entry: ScheduleIndexEntry) -> None:+ self.upsert_many([entry])++ def upsert_many(self, entries: Sequence[ScheduleIndexEntry]) -> None:+ if not entries:+ return+ rows = [+ {**e.model_dump(), "organization_id": e.organization_id or None}+ for e in entries+ ]+ # delete-then-insert is not conflict-safe across replicas: under READ+ # COMMITTED a concurrent backfill's uncommitted rows are invisible to+ # the delete and then collide on the primary key. One retry after the+ # other transaction commits resolves it.+ for attempt in (1, 2):+ try:+ with self._engine.begin() as conn:+ for start in range(0, len(rows), _WRITE_BATCH_SIZE):+ batch = rows[start : start + _WRITE_BATCH_SIZE]+ conn.execute(+ delete(self._table).where(+ self._table.c.job_id.in_([r["job_id"] for r in batch])+ )+ )+ for start in range(0, len(rows), _WRITE_BATCH_SIZE):+ conn.execute(+ insert(self._table), rows[start : start + _WRITE_BATCH_SIZE]+ )+ return+ except IntegrityError:+ if attempt == 2:+ raise++ def delete(self, job_id: str) -> None:+ self.delete_many([job_id])++ def delete_many(self, job_ids: Sequence[str]) -> None:+ if not job_ids:+ return+ with self._engine.begin() as conn:+ for start in range(0, len(job_ids), _WRITE_BATCH_SIZE):+ conn.execute(+ delete(self._table).where(+ self._table.c.job_id.in_(+ job_ids[start : start + _WRITE_BATCH_SIZE]+ )+ )+ )++ def all_job_ids(self) -> set[str]:+ with self._engine.connect() as conn:+ return {row[0] for row in conn.execute(select(self._table.c.job_id))}++ def candidate_job_ids(+ self,+ *,+ user_id: str | None = None,+ graph_id: str | None = None,+ session_id: str | None = None,+ kind: str | None = None,+ organization_id: str | None = None,+ ) -> list[str] | None:+ """Job ids that *may* satisfy the given filters, or ``None`` when no+ identity filter was given (the trusted-global case, which must keep+ seeing rows the index doesn't track).++ The set is a deliberate superset of the exact visibility rules:+ org-scoped reads return every row tagged with the org (team and+ expert trimming happens in the caller's predicate, which re-checks+ each loaded job), because encoding those rules here would duplicate+ the authorization logic in two places.+ """+ clauses = []+ if kind is not None:+ clauses.append(self._table.c.kind == kind)+ if graph_id is not None:+ clauses.append(self._table.c.graph_id == graph_id)+ if session_id is not None:+ clauses.append(self._table.c.session_id == session_id)+ ownership = []+ if user_id is not None:+ ownership.append(self._table.c.user_id == user_id)+ if organization_id is not None:+ ownership.append(self._table.c.organization_id == organization_id)+ if ownership:+ clauses.append(or_(*ownership))++ if not (graph_id or session_id or ownership):+ # ``kind`` alone isn't an identity filter — a kind-only read is+ # a global listing and must include un-indexed rows.+ return None++ query = select(self._table.c.job_id)+ for clause in clauses:+ query = query.where(clause)+ with self._engine.connect() as conn:+ return [row[0] for row in conn.execute(query)]diff --git a/autogpt_platform/backend/backend/executor/schedule_index_test.py b/autogpt_platform/backend/backend/executor/schedule_index_test.pynew file mode 100644index 0000000000..602313f9b7--- /dev/null+++ b/autogpt_platform/backend/backend/executor/schedule_index_test.py@@ -0,0 +1,208 @@+"""Unit tests for the schedule index (in-memory SQLite, no infra)."""++import sqlite3+from unittest.mock import patch++import pytest+from sqlalchemy import create_engine+from sqlalchemy.exc import IntegrityError++from backend.executor.schedule_index import ScheduleIndex, ScheduleIndexEntry+++def _index() -> ScheduleIndex:+ connection = sqlite3.connect(":memory:")+ connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 999)+ index = ScheduleIndex(create_engine("sqlite://", creator=lambda: connection))+ index.ensure_table()+ return index+++def _entry(job_id: str, **overrides) -> ScheduleIndexEntry:+ fields = {+ "job_id": job_id,+ "user_id": "user-1",+ "kind": "graph",+ "graph_id": "graph-1",+ }+ fields.update(overrides)+ return ScheduleIndexEntry(**fields)+++# ---------------------------------------------------------------------------+# writes+# ---------------------------------------------------------------------------+++def test_upsert_inserts_and_replaces_on_same_job_id():+ index = _index()+ index.upsert(_entry("j1", user_id="user-1"))+ index.upsert(_entry("j1", user_id="user-2"))++ assert index.candidate_job_ids(user_id="user-1") == []+ assert index.candidate_job_ids(user_id="user-2") == ["j1"]+ assert index.all_job_ids() == {"j1"}+++def test_upsert_many_and_delete_many():+ index = _index()+ index.upsert_many([_entry("j1"), _entry("j2"), _entry("j3")])+ index.delete_many(["j1", "j3"])+ assert index.all_job_ids() == {"j2"}+++def test_upsert_many_replaces_more_rows_than_the_bind_limit():+ index = _index()+ entries = [_entry(f"j{i}") for i in range(2001)]+ index.upsert_many(entries)+ index.upsert(_entry("untouched"))++ index.upsert_many(+ [entry.model_copy(update={"user_id": "user-2"}) for entry in entries]+ )++ assert index.candidate_job_ids(user_id="user-1") == ["untouched"]+ assert set(index.candidate_job_ids(user_id="user-2") or []) == {+ entry.job_id for entry in entries+ }+++def test_delete_many_removes_more_rows_than_the_bind_limit():+ index = _index()+ job_ids = [f"j{i}" for i in range(2001)]+ for job_id in job_ids:+ index.upsert(_entry(job_id))+ index.upsert(_entry("untouched"))++ index.delete_many(job_ids)++ assert index.all_job_ids() == {"untouched"}+++def test_upsert_many_rolls_back_all_batches_after_retry_exhaustion():+ index = _index()+ entries = [_entry(f"j{i}") for i in range(2001)]+ for entry in entries:+ index.upsert(entry)+ replacements = [entry.model_copy(update={"user_id": "user-2"}) for entry in entries]++ with patch.object(index._engine, "begin", wraps=index._engine.begin) as begin:+ with pytest.raises(IntegrityError):+ index.upsert_many([*replacements, replacements[0]])++ assert begin.call_count == 2+ assert index.candidate_job_ids(user_id="user-2") == []+ assert set(index.candidate_job_ids(user_id="user-1") or []) == {+ entry.job_id for entry in entries+ }+++def test_upsert_many_retries_a_transient_integrity_error():+ index = _index()+ conflict = IntegrityError(None, None, sqlite3.IntegrityError("concurrent insert"))+ with patch.object(+ index._engine, "begin", side_effect=[conflict, index._engine.begin()]+ ):+ index.upsert_many([_entry("j1"), _entry("j2")])++ assert index.all_job_ids() == {"j1", "j2"}+++def test_delete_missing_row_is_a_noop():+ index = _index()+ index.delete("nope")+ assert index.all_job_ids() == set()+++def test_empty_batches_are_noops():+ index = _index()+ index.upsert_many([])+ index.delete_many([])+ assert index.all_job_ids() == set()+++def test_empty_string_organization_is_stored_as_null():+ # Legacy GraphExecutionJobArgs default organization_id to "" — those+ # rows must never match an org-scoped query.+ index = _index()+ index.upsert(_entry("j1", organization_id=""))+ assert index.candidate_job_ids(organization_id="") == []+ assert index.candidate_job_ids(user_id="user-1") == ["j1"]+++# ---------------------------------------------------------------------------+# candidate queries+# ---------------------------------------------------------------------------+++def test_candidates_by_user():+ index = _index()+ index.upsert_many(+ [+ _entry("j1", user_id="a"),+ _entry("j2", user_id="b"),+ _entry("j3", user_id="a"),+ ]+ )+ assert sorted(index.candidate_job_ids(user_id="a") or []) == ["j1", "j3"]+++def test_candidates_by_graph_and_session():+ index = _index()+ index.upsert_many(+ [+ _entry("j1", graph_id="g1"),+ _entry("j2", graph_id="g2"),+ _entry("j3", kind="copilot_turn", graph_id=None, session_id="s1"),+ ]+ )+ assert index.candidate_job_ids(graph_id="g1") == ["j1"]+ assert index.candidate_job_ids(session_id="s1") == ["j3"]+++def test_kind_narrows_but_is_not_an_identity_filter():+ index = _index()+ index.upsert_many(+ [+ _entry("j1", user_id="a", kind="graph"),+ _entry("j2", user_id="a", kind="copilot_turn", graph_id=None),+ ]+ )+ assert index.candidate_job_ids(user_id="a", kind="graph") == ["j1"]+ # kind alone is a global listing: the index must decline (return None)+ # so the caller falls back to the full scan that includes un-indexed rows.+ assert index.candidate_job_ids(kind="graph") is None+++def test_no_filters_returns_none_for_full_scan_fallback():+ index = _index()+ index.upsert(_entry("j1"))+ assert index.candidate_job_ids() is None+++def test_org_scope_is_own_rows_or_org_rows():+ index = _index()+ index.upsert_many(+ [+ _entry("j1", user_id="me", organization_id=None),+ _entry("j2", user_id="teammate", organization_id="org-1"),+ _entry("j3", user_id="stranger", organization_id="org-2"),+ _entry("j4", user_id="teammate", organization_id=None),+ ]+ )+ got = index.candidate_job_ids(user_id="me", organization_id="org-1")+ # Superset semantics: own rows + all org rows; team/expert trimming is+ # the caller's predicate's job.+ assert sorted(got or []) == ["j1", "j2"]+++def test_dimension_filters_are_anded():+ index = _index()+ index.upsert_many(+ [+ _entry("j1", user_id="a", graph_id="g1"),+ _entry("j2", user_id="a", graph_id="g2"),+ _entry("j3", user_id="b", graph_id="g1"),+ ]+ )+ assert index.candidate_job_ids(user_id="a", graph_id="g1") == ["j1"]diff --git a/autogpt_platform/backend/backend/executor/schedule_index_test_helpers.py b/autogpt_platform/backend/backend/executor/schedule_index_test_helpers.pynew file mode 100644index 0000000000..a5cf30be48--- /dev/null+++ b/autogpt_platform/backend/backend/executor/schedule_index_test_helpers.py@@ -0,0 +1,48 @@+"""Fixtures for scheduler index integration tests."""++from datetime import datetime, timezone+from unittest.mock import MagicMock++from sqlalchemy import create_engine++from backend.executor.schedule_index import ScheduleIndex+from backend.executor.scheduler import Scheduler+++def _scheduler_with_index() -> Scheduler:+ scheduler = Scheduler(register_system_tasks=False)+ scheduler._schedule_index = ScheduleIndex(create_engine("sqlite://"))+ scheduler._schedule_index.ensure_table()+ scheduler._schedule_index_ready = True+ scheduler.scheduler = MagicMock()+ # Non-paused listings read the jobstore with a SQL next_run_time filter;+ # serve them from the same stubbed scan the tests drive.+ scheduler._execution_jobstore = MagicMock()+ scheduler._execution_jobstore._get_jobs.side_effect = lambda *_: [+ job for job in scheduler.scheduler.get_jobs() if job.next_run_time is not None+ ]+ return scheduler+++def _graph_job_kwargs(schedule_id: str, user_id: str, graph_id: str) -> dict:+ return {+ "kind": "graph",+ "schedule_id": schedule_id,+ "user_id": user_id,+ "graph_id": graph_id,+ "graph_version": 1,+ "cron": "0 0 * * *",+ "input_data": {},+ "input_credentials": {},+ }+++def _mock_job(kwargs: dict) -> MagicMock:+ job = MagicMock()+ job.kwargs = kwargs+ job.id = kwargs.get("schedule_id", "fake-id")+ job.name = "fake-name"+ job.next_run_time = datetime(2026, 5, 22, 10, 0, tzinfo=timezone.utc)+ job.trigger = MagicMock()+ job.trigger.timezone = "UTC"+ return jobdiff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex c4662cee2a..9717b3ce92 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -6,6 +6,8 @@ import reimport threadingimport timeimport uuid+from collections.abc import Callable+from concurrent.futures import Futurefrom datetime import datetime, timedelta, timezonefrom enum import Enumfrom typing import Annotated, Literal, Optional, Union@@ -48,6 +50,7 @@ from backend.data.schedule import normalize_schedule_namefrom backend.executor import schedule_eventsfrom backend.executor import utils as execution_utilsfrom backend.executor.jobstore import ResilientSQLAlchemyJobStore+from backend.executor.schedule_index import ScheduleIndex, ScheduleIndexEntryfrom backend.monitoring import (flush_matured_alerts,report_block_error_rates,@@ -179,6 +182,23 @@ def job_max_instances_listener(event):_event_loop: asyncio.AbstractEventLoop | None = None_event_loop_thread: threading.Thread | None = None+# The running Scheduler service instance, so module-level job functions+# (which APScheduler pickles by reference) can reach instance state — the+# same in-process shortcut the event-loop globals above provide.+_scheduler_instance: "Scheduler | None" = None+++def reconcile_schedule_index():+ """Daily system task: rebuild the schedule index from the jobstore.++ Heals rows missed by best-effort writes and clears rows left behind by+ fired one-shots that only APScheduler's auto-removal saw.+ """+ if _scheduler_instance is None:+ logger.warning("Schedule index reconcile skipped: scheduler not running")+ return+ _scheduler_instance._reconcile_schedule_index()+@func_retrydef get_event_loop():@@ -1649,6 +1669,11 @@ def ensure_embeddings_coverage():# bounded rather than left to grow with the backlog._PARKED_SCAN_LIMIT = 1000+# A rolling deploy runs the outgoing pod beside this one until it exits (prod:+# ~10 s to ready, a 15 s preStop, a 300 s grace). The index is trusted only after+# a reconcile that began later, so rows that pod wrote unindexed are in it.+_SCHEDULE_INDEX_SETTLE_S = 600+class Jobstores(Enum):EXECUTION = "execution"@@ -1937,10 +1962,40 @@ def _job_to_info(return None+def _index_entry(+ job_id: str, args: GraphExecutionJobArgs | CopilotTurnJobArgs+) -> ScheduleIndexEntry:+ """Build the index row for a schedule.++ *job_id* is passed separately because it must be the jobstore key+ (``job.id``): legacy rows can carry ``schedule_id=None`` in their kwargs+ even though the job itself is addressable.+ """+ return ScheduleIndexEntry(+ job_id=job_id,+ user_id=args.user_id,+ kind=args.kind,+ graph_id=args.graph_id if args.kind == "graph" else None,+ session_id=args.session_id if args.kind == "copilot_turn" else None,+ organization_id=args.organization_id or None,+ team_id=args.team_id,+ expert_id=args.expert_id,+ )++class Scheduler(AppService):scheduler: BackgroundScheduler_persistent_jobstores: dict[str, ResilientSQLAlchemyJobStore] = {}+ # Sidecar index for filtered schedule reads. ``None`` (not set up or+ # setup failed) or ``ready=False`` (backfill not yet complete) both+ # fall back to the full jobstore scan, so the index is never+ # load-bearing. Class-level defaults like the jobs cache below, so+ # every construction path has them.+ _schedule_index: ScheduleIndex | None = None+ _schedule_index_ready: bool = False+ _schedule_index_failure_version: int = 0+def __init__(self, register_system_tasks: bool = True):self.register_system_tasks = register_system_tasks@@ -1990,13 +2045,28 @@ class Scheduler(AppService):# Configure executors to limit concurrency without skipping jobsfrom apscheduler.executors.pool import ThreadPoolExecutor+ # Shared between the EXECUTION jobstore and the schedule index so+ # both live in the same pool/database.+ execution_engine = create_engine(+ url=db_url,+ pool_size=self.db_pool_size(),+ max_overflow=0,+ )+ self._schedule_index = ScheduleIndex(execution_engine, schema=db_schema)+ try:+ self._schedule_index.ensure_table()+ except Exception:+ # Two replicas racing create_all(), or a transient DB error. Keep+ # the index so writes still go through once the table exists;+ # every reconcile retries the setup.+ logger.exception(+ "Schedule index setup failed; filtered schedule reads fall back "+ "to the full jobstore scan until the backfill retries it"+ )+self._persistent_jobstores = {Jobstores.EXECUTION.value: ResilientSQLAlchemyJobStore(- engine=create_engine(- url=db_url,- pool_size=self.db_pool_size(),- max_overflow=0,- ),+ engine=execution_engine,metadata=MetaData(schema=db_schema),# this one is pre-existing so it keeps the# default table name.@@ -2203,6 +2273,20 @@ class Scheduler(AppService):jobstore=Jobstores.EXECUTION.value,)+ # Schedule Index Reconcile - Every 24 hours+ # Safety net for the best-effort index writes: re-adds rows a+ # failed write missed and clears rows for jobs APScheduler+ # auto-removed (fired one-shots) outside our delete path.+ self.scheduler.add_job(+ reconcile_schedule_index,+ id="reconcile_schedule_index",+ trigger="interval",+ hours=24,+ replace_existing=True,+ max_instances=1,+ jobstore=Jobstores.EXECUTION.value,+ )+self.scheduler.add_listener(job_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)self.scheduler.add_listener(job_missed_listener, EVENT_JOB_MISSED)self.scheduler.add_listener(job_max_instances_listener, EVENT_JOB_MAX_INSTANCES)@@ -2213,6 +2297,10 @@ class Scheduler(AppService):self._register_posthog_lifecycle_sweep()self._report_parked_jobs()+ global _scheduler_instance+ _scheduler_instance = self+ self._start_schedule_index_backfill()+# Keep the service running since BackgroundScheduler doesn't blocksuper().run_service()@@ -2263,7 +2351,7 @@ class Scheduler(AppService):self,*,dispatch_func,- job_args: Union[GraphExecutionJobArgs, CopilotTurnJobArgs],+ job_args: GraphExecutionJobArgs | CopilotTurnJobArgs,trigger,name: str | None,) -> JobObj:@@ -2284,6 +2372,10 @@ class Scheduler(AppService):replace_existing=True,id=job_args.schedule_id,)+ # Index AFTER add_job: a pre-add row would race the read path's+ # dangling-row cleanup (row visible, job not yet persisted → row+ # deleted → schedule invisible until reconcile).+ self._upsert_schedule_index_row(job.id, job_args)# Invalidate the read cache so the new job shows up on the next# ``get_execution_schedules`` call instead of waiting up to# ``_JOBS_CACHE_TTL_S`` seconds for the cached list to expire.@@ -2443,6 +2535,7 @@ class Scheduler(AppService):job, info = self._authorized_job(schedule_id, user_id, action="delete")logger.info(f"Deleting job {schedule_id} (kind={info.kind})")job.remove()+ self._delete_schedule_index_row(schedule_id)# Invalidate the read cache so the deletion shows up immediately# on the next ``get_execution_schedules`` call.self._invalidate_jobs_cache()@@ -2537,100 +2630,326 @@ class Scheduler(AppService):if isinstance(info, GraphExecutionJobInfo)]- # Process-wide cache for ``scheduler.get_jobs(EXECUTION)`` — the fully- # unfiltered row set, including paused schedules and already-fired- # one-shot jobs. APScheduler has no SQL-level user_id / kind filter- # either way — it loads every row and unpickles each ``job.kwargs`` in- # Python — so this is now only worth paying for on the rare- # ``include_paused=True`` lifecycle lookups; see the sibling- # ``_get_active_jobs_cached`` below for the path everything else takes.- # Mutations (`add_*_schedule`, `delete_schedule`) clear both caches so- # user-visible latency on writes is unchanged.+ # Two cached reads of the EXECUTION jobstore (``_read_job_list``):+ # ``_get_jobs_cached`` unpickles every row, paused and fired-once included,+ # for the ``include_paused=True`` lookups; ``_get_active_jobs_cached`` filters+ # ``next_run_time IS NOT NULL`` in SQL, since the table keeps paused and+ # fired rows forever. Mutations call ``_invalidate_jobs_cache``._JOBS_CACHE_TTL_S = 5.0- _jobs_cache: list[JobObj] | None = None- _jobs_cache_expires_at: float = 0.0- # Serialises (a) concurrent cache misses so the slow ``get_jobs``- # unpickle runs only once per TTL, and (b) the read/invalidate race- # where an invalidation between a thread's slow read and its- # cache-write would otherwise leave a just-invalidated cache holding- # the pre-mutation list. Threading rather than asyncio because the- # APScheduler ``BackgroundScheduler`` thread + the Pyro RPC workers- # both call into this method.+ # Process-wide; guards both lists and the version. A threading lock because+ # APScheduler's thread and the Pyro RPC workers both read._jobs_cache_lock = threading.Lock()- # Monotonically increasing version stamp; bumped by every- # invalidation. A reader captures the version BEFORE it runs the- # slow query and only writes back if the version is unchanged on- # completion — this kills the race where invalidate fires while a- # slow read is in flight.+ # Bumped by every invalidation. A read that started before one must not+ # write its result back: it may predate the mutation._jobs_cache_version: int = 0+ _job_lists: "dict[str, _JobList] | None" = Nonedef _get_jobs_cached(self) -> list[JobObj]:+ return self._read_job_list(+ "all", lambda: self.scheduler.get_jobs(jobstore=Jobstores.EXECUTION.value)+ )++ def _get_active_jobs_cached(self) -> list[JobObj]:+ return self._read_job_list(+ "active",+ lambda: self._execution_jobstore._get_jobs(+ self._execution_jobstore.jobs_t.c.next_run_time.isnot(None)+ ),+ # The one scheduler metric with an alert on it. Only an accepted+ # read sets it, so an invalidated one cannot overwrite a newer count.+ on_accept=lambda jobs: SCHEDULER_JOBS.labels(+ job_type="execution", status="scheduled"+ ).set(len(jobs)),+ )++ def _read_job_list(+ self,+ key: Literal["all", "active"],+ fetch: Callable[[], list[JobObj]],+ on_accept: Callable[[list[JobObj]], None] | None = None,+ ) -> list[JobObj]:+ """The cached list for *key*, with at most one fetch in flight.++ Callers of a cold or invalidated list wait for that one fetch. An+ expired list is returned as it is while a background fetch replaces+ it, so no caller waits a scan that an older list can answer.+ """with self._jobs_cache_lock:- now = time.monotonic()- if self._jobs_cache is not None and now < self._jobs_cache_expires_at:- return self._jobs_cache- version_at_start = self._jobs_cache_version- # Drop the lock for the heavy I/O so unrelated writers (which- # only take the lock briefly inside ``_invalidate_jobs_cache``)- # don't queue behind a slow scheduler query.- jobs = self.scheduler.get_jobs(jobstore=Jobstores.EXECUTION.value)+ entry = self._job_list(key)+ if entry.jobs is not None and time.monotonic() < entry.expires_at:+ return entry.jobs+ stale = entry.jobs+ version = self._jobs_cache_version+ pending = entry.refresh+ starts_refresh = pending is None+ if pending is None:+ pending = entry.refresh = Future()+ if starts_refresh:+ args = (entry, pending, version, fetch, on_accept)+ if stale is None:+ self._refresh_job_list(*args)+ else:+ pending.add_done_callback(_log_refresh_failure)+ threading.Thread(+ target=self._refresh_job_list,+ args=args,+ daemon=True,+ name="ScheduleListRefresh",+ ).start()+ return stale if stale is not None else pending.result()++ def _refresh_job_list(+ self,+ entry: "_JobList",+ pending: "Future[list[JobObj]]",+ version: int,+ fetch: Callable[[], list[JobObj]],+ on_accept: Callable[[list[JobObj]], None] | None,+ ) -> None:+ try:+ jobs = fetch()+ except BaseException as e:+ with self._jobs_cache_lock:+ if entry.refresh is pending:+ entry.refresh = None+ pending.set_exception(e)+ return+ try:+ with self._jobs_cache_lock:+ if entry.refresh is pending:+ entry.refresh = None+ if self._jobs_cache_version == version:+ entry.jobs = jobs+ entry.expires_at = time.monotonic() + self._JOBS_CACHE_TTL_S+ if on_accept is not None:+ on_accept(jobs)+ finally:+ # Waiters block on this; nothing after the fetch may strand them.+ pending.set_result(jobs)++ def _job_list(self, key: Literal["all", "active"]) -> "_JobList":+ """The caller holds ``_jobs_cache_lock``."""+ if self._job_lists is None:+ self._job_lists = {"all": _JobList(), "active": _JobList()}+ return self._job_lists[key]++ def _invalidate_jobs_cache(self) -> None:with self._jobs_cache_lock:- # If an invalidation happened while we were querying, the- # list we just fetched might already be stale. Skip the- # write-back; the next caller will re-query.- if self._jobs_cache_version == version_at_start:- self._jobs_cache = jobs- self._jobs_cache_expires_at = time.monotonic() + self._JOBS_CACHE_TTL_S- return jobs+ # Dropping the in-flight fetch too sends the next caller to a fresh+ # read rather than to one that began before this write.+ for entry in (self._job_lists or {}).values():+ entry.jobs = None+ entry.expires_at = 0.0+ entry.refresh = None+ self._jobs_cache_version += 1- # Second cache, keyed off the same lock/version, for the ``next_run_time- # IS NOT NULL`` (non-paused) rows only. This is what every caller except- # the pause/resume lifecycle lookups (``include_paused=True``) actually- # wants, and unlike ``_get_jobs_cached`` it pushes that filter down to- # SQL instead of unpickling every paused/already-fired row in Python- # only to throw it away — ``apscheduler_jobs`` accumulates those forever- # (nothing deletes a paused or fired-once job), so on a table with a- # meaningful history the unfiltered scan is what Sentry was flagging as- # a slow, unbounded query. ``next_run_time`` already carries a btree- # index from APScheduler's own table definition, so this needs no- # schema change.- _active_jobs_cache: list[JobObj] | None = None- _active_jobs_cache_expires_at: float = 0.0+ # --- schedule index (see backend/executor/schedule_index.py) ---+ #+ # The index narrows filtered reads to a handful of point lookups+ # instead of unpickling the whole jobstore. It is candidates-only:+ # every job it nominates still flows through the same predicate as+ # the full-scan path, so index staleness can hide a schedule at+ # worst (healed by backfill/reconcile), never leak or corrupt one.- def _get_active_jobs_cached(self) -> list[JobObj]:- with self._jobs_cache_lock:- now = time.monotonic()- if (- self._active_jobs_cache is not None- and now < self._active_jobs_cache_expires_at- ):- return self._active_jobs_cache- version_at_start = self._jobs_cache_version- jobs = self._execution_jobstore._get_jobs(- self._execution_jobstore.jobs_t.c.next_run_time.isnot(None)+ def _get_schedule_jobs(+ self,+ *,+ user_id: str | None,+ graph_id: str | None,+ session_id: str | None,+ kind: str | None,+ organization_id: str | None,+ include_paused: bool = False,+ ) -> list[JobObj]:+ """Jobs to run the ``get_execution_schedules`` predicate over.++ Index-narrowed when a usable index exists and an identity filter+ was given; the full (cached) jobstore scan otherwise.+ """+ full_scan = (+ self._get_jobs_cached if include_paused else self._get_active_jobs_cached)+ index = self._schedule_indexwith self._jobs_cache_lock:- if self._jobs_cache_version == version_at_start:- self._active_jobs_cache = jobs- self._active_jobs_cache_expires_at = (- time.monotonic() + self._JOBS_CACHE_TTL_S- )- # The one scheduler metric with an alert on it was never set.- # Only an accepted read may publish it: a read that was- # invalidated mid-query is stale by definition and must not- # overwrite a newer count another reader has already set.- SCHEDULER_JOBS.labels(job_type="execution", status="scheduled").set(- len(jobs)+ ready = self._schedule_index_ready+ failure_version = self._schedule_index_failure_version+ if index is None or not ready:+ return full_scan()+ try:+ job_ids = index.candidate_job_ids(+ user_id=user_id,+ graph_id=graph_id,+ session_id=session_id,+ kind=kind,+ organization_id=organization_id,+ )+ except Exception:+ logger.warning(+ "Schedule index query failed; falling back to full scan",+ exc_info=True,+ )+ return full_scan()+ if job_ids is None: # no identity filter — trusted-global listing+ return full_scan()++ jobs: list[JobObj] = []+ dangling: list[str] = []+ for job_id in job_ids:+ try:+ job = self.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)+ except Exception:+ logger.warning(+ "Schedule %s could not be loaded; falling back to full scan",+ job_id,+ exc_info=True,)+ return full_scan()+ if job is None:+ dangling.append(job_id)+ continue+ if job.next_run_time is None and not include_paused:+ continue+ jobs.append(job)+ with self._jobs_cache_lock:+ ready = (+ self._schedule_index_ready+ and self._schedule_index_failure_version == failure_version+ )+ if not ready:+ return full_scan()+ if self.scheduler.running:+ self._delete_schedule_index_rows(dangling)return jobs- def _invalidate_jobs_cache(self) -> None:+ def _upsert_schedule_index_row(+ self,+ job_id: str,+ job_args: GraphExecutionJobArgs | CopilotTurnJobArgs,+ ) -> None:+ """Keep successful mutations visible through full scans after an index failure."""+ if self._schedule_index is None:+ return+ try:+ self._schedule_index.upsert(_index_entry(job_id, job_args))+ except Exception:+ with self._jobs_cache_lock:+ self._schedule_index_ready = False+ self._schedule_index_failure_version += 1+ logger.exception(+ f"Failed to index schedule {job_id}; filtered listings will "+ "use the full jobstore scan until a successful reconcile"+ )++ def _delete_schedule_index_row(self, job_id: str) -> None:+ self._delete_schedule_index_rows([job_id])++ def _delete_schedule_index_rows(self, job_ids: list[str]) -> None:+ if self._schedule_index is None or not job_ids:+ return+ try:+ self._schedule_index.delete_many(job_ids)+ except Exception:+ logger.warning("Failed to remove stale schedule index rows", exc_info=True)++ def _mark_schedule_index_ready(self, failure_version: int) -> None:with self._jobs_cache_lock:- self._jobs_cache = None- self._jobs_cache_expires_at = 0.0- self._active_jobs_cache = None- self._active_jobs_cache_expires_at = 0.0- self._jobs_cache_version += 1+ if self._schedule_index_failure_version == failure_version:+ self._schedule_index_ready = True++ def _reconcile_schedule_index(self) -> None:+ """Rebuild the index from a full jobstore scan (startup + daily)."""+ index = self._schedule_index+ if index is None:+ return+ if not self.scheduler.running:+ # BaseScheduler.shutdown() flips the state outside the jobstore+ # lock, after which get_jobs()/get_job() answer "nothing" for+ # everything. Reconciling on that view would wipe the shared+ # index for every replica.+ logger.warning("Scheduler is not running; skipping index reconcile")+ return+ # Idempotent; repairs a table whose creation failed at startup.+ index.ensure_table()+ with self._jobs_cache_lock:+ failure_version = self._schedule_index_failure_version+ jobs = self.scheduler.get_jobs(jobstore=Jobstores.EXECUTION.value)+ if not jobs and failure_version:+ return+ entries = []+ for job in jobs:+ info = _job_to_info(job)+ if info is None:+ # System/maintenance jobs and corrupted rows aren't+ # user-listable, so they don't get index rows either.+ continue+ entries.append(_index_entry(job.id, info))+ index.upsert_many(entries)++ # Clear rows whose job is gone — but verify each against the live+ # jobstore first: a schedule added while this scan ran is in the+ # index and not in our snapshot, and must survive.+ stale = index.all_job_ids() - {e.job_id for e in entries}+ if not self.scheduler.running or (not jobs and stale):+ # A shutdown raced the scan, or the raw scan came back empty while+ # the index still has rows: neither is evidence the rows are+ # stale, so keep them for the next reconcile. (Test the raw scan,+ # not `entries`: a jobstore holding only system jobs is a+ # legitimate empty listing whose stale rows must still go.)+ logger.warning(+ "Skipping stale-row sweep: scan returned %d jobs against %d "+ "index rows (running=%s)",+ len(jobs),+ len(stale),+ self.scheduler.running,+ )+ self._mark_schedule_index_ready(failure_version)+ return+ confirmed_gone = []+ for job_id in stale:+ try:+ if (+ self.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)+ is None+ ):+ confirmed_gone.append(job_id)+ except Exception:+ # A row that cannot be reconstituted is not proof the job is+ # gone; leave it for the full scan's own cleanup.+ logger.warning(+ "Could not load job %s while reconciling the index",+ job_id,+ exc_info=True,+ )+ if not self.scheduler.running:+ return+ index.delete_many(confirmed_gone)++ self._mark_schedule_index_ready(failure_version)+ logger.info(+ f"Schedule index reconciled: {len(entries)} rows, "+ f"{len(confirmed_gone)} stale rows removed"+ )++ def _start_schedule_index_backfill(self) -> None:+ if self._schedule_index is None:+ return++ def _run():+ try:+ time.sleep(_SCHEDULE_INDEX_SETTLE_S)+ self._reconcile_schedule_index()+ except Exception:+ logger.exception(+ "Schedule index backfill failed; filtered schedule reads "+ "will use the full jobstore scan until the next reconcile"+ )++ threading.Thread(target=_run, daemon=True, name="ScheduleIndexBackfill").start()++ @expose+ def execute_reconcile_schedule_index(self):+ """Manually trigger a schedule index rebuild."""+ return reconcile_schedule_index()@exposedef get_execution_schedules(@@ -2665,10 +2984,13 @@ class Scheduler(AppService):Fired one-shot jobs share the same null marker, so they resurface too— filter by kind/id if that matters to the caller."""- jobs: list[JobObj] = (- self._get_jobs_cached()- if include_paused- else self._get_active_jobs_cached()+ jobs: list[JobObj] = self._get_schedule_jobs(+ user_id=user_id,+ graph_id=graph_id,+ session_id=session_id,+ kind=kind,+ organization_id=organization_id,+ include_paused=include_paused,)results: list[Union[GraphExecutionJobInfo, CopilotTurnJobInfo]] = []for job in jobs:@@ -3169,6 +3491,23 @@ class Scheduler(AppService):return result.model_dump(mode="json")+class _JobList:+ """One cached job list, when it expires, and the fetch replacing it."""++ def __init__(self) -> None:+ self.jobs: list[JobObj] | None = None+ self.expires_at = 0.0+ self.refresh: Future[list[JobObj]] | None = None+++def _log_refresh_failure(refresh: "Future[list[JobObj]]") -> None:+ if (error := refresh.exception()) is not None:+ logger.warning(+ "Background schedule list refresh failed; serving the expired list",+ exc_info=error,+ )++class SchedulerClient(AppServiceClient):@classmethoddef get_service_type(cls):diff --git a/autogpt_platform/backend/backend/executor/scheduler_index_read_test.py b/autogpt_platform/backend/backend/executor/scheduler_index_read_test.pynew file mode 100644index 0000000000..00ae786ef8--- /dev/null+++ b/autogpt_platform/backend/backend/executor/scheduler_index_read_test.py@@ -0,0 +1,271 @@+"""Indexed scheduler reads and visibility predicates with a real SQLite index."""++from unittest.mock import MagicMock, patch++from backend.executor.schedule_index_test_helpers import (+ _graph_job_kwargs,+ _mock_job,+ _scheduler_with_index,+)+from backend.executor.scheduler import (+ CopilotTurnJobArgs,+ GraphExecutionJobArgs,+ Scheduler,+ _index_entry,+)+++def test_index_entry_from_graph_args_normalizes_empty_org():+ args = GraphExecutionJobArgs(**_graph_job_kwargs("s1", "u1", "g1"))+ entry = _index_entry("job-id-not-schedule-id", args)+ # job_id comes from the jobstore key, not kwargs — legacy rows can have+ # schedule_id=None while still being addressable by job.id.+ assert entry.job_id == "job-id-not-schedule-id"+ assert entry.user_id == "u1"+ assert entry.kind == "graph"+ assert entry.graph_id == "g1"+ assert entry.session_id is None+ assert entry.organization_id is None # "" default → NULL+++def test_index_entry_from_copilot_args_carries_session():+ args = CopilotTurnJobArgs(+ schedule_id="s1",+ user_id="u1",+ session_id="sess-1",+ message="m",+ cron="0 0 * * *",+ organization_id="org-1",+ )+ entry = _index_entry("s1", args)+ assert entry.kind == "copilot_turn"+ assert entry.graph_id is None+ assert entry.session_id == "sess-1"+ assert entry.organization_id == "org-1"+++def test_get_schedule_jobs_falls_back_without_index_or_before_backfill():+ scheduler = Scheduler(register_system_tasks=False)+ scheduler.scheduler = MagicMock()+ full_scan = [MagicMock()]+ with patch.object(+ scheduler, "_get_active_jobs_cached", return_value=full_scan+ ) as scan:+ assert (+ scheduler._get_schedule_jobs(+ user_id="u1",+ graph_id=None,+ session_id=None,+ kind=None,+ organization_id=None,+ )+ == full_scan+ )+ scan.assert_called_once()++ scheduler = _scheduler_with_index()+ scheduler._schedule_index_ready = False+ with patch.object(+ scheduler, "_get_active_jobs_cached", return_value=full_scan+ ) as scan:+ assert (+ scheduler._get_schedule_jobs(+ user_id="u1",+ graph_id=None,+ session_id=None,+ kind=None,+ organization_id=None,+ )+ == full_scan+ )+ scan.assert_called_once()+++def test_get_schedule_jobs_unfiltered_read_uses_full_scan():+ scheduler = _scheduler_with_index()+ full_scan = [MagicMock()]+ with patch.object(+ scheduler, "_get_active_jobs_cached", return_value=full_scan+ ) as scan:+ assert (+ scheduler._get_schedule_jobs(+ user_id=None,+ graph_id=None,+ session_id=None,+ kind="graph", # kind alone is not an identity filter+ organization_id=None,+ )+ == full_scan+ )+ scan.assert_called_once()+++def test_get_schedule_jobs_loads_candidates_and_drops_dangling_rows():+ scheduler = _scheduler_with_index()+ live_args = GraphExecutionJobArgs(**_graph_job_kwargs("live", "u1", "g1"))+ gone_args = GraphExecutionJobArgs(**_graph_job_kwargs("gone", "u1", "g1"))+ assert scheduler._schedule_index is not None+ scheduler._schedule_index.upsert_many(+ [_index_entry("live", live_args), _index_entry("gone", gone_args)]+ )++ live_job = _mock_job(live_args.model_dump(mode="json"))+ scheduler.scheduler.get_job.side_effect = lambda job_id, jobstore=None: (+ live_job if job_id == "live" else None+ )++ with patch.object(scheduler, "_get_active_jobs_cached") as scan:+ jobs = scheduler._get_schedule_jobs(+ user_id="u1",+ graph_id=None,+ session_id=None,+ kind=None,+ organization_id=None,+ )+ scan.assert_not_called()+ assert jobs == [live_job]+ # The dangling row (fired one-shot removed by APScheduler) is cleaned up.+ assert scheduler._schedule_index.all_job_ids() == {"live"}+++def test_get_schedule_jobs_index_error_falls_back_to_full_scan():+ scheduler = _scheduler_with_index()+ full_scan = [MagicMock()]+ assert scheduler._schedule_index is not None+ with (+ patch.object(+ scheduler._schedule_index,+ "candidate_job_ids",+ side_effect=RuntimeError("db down"),+ ),+ patch.object(+ scheduler, "_get_active_jobs_cached", return_value=full_scan+ ) as scan,+ ):+ assert (+ scheduler._get_schedule_jobs(+ user_id="u1",+ graph_id=None,+ session_id=None,+ kind=None,+ organization_id=None,+ )+ == full_scan+ )+ scan.assert_called_once()+++def test_get_execution_schedules_via_index_applies_exact_predicate():+ """Index candidates still flow through the ownership predicate."""+ scheduler = _scheduler_with_index()+ mine = GraphExecutionJobArgs(**_graph_job_kwargs("mine", "u1", "g1"))+ assert scheduler._schedule_index is not None+ scheduler._schedule_index.upsert(_index_entry("mine", mine))++ job = _mock_job(mine.model_dump(mode="json"))+ scheduler.scheduler.get_job.return_value = job++ with patch.object(scheduler, "_get_active_jobs_cached") as scan:+ results = scheduler.get_execution_schedules(user_id="u1")+ scan.assert_not_called()+ assert [r.id for r in results] == ["mine"]++ # A stale/hostile index row for another user's schedule is trimmed by+ # the predicate even though the index nominated it.+ other = GraphExecutionJobArgs(**_graph_job_kwargs("other", "u2", "g2"))+ scheduler._schedule_index.upsert(+ _index_entry("other", other).model_copy(update={"user_id": "u1"})+ )+ scheduler.scheduler.get_job.side_effect = lambda job_id, jobstore=None: (+ job if job_id == "mine" else _mock_job(other.model_dump(mode="json"))+ )+ results = scheduler.get_execution_schedules(user_id="u1")+ assert [r.id for r in results] == ["mine"]+++def test_indexed_listing_hides_a_paused_schedule_unless_asked():+ """The active-jobs scan drops paused rows in SQL; the index path loads+ candidates by id, so it has to drop them itself."""+ scheduler = _scheduler_with_index()+ args = GraphExecutionJobArgs(**_graph_job_kwargs("paused", "u1", "g1"))+ assert scheduler._schedule_index is not None+ scheduler._schedule_index.upsert(_index_entry("paused", args))+ job = _mock_job(args.model_dump(mode="json"))+ job.next_run_time = None+ scheduler.scheduler.get_job.return_value = job++ assert scheduler.get_execution_schedules(user_id="u1") == []+ assert [+ s.id+ for s in scheduler.get_execution_schedules(user_id="u1", include_paused=True)+ ] == ["paused"]+++def test_persist_schedule_writes_index_row_and_delete_removes_it():+ scheduler = _scheduler_with_index()+ args = GraphExecutionJobArgs(**_graph_job_kwargs("s1", "u1", "g1"))+ job = _mock_job(args.model_dump(mode="json"))+ scheduler.scheduler.add_job.return_value = job++ scheduler._persist_schedule(+ dispatch_func=MagicMock(),+ job_args=args,+ trigger=MagicMock(),+ name="n",+ )+ assert scheduler._schedule_index is not None+ assert scheduler._schedule_index.all_job_ids() == {"s1"}++ with patch.object(+ scheduler, "_authorized_job", return_value=(job, MagicMock(kind="graph"))+ ):+ scheduler.delete_graph_execution_schedule("s1", "u1")+ assert scheduler._schedule_index.all_job_ids() == set()+++def test_persist_schedule_survives_index_write_failure():+ scheduler = _scheduler_with_index()+ args = GraphExecutionJobArgs(**_graph_job_kwargs("s1", "u1", "g1"))+ scheduler.scheduler.add_job.return_value = _mock_job(args.model_dump(mode="json"))+ assert scheduler._schedule_index is not None+ with patch.object(+ scheduler._schedule_index, "upsert", side_effect=RuntimeError("db down")+ ):+ job = scheduler._persist_schedule(+ dispatch_func=MagicMock(),+ job_args=args,+ trigger=MagicMock(),+ name="n",+ )+ assert job is not None # schedule creation must not fail on index errors+++def test_reconcile_rebuilds_index_and_keeps_racing_rows():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None++ in_store = GraphExecutionJobArgs(**_graph_job_kwargs("in-store", "u1", "g1"))+ store_jobs = [+ _mock_job(in_store.model_dump(mode="json")),+ _mock_job({"user_id": "u1"}), # maintenance job: unparseable → skipped+ ]+ scheduler.scheduler.get_jobs.return_value = store_jobs++ # Row for a job that fired and was auto-removed (confirmed gone) …+ fired = GraphExecutionJobArgs(**_graph_job_kwargs("fired", "u1", "g1"))+ index.upsert(_index_entry("fired", fired))+ # … and a row added concurrently with the scan: not in the snapshot but+ # still present in the live store — it must survive the reconcile.+ racing = GraphExecutionJobArgs(**_graph_job_kwargs("racing", "u1", "g1"))+ index.upsert(_index_entry("racing", racing))+ racing_job = _mock_job(racing.model_dump(mode="json"))+ scheduler.scheduler.get_job.side_effect = lambda job_id, jobstore=None: (+ racing_job if job_id == "racing" else None+ )++ scheduler._schedule_index_ready = False+ scheduler._reconcile_schedule_index()++ assert index.all_job_ids() == {"in-store", "racing"}+ assert scheduler._schedule_index_ready is Truediff --git a/autogpt_platform/backend/backend/executor/scheduler_index_recovery_test.py b/autogpt_platform/backend/backend/executor/scheduler_index_recovery_test.pynew file mode 100644index 0000000000..1326841456--- /dev/null+++ b/autogpt_platform/backend/backend/executor/scheduler_index_recovery_test.py@@ -0,0 +1,266 @@+"""Recovery from index write and authoritative job lookup failures."""++from unittest.mock import MagicMock, patch++import pytest+from sqlalchemy import create_engine++from backend.executor.schedule_index import ScheduleIndex+from backend.executor.schedule_index_test_helpers import (+ _graph_job_kwargs,+ _mock_job,+ _scheduler_with_index,+)+from backend.executor.scheduler import (+ _SCHEDULE_INDEX_SETTLE_S,+ GraphExecutionJobArgs,+ _index_entry,+)+++def test_startup_backfill_waits_out_the_outgoing_pod_before_trusting_the_index():+ """The pod being replaced keeps writing schedules until it exits, without+ index rows if it predates the index or its index write failed. A reconcile+ that snapshots the jobstore before then would leave those rows hidden."""+ scheduler = _scheduler_with_index()+ scheduler._schedule_index_ready = False+ steps: list[object] = []+ with (+ patch(+ "backend.executor.scheduler.threading.Thread",+ side_effect=lambda target, **_: MagicMock(start=target),+ ),+ patch("backend.executor.scheduler.time.sleep", side_effect=steps.append),+ patch.object(+ scheduler,+ "_reconcile_schedule_index",+ side_effect=lambda: steps.append("reconcile"),+ ),+ ):+ scheduler._start_schedule_index_backfill()++ assert steps == [_SCHEDULE_INDEX_SETTLE_S, "reconcile"]+ assert scheduler._schedule_index_ready is False+++def test_reconcile_creates_an_index_table_that_failed_at_startup():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index = ScheduleIndex(create_engine("sqlite://"))+ args = GraphExecutionJobArgs(**_graph_job_kwargs("s1", "u1", "g1"))+ scheduler.scheduler.get_jobs.return_value = [+ _mock_job(args.model_dump(mode="json"))+ ]+ scheduler._schedule_index_ready = False++ scheduler._reconcile_schedule_index()++ assert index.all_job_ids() == {"s1"}+ assert scheduler._schedule_index_ready is True+++def test_failed_upsert_keeps_persisted_schedule_visible_until_reconcile():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ args = GraphExecutionJobArgs(**_graph_job_kwargs("s1", "u1", "g1"))+ job = _mock_job(args.model_dump(mode="json"))+ scheduler.scheduler.get_jobs.return_value = []+ assert scheduler._get_active_jobs_cached() == []+ scheduler.scheduler.add_job.return_value = job+ scheduler.scheduler.get_jobs.return_value = [job]+ scheduler.scheduler.get_job.return_value = job++ with patch.object(index, "upsert", side_effect=RuntimeError("transient write")):+ assert (+ scheduler._persist_schedule(+ dispatch_func=MagicMock(), job_args=args, trigger=MagicMock(), name="n"+ )+ == job+ )++ assert index.all_job_ids() == set()+ assert scheduler._schedule_index_ready is False+ assert [s.id for s in scheduler.get_execution_schedules(user_id="u1")] == ["s1"]+ assert scheduler.get_execution_schedules(user_id="other") == []++ scheduler._reconcile_schedule_index()+ assert scheduler._schedule_index_ready is True+ assert index.all_job_ids() == {"s1"}+ with patch.object(scheduler, "_get_active_jobs_cached") as full_scan:+ assert [s.id for s in scheduler.get_execution_schedules(user_id="u1")] == ["s1"]+ full_scan.assert_not_called()+++def test_transient_point_lookup_preserves_live_row_and_listing():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ args = GraphExecutionJobArgs(**_graph_job_kwargs("s1", "u1", "g1"))+ job = _mock_job(args.model_dump(mode="json"))+ index.upsert(_index_entry("s1", args))+ scheduler.scheduler.get_job.side_effect = [RuntimeError("database timeout"), job]+ scheduler.scheduler.get_jobs.return_value = [job]++ assert [s.id for s in scheduler.get_execution_schedules(user_id="u1")] == ["s1"]+ assert index.all_job_ids() == {"s1"}+ with patch.object(scheduler, "_get_active_jobs_cached") as full_scan:+ assert [s.id for s in scheduler.get_execution_schedules(user_id="u1")] == ["s1"]+ full_scan.assert_not_called()+++def test_reconcile_does_not_trust_snapshot_before_failed_upsert():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ old_args = GraphExecutionJobArgs(**_graph_job_kwargs("old", "u1", "g1"))+ new_args = GraphExecutionJobArgs(**_graph_job_kwargs("new", "u1", "g1"))+ old_job = _mock_job(old_args.model_dump(mode="json"))+ new_job = _mock_job(new_args.model_dump(mode="json"))+ scheduler.scheduler.add_job.return_value = new_job+ scheduler.scheduler.get_job.return_value = old_job++ def scan_before_concurrent_write(*args, **kwargs):+ with patch.object(index, "upsert", side_effect=RuntimeError("write failed")):+ scheduler._persist_schedule(+ dispatch_func=MagicMock(),+ job_args=new_args,+ trigger=MagicMock(),+ name="new",+ )+ return [old_job]++ scheduler.scheduler.get_jobs.side_effect = scan_before_concurrent_write+ scheduler._reconcile_schedule_index()+ assert index.all_job_ids() == {"old"}+ assert scheduler._schedule_index_ready is False++ scheduler.scheduler.get_jobs.side_effect = None+ scheduler.scheduler.get_jobs.return_value = [old_job, new_job]+ assert {s.id for s in scheduler.get_execution_schedules(user_id="u1")} == {+ "old",+ "new",+ }+ scheduler._reconcile_schedule_index()+ assert scheduler._schedule_index_ready is True+ assert index.all_job_ids() == {"old", "new"}+++def test_empty_scan_reconcile_preserves_existing_index_rows():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ args = GraphExecutionJobArgs(**_graph_job_kwargs("s1", "u1", "g1"))+ index.upsert(_index_entry("s1", args))+ scheduler._schedule_index_ready = False+ scheduler.scheduler.get_jobs.return_value = []+ scheduler.scheduler.get_job.return_value = None++ scheduler._reconcile_schedule_index()++ assert index.all_job_ids() == {"s1"}+ assert scheduler._schedule_index_ready is True+ scheduler.scheduler.get_job.assert_not_called()+++def test_system_only_scan_drops_definitively_missing_index_rows():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ args = GraphExecutionJobArgs(**_graph_job_kwargs("gone", "u1", "g1"))+ index.upsert(_index_entry("gone", args))+ scheduler.scheduler.get_jobs.return_value = [_mock_job({"user_id": "u1"})]+ scheduler.scheduler.get_job.return_value = None++ scheduler._reconcile_schedule_index()++ assert index.all_job_ids() == set()+ assert scheduler._schedule_index_ready is True+++def test_failed_write_during_candidate_lookup_falls_back():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ args = GraphExecutionJobArgs(**_graph_job_kwargs("s1", "u1", "g1"))+ job = _mock_job(args.model_dump(mode="json"))+ scheduler.scheduler.add_job.return_value = job+ scheduler.scheduler.get_jobs.return_value = [job]++ def query_before_failed_write(**kwargs):+ with patch.object(index, "upsert", side_effect=RuntimeError("write failed")):+ scheduler._persist_schedule(+ dispatch_func=MagicMock(), job_args=args, trigger=MagicMock(), name="n"+ )+ return []++ with patch.object(+ index, "candidate_job_ids", side_effect=query_before_failed_write+ ):+ assert [s.id for s in scheduler.get_execution_schedules(user_id="u1")] == ["s1"]++[email protected]("has_existing_row", [True, False])+def test_empty_scan_does_not_reenable_index_after_known_failed_write(has_existing_row):+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ old_args = GraphExecutionJobArgs(**_graph_job_kwargs("old", "u1", "g1"))+ new_args = GraphExecutionJobArgs(**_graph_job_kwargs("new", "u1", "g1"))+ if has_existing_row:+ index.upsert(_index_entry("old", old_args))+ with patch.object(index, "upsert", side_effect=RuntimeError("write failed")):+ scheduler._upsert_schedule_index_row("new", new_args)+ scheduler.scheduler.get_jobs.return_value = []++ scheduler._reconcile_schedule_index()++ assert scheduler._schedule_index_ready is False+ assert index.all_job_ids() == ({"old"} if has_existing_row else set())+++def test_corrupt_job_falls_back_then_cleans_up_after_definitive_absence():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ args = GraphExecutionJobArgs(**_graph_job_kwargs("bad", "u1", "g1"))+ index.upsert(_index_entry("bad", args))+ scheduler.scheduler.get_job.side_effect = [LookupError("renamed callable"), None]+ scheduler.scheduler.get_jobs.return_value = []++ assert scheduler.get_execution_schedules(user_id="u1") == []+ assert index.all_job_ids() == {"bad"}+ assert scheduler.get_execution_schedules(user_id="u1") == []+ assert index.all_job_ids() == set()+++def test_shutdown_during_candidate_lookup_keeps_shared_index_rows():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ args = GraphExecutionJobArgs(**_graph_job_kwargs("live", "u1", "g1"))+ index.upsert(_index_entry("live", args))++ def stopping_scheduler(*args, **kwargs):+ scheduler.scheduler.running = False+ return None++ scheduler.scheduler.get_job.side_effect = stopping_scheduler+ scheduler.get_execution_schedules(user_id="u1")+ assert index.all_job_ids() == {"live"}+++def test_shutdown_during_reconcile_verification_keeps_shared_index_rows():+ scheduler = _scheduler_with_index()+ index = scheduler._schedule_index+ assert index is not None+ args = GraphExecutionJobArgs(**_graph_job_kwargs("live", "u1", "g1"))+ index.upsert(_index_entry("live", args))+ scheduler.scheduler.get_jobs.return_value = [_mock_job({"user_id": "u1"})]++ def stopping_scheduler(*args, **kwargs):+ scheduler.scheduler.running = False+ return None++ scheduler.scheduler.get_job.side_effect = stopping_scheduler+ scheduler._reconcile_schedule_index()+ assert index.all_job_ids() == {"live"}diff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.pyindex 7faadc7e24..70d9ecff47 100644--- a/autogpt_platform/backend/backend/executor/scheduler_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py@@ -906,9 +906,6 @@ def test_get_active_jobs_cached_sets_the_scheduler_jobs_gauge():from backend.executor.scheduler import Schedulers = Scheduler.__new__(Scheduler)- s._active_jobs_cache = None- s._active_jobs_cache_expires_at = 0.0- s._jobs_cache_version = 0s._execution_jobstore = MagicMock()s._execution_jobstore._get_jobs.return_value = [object(), object(), object()]@@ -928,9 +925,6 @@ def test_stale_read_invalidated_mid_query_does_not_overwrite_the_gauge():from backend.executor.scheduler import Schedulers = Scheduler.__new__(Scheduler)- s._active_jobs_cache = None- s._active_jobs_cache_expires_at = 0.0- s._jobs_cache_version = 0s._execution_jobstore = MagicMock()# A fresh, accepted read publishes 5.@@ -946,8 +940,7 @@ def test_stale_read_invalidated_mid_query_does_not_overwrite_the_gauge():# Now a read whose query is interrupted by an invalidation: it returns a# different count, but the version moved, so the cache write is skipped.- s._active_jobs_cache = None- s._active_jobs_cache_expires_at = 0.0+ s._invalidate_jobs_cache()def _slow_query_then_invalidated(*_args, **_kwargs):s._invalidate_jobs_cache()@@ -955,5 +948,7 @@ def test_stale_read_invalidated_mid_query_does_not_overwrite_the_gauge():s._execution_jobstore._get_jobs.side_effect = _slow_query_then_invalidatedassert len(s._get_active_jobs_cached()) == 2 # caller still gets the list- assert s._active_jobs_cache is None # write-back was rejected- assert gauge() == 5 # and so was the gauge update+ assert gauge() == 5 # but the gauge update was rejected+ s._execution_jobstore._get_jobs.side_effect = None+ s._execution_jobstore._get_jobs.return_value = [object()] * 7+ assert len(s._get_active_jobs_cached()) == 7 # and so was the cache writediff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.pyindex 5b8e62f20f..31eea8a287 100644--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py@@ -7,6 +7,8 @@ backend test job (and counted by codecov), not just the integration suite.import asyncioimport logging+import threading+import timefrom datetime import datetime, timedelta, timezonefrom typing import NamedTuplefrom unittest.mock import AsyncMock, MagicMock, patch@@ -2381,3 +2383,206 @@ class TestPostHogLifecycleSweepRegistration:assert not getattr(Scheduler._register_posthog_lifecycle_sweep, EXPOSED_FLAG, False)+++class TestScheduleListCache:+ """One jobstore read per cache miss, and no caller waits a scan while an+ expired list can answer it. Both cached reads share the logic, so every+ test runs against each."""++ @pytest.fixture(params=["all", "active"])+ def cache(self, request, monkeypatch: pytest.MonkeyPatch):+ sched = Scheduler(register_system_tasks=False)+ sched.scheduler = MagicMock()+ sched._execution_jobstore = MagicMock()+ fetch = _GatedFetch()+ if request.param == "all":+ sched.scheduler.get_jobs = fetch+ read = sched._get_jobs_cached+ else:+ sched._execution_jobstore._get_jobs = fetch+ read = sched._get_active_jobs_cached+ # Every list expires the moment it is cached, so each read after the+ # first is an expired one.+ monkeypatch.setattr(Scheduler, "_JOBS_CACHE_TTL_S", 0.0)+ monkeypatch.setattr(Scheduler, "_jobs_cache_lock", _CountingLock())+ yield sched, read, fetch+ fetch.release_all()++ def test_eight_misses_run_one_query(self, cache):+ sched, read, fetch = cache+ results = _read_concurrently(read, fetch, callers=8)++ assert fetch.calls == 1+ assert all(r is results[0] for r in results)++ def test_a_failed_query_fails_every_waiter_once(self, cache):+ sched, read, fetch = cache+ fetch.fail_with = RuntimeError("QueuePool limit reached")++ results = _read_concurrently(read, fetch, callers=8)++ assert fetch.calls == 1+ assert all(isinstance(r, RuntimeError) for r in results)++ def test_an_expired_list_is_served_while_one_refresh_runs(self, cache):+ sched, read, fetch = cache+ fetch.release(1)+ old = read()++ assert _within(read) is old # starts the refresh, does not wait for it+ fetch.wait_for_calls(2)+ assert _within(read) is old+ assert fetch.calls == 2++ fetch.release(2)+ _join_refreshes()+ assert _within(read) == ["list-2"]++ def test_a_failed_refresh_keeps_the_old_list_and_retries(self, cache):+ sched, read, fetch = cache+ fetch.release(1)+ old = read()+ fetch.fail_with = RuntimeError("database timeout")++ assert _within(read) is old+ fetch.release(2)+ _join_refreshes()+ assert _within(read) is old # and starts refresh #3: #2 did not wedge it+ fetch.wait_for_calls(3)++ def test_an_invalidation_is_never_answered_from_the_old_list(self, cache):+ sched, read, fetch = cache+ fetch.release(1)+ old = read()+ assert _within(read) is old+ fetch.wait_for_calls(2) # refresh #2 started before the write++ sched._invalidate_jobs_cache()+ after = _start(read)+ fetch.wait_for_calls(3) # a fresh read, not a wait on refresh #2+ assert after.is_alive()++ fetch.release(3)+ assert after.result() == ["list-3"]+ fetch.release(2)+ _join_refreshes()+ # Refresh #2 read before the write, so it must not land in the cache.+ assert _within(read) == ["list-3"]++ def test_a_failing_gauge_update_still_answers_the_waiters(self, monkeypatch):+ sched = Scheduler(register_system_tasks=False)+ sched._execution_jobstore = MagicMock()+ fetch = sched._execution_jobstore._get_jobs = _GatedFetch()+ monkeypatch.setattr(Scheduler, "_jobs_cache_lock", _CountingLock())+ monkeypatch.setattr(+ f"{_SCHEDULER_PATH}.SCHEDULER_JOBS.labels",+ MagicMock(side_effect=RuntimeError("registry")),+ )++ results = _read_concurrently(sched._get_active_jobs_cached, fetch, callers=8)++ assert fetch.calls == 1+ assert sum(isinstance(r, RuntimeError) for r in results) == 1 # the fetcher+ assert [r for r in results if isinstance(r, list)] == [["list-1"]] * 7+++class _GatedFetch:+ """A jobstore read that counts its calls and holds each until released."""++ def __init__(self) -> None:+ self.calls = 0+ self.fail_with: Exception | None = None+ self._lock = threading.Lock()+ self._gates: dict[int, threading.Event] = {}++ def __call__(self, *_args, **_kwargs) -> list[str]:+ with self._lock:+ self.calls += 1+ call = self.calls+ failure = self.fail_with+ self._gate(call).wait(timeout=10)+ if failure is not None:+ raise failure+ return [f"list-{call}"]++ def release(self, call: int) -> None:+ self._gate(call).set()++ def release_all(self) -> None:+ for call in range(1, self.calls + 2):+ self.release(call)++ def wait_for_calls(self, count: int) -> None:+ _wait_until(lambda: self.calls >= count)++ def _gate(self, call: int) -> threading.Event:+ with self._lock:+ return self._gates.setdefault(call, threading.Event())+++class _CountingLock:+ def __init__(self) -> None:+ self._lock = threading.Lock()+ self.acquired = 0++ def __enter__(self) -> "_CountingLock":+ self._lock.acquire()+ self.acquired += 1+ return self++ def __exit__(self, *_exc) -> None:+ self._lock.release()+++class _Call(threading.Thread):+ def __init__(self, fn) -> None:+ super().__init__(daemon=True)+ self._fn = fn+ self._outcome: object = None++ def run(self) -> None:+ try:+ self._outcome = self._fn()+ except Exception as e:+ self._outcome = e++ def result(self, timeout: float = 5.0) -> object:+ self.join(timeout)+ assert not self.is_alive(), "the read is still waiting"+ return self._outcome+++def _start(fn) -> _Call:+ call = _Call(fn)+ call.start()+ return call+++def _within(fn, timeout: float = 2.0) -> object:+ return _start(fn).result(timeout)+++def _read_concurrently(read, fetch: _GatedFetch, callers: int) -> list[object]:+ lock = Scheduler._jobs_cache_lock+ assert isinstance(lock, _CountingLock)+ calls = [_start(read) for _ in range(callers)]+ # Each caller takes the lock once to check the cache; the fetcher's+ # write-back is held behind the gate, so this counts arrivals only.+ _wait_until(lambda: lock.acquired >= callers)+ fetch.release_all()+ return [call.result() for call in calls]+++def _join_refreshes() -> None:+ for thread in threading.enumerate():+ if thread.name == "ScheduleListRefresh":+ thread.join(timeout=5)+ assert not thread.is_alive(), "a background refresh is still running"+++def _wait_until(condition, timeout: float = 5.0) -> None:+ deadline = time.monotonic() + timeout+ while not condition():+ assert time.monotonic() < deadline, "timed out"+ time.sleep(0.01)