AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): park unrestorable scheduler jobs instead of deleting them (#14218)
Reinier van der Leere1b198c0617a
diff --git a/autogpt_platform/backend/backend/executor/jobstore.py b/autogpt_platform/backend/backend/executor/jobstore.pynew file mode 100644index 0000000000..1866c8c027--- /dev/null+++ b/autogpt_platform/backend/backend/executor/jobstore.py@@ -0,0 +1,176 @@+import logging+import pickle++from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore+from sqlalchemy import and_, select++logger = logging.getLogger(__name__)+++class ResilientSQLAlchemyJobStore(SQLAlchemyJobStore):+ """Parks jobs it cannot restore instead of deleting them.++ Upstream ``_get_jobs`` DELETEs any row whose ``job_state`` fails to load,+ so a deploy that renames or removes a symbol a persisted job references+ destroys the user's schedule with no recovery path — the table carries no+ audit trail. Parking clears ``next_run_time`` (APScheduler's own "paused"+ marker), which keeps ``job_state`` intact for repair and stops+ ``get_due_jobs`` returning the row, so one bad job cannot spin the+ scheduler.++ Parked jobs do not run. ``get_parked_job_ids`` is the operator surface+ that keeps that from being silent, and ``reconcile_repaired_jobs``+ finishes the job once the payload is repaired.+ """++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self._parked_ids: set[str] = set()++ def _get_jobs(self, *conditions):+ jobs = []+ selectable = select(+ self.jobs_t.c.id, self.jobs_t.c.job_state, self.jobs_t.c.next_run_time+ ).order_by(self.jobs_t.c.next_run_time)+ selectable = selectable.where(and_(*conditions)) if conditions else selectable+ unrestorable: list = []++ with self.engine.begin() as connection:+ for row in connection.execute(selectable):+ try:+ jobs.append(self._reconstitute_job(row.job_state))+ except (KeyboardInterrupt, SystemExit):+ raise+ except BaseException as e:+ unrestorable.append((row, e))++ for row, error in unrestorable:+ if not self._park(connection, row):+ continue+ # Unconditioned _get_jobs still returns parked rows, so+ # without this every get_jobs() call re-reports them.+ if row.id not in self._parked_ids:+ self._parked_ids.add(row.id)+ self._logger.error(+ 'Unable to restore job "%s" -- parking it. The row is '+ "kept; repair it and set next_run_time to resume.",+ row.id,+ exc_info=error,+ )++ return jobs++ def _park(self, connection, row) -> bool:+ """Clear ``next_run_time`` for the row we read, and say whether we did.++ Matching the scanned ``job_state`` too keeps a repair or resume that+ landed since the scan from being silently undone.+ """+ result = connection.execute(+ self.jobs_t.update()+ .where(+ and_(+ self.jobs_t.c.id == row.id,+ self.jobs_t.c.job_state == row.job_state,+ self.jobs_t.c.next_run_time.is_not(None),+ )+ )+ .values(next_run_time=None)+ )+ return result.rowcount == 1++ def get_parked_job_ids(self, limit: int | None = None) -> list[str]:+ """Ids of rows that are paused *and* still unrestorable.++ A user-paused job also has ``next_run_time = NULL`` but restores+ fine, so attempting the restore is what separates the two. *limit*+ caps the rows scanned: paused and fired-once rows accumulate+ forever, and startup must not pay for the whole backlog.+ """+ selectable = select(self.jobs_t.c.id, self.jobs_t.c.job_state).where(+ self.jobs_t.c.next_run_time.is_(None)+ )+ if limit is not None:+ selectable = selectable.limit(limit)+ parked = []+ with self.engine.begin() as connection:+ for row in connection.execute(selectable):+ try:+ self._reconstitute_job(row.job_state)+ except (KeyboardInterrupt, SystemExit):+ raise+ except BaseException:+ parked.append(row.id)+ return parked++ def reconcile_repaired_jobs(self, batch_size: int = 500) -> list[str]:+ """Finish parking rows whose payload has since been repaired.++ Parking can only clear the ``next_run_time`` COLUMN: the row is by+ definition one we could not deserialize, so the copy inside+ ``job_state`` keeps its old value. Once the payload is repaired the+ two disagree, and the row is stranded — it deserializes, so it is no+ longer reported as parked; the column still reads paused, so+ ``get_due_jobs`` skips it; and ``resume`` reads the stale non-None+ copy and refuses. Writing that copy back to None makes it an+ ordinary paused job, which resume revives.++ ``jobstore_backfill`` already repairs the rows it rewrites, so this is+ the path for a repair made some other way, and it is not run at+ startup. It walks the whole paused set in id order, a batch per+ transaction: a cap alone would keep re-reading the first page and+ never reach a repaired row behind a backlog nothing deletes, while one+ transaction over the lot would hold locks across every row it writes.+ """+ healed: list[str] = []+ after = ""+ while True:+ selectable = (+ select(self.jobs_t.c.id, self.jobs_t.c.job_state)+ .where(+ and_(+ self.jobs_t.c.next_run_time.is_(None),+ self.jobs_t.c.id > after,+ )+ )+ .order_by(self.jobs_t.c.id)+ .limit(batch_size)+ )+ rows = self._reconcile_batch(selectable, healed)+ if len(rows) < batch_size:+ return healed+ after = rows[-1]++ def _reconcile_batch(self, selectable, healed: list[str]) -> list[str]:+ """One batch in one transaction; returns the ids it read, in order."""+ seen = []+ with self.engine.begin() as connection:+ for row in connection.execute(selectable):+ seen.append(row.id)+ try:+ job = self._reconstitute_job(row.job_state)+ except (KeyboardInterrupt, SystemExit):+ raise+ except BaseException:+ continue # still unrestorable: genuinely parked+ if job.next_run_time is None:+ continue # an ordinary paused job; nothing to reconcile+ job.next_run_time = None+ result = connection.execute(+ self.jobs_t.update()+ .where(+ and_(+ self.jobs_t.c.id == row.id,+ self.jobs_t.c.job_state == row.job_state,+ self.jobs_t.c.next_run_time.is_(None),+ )+ )+ .values(+ job_state=pickle.dumps(job.__getstate__(), self.pickle_protocol)+ )+ )+ if result.rowcount != 1:+ continue # resumed or repaired under the scan; leave it be+ self._parked_ids.discard(row.id)+ healed.append(row.id)+ return seendiff --git a/autogpt_platform/backend/backend/executor/jobstore_backfill.py b/autogpt_platform/backend/backend/executor/jobstore_backfill.pynew file mode 100644index 0000000000..471ad7b8c3--- /dev/null+++ b/autogpt_platform/backend/backend/executor/jobstore_backfill.py@@ -0,0 +1,178 @@+"""Rewrite enum members pickled into persisted APScheduler job arguments.++Job kwargs written before #13190 used ``model_dump()`` rather than+``model_dump(mode="json")``, so real Enum members were pickled into+``apscheduler_jobs``. ``ProviderName`` is one of them, and an Enum pickles by+value: unpickling calls ``ProviderName("github")``, which raises the moment a+member is renamed or dropped — and APScheduler's answer to a job it cannot+restore is to destroy it (SENTRY-1392).++This converts those members to their plain values, which the dispatch+functions' pydantic models coerce back on the way in. It must run while every+member still resolves, i.e. BEFORE any deploy that removes one.++ poetry run python -m backend.executor.jobstore_backfill # dry run+ poetry run python -m backend.executor.jobstore_backfill --apply+"""++import argparse+import logging+import os+import pickle+import sys+from enum import Enum+from typing import Any++from sqlalchemy import MetaData, Table, create_engine, select+from sqlalchemy.exc import NoSuchTableError++logger = logging.getLogger(__name__)++TABLES = ("apscheduler_jobs", "apscheduler_jobs_batched_notifications")+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument(+ "--apply",+ action="store_true",+ help="write the changes; without it the run only reports",+ )+ args = parser.parse_args()+ # Imported here: scheduler pulls in the whole backend, and this+ # module is imported by tests that need none of it.+ from backend.executor.scheduler import _extract_schema_from_url++ logging.basicConfig(level=logging.INFO, format="%(message)s")++ database_url = os.getenv("DIRECT_URL")+ if not database_url:+ logger.error("DIRECT_URL is not set")+ return 2++ schema, url = _extract_schema_from_url(database_url)+ engine = create_engine(url)+ metadata = MetaData(schema=schema)++ total_changed = total_unreadable = total_skipped = 0+ for tablename in TABLES:+ changed, unreadable, skipped = _normalize_table(+ engine, metadata, tablename, args.apply+ )+ total_changed += changed+ total_unreadable += unreadable+ total_skipped += skipped++ verb = "rewrote" if args.apply else "would rewrite"+ logger.info(f"\n{verb} {total_changed} row(s); {total_unreadable} unreadable")+ if not args.apply and total_changed:+ logger.info("re-run with --apply to write")+ if total_skipped:+ # The run is incomplete; saying so here is the point, since this is the+ # line an operator reads to decide the backfill is done.+ logger.warning(+ f"{total_skipped} row(s) changed under the scan and were NOT rewritten; "+ "re-run to pick them up"+ )+ return 1+ return 0+++def _normalize_table(+ engine, metadata, tablename: str, apply: bool+) -> tuple[int, int, int]:+ try:+ table = Table(tablename, metadata, autoload_with=engine)+ except NoSuchTableError:+ logger.info(f"{tablename}: skipped (no such table)")+ return 0, 0, 0++ changed = unreadable = skipped = 0+ with engine.begin() as connection:+ rows = list(+ connection.execute(+ select(table.c.id, table.c.job_state, table.c.next_run_time)+ )+ )+ for row in rows:+ try:+ state = pickle.loads(row.job_state)+ except (KeyboardInterrupt, SystemExit):+ raise+ except BaseException as e:+ # Already unrestorable; parking/repair is a separate concern.+ logger.warning(f"{tablename}: {row.id} is unreadable ({e})")+ unreadable += 1+ continue++ args, kwargs = state.get("args", ()), state.get("kwargs", {})+ # Not an equality check: a str-based Enum compares equal to its own+ # value, so the stripped copy would look unchanged and be skipped.+ if not (_has_enum(args) or _has_enum(kwargs)):+ continue++ new_state = dict(state)+ new_state["args"] = _strip_enums(args)+ new_state["kwargs"] = _strip_enums(kwargs)+ if row.next_run_time is None:+ # The row is paused, and parking is what pauses an unrestorable+ # one -- but parking could only clear the COLUMN, so this copy+ # still holds the old time. Leaving them disagreed strands the+ # row: nothing reports it, nothing runs it, resume refuses it.+ new_state["next_run_time"] = None++ logger.info(f"{tablename}: {row.id} carries pickled enum(s)")+ if not apply:+ changed += 1+ continue++ result = connection.execute(+ table.update()+ .where(+ table.c.id == row.id,+ table.c.job_state == row.job_state,+ )+ .values(job_state=pickle.dumps(new_state, pickle.HIGHEST_PROTOCOL))+ )+ if result.rowcount == 1:+ changed += 1+ else:+ skipped += 1+ logger.warning(+ f"{tablename}: {row.id} changed under the scan; left alone, "+ "re-run to pick it up"+ )++ suffix = f", {skipped} skipped (changed under the scan)" if skipped else ""+ logger.info(+ f"{tablename}: {len(rows)} row(s) scanned, {changed} to rewrite{suffix}"+ )+ return changed, unreadable, skipped+++def _has_enum(value: Any) -> bool:+ """Whether an Enum member is hiding anywhere in *value*."""+ if isinstance(value, Enum):+ return True+ if isinstance(value, dict):+ return any(_has_enum(k) or _has_enum(v) for k, v in value.items())+ if isinstance(value, (list, tuple)):+ return any(_has_enum(v) for v in value)+ return False+++def _strip_enums(value: Any) -> Any:+ """Replace Enum members with their values, recursively."""+ if isinstance(value, Enum):+ return value.value+ if isinstance(value, dict):+ return {_strip_enums(k): _strip_enums(v) for k, v in value.items()}+ if isinstance(value, list):+ return [_strip_enums(v) for v in value]+ if isinstance(value, tuple):+ return tuple(_strip_enums(v) for v in value)+ return value+++if __name__ == "__main__":+ sys.exit(main())diff --git a/autogpt_platform/backend/backend/executor/jobstore_backfill_test.py b/autogpt_platform/backend/backend/executor/jobstore_backfill_test.pynew file mode 100644index 0000000000..2de25f7a1d--- /dev/null+++ b/autogpt_platform/backend/backend/executor/jobstore_backfill_test.py@@ -0,0 +1,252 @@+import os+import pickle+import tempfile+from contextlib import contextmanager+from enum import Enum+from unittest.mock import patch++import pytest+from sqlalchemy import (+ Column,+ Float,+ LargeBinary,+ MetaData,+ Table,+ Unicode,+ create_engine,+ select,+)+from sqlalchemy.exc import OperationalError++from backend.executor.jobstore_backfill import _has_enum, _normalize_table, _strip_enums++_BACKFILL = "backend.executor.jobstore_backfill"++TABLE = "apscheduler_jobs"+++class Provider(str, Enum):+ GITHUB = "github"+++class _Dropped(Enum):+ GONE = "gone"+++# Pickled while GONE exists, then rebound without it — an unreadable row.+UNREADABLE = pickle.dumps({"kwargs": {"x": _Dropped.GONE}}, pickle.HIGHEST_PROTOCOL)+++class _Dropped(Enum): # noqa: F811 - deliberate rebind, drops GONE+ KEPT = "kept"+++def test_str_enum_is_detected_despite_comparing_equal_to_its_value():+ """The bug this guards: Provider.GITHUB == "github" is True, so an+ equality check would call a poisoned row unchanged and skip it."""+ assert Provider.GITHUB == "github"+ assert _has_enum({"provider": Provider.GITHUB})+ assert not _has_enum({"provider": "github"})+++def test_dry_run_reports_without_mutating():+ with _db() as engine:+ _insert(engine, "job1", {"provider": Provider.GITHUB})+ before = _job_state(engine, "job1")++ changed, unreadable, _ = _normalize_table(+ engine, MetaData(), TABLE, apply=False+ )++ assert (changed, unreadable) == (1, 0)+ assert _job_state(engine, "job1") == before+++def test_apply_rewrites_the_enum_to_a_plain_value():+ with _db() as engine:+ _insert(engine, "job1", {"provider": Provider.GITHUB})++ changed, _, _ = _normalize_table(engine, MetaData(), TABLE, apply=True)++ assert changed == 1+ kwargs = pickle.loads(_job_state(engine, "job1"))["kwargs"]+ assert kwargs == {"provider": "github"}+ assert type(kwargs["provider"]) is str+ assert not isinstance(kwargs["provider"], Enum)+++def test_apply_is_idempotent():+ with _db() as engine:+ _insert(engine, "job1", {"provider": Provider.GITHUB})+ _normalize_table(engine, MetaData(), TABLE, apply=True)+ after_first = _job_state(engine, "job1")++ changed, _, _ = _normalize_table(engine, MetaData(), TABLE, apply=True)++ assert changed == 0+ assert _job_state(engine, "job1") == after_first+++def test_nested_and_sequence_enums_are_rewritten():+ with _db() as engine:+ _insert(+ engine,+ "job1",+ {"creds": {"a": {"provider": Provider.GITHUB}}, "seq": [Provider.GITHUB]},+ )++ _normalize_table(engine, MetaData(), TABLE, apply=True)++ kwargs = pickle.loads(_job_state(engine, "job1"))["kwargs"]+ assert kwargs == {"creds": {"a": {"provider": "github"}}, "seq": ["github"]}+ assert type(kwargs["creds"]["a"]["provider"]) is str+ assert type(kwargs["seq"][0]) is str+++def test_unreadable_row_is_counted_and_left_untouched():+ """A backfill that mangles a row it cannot read is worse than one that skips."""+ with _db() as engine:+ _insert_raw(engine, "broken", UNREADABLE)+ _insert(engine, "job1", {"provider": Provider.GITHUB})++ changed, unreadable, _ = _normalize_table(engine, MetaData(), TABLE, apply=True)++ assert (changed, unreadable) == (1, 1)+ assert _job_state(engine, "broken") == UNREADABLE+++def test_rows_without_enums_are_left_alone():+ with _db() as engine:+ _insert(engine, "job1", {"user_id": "u1", "cron": "0 * * * *"})+ before = _job_state(engine, "job1")++ changed, unreadable, _ = _normalize_table(engine, MetaData(), TABLE, apply=True)++ assert (changed, unreadable) == (0, 0)+ assert _job_state(engine, "job1") == before+++def test_missing_table_is_skipped_not_raised():+ with _db() as engine:+ assert _normalize_table(engine, MetaData(), "no_such_table", apply=True) == (+ 0,+ 0,+ 0,+ )+++def test_a_reflection_failure_is_not_reported_as_a_missing_table():+ """A missing table is a skip; anything else must fail loudly. This script+ gates a deploy, so a connection or permission error exiting 0 would read as+ 'nothing to rewrite'."""+ engine = create_engine("sqlite:////nonexistent-dir/nope.sqlite")++ with pytest.raises(OperationalError):+ _normalize_table(engine, MetaData(), TABLE, apply=True)+++def test_a_row_that_changed_under_the_scan_is_left_alone():+ """The scheduler rewrites job_state whenever a job fires, and READ+ COMMITTED lets that land between the scan and the update."""+ with _db() as engine:+ _insert(engine, "job1", {"kwargs": {"provider": Provider.GITHUB}})+ newer = pickle.dumps({"kwargs": {"written": "by the scheduler"}})++ # Stand in for the concurrent writer: the value in the table is no+ # longer the one the scan read, so the predicate must not match.+ def _interfere(value):+ with engine.begin() as conn:+ conn.execute(+ Table(TABLE, MetaData(), autoload_with=engine)+ .update()+ .where(Column("id", Unicode(191)) == "job1")+ .values(job_state=newer)+ )+ return _strip_enums(value)++ with patch(f"{_BACKFILL}._strip_enums", side_effect=_interfere):+ changed, unreadable, skipped = _normalize_table(+ engine, MetaData(), TABLE, apply=True+ )++ assert (changed, skipped) == (0, 1)+ assert _job_state(engine, "job1") == newer+++def test_repairing_a_parked_row_also_clears_its_pickled_next_run_time():+ """A parked row is paused in the COLUMN only, because parking could not+ deserialize it to rewrite the copy. Repairing it without clearing that copy+ strands the row — nothing reports it, nothing runs it, resume refuses it —+ so the repair has to finish the job."""+ with _db() as engine:+ _insert_raw(engine, "parked", _state(1234.0), next_run_time=None)++ changed, _, _ = _normalize_table(engine, MetaData(), TABLE, apply=True)++ assert changed == 1+ assert pickle.loads(_job_state(engine, "parked"))["next_run_time"] is None+++def test_repairing_a_running_row_leaves_its_next_run_time_alone():+ with _db() as engine:+ _insert_raw(engine, "live", _state(1234.0), next_run_time=99.0)++ _normalize_table(engine, MetaData(), TABLE, apply=True)++ assert pickle.loads(_job_state(engine, "live"))["next_run_time"] == 1234.0+++@contextmanager+def _db():+ fd, path = tempfile.mkstemp(suffix=".sqlite")+ os.close(fd)+ engine = create_engine(f"sqlite:///{path}")+ metadata = MetaData()+ Table(+ TABLE,+ metadata,+ Column("id", Unicode(191), primary_key=True),+ Column("next_run_time", Float(25), index=True),+ Column("job_state", LargeBinary, nullable=False),+ )+ metadata.create_all(engine)+ try:+ yield engine+ finally:+ engine.dispose()+ os.unlink(path)+++def _state(next_run_time: float) -> bytes:+ return pickle.dumps(+ {+ "id": "x",+ "args": (),+ "kwargs": {"provider": Provider.GITHUB},+ "next_run_time": next_run_time,+ },+ pickle.HIGHEST_PROTOCOL,+ )+++def _insert(engine, job_id: str, kwargs: dict) -> None:+ state = {"id": job_id, "args": (), "kwargs": kwargs}+ _insert_raw(engine, job_id, pickle.dumps(state, pickle.HIGHEST_PROTOCOL))+++def _insert_raw(engine, job_id: str, job_state: bytes, next_run_time=1.0) -> None:+ table = Table(TABLE, MetaData(), autoload_with=engine)+ with engine.begin() as conn:+ conn.execute(+ table.insert().values(+ id=job_id, next_run_time=next_run_time, job_state=job_state+ )+ )+++def _job_state(engine, job_id: str) -> bytes:+ table = Table(TABLE, MetaData(), autoload_with=engine)+ with engine.begin() as conn:+ return conn.execute(+ select(table.c.job_state).where(table.c.id == job_id)+ ).scalar()diff --git a/autogpt_platform/backend/backend/executor/jobstore_test.py b/autogpt_platform/backend/backend/executor/jobstore_test.pynew file mode 100644index 0000000000..018cceacae--- /dev/null+++ b/autogpt_platform/backend/backend/executor/jobstore_test.py@@ -0,0 +1,383 @@+import os+import pickle+import tempfile+from contextlib import contextmanager+from enum import Enum+from unittest.mock import MagicMock, patch++import pytest+from apscheduler.schedulers.background import BackgroundScheduler+from sqlalchemy import select++from backend.executor.jobstore import ResilientSQLAlchemyJobStore+from backend.executor.jobstore_backfill import _strip_enums+from backend.executor.scheduler import Jobstores, Scheduler++_SCHEDULER_PATH = "backend.executor.scheduler"+++class _RemovedEnum(Enum):+ GONE = "gone"+++# Pickled while GONE exists, then the class is rebound without it — the exact+# shape of SENTRY-1392, where a deploy dropped a NotificationType member.+POISONED_STATE = pickle.dumps(+ {"kwargs": {"x": _RemovedEnum.GONE}}, pickle.HIGHEST_PROTOCOL+)+++class _RemovedEnum(Enum): # noqa: F811 - deliberate rebind, drops GONE+ KEPT = "kept"+++def noop():+ pass+++def test_poison_reproduces_the_production_error():+ """The poison must fail the way production did — on the missing enum member,+ not merely as invalid bytes. Which exception carries that is the+ interpreter's business: 3.11.2 reduces a plain Enum member by NAME and+ raises AttributeError, every later build reduces by value and raises+ ValueError, so pinning one wording fails on the other."""+ with pytest.raises((ValueError, AttributeError)) as caught:+ pickle.loads(POISONED_STATE)++ assert "GONE" in str(caught.value) or "gone" in str(caught.value)+++def test_unrestorable_job_is_parked_not_deleted():+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="good")+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ _poison(store, "poisoned")++ jobs = store._get_jobs()++ assert [j.id for j in jobs] == ["good"]+ # The row survives — this is the whole point of the fix.+ assert _ids(store) == {"good", "poisoned"}+ assert _next_run_time(store, "poisoned") is None+ assert _next_run_time(store, "good") is not None+++def test_active_jobs_read_parks_the_row_then_stops_unpickling_it():+ """The ``next_run_time IS NOT NULL`` listing path meets a poisoned row+ first, because parking is what makes that column NULL."""+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="good")+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ _poison(store, "poisoned")+ active = store.jobs_t.c.next_run_time.isnot(None)++ assert [j.id for j in store._get_jobs(active)] == ["good"]+ assert _ids(store) == {"good", "poisoned"}+ assert _next_run_time(store, "poisoned") is None++ with patch.object(+ store, "_reconstitute_job", wraps=store._reconstitute_job+ ) as restore:+ assert [j.id for j in store._get_jobs(active)] == ["good"]+ # The filter now excludes the parked row in SQL, so the second read+ # never fetches it — one restore, for "good".+ assert restore.call_count == 1+++def test_scheduler_listing_path_parks_an_unrestorable_schedule():+ """The seam between the two fixes: the listing read is filtered at the SQL+ level (#14439) and must still park rather than delete (#14218), so the+ jobstore behind it has to be the resilient one."""+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ _poison(store, "poisoned")++ with _scheduler_wired_to(store) as sched:+ assert (+ sched._execution_jobstore+ is sched._persistent_jobstores[Jobstores.EXECUTION.value]+ )+ assert sched.get_execution_schedules() == []++ assert _ids(store) == {"poisoned"}+ assert _next_run_time(store, "poisoned") is None+ # Parked by that very read, through this instance — not a second one+ # the operator surface would be reporting on separately.+ assert store._parked_ids == {"poisoned"}+ assert sched.get_parked_jobs()[Jobstores.EXECUTION.value] == ["poisoned"]+++def test_repaired_job_can_be_resumed_and_then_fires():+ """Park, repair, resume, run. Parking can only clear the COLUMN, so without+ reconciliation the repaired row is stranded: it reports healthy, never comes+ due, and resume refuses it because the pickled copy is still set."""+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ healthy_state = _job_state(store, "poisoned")+ _poison(store, "poisoned")+ store._get_jobs()+ assert store.get_parked_job_ids() == ["poisoned"]++ # Repair rewrites job_state and leaves its next_run_time untouched,+ # which is exactly what jobstore_backfill does.+ _set_job_state(store, "poisoned", healthy_state)+ assert store.get_parked_job_ids() == []+ assert store._get_jobs()[0].next_run_time is not None # stranded+ assert _next_run_time(store, "poisoned") is None++ assert store.reconcile_repaired_jobs() == ["poisoned"]++ # Now an ordinary paused job: both copies agree, so resume revives it.+ assert store._get_jobs()[0].next_run_time is None+ scheduler.resume_job("poisoned")+ assert _next_run_time(store, "poisoned") is not None+ assert [j.id for j in store.get_due_jobs(_utc(2**31 - 1))] == ["poisoned"]+++def test_reconcile_leaves_a_user_paused_job_alone():+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="paused")+ scheduler.pause_job("paused")++ assert store.reconcile_repaired_jobs() == []+ assert _next_run_time(store, "paused") is None+++def test_reconcile_leaves_a_still_broken_job_parked():+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ _poison(store, "poisoned")+ store._get_jobs()++ assert store.reconcile_repaired_jobs() == []+ assert store.get_parked_job_ids() == ["poisoned"]+++def test_reconcile_walks_past_its_batch_size():+ """Batching bounds the transaction, not the work: every stranded row must+ still be reached, which a plain cap would not do."""+ with _store() as (store, scheduler):+ for i in range(7):+ scheduler.add_job(noop, "interval", seconds=3600, id=f"j{i}")+ healthy = _job_state(store, f"j{i}")+ _poison(store, f"j{i}")+ store._get_jobs()+ _set_job_state(store, f"j{i}", healthy)++ healed = store.reconcile_repaired_jobs(batch_size=2)++ assert sorted(healed) == [f"j{i}" for i in range(7)]+ assert all(_next_run_time(store, f"j{i}") is None for i in range(7))+ assert store.reconcile_repaired_jobs(batch_size=2) == []+++def test_a_row_repaired_under_the_scan_is_not_parked():+ """Parking must not undo a repair that landed after the row was read."""+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ healthy_state = _job_state(store, "poisoned")+ _poison(store, "poisoned")++ def _repair_midway(job_state):+ _set_job_state(store, "poisoned", healthy_state)+ raise ValueError("still poisoned as far as this scan knows")++ with patch.object(store, "_reconstitute_job", side_effect=_repair_midway):+ store._get_jobs()++ # The repair survives: the row is runnable, not re-parked.+ assert _next_run_time(store, "poisoned") is not None+ assert store.get_parked_job_ids() == []+++def test_reconcile_leaves_a_row_that_changed_under_the_scan_alone():+ """A resume landing mid-reconcile must not be reverted to paused."""+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ healthy_state = _job_state(store, "poisoned")+ _poison(store, "poisoned")+ store._get_jobs()+ _set_job_state(store, "poisoned", healthy_state)++ real = store._reconstitute_job++ def _resume_midway(job_state):+ with store.engine.begin() as conn:+ conn.execute(+ store.jobs_t.update()+ .where(store.jobs_t.c.id == "poisoned")+ .values(next_run_time=1.0)+ )+ return real(job_state)++ with patch.object(store, "_reconstitute_job", side_effect=_resume_midway):+ assert store.reconcile_repaired_jobs() == []++ assert _next_run_time(store, "poisoned") == 1.0+++def test_parked_scan_limit_bounds_the_rows_read():+ with _store() as (store, scheduler):+ for i in range(5):+ scheduler.add_job(noop, "interval", seconds=3600, id=f"j{i}")+ _poison(store, f"j{i}")+ store._get_jobs()++ assert len(store.get_parked_job_ids()) == 5+ assert len(store.get_parked_job_ids(limit=2)) == 2++[email protected]("filtered", [False, True], ids=["all", "active_only"])+def test_upstream_jobstore_would_have_deleted_the_row(filtered: bool):+ """Control: pins the upstream behaviour this subclass exists to prevent, on+ both reads — including the active-only one the listing path now issues."""+ from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore++ with _store(cls=SQLAlchemyJobStore) as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ _poison(store, "poisoned")++ conditions = (store.jobs_t.c.next_run_time.isnot(None),) if filtered else ()+ store._get_jobs(*conditions)++ assert _ids(store) == set()+++def test_parked_job_is_never_returned_as_due():+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=1, id="poisoned")+ _poison(store, "poisoned")+ store._get_jobs()++ far_future = _utc(2**31 - 1)+ assert store.get_due_jobs(far_future) == []+ assert _ids(store) == {"poisoned"}+++def test_get_parked_job_ids_ignores_a_healthy_paused_job():+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="paused")+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ scheduler.pause_job("paused")+ _poison(store, "poisoned")+ store._get_jobs()++ # A paused job also has next_run_time NULL, so only the restore+ # attempt separates it from a parked one.+ assert store.get_parked_job_ids() == ["poisoned"]+++def test_repeated_loads_report_each_bad_job_once():+ with _store() as (store, scheduler):+ scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")+ _poison(store, "poisoned")++ store._get_jobs()+ store._get_jobs()++ assert store._parked_ids == {"poisoned"}+++def test_strip_enums_recurses_through_containers():+ class Provider(str, Enum):+ GITHUB = "github"++ stripped = _strip_enums(+ {"creds": {"a": {"provider": Provider.GITHUB}}, "list": [Provider.GITHUB]}+ )++ assert stripped == {"creds": {"a": {"provider": "github"}}, "list": ["github"]}+ assert not isinstance(stripped["list"][0], Enum)+++def test_strip_enums_leaves_plain_values_alone():+ payload = {"user_id": "u1", "graph_version": 3, "cron": "0 * * * *"}+ assert _strip_enums(payload) == payload+++@contextmanager+def _store(cls=ResilientSQLAlchemyJobStore):+ fd, path = tempfile.mkstemp(suffix=".sqlite")+ os.close(fd)+ store = cls(url=f"sqlite:///{path}")+ scheduler = BackgroundScheduler()+ scheduler.add_jobstore(store, "default")+ scheduler.start(paused=True)+ try:+ yield store, scheduler+ finally:+ scheduler.shutdown(wait=False)+ os.unlink(path)+++@contextmanager+def _scheduler_wired_to(store):+ """Run ``Scheduler.run_service`` far enough to wire *store* in as the+ EXECUTION jobstore, with every other dependency stubbed out."""+ spare = MagicMock(get_parked_job_ids=MagicMock(return_value=[]))+ with (+ patch(f"{_SCHEDULER_PATH}.BackgroundScheduler", return_value=MagicMock()),+ patch(f"{_SCHEDULER_PATH}.load_dotenv"),+ patch(f"{_SCHEDULER_PATH}._init_launchdarkly_for_scheduler"),+ patch(f"{_SCHEDULER_PATH}.asyncio.new_event_loop", return_value=MagicMock()),+ patch(f"{_SCHEDULER_PATH}.threading.Thread", return_value=MagicMock()),+ patch(f"{_SCHEDULER_PATH}.create_engine", return_value=MagicMock()),+ patch(+ f"{_SCHEDULER_PATH}.ResilientSQLAlchemyJobStore",+ side_effect=[store, spare],+ ),+ patch(f"{_SCHEDULER_PATH}.MemoryJobStore", return_value=MagicMock()),+ patch(+ f"{_SCHEDULER_PATH}._extract_schema_from_url",+ return_value=("public", "sqlite://"),+ ),+ patch(f"{_SCHEDULER_PATH}.ensure_embeddings_coverage"),+ patch("backend.util.service.AppService.run_service"),+ ):+ sched = Scheduler(register_system_tasks=False)+ sched.run_service()+ sched._invalidate_jobs_cache()+ yield sched+++def _job_state(store, job_id: str):+ with store.engine.begin() as conn:+ return conn.execute(+ select(store.jobs_t.c.job_state).where(store.jobs_t.c.id == job_id)+ ).scalar()+++def _set_job_state(store, job_id: str, job_state) -> None:+ with store.engine.begin() as conn:+ conn.execute(+ store.jobs_t.update()+ .where(store.jobs_t.c.id == job_id)+ .values(job_state=job_state)+ )+++def _poison(store, job_id: str) -> None:+ with store.engine.begin() as conn:+ conn.execute(+ store.jobs_t.update()+ .where(store.jobs_t.c.id == job_id)+ .values(job_state=POISONED_STATE)+ )+++def _ids(store) -> set[str]:+ with store.engine.begin() as conn:+ return {r.id for r in conn.execute(select(store.jobs_t.c.id))}+++def _next_run_time(store, job_id: str):+ with store.engine.begin() as conn:+ return conn.execute(+ select(store.jobs_t.c.next_run_time).where(store.jobs_t.c.id == job_id)+ ).scalar()+++def _utc(timestamp: float):+ from datetime import datetime, timezone++ return datetime.fromtimestamp(timestamp, tz=timezone.utc)diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex 9b013d0c15..337c73c3a6 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -19,7 +19,6 @@ from apscheduler.events import ()from apscheduler.job import Job as JobObjfrom apscheduler.jobstores.memory import MemoryJobStore-from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStorefrom apscheduler.schedulers.background import BackgroundSchedulerfrom apscheduler.triggers.cron import CronTriggerfrom apscheduler.triggers.date import DateTrigger@@ -44,6 +43,7 @@ from backend.data.execution import ExecutionTrigger, GraphExecutionWithNodesfrom backend.data.model import CredentialsMetaInput, GraphInputfrom backend.executor import schedule_eventsfrom backend.executor import utils as execution_utils+from backend.executor.jobstore import ResilientSQLAlchemyJobStorefrom backend.monitoring import (flush_matured_alerts,report_block_error_rates,@@ -1467,6 +1467,11 @@ def ensure_embeddings_coverage():# Monitoring functions are now imported from monitoring module+# Paused and fired-once rows are never deleted, so every scan over them is+# bounded rather than left to grow with the backlog.+_PARKED_SCAN_LIMIT = 1000++class Jobstores(Enum):EXECUTION = "execution"BATCHED_NOTIFICATIONS = "batched_notifications"@@ -1746,6 +1751,7 @@ def _job_to_info(class Scheduler(AppService):scheduler: BackgroundScheduler+ _persistent_jobstores: dict[str, ResilientSQLAlchemyJobStore] = {}def __init__(self, register_system_tasks: bool = True):self.register_system_tasks = register_system_tasks@@ -1796,20 +1802,35 @@ class Scheduler(AppService):# Configure executors to limit concurrency without skipping jobsfrom apscheduler.executors.pool import ThreadPoolExecutor- # Kept as a named reference (rather than only living inside the- # ``jobstores=`` dict below) so ``_get_active_jobs_cached`` can query- # its table directly with a server-side filter — see that method for- # why the stock ``get_all_jobs()`` isn't enough.- self._execution_jobstore = SQLAlchemyJobStore(- engine=create_engine(- url=db_url,- pool_size=self.db_pool_size(),- max_overflow=0,+ self._persistent_jobstores = {+ Jobstores.EXECUTION.value: ResilientSQLAlchemyJobStore(+ engine=create_engine(+ url=db_url,+ pool_size=self.db_pool_size(),+ max_overflow=0,+ ),+ metadata=MetaData(schema=db_schema),+ # this one is pre-existing so it keeps the+ # default table name.+ tablename="apscheduler_jobs",),- metadata=MetaData(schema=db_schema),- # this one is pre-existing so it keeps the default table name.- tablename="apscheduler_jobs",- )+ Jobstores.BATCHED_NOTIFICATIONS.value: ResilientSQLAlchemyJobStore(+ engine=create_engine(+ url=db_url,+ pool_size=self.db_pool_size(),+ max_overflow=0,+ ),+ metadata=MetaData(schema=db_schema),+ tablename="apscheduler_jobs_batched_notifications",+ ),+ }+ # Named reference so ``_get_active_jobs_cached`` can query the table+ # directly with a server-side filter — see that method for why the+ # stock ``get_all_jobs()`` isn't enough. It must stay the same+ # instance as the dict entry above: the filtered read goes through+ # ``ResilientSQLAlchemyJobStore._get_jobs``, which parks an+ # unrestorable row rather than deleting it.+ self._execution_jobstore = self._persistent_jobstores[Jobstores.EXECUTION.value]self.scheduler = BackgroundScheduler(executors={@@ -1823,16 +1844,7 @@ class Scheduler(AppService):"misfire_grace_time": None, # No time limit for missed jobs},jobstores={- Jobstores.EXECUTION.value: self._execution_jobstore,- Jobstores.BATCHED_NOTIFICATIONS.value: SQLAlchemyJobStore(- engine=create_engine(- url=db_url,- pool_size=self.db_pool_size(),- max_overflow=0,- ),- metadata=MetaData(schema=db_schema),- tablename="apscheduler_jobs_batched_notifications",- ),+ **self._persistent_jobstores,# These don't really need persistenceJobstores.WEEKLY_NOTIFICATIONS.value: MemoryJobStore(),},@@ -1991,10 +2003,33 @@ class Scheduler(AppService):self.scheduler.add_listener(job_missed_listener, EVENT_JOB_MISSED)self.scheduler.add_listener(job_max_instances_listener, EVENT_JOB_MAX_INSTANCES)self.scheduler.start()+ self._report_parked_jobs()# Keep the service running since BackgroundScheduler doesn't blocksuper().run_service()+ # Paused and fired-once rows are never deleted, so this scan is bounded+ # rather than left to grow with the backlog — startup precedes the RPC port.+ def _report_parked_jobs(self) -> None:+ """Parking is recoverable but silent — startup has to say it happened."""+ for alias, store in self._persistent_jobstores.items():+ try:+ parked = store.get_parked_job_ids(limit=_PARKED_SCAN_LIMIT)+ except Exception as e:+ logger.error(f"Could not check jobstore '{alias}' for parked jobs: {e}")+ continue+ if len(parked) == _PARKED_SCAN_LIMIT:+ logger.error(+ f"at least {len(parked)} job(s) in jobstore '{alias}' are PARKED "+ "and will not run until repaired; the startup scan stopped at "+ f"its cap: {parked}"+ )+ elif parked:+ logger.error(+ f"{len(parked)} job(s) in jobstore '{alias}' are PARKED and will "+ f"not run until repaired: {parked}"+ )+def cleanup(self):if self.scheduler:logger.info("⏳ Shutting down scheduler...")@@ -2465,6 +2500,33 @@ class Scheduler(AppService):def execute_send_due_briefings(self):send_due_briefings()+ @expose+ def get_parked_jobs(+ self, limit: int | None = _PARKED_SCAN_LIMIT+ ) -> dict[str, list[str]]:+ """Job ids the scheduler could not restore, per jobstore.++ Bounded by default: paused and fired-once rows accumulate forever and+ each one read here is deserialized. Pass ``limit=None`` for the whole+ set, accepting a scan proportional to that backlog.+ """+ return {+ alias: store.get_parked_job_ids(limit=limit)+ for alias, store in self._persistent_jobstores.items()+ }++ @expose+ def reconcile_parked_jobs(self) -> dict[str, list[str]]:+ """Make repaired rows resumable again, per jobstore.++ For a repair made without ``jobstore_backfill``, which already does+ this for the rows it rewrites.+ """+ return {+ alias: store.reconcile_repaired_jobs()+ for alias, store in self._persistent_jobstores.items()+ }+@exposedef execute_report_late_executions(self):return report_late_executions()@@ -2879,6 +2941,9 @@ class SchedulerClient(AppServiceClient):# Polymorphic list — preferred for new callers; returns both kinds.get_execution_schedules = endpoint_to_async(Scheduler.get_execution_schedules)+ get_parked_jobs = endpoint_to_async(Scheduler.get_parked_jobs)+ reconcile_parked_jobs = endpoint_to_async(Scheduler.reconcile_parked_jobs)+add_community_rebuild_schedule = endpoint_to_async(Scheduler.add_community_rebuild_schedule)diff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.pyindex 1afcb46990..99b1b5a7e9 100644--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py@@ -1521,7 +1521,10 @@ def _registered_jobs(monkeypatch, interval_hours: int) -> _StartupRun:patch(f"{_SCHEDULER_PATH}.asyncio.new_event_loop", return_value=MagicMock()),patch(f"{_SCHEDULER_PATH}.threading.Thread", return_value=MagicMock()),patch(f"{_SCHEDULER_PATH}.create_engine", return_value=MagicMock()),- patch(f"{_SCHEDULER_PATH}.SQLAlchemyJobStore", return_value=MagicMock()),+ patch(+ f"{_SCHEDULER_PATH}.ResilientSQLAlchemyJobStore",+ return_value=MagicMock(get_parked_job_ids=MagicMock(return_value=[])),+ ),patch(f"{_SCHEDULER_PATH}.MemoryJobStore", return_value=MagicMock()),patch(f"{_SCHEDULER_PATH}._extract_schema_from_url",