Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

dx(backend/executor): keep the test server's scheduler jobs out of the platform database (#15702)

Reinier van der Leerc46883f2a7a6
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index f4aacb61ef..9ad7a1dc33 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -4,6 +4,7 @@ import hashlib
import logging
import os
import re
+import tempfile
import threading
import time
import uuid
@@ -2684,9 +2685,15 @@ class Scheduler(AppService):
_schedule_index: ScheduleIndex | None = None
_schedule_index_ready: bool = False
_schedule_index_failure_version: int = 0
+ _throwaway_dir: tempfile.TemporaryDirectory | None = None
- def __init__(self, register_system_tasks: bool = True):
+ def __init__(
+ self, register_system_tasks: bool = True, throwaway_jobstores: bool = False
+ ):
self.register_system_tasks = register_system_tasks
+ # Test servers: jobs a real scheduler saved in the platform database
+ # must not run in them, and their tests' schedules must not land there.
+ self.throwaway_jobstores = throwaway_jobstores
@classmethod
def get_port(cls) -> int:
@@ -2725,7 +2732,7 @@ class Scheduler(AppService):
)
_event_loop_thread.start()
- db_schema, db_url = _extract_schema_from_url(os.getenv("DIRECT_URL"))
+ db_schema, db_url = self._jobstore_database()
# Configure executors to limit concurrency without skipping jobs
from apscheduler.executors.pool import ThreadPoolExecutor
@@ -2994,6 +3001,16 @@ class Scheduler(AppService):
# Keep the service running since BackgroundScheduler doesn't block
super().run_service()
+ def _jobstore_database(self) -> tuple[str | None, str]:
+ """Schema and URL of the jobstores and schedule index: the platform
+ database, or a SQLite file of this process's own, removed at cleanup."""
+ if not self.throwaway_jobstores:
+ return _extract_schema_from_url(os.getenv("DIRECT_URL"))
+ self._throwaway_dir = tempfile.TemporaryDirectory(
+ prefix="scheduler-jobstores-", ignore_cleanup_errors=True
+ )
+ return None, f"sqlite:///{self._throwaway_dir.name}/jobs.sqlite"
+
def _register_idle_credential_sweep(self) -> None:
"""Keep an existing daily job's overdue run or explicit pause intact."""
job_id = "refresh_near_expiry_credentials"
@@ -3053,6 +3070,11 @@ class Scheduler(AppService):
logger.info("⏳ Waiting for event loop thread to finish...")
_event_loop_thread.join(timeout=SCHEDULER_OPERATION_TIMEOUT_SECONDS)
+ # Explicitly: a forked service process ends in os._exit, which skips the
+ # directory's own finalizer.
+ if self._throwaway_dir is not None:
+ self._throwaway_dir.cleanup()
+
super().cleanup()
def _persist_schedule(
diff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.py
index 4f7b3a094d..7eec794e36 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py
@@ -75,7 +75,7 @@ async def test_paused_schedule_excluded_unless_include_paused(server: SpinTestSe
(Scheduler._get_active_jobs_cached), not just via the old in-Python
check — and ``include_paused=True`` must still find it through the
unfiltered ``_get_jobs_cached`` path used by pause/resume lifecycle
- callers. Exercised against the real apscheduler_jobs table."""
+ callers. Exercised against a real SQL jobstore table."""
await db.connect()
test_user = await create_test_user()
test_graph = await server.agent_server.test_create_graph(
diff --git a/autogpt_platform/backend/backend/executor/scheduler_throwaway_jobstores_test.py b/autogpt_platform/backend/backend/executor/scheduler_throwaway_jobstores_test.py
new file mode 100644
index 0000000000..6760d78336
--- /dev/null
+++ b/autogpt_platform/backend/backend/executor/scheduler_throwaway_jobstores_test.py
@@ -0,0 +1,146 @@
+"""The test server's scheduler never runs or rewrites the jobs a real
+scheduler saved in the platform database, and saves nothing there itself."""
+
+import os
+import threading
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from unittest.mock import patch
+
+import pytest
+from apscheduler.job import Job
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.interval import IntervalTrigger
+from sqlalchemy import Engine, MetaData, create_engine, text
+
+from backend.executor import scheduler as scheduler_module
+from backend.executor.jobstore import ResilientSQLAlchemyJobStore
+from backend.executor.scheduler import Jobstores, Scheduler, _extract_schema_from_url
+from backend.util.test import SpinTestServer
+
+_EXECUTION = Jobstores.EXECUTION.value
+_RUNS: list[str] = []
+_MARKER_RAN = threading.Event()
+
+
+def record_run(name: str) -> None:
+ _RUNS.append(name)
+ if name == "marker":
+ _MARKER_RAN.set()
+
+
+def _postgres_url() -> str | None:
+ direct = os.getenv("DIRECT_URL") or ""
+ return direct if direct.startswith("postgresql") else None
+
+
[email protected](_postgres_url() is None, reason="needs the PostgreSQL test DB")
+def test_the_test_servers_scheduler_leaves_saved_jobs_alone(
+ platform_db: tuple[Engine, str], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ engine, schema = platform_db
+ _save_overdue_job(engine, schema)
+ saved = _rows(engine, schema)
+ scheduler = SpinTestServer().scheduler
+
+ with _serving(scheduler, monkeypatch):
+ # Recurring, so a marker the platform database got keeps its row there.
+ scheduler.scheduler.add_job(
+ record_run,
+ "interval",
+ hours=6,
+ args=["marker"],
+ id="marker",
+ next_run_time=datetime.now(timezone.utc),
+ jobstore=_EXECUTION,
+ )
+ # Due before the marker, so a loaded saved job is submitted first and
+ # cleanup's shutdown(wait=True) lets it finish.
+ assert _MARKER_RAN.wait(10)
+ database = scheduler._execution_jobstore.engine.url.database
+
+ assert _RUNS == ["marker"]
+ assert _rows(engine, schema) == saved
+ assert database and not os.path.exists(database)
+
+
+def platform_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[Engine, str]]:
+ """A schema of its own in the test PostgreSQL, standing in for a platform
+ database a real scheduler has used; DIRECT_URL points the scheduler at it."""
+ _RUNS.clear()
+ _MARKER_RAN.clear()
+ url = _extract_schema_from_url(_postgres_url())[1]
+ schema = f"saved_jobs_{uuid.uuid4().hex[:12]}"
+ engine = create_engine(url)
+ with engine.begin() as conn:
+ conn.execute(text(f'CREATE SCHEMA "{schema}"'))
+ monkeypatch.setenv(
+ "DIRECT_URL", f"{url}{'&' if '?' in url else '?'}schema={schema}"
+ )
+ try:
+ yield engine, schema
+ finally:
+ with engine.begin() as conn:
+ conn.execute(text(f'DROP SCHEMA "{schema}" CASCADE'))
+ engine.dispose()
+
+
+def _save_overdue_job(engine: Engine, schema: str) -> None:
+ """An hour-overdue interval job with the scheduler's job defaults, written
+ to the store directly: shutting down a paused scheduler processes due jobs."""
+ store = ResilientSQLAlchemyJobStore(
+ engine=engine, metadata=MetaData(schema=schema), tablename="apscheduler_jobs"
+ )
+ seeder = BackgroundScheduler(timezone=timezone.utc)
+ store.start(seeder, _EXECUTION)
+ store.add_job(
+ Job(
+ seeder,
+ id="saved",
+ func=record_run,
+ args=("saved",),
+ kwargs={},
+ trigger=IntervalTrigger(hours=6, timezone=timezone.utc),
+ next_run_time=datetime.now(timezone.utc) - timedelta(hours=1),
+ executor="default",
+ coalesce=True,
+ misfire_grace_time=None,
+ max_instances=1000,
+ )
+ )
+
+
+def _rows(engine: Engine, schema: str) -> list[tuple]:
+ with engine.connect() as conn:
+ return [
+ (row.id, row.next_run_time, bytes(row.job_state))
+ for row in conn.execute(
+ text(
+ "SELECT id, next_run_time, job_state "
+ f'FROM "{schema}".apscheduler_jobs ORDER BY id'
+ )
+ )
+ ]
+
+
+@contextmanager
+def _serving(service: Scheduler, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+ """``run_service`` up to where the process would serve RPCs, then the
+ service's own cleanup; the module globals it sets are restored after."""
+ for name in ("_event_loop", "_event_loop_thread", "_scheduler_instance"):
+ monkeypatch.setattr(scheduler_module, name, getattr(scheduler_module, name))
+ with (
+ patch.object(scheduler_module, "load_dotenv"),
+ patch("backend.util.service.AppService.run_service"),
+ patch("backend.util.service.AppService.cleanup"),
+ ):
+ service.run_service()
+ try:
+ yield
+ finally:
+ service.cleanup()
+ if scheduler_module._event_loop is not None:
+ scheduler_module._event_loop.close()
diff --git a/autogpt_platform/backend/backend/util/test.py b/autogpt_platform/backend/backend/util/test.py
index 6a26a80efc..765a203861 100644
--- a/autogpt_platform/backend/backend/util/test.py
+++ b/autogpt_platform/backend/backend/util/test.py
@@ -31,7 +31,9 @@ class SpinTestServer:
self.db_api = DatabaseManager()
self.exec_manager = ExecutionManager()
self.agent_server = AgentServer()
- self.scheduler = Scheduler(register_system_tasks=False)
+ self.scheduler = Scheduler(
+ register_system_tasks=False, throwaway_jobstores=True
+ )
self.notif_manager = NotificationManager()
@staticmethod