Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): pass the liveness probe before loading the block registry (#15928)

Reinier van der Leer964bc78e2656
diff --git a/autogpt_platform/backend/backend/copilot/executor/manager.py b/autogpt_platform/backend/backend/copilot/executor/manager.py
index 0065a9dab5..447935c962 100644
--- a/autogpt_platform/backend/backend/copilot/executor/manager.py
+++ b/autogpt_platform/backend/backend/copilot/executor/manager.py
@@ -117,6 +117,18 @@ class CoPilotExecutor(AppProcess):
logger.info(f"Pod assigned executor_id: {self.executor_id}")
logger.info(f"Spawn max-{self.pool_size} workers...")
+ # First: the liveness probe connects here and the loads below take seconds.
+ # Work comes only through RabbitMQ, so an early open routes nothing to us.
+ # Deliberate reuse of pyro_host: despite the legacy name it is the
+ # bind address for every service's internal listener (see
+ # backend.util.service). Metrics follow the same interface as the RPC
+ # server — 0.0.0.0 under docker-compose (PYRO_HOST is set there for
+ # cross-container scraping), loopback in the single-container runtime.
+ start_http_server(
+ settings.config.copilot_executor_port,
+ addr=settings.config.pyro_host,
+ )
+
_load_catalog()
# Load the block registry before taking turns. A turn's first tool call
# loads it otherwise, and the collections over the grown heap stall every turn.
@@ -136,15 +148,6 @@ class CoPilotExecutor(AppProcess):
pool_size_gauge.set(self.pool_size)
self._update_metrics()
- # Deliberate reuse of pyro_host: despite the legacy name it is the
- # bind address for every service's internal listener (see
- # backend.util.service). Metrics follow the same interface as the RPC
- # server — 0.0.0.0 under docker-compose (PYRO_HOST is set there for
- # cross-container scraping), loopback in the single-container runtime.
- start_http_server(
- settings.config.copilot_executor_port,
- addr=settings.config.pyro_host,
- )
self.cancel_thread.start()
self.run_thread.start()
diff --git a/autogpt_platform/backend/backend/copilot/executor/manager_block_warmup_test.py b/autogpt_platform/backend/backend/copilot/executor/manager_block_warmup_test.py
index 20e663b857..631f1a8b7b 100644
--- a/autogpt_platform/backend/backend/copilot/executor/manager_block_warmup_test.py
+++ b/autogpt_platform/backend/backend/copilot/executor/manager_block_warmup_test.py
@@ -1,4 +1,5 @@
-"""The copilot executor must load the block registry before it takes any turn."""
+"""The copilot executor must open its probe port, then load the block registry
+before it takes any turn."""
from unittest.mock import MagicMock, patch
@@ -11,9 +12,10 @@ class _StopRunLoop(Exception):
pass
-def test_run_loads_blocks_and_freezes_the_heap_before_taking_turns() -> None:
- """Otherwise a turn's first tool call loads every block, and the full
- collections over the grown heap stall every turn on the pod for seconds."""
+def test_run_opens_its_port_before_loading_blocks_and_taking_turns() -> None:
+ """Without the load a turn's first tool call loads every block, and the full
+ collections over the grown heap stall every turn on the pod for seconds. The
+ liveness probe connects to the port, so it must not wait for the loads."""
executor = CoPilotExecutor()
order: list[str] = []
executor._cancel_thread = MagicMock()
@@ -26,17 +28,30 @@ def test_run_loads_blocks_and_freezes_the_heap_before_taking_turns() -> None:
return {}
with (
- patch("backend.copilot.executor.manager._load_catalog"),
+ patch(
+ "backend.copilot.executor.manager._load_catalog",
+ side_effect=lambda: order.append("load_catalog"),
+ ),
patch("backend.copilot.executor.manager.get_blocks", side_effect=load_blocks),
patch(
"backend.copilot.executor.manager.freeze_startup_heap",
side_effect=lambda: order.append("freeze_heap"),
),
- patch("backend.copilot.executor.manager.start_http_server"),
+ patch(
+ "backend.copilot.executor.manager.start_http_server",
+ side_effect=lambda *_, **__: order.append("open_port"),
+ ),
patch.object(CoPilotExecutor, "_update_metrics"),
patch("backend.copilot.executor.manager.time.sleep", side_effect=_StopRunLoop),
pytest.raises(_StopRunLoop),
):
executor.run()
- assert order == ["load_blocks", "freeze_heap", "cancel_thread", "run_thread"]
+ assert order == [
+ "open_port",
+ "load_catalog",
+ "load_blocks",
+ "freeze_heap",
+ "cancel_thread",
+ "run_thread",
+ ]
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index 1ecf90d0bb..2346ad22b0 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -140,6 +140,7 @@ from backend.util.exceptions import (
UserPaywalledError,
)
from backend.util.logging import PrefixFilter
+from backend.util.process import freeze_startup_heap
from backend.util.retry import func_retry
from backend.util.service import (
AppService,
@@ -2859,6 +2860,7 @@ class Scheduler(AppService):
_schedule_index_ready: bool = False
_schedule_index_failure_version: int = 0
_throwaway_dir: tempfile.TemporaryDirectory | None = None
+ _block_load_error: Exception | None = None
def __init__(
self, register_system_tasks: bool = True, throwaway_jobstores: bool = False
@@ -2888,21 +2890,16 @@ class Scheduler(AppService):
# Normal operation - check if scheduler is running
if not self.scheduler.running:
raise UnhealthyServiceError("Scheduler is not running")
+ if self._block_load_error is not None:
+ raise UnhealthyServiceError(
+ f"Loading the block registry failed: {self._block_load_error}"
+ )
return await super().health_check()
def run_service(self):
load_dotenv()
- # Load the block registry before serving. Otherwise the first add-schedule
- # loads it inside graph validation, and the scheduler freezes meanwhile.
- load_started = time.monotonic()
- block_count = len(get_blocks())
- logger.info(
- f"[{self.service_name}] Loaded {block_count} blocks in "
- f"{time.monotonic() - load_started:.1f}s"
- )
-
# Initialize the event loop for async jobs
global _event_loop
_event_loop = asyncio.new_event_loop()
@@ -3194,10 +3191,40 @@ class Scheduler(AppService):
self._register_memory_cron_reconcile()
self._report_parked_jobs()
self._start_schedule_index_backfill()
+ threading.Thread(
+ target=self._load_blocks_once_serving,
+ daemon=True,
+ name="SchedulerBlockLoad",
+ ).start()
# Keep the service running since BackgroundScheduler doesn't block
super().run_service()
+ def _load_blocks_once_serving(self) -> None:
+ """Load the block registry once the RPC server is up, then freeze it.
+
+ The first add-schedule loads it inside graph validation otherwise. Not in
+ ``run_service``: the server starts only after it, on the loop it hands over to.
+ """
+ while not (self.http_server and self.http_server.started):
+ if not self.scheduler.running:
+ return
+ time.sleep(0.1)
+ load_started = time.monotonic()
+ try:
+ block_count = len(get_blocks())
+ except Exception as e:
+ # Unhealthy, so the pod restarts as it did when this ran before serving.
+ self._block_load_error = e
+ logger.exception(f"[{self.service_name}] Loading the block registry failed")
+ return
+ logger.info(
+ f"[{self.service_name}] Loaded {block_count} blocks in "
+ f"{time.monotonic() - load_started:.1f}s"
+ )
+ # The serving lifespan froze the heap before the blocks existed.
+ freeze_startup_heap()
+
def _jobstore_database(self) -> tuple[str | None, str]:
"""Schema and URL of the jobstores and schedule index: the platform
database, or a SQLite file of this process's own, removed at cleanup."""
diff --git a/autogpt_platform/backend/backend/executor/scheduler_block_warmup_test.py b/autogpt_platform/backend/backend/executor/scheduler_block_warmup_test.py
new file mode 100644
index 0000000000..ff27db9ca2
--- /dev/null
+++ b/autogpt_platform/backend/backend/executor/scheduler_block_warmup_test.py
@@ -0,0 +1,92 @@
+"""The scheduler must answer its liveness probe before it loads the block registry."""
+
+import asyncio
+import threading
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from backend.executor import scheduler as scheduler_module
+from backend.executor.scheduler import Scheduler
+from backend.util.service import UnhealthyServiceError
+
+
+def test_blocks_load_once_the_rpc_server_serves_then_the_heap_freezes(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The RPC server starts only after run_service hands over to its event loop,
+ so a load inside run_service held the liveness probe for the 10-17 s it takes
+ on a pod, and Kubernetes killed the scheduler before it ever answered."""
+ order: list[str] = []
+ frozen = threading.Event()
+
+ def load_blocks() -> dict:
+ serving = "serving" in order
+ order.append("load_blocks" if serving else "load_blocks_before_serving")
+ return {}
+
+ def freeze_heap() -> None:
+ order.append("freeze_heap")
+ frozen.set()
+
+ with _run_service(monkeypatch, order, load_blocks, freeze_heap):
+ frozen.wait(5)
+
+ assert order == ["serving", "load_blocks", "freeze_heap"]
+
+
+async def test_a_failed_block_load_makes_the_scheduler_unhealthy(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """So the pod restarts, as it did when the load ran before serving, instead of
+ staying healthy while every schedule call retries the failing load."""
+
+ def load_blocks() -> dict:
+ raise ImportError("a block failed to import")
+
+ with _run_service(monkeypatch, [], load_blocks, MagicMock()) as service:
+ for _ in range(100):
+ try:
+ await service.health_check()
+ except UnhealthyServiceError:
+ return
+ await asyncio.sleep(0.05)
+
+ pytest.fail("the scheduler stayed healthy after its block load failed")
+
+
+@contextmanager
+def _run_service(
+ monkeypatch: pytest.MonkeyPatch,
+ order: list[str],
+ load_blocks: Callable[[], dict],
+ freeze_heap: Callable[[], None],
+) -> Iterator[Scheduler]:
+ """``run_service`` with throwaway jobstores, its RPC server marked serving at
+ the hand-over to the event loop, then the service's own cleanup."""
+ for name in ("_event_loop", "_event_loop_thread", "_scheduler_instance"):
+ monkeypatch.setattr(scheduler_module, name, getattr(scheduler_module, name))
+ service = Scheduler(register_system_tasks=False, throwaway_jobstores=True)
+ server = MagicMock(started=False)
+ service.http_server = server
+
+ def serve() -> None:
+ order.append("serving")
+ server.started = True
+
+ with (
+ patch.object(scheduler_module, "load_dotenv"),
+ patch.object(scheduler_module, "get_blocks", side_effect=load_blocks),
+ patch.object(scheduler_module, "freeze_startup_heap", side_effect=freeze_heap),
+ patch("backend.util.service.AppService.run_service", side_effect=serve),
+ patch("backend.util.service.AppService.cleanup"),
+ ):
+ service.run_service()
+ try:
+ yield service
+ finally:
+ service.cleanup()
+ if scheduler_module._event_loop is not None:
+ scheduler_module._event_loop.close()