Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/executor): bind the scheduler's RPC port before the embedding backfill (#14451)

Reinier van der Leer93f64d30380e
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index d071b64475..fd757d45a8 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -1889,8 +1889,13 @@ class Scheduler(AppService):
id="ensure_embeddings_coverage",
trigger="interval",
hours=6,
+ # Due now rather than called inline below: run_service() is what
+ # starts the event loop uvicorn binds the RPC port on.
+ next_run_time=datetime.now(timezone.utc),
replace_existing=True,
max_instances=1, # Prevent overlapping runs
+ misfire_grace_time=None,
+ coalesce=True,
jobstore=Jobstores.EXECUTION.value,
)
@@ -1912,18 +1917,6 @@ class Scheduler(AppService):
self.scheduler.add_listener(job_max_instances_listener, EVENT_JOB_MAX_INSTANCES)
self.scheduler.start()
- # Run embedding backfill immediately on startup
- # This ensures blocks/docs are searchable right away, not after 6 hours
- # Safe to run on multiple pods - uses upserts and checks for existing embeddings
- if self.register_system_tasks:
- logger.info("Running embedding backfill on startup...")
- try:
- result = ensure_embeddings_coverage()
- logger.info(f"Startup embedding backfill complete: {result}")
- except Exception as e:
- logger.error(f"Startup embedding backfill failed: {e}")
- # Don't fail startup - the scheduled job will retry later
-
# Keep the service running since BackgroundScheduler doesn't block
super().run_service()
diff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
index afaa0d5c7b..30e6b1d623 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
@@ -8,6 +8,7 @@ backend test job (and counted by codecov), not just the integration suite.
import asyncio
import logging
from datetime import datetime, timedelta, timezone
+from typing import NamedTuple
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -1486,14 +1487,23 @@ def test_add_graph_schedule_keeps_autopilot_tenancy():
# ---------------------------------------------------------------------------
-def _registered_jobs(monkeypatch, interval_hours: int) -> list:
+class _StartupRun(NamedTuple):
+ add_job_calls: list
+ embedding_backfill: MagicMock
+ backfill_calls_at_readiness: int
+
+
+def _registered_jobs(monkeypatch, interval_hours: int) -> _StartupRun:
"""Drive ``Scheduler.run_service`` with every heavy dependency stubbed and
- a mock APScheduler, returning the list of ``add_job`` mock calls."""
+ a mock APScheduler, recording what it registered and what it ran before
+ handing over to ``AppService.run_service``."""
monkeypatch.setattr(
f"{_SCHEDULER_PATH}.config.stripe_tier_reconcile_interval_hours",
interval_hours,
)
mock_scheduler = MagicMock()
+ embedding_backfill = MagicMock(return_value=None)
+ at_readiness = []
with (
patch(f"{_SCHEDULER_PATH}.BackgroundScheduler", return_value=mock_scheduler),
patch(f"{_SCHEDULER_PATH}.load_dotenv"),
@@ -1506,12 +1516,17 @@ def _registered_jobs(monkeypatch, interval_hours: int) -> list:
f"{_SCHEDULER_PATH}._extract_schema_from_url",
return_value=("public", "sqlite://"),
),
- patch(f"{_SCHEDULER_PATH}.ensure_embeddings_coverage", return_value=None),
+ patch(f"{_SCHEDULER_PATH}.ensure_embeddings_coverage", embedding_backfill),
# super().run_service() blocks forever keeping the service alive; no-op it.
- patch("backend.util.service.AppService.run_service", return_value=None),
+ patch(
+ "backend.util.service.AppService.run_service",
+ side_effect=lambda: at_readiness.append(embedding_backfill.call_count),
+ ),
):
Scheduler(register_system_tasks=True).run_service()
- return mock_scheduler.add_job.call_args_list
+ return _StartupRun(
+ mock_scheduler.add_job.call_args_list, embedding_backfill, at_readiness[0]
+ )
def test_reconcile_stripe_tiers_job_registered_with_interval_and_single_instance(
@@ -1519,7 +1534,7 @@ def test_reconcile_stripe_tiers_job_registered_with_interval_and_single_instance
):
"""The sweep must be registered as an interval job keyed off the configured
interval setting and capped to a single concurrent instance."""
- calls = _registered_jobs(monkeypatch, interval_hours=6)
+ calls = _registered_jobs(monkeypatch, interval_hours=6).add_job_calls
matches = [c for c in calls if c.args and c.args[0] is reconcile_stripe_tiers]
assert len(matches) == 1
@@ -1533,11 +1548,40 @@ def test_reconcile_stripe_tiers_job_registered_with_interval_and_single_instance
def test_reconcile_stripe_tiers_interval_follows_config_setting(monkeypatch):
"""Changing the configured interval changes the registered ``seconds``."""
- calls = _registered_jobs(monkeypatch, interval_hours=12)
+ calls = _registered_jobs(monkeypatch, interval_hours=12).add_job_calls
match = next(c for c in calls if c.args and c.args[0] is reconcile_stripe_tiers)
assert match.kwargs["seconds"] == 12 * 3600
+def test_embedding_backfill_does_not_delay_rpc_readiness(monkeypatch):
+ """``AppService.run_service`` starts the event loop uvicorn is scheduled
+ onto, so anything run before it keeps the RPC port closed. The backfill
+ takes minutes on a fresh stack; it must not sit on that path."""
+ run = _registered_jobs(monkeypatch, interval_hours=6)
+
+ assert run.backfill_calls_at_readiness == 0
+
+
+def test_embedding_backfill_is_registered_to_run_immediately(monkeypatch):
+ """Dropping the startup call must not delay coverage: the six-hourly job
+ is due now, and cannot be skipped as a misfire however late it starts."""
+ before = datetime.now(timezone.utc)
+ run = _registered_jobs(monkeypatch, interval_hours=6)
+
+ jobs = [
+ c for c in run.add_job_calls if c.kwargs["id"] == "ensure_embeddings_coverage"
+ ]
+ assert len(jobs) == 1
+ job = jobs[0]
+ assert job.args[0] is run.embedding_backfill
+ assert before <= job.kwargs["next_run_time"] <= datetime.now(timezone.utc)
+ assert job.kwargs["trigger"] == "interval"
+ assert job.kwargs["hours"] == 6
+ assert job.kwargs["max_instances"] == 1
+ assert job.kwargs["misfire_grace_time"] is None
+ assert job.kwargs["coalesce"] is True
+
+
class TestScheduleOrgVisibility:
"""Org/team visibility filtering in get_execution_schedules."""