Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend/copilot): durable per-scope schedule registry for dream and community jobs with lifecycle hooks and backfill (#14960)

Nicholas Tindle98dd8e182f09
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.py
index f6a4074a44..80efae1bf3 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py
@@ -1866,7 +1866,7 @@ async def stream_chat_post(
# Fire-and-forget; per-user Redis dedup inside the helper provides
# cross-process / cross-restart idempotency. Same pattern as
- # graphiti/ingest.py's ensure_dream_system_scheduled registration.
+ # graphiti/ingest.py's ensure_scope_scheduled registration.
from backend.copilot.briefing.scheduling import ensure_morning_briefing_scheduled
# Spawned through the shared helper: the loop only holds a weak
diff --git a/autogpt_platform/backend/backend/api/features/experts/experts_db.py b/autogpt_platform/backend/backend/api/features/experts/experts_db.py
index de0d5916ed..9367e15a60 100644
--- a/autogpt_platform/backend/backend/api/features/experts/experts_db.py
+++ b/autogpt_platform/backend/backend/api/features/experts/experts_db.py
@@ -96,6 +96,7 @@ from backend.api.features.store.categories import category_match_values
from backend.blocks import get_output_block_ids
from backend.copilot.autopilot_mode import AUTOPILOT_MODES, AutopilotMode
from backend.copilot.briefing.outcome import DEFAULT_AGENT_NAME, run_link
+from backend.copilot.dream.registry import ensure_expert_scheduled, sync_expert_scope
from backend.copilot.tools.skills import (
BuiltInSkillError,
SkillNotFoundError,
@@ -1024,7 +1025,10 @@ async def _hire_expert_impl(
expert, state = await _reserve_hired_expert(user_id, template_id, create_data)
if state == "revived":
+ # Resuming the revived expert's schedules resumes its memory crons.
expert = await _resume_revived_hire(expert)
+ elif state == "created":
+ _schedule_expert_memory(user_id, expert.id)
# A hire that runs setup is counted by its job, once its preloads are
# known; an idempotent re-hire of an already-active expert is not a hire.
if state == "created" or await _claim_setup(expert.id):
@@ -1045,6 +1049,20 @@ async def _hire_expert_impl(
return HireResult(expert=_to_model(expert))
+def _schedule_expert_memory(user_id: str, expert_id: str) -> None:
+ """Register a new expert's memory crons without holding up its hire.
+
+ Safe to race an archive or a pause: registration never resumes a paused
+ scope, whenever it lands. Crons registered for a scope paused meanwhile
+ are held back by the fire gate and removed by the reconcile pass.
+ Only revive and the resume route resume.
+ """
+ spawn_background_task(
+ ensure_expert_scheduled(user_id, expert_id),
+ name=f"expert-memory-schedule-{expert_id}",
+ )
+
+
def _emit_expert_hired(
user_id: str,
expert: prisma.models.Expert,
@@ -1371,6 +1389,7 @@ async def create_raised_expert(
weekly_budget=weekly_budget,
skills=resolved.skill_names,
)
+ _schedule_expert_memory(user_id, expert.id)
failed_skill_installs = await raise_attachments.install_marketplace_skills(
user_id, expert.id, resolved.skills
)
@@ -2379,8 +2398,9 @@ async def archive_expert(user_id: str, expert_id: str) -> None:
# Pause BEFORE flipping isArchived: pause_expert_schedules refuses
# archived rows, and pausing first still records the pause event + stamp
# for the archive. A nonexistent/foreign expert makes the pause a no-op
- # and the archive update below raises the 404.
- await scheduling.pause_expert_schedules(
+ # and the archive update below raises the 404. The pause also pauses the
+ # expert's memory crons.
+ paused = await scheduling.pause_expert_schedules(
user_id, expert_id, reason="Expert archived"
)
updated = await prisma.models.Expert.prisma().update_many(
@@ -2406,6 +2426,10 @@ async def archive_expert(user_id: str, expert_id: str) -> None:
raise ExpertNotFoundError(expert_id)
# Re-archiving is an idempotent no-op; the funnel counts each firing once.
return
+ if not paused:
+ # Already paused before the archive (a budget breach), so the pause
+ # above did nothing: make sure the memory crons stop too.
+ await sync_expert_scope(user_id, expert_id, active=False)
emit_funnel_event(user_id, PostHogEvent.EXPERT_FIRED, {"expert_id": expert_id})
try:
await scheduling.detach_expert_triggers(user_id, expert_id)
diff --git a/autogpt_platform/backend/backend/api/features/experts/memory_schedule_hooks_test.py b/autogpt_platform/backend/backend/api/features/experts/memory_schedule_hooks_test.py
new file mode 100644
index 0000000000..4a111698fb
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/experts/memory_schedule_hooks_test.py
@@ -0,0 +1,270 @@
+"""The expert lifecycle keeps the expert's memory crons in step with it: a new
+hire or raise registers them in the background (so the hire returns at once)
+and never resumes a scope paused meanwhile, archiving pauses them (also for an
+expert whose schedules were already paused), and none of the hooks can hold an
+API request past the registry deadline. Pause and resume themselves are
+pinned in ``scheduling_test.py``; the registry in ``copilot/dream/``."""
+
+import asyncio
+import time
+from collections.abc import Callable, Coroutine, Iterator
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import prisma.models
+import pytest
+
+from backend.api.features.experts import experts_db, scheduling
+from backend.copilot.dream import registry
+from backend.copilot.dream.registry_fakes_test import (
+ EXPERT,
+ PAUSED,
+ USER,
+ WIPED,
+ Hang,
+ install_registry_env,
+ use_short_deadline,
+)
+
+TEMPLATE = SimpleNamespace(
+ id="template-1",
+ isTemplate=True,
+ name="Maria",
+ avatarUrl=None,
+ color="",
+ role="Marketing Specialist",
+ jobTitle=None,
+ tagline=None,
+ bio=None,
+ skills=[],
+ categories=[],
+ identity="You are Maria.",
+ voicePreferences=None,
+ boundaries=None,
+ toolProfile=None,
+ Workflows=[],
+)
+
+
+def sync(mocker) -> AsyncMock:
+ return mocker.patch.object(experts_db, "sync_expert_scope", new=AsyncMock())
+
+
+def register(mocker) -> AsyncMock:
+ return mocker.patch.object(experts_db, "ensure_expert_scheduled", new=AsyncMock())
+
+
+def spawned(mocker) -> Iterator[dict[str, Any]]:
+ """The background tasks the code under test spawned, by name, kept for
+ the test to await; whatever it leaves is closed afterwards."""
+ tasks: dict[str, Any] = {}
+
+ def spawn(coro: Any, *, name: str) -> None:
+ tasks[name] = coro
+
+ mocker.patch.object(experts_db, "spawn_background_task", side_effect=spawn)
+ yield tasks
+ for coro in tasks.values():
+ coro.close()
+
+
+def _hire(mocker, state: str) -> None:
+ expert_client = SimpleNamespace(find_first=AsyncMock(return_value=TEMPLATE))
+ mocker.patch.object(prisma.models.Expert, "prisma", return_value=expert_client)
+ mocker.patch.object(
+ experts_db,
+ "_reserve_hired_expert",
+ new=AsyncMock(return_value=(SimpleNamespace(id="expert-1"), state)),
+ )
+ mocker.patch.object(
+ experts_db,
+ "_resume_revived_hire",
+ new=AsyncMock(return_value=SimpleNamespace(id="expert-1")),
+ )
+ mocker.patch.object(experts_db, "_claim_setup", new=AsyncMock(return_value=False))
+ mocker.patch.object(experts_db, "_run_hire_setup", new=MagicMock())
+ mocker.patch.object(experts_db, "emit_funnel_event")
+ mocker.patch.object(experts_db, "_emit_expert_hired")
+ mocker.patch.object(experts_db, "_to_model")
+ mocker.patch.object(experts_db, "HireResult")
+
+
+async def test_a_new_hire_registers_its_memory_scope_in_the_background(
+ mocker, sync, register, spawned
+):
+ _hire(mocker, "created")
+
+ await experts_db._hire_expert_impl("owner-1", "template-1", None, None)
+
+ register.assert_not_awaited() # the hire did not wait on it
+ await spawned["expert-memory-schedule-expert-1"]
+ register.assert_awaited_once_with("owner-1", "expert-1")
+ sync.assert_not_awaited() # registering, not resuming
+
+
[email protected]("state", ["revived", "existing"])
+async def test_revive_and_idempotent_rehire_leave_it_to_their_own_paths(
+ mocker, sync, register, spawned, state: str
+):
+ """A revive resumes the memory crons through ``resume_expert_schedules``;
+ re-hiring an active expert changes nothing."""
+ _hire(mocker, state)
+
+ await experts_db._hire_expert_impl("owner-1", "template-1", None, None)
+
+ sync.assert_not_awaited()
+ register.assert_not_awaited()
+ assert "expert-memory-schedule-expert-1" not in spawned
+
+
[email protected]("stop", ["pause", "wipe"])
+async def test_a_hire_registration_landing_after_a_pause_leaves_it_stopped(
+ monkeypatch, stop: str
+):
+ """Codex's reproduction: whatever the hire spawns for the memory crons is
+ held back until the scope was paused (or wiped), then run. It must leave
+ the scope as it found it, with no crons, rather than resume it."""
+ env = install_registry_env(monkeypatch)
+ held: list[Coroutine[Any, Any, Any]] = []
+
+ def hold(coro: Coroutine[Any, Any, Any], *, name: str) -> None:
+ held.append(coro)
+
+ monkeypatch.setattr(experts_db, "spawn_background_task", hold)
+ experts_db._schedule_expert_memory(USER, "expert-1")
+ stop_scope = registry.pause_scope if stop == "pause" else registry.mark_wiped
+ assert await stop_scope(EXPERT)
+
+ await asyncio.wait_for(held[0], 1) # the hire's registration lands late
+
+ assert env.db.rows[EXPERT.scope_key].state == (PAUSED if stop == "pause" else WIPED)
+ assert env.scheduler.jobs == {}
+ assert env.adds() == 0
+
+
+async def test_a_raised_expert_registers_its_memory_scope(mocker, register, spawned):
+ resolved = SimpleNamespace(
+ skill_names=[], skills=[], library_skill_names=[], workflows=[]
+ )
+ attachments = experts_db.raise_attachments
+ mocker.patch.object(
+ attachments, "resolve_attachments", new=AsyncMock(return_value=resolved)
+ )
+ mocker.patch.object(
+ attachments, "install_marketplace_skills", new=AsyncMock(return_value=[])
+ )
+ mocker.patch.object(
+ attachments, "install_workflows", new=AsyncMock(return_value=[])
+ )
+ mocker.patch.object(
+ experts_db,
+ "_create_raised_expert_row",
+ new=AsyncMock(return_value=SimpleNamespace(id="expert-1", skills=[])),
+ )
+ mocker.patch.object(
+ experts_db, "_copy_library_skills", new=AsyncMock(return_value=[])
+ )
+ mocker.patch.object(experts_db, "_to_model")
+ mocker.patch.object(experts_db, "RaiseResult")
+
+ await experts_db.create_raised_expert("owner-1", "Nova", None, None)
+
+ await spawned["expert-memory-schedule-expert-1"]
+ register.assert_awaited_once_with("owner-1", "expert-1")
+
+
+def _archive(mocker, *, paused: bool, archived_rows: int) -> None:
+ mocker.patch.object(
+ scheduling, "pause_expert_schedules", new=AsyncMock(return_value=paused)
+ )
+ expert_client = SimpleNamespace(
+ update_many=AsyncMock(return_value=archived_rows),
+ find_first=AsyncMock(return_value=SimpleNamespace(id="expert-1")),
+ )
+ mocker.patch.object(prisma.models.Expert, "prisma", return_value=expert_client)
+ mocker.patch.object(scheduling, "detach_expert_triggers", new=AsyncMock())
+ mocker.patch.object(experts_db, "emit_funnel_event")
+
+
+async def test_archiving_an_already_paused_expert_still_pauses_its_memory(mocker, sync):
+ """A budget breach paused the schedules earlier, so the archive's own
+ pause is a no-op — the memory crons must stop anyway."""
+ _archive(mocker, paused=False, archived_rows=1)
+
+ await experts_db.archive_expert("owner-1", "expert-1")
+
+ sync.assert_awaited_once_with("owner-1", "expert-1", active=False)
+
+
+async def test_archive_leaves_the_memory_pause_to_the_schedule_pause(mocker, sync):
+ """``pause_expert_schedules`` pauses the memory crons itself when it
+ pauses anything; the archive does not pause them twice."""
+ _archive(mocker, paused=True, archived_rows=1)
+
+ await experts_db.archive_expert("owner-1", "expert-1")
+
+ sync.assert_not_awaited()
+
+
+async def test_re_archiving_changes_nothing(mocker, sync):
+ _archive(mocker, paused=False, archived_rows=0)
+
+ await experts_db.archive_expert("owner-1", "expert-1")
+
+ sync.assert_not_awaited()
+
+
[email protected]("hook", ["pause", "resume", "archive"])
+async def test_a_lifecycle_request_is_not_held_up_by_a_hanging_scheduler(
+ mocker, monkeypatch, hook: str
+):
+ """The API routes behind pause, resume and archive (and revive, which
+ resumes) finish their own writes and return within the registry
+ deadline; the scheduler calls they gave up on are cancelled."""
+ env = install_registry_env(monkeypatch)
+ use_short_deadline(monkeypatch)
+ hang = Hang()
+ for rpc in (
+ "remove_scope_memory_jobs",
+ "add_scope_community_rebuild_schedule",
+ "add_scope_nightly_batch_schedule",
+ ):
+ monkeypatch.setattr(env.scheduler, rpc, hang.forever)
+ expert_table = SimpleNamespace(
+ update_many=AsyncMock(return_value=1),
+ find_first=AsyncMock(return_value=SimpleNamespace(id="expert-1")),
+ )
+ mocker.patch.object(prisma.models.Expert, "prisma", return_value=expert_table)
+ mocker.patch.object(
+ prisma.models.ExpertPauseEvent,
+ "prisma",
+ return_value=SimpleNamespace(create=AsyncMock(), update_many=AsyncMock()),
+ )
+ mocker.patch.object(scheduling, "reset_weekly_spend", new=AsyncMock())
+ mocker.patch.object(scheduling, "detach_expert_triggers", new=AsyncMock())
+ mocker.patch.object(experts_db, "emit_funnel_event")
+ hooks: dict[str, Callable[[], Coroutine[Any, Any, Any]]] = {
+ "pause": lambda: scheduling.pause_expert_schedules(USER, "expert-1", "test"),
+ "resume": lambda: scheduling.resume_expert_schedules(USER, "expert-1"),
+ "archive": lambda: experts_db.archive_expert(USER, "expert-1"),
+ }
+ started = time.monotonic()
+
+ await hooks[hook]()
+
+ assert time.monotonic() - started < 1.0
+ expert_table.update_many.assert_awaited() # the lifecycle write landed
+ assert hang.started >= 1
+ assert hang.cancelled == hang.started
diff --git a/autogpt_platform/backend/backend/api/features/experts/scheduling.py b/autogpt_platform/backend/backend/api/features/experts/scheduling.py
index cfd7bf7bf2..ec69f98ae0 100644
--- a/autogpt_platform/backend/backend/api/features/experts/scheduling.py
+++ b/autogpt_platform/backend/backend/api/features/experts/scheduling.py
@@ -19,6 +19,7 @@ from backend.api.features.experts import routine_jobs
from backend.api.features.experts.errors import ExpertScheduleCleanupError
from backend.api.features.experts.models import ExpertDetachPreview
from backend.copilot import db as chat_db
+from backend.copilot.dream.registry import sync_expert_scope
from backend.data.expert_spend import get_weekly_spend, reset_weekly_spend
from backend.data.model import CredentialsMetaInput
from backend.data.user import get_user_by_id
@@ -434,6 +435,8 @@ async def reattach_expert_triggers(user_id: str, expert_id: str) -> None:
async def pause_expert_schedules(user_id: str, expert_id: str, reason: str) -> bool:
"""Pause the expert's scheduled/triggered runs (chat is untouched) and
log the pause. Returns False when already paused (no double events).
+ Its memory crons (nightly dream, weekly community rebuild) pause with
+ them; that part is bounded by the registry deadline and fails soft.
Refuses archived experts so a pause can't silently mutate a row the rest
of the API reports as not-found; the archive flow itself pauses BEFORE
@@ -454,6 +457,7 @@ async def pause_expert_schedules(user_id: str, expert_id: str, reason: str) -> b
await prisma.models.ExpertPauseEvent.prisma().create(
data={"expertId": expert_id, "reason": reason}
)
+ await sync_expert_scope(user_id, expert_id, active=False)
return True
@@ -470,7 +474,11 @@ async def resume_expert_schedules(user_id: str, expert_id: str) -> bool:
Refuses archived experts: resuming one would un-pause schedules for an
expert every other surface 404s on (the route would then report 404
anyway, AFTER the mutation already landed). Revival goes through the
- re-hire flow, which clears ``isArchived`` before resuming."""
+ re-hire flow, which clears ``isArchived`` before resuming.
+
+ The expert's memory crons resume with the schedules (registered if the
+ expert never had any); that part is bounded by the registry deadline
+ and fails soft."""
updated = await prisma.models.Expert.prisma().update_many(
where={
"id": expert_id,
@@ -488,6 +496,7 @@ async def resume_expert_schedules(user_id: str, expert_id: str) -> bool:
where={"expertId": expert_id, "clearedAt": None},
data={"clearedAt": datetime.now(timezone.utc)},
)
+ await sync_expert_scope(user_id, expert_id, active=True)
return True
diff --git a/autogpt_platform/backend/backend/api/features/experts/scheduling_test.py b/autogpt_platform/backend/backend/api/features/experts/scheduling_test.py
index c1ae1f7c58..5567c6f1e0 100644
--- a/autogpt_platform/backend/backend/api/features/experts/scheduling_test.py
+++ b/autogpt_platform/backend/backend/api/features/experts/scheduling_test.py
@@ -127,6 +127,53 @@ async def test_resume_only_mutates_private_expert(mocker) -> None:
reset_spend.assert_not_awaited()
+async def test_pausing_an_expert_pauses_its_memory_crons(mocker) -> None:
+ expert_client = mocker.MagicMock()
+ expert_client.update_many = AsyncMock(return_value=1)
+ pause_event_client = mocker.MagicMock()
+ pause_event_client.create = AsyncMock()
+ mocker.patch.object(prisma.models.Expert, "prisma", return_value=expert_client)
+ mocker.patch.object(
+ prisma.models.ExpertPauseEvent, "prisma", return_value=pause_event_client
+ )
+ sync = mocker.patch.object(scheduling, "sync_expert_scope", new=AsyncMock())
+
+ assert await scheduling.pause_expert_schedules("owner", "expert-1", "budget")
+
+ sync.assert_awaited_once_with("owner", "expert-1", active=False)
+
+
+async def test_a_refused_pause_leaves_the_memory_crons_alone(mocker) -> None:
+ expert_client = mocker.MagicMock()
+ expert_client.update_many = AsyncMock(return_value=0)
+ mocker.patch.object(prisma.models.Expert, "prisma", return_value=expert_client)
+ sync = mocker.patch.object(scheduling, "sync_expert_scope", new=AsyncMock())
+
+ assert not await scheduling.pause_expert_schedules("owner", "expert-1", "x")
+
+ sync.assert_not_awaited()
+
+
+async def test_resuming_an_expert_resumes_its_memory_crons(mocker) -> None:
+ expert_client = mocker.MagicMock()
+ expert_client.update_many = AsyncMock(return_value=1)
+ pause_event_client = mocker.MagicMock()
+ pause_event_client.update_many = AsyncMock()
+ mocker.patch.object(prisma.models.Expert, "prisma", return_value=expert_client)
+ mocker.patch.object(
+ prisma.models.ExpertPauseEvent, "prisma", return_value=pause_event_client
+ )
+ mocker.patch.object(scheduling, "reset_weekly_spend", new=AsyncMock())
+ sync = mocker.patch.object(scheduling, "sync_expert_scope", new=AsyncMock())
+
+ assert await scheduling.resume_expert_schedules("owner", "expert-1")
+
+ sync.assert_awaited_once_with("owner", "expert-1", active=True)
+
+
@pytest.mark.asyncio
async def test_budget_gate_fails_closed_for_non_private_expert(mocker) -> None:
expert_client = mocker.MagicMock()
diff --git a/autogpt_platform/backend/backend/copilot/dream/deadline.py b/autogpt_platform/backend/backend/copilot/dream/deadline.py
new file mode 100644
index 0000000000..c4c80a5e6f
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/deadline.py
@@ -0,0 +1,47 @@
+"""How long the schedule registry waits on any one call to a dependency.
+
+The registry's calls are best-effort, but its callers are not: a hire, an
+archive, a resume route or a cron gate must not hang on them. The
+DatabaseManager client retries for minutes and the scheduler client waits up
+to 300 s per request, so every registry call runs under this one deadline.
+On timeout the call is cancelled and ``TimeoutError`` raised, which each
+caller's fail-soft handler treats like any other failure. A timeout does not
+prove the call had no effect: a request the server already received may
+still land. Anything awaited inside a deadline that bounds itself must do so
+with ``asyncio.timeout`` too (see :func:`within_deadline`).
+"""
+
+import sys
+
+if sys.version_info >= (3, 11):
+ from asyncio import timeout
+else:
+ from async_timeout import timeout
+from typing import Awaitable, TypeVar
+
+REGISTRY_CALL_TIMEOUT_SECONDS = 10.0
+
+# The scheduler's sync cron bodies reach the registry through ``run_async``;
+# its own bound sits just past the registry's so the registry fails soft
+# first and the bridge is only a backstop.
+BRIDGE_MARGIN_SECONDS = 5.0
+
+T = TypeVar("T")
+
+
+async def within_deadline(call: Awaitable[T]) -> T:
+ """``call`` under the registry deadline; cancelled on timeout.
+
+ ``asyncio.timeout`` rather than ``asyncio.wait_for``: these deadlines
+ nest (an API hook bounds a whole change whose calls are bounded too),
+ and on Python 3.11 an outer deadline that expires just as an inner
+ ``wait_for`` completes has its cancellation swallowed, so the change
+ would run on past it.
+ """
+ async with timeout(REGISTRY_CALL_TIMEOUT_SECONDS):
+ return await call
+
+
+def bridge_timeout() -> float:
+ """The ``run_async`` bound for a registry call made from a cron body."""
+ return REGISTRY_CALL_TIMEOUT_SECONDS + BRIDGE_MARGIN_SECONDS
diff --git a/autogpt_platform/backend/backend/copilot/dream/deadline_test.py b/autogpt_platform/backend/backend/copilot/dream/deadline_test.py
new file mode 100644
index 0000000000..e902ed6d66
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/deadline_test.py
@@ -0,0 +1,51 @@
+import asyncio
+
+import pytest
+
+from . import deadline
+
+
+async def test_deadline_cancels_work(monkeypatch):
+ monkeypatch.setattr(deadline, "REGISTRY_CALL_TIMEOUT_SECONDS", 0.01)
+ cancelled = asyncio.Event()
+
+ async def work():
+ try:
+ await asyncio.sleep(10)
+ finally:
+ cancelled.set()
+
+ with pytest.raises((TimeoutError, asyncio.TimeoutError)):
+ await deadline.within_deadline(work())
+ assert cancelled.is_set()
+
+
+async def test_nested_deadline_preserves_outer_cancellation(monkeypatch):
+ monkeypatch.setattr(deadline, "REGISTRY_CALL_TIMEOUT_SECONDS", 0.01)
+ finished = False
+
+ async def nested():
+ nonlocal finished
+ monkeypatch.setattr(deadline, "REGISTRY_CALL_TIMEOUT_SECONDS", 10)
+ await deadline.within_deadline(asyncio.sleep(10))
+ finished = True
+
+ with pytest.raises((TimeoutError, asyncio.TimeoutError)):
+ await deadline.within_deadline(nested())
+ assert not finished
+
+
+async def test_deadline_preserves_external_cancellation():
+ task = asyncio.create_task(deadline.within_deadline(asyncio.sleep(10)))
+ await asyncio.sleep(0)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+
+async def test_deadline_returns_result():
+ assert await deadline.within_deadline(asyncio.sleep(0, result=42)) == 42
diff --git a/autogpt_platform/backend/backend/copilot/dream/nightly_batch.py b/autogpt_platform/backend/backend/copilot/dream/nightly_batch.py
index fa80c231dd..14883f357d 100644
--- a/autogpt_platform/backend/backend/copilot/dream/nightly_batch.py
+++ b/autogpt_platform/backend/backend/copilot/dream/nightly_batch.py
@@ -1,7 +1,8 @@
-"""Nightly batch-submit cron body — fans out per-user batch-family work.
+"""Nightly batch-submit cron body — fans out one scope's batch-family work.
-One per-user APScheduler cron (``dream_nightly_batch_{user_id}``)
-fires at user-local 03:00 daily and calls :func:`run_nightly_batch_submit`.
+One APScheduler cron per memory scope (``dream_nightly_batch_{scope_key}``;
+the account's scope key is its user id) fires at owner-local 03:00 daily
+and calls :func:`run_nightly_batch_submit`.
The function sequentially invokes each enabled "batch-family"
submitter:
@@ -65,6 +66,8 @@ class NightlyBatchResult(BaseModel):
"""
user_id: str
+ # The expert whose memory scope this pass ran on; None for the account.
+ expert_id: str | None = None
nightly_id: str = Field(
description=(
"UUID correlating every per-submitter cost-log row for "
@@ -99,8 +102,14 @@ class NightlyBatchResult(BaseModel):
error: str | None = None
-async def run_nightly_batch_submit(user_id: str) -> NightlyBatchResult:
- """Fan out per-user nightly batch-family submissions, in order.
+async def run_nightly_batch_submit(
+ user_id: str, *, expert_id: str | None = None
+) -> NightlyBatchResult:
+ """Fan out one memory scope's nightly batch-family submissions, in order.
+
+ ``expert_id`` picks the expert's scope instead of the account's; the
+ budget check stays on the owner either way, since an expert's dreams
+ are paid from the owner's allowance.
Today the order is dream-pass → ratification-supersession-sweep.
Future additions land as new sequential calls between these two
@@ -132,6 +141,7 @@ async def run_nightly_batch_submit(user_id: str) -> NightlyBatchResult:
if budget_skip == "rate_limit_unavailable":
return NightlyBatchResult(
user_id=user_id,
+ expert_id=expert_id,
nightly_id=nightly_id,
started_at=started_at,
completed_at=completed_at,
@@ -140,6 +150,7 @@ async def run_nightly_batch_submit(user_id: str) -> NightlyBatchResult:
)
return NightlyBatchResult(
user_id=user_id,
+ expert_id=expert_id,
nightly_id=nightly_id,
started_at=started_at,
completed_at=completed_at,
@@ -149,7 +160,10 @@ async def run_nightly_batch_submit(user_id: str) -> NightlyBatchResult:
)
result = NightlyBatchResult(
- user_id=user_id, nightly_id=nightly_id, started_at=started_at
+ user_id=user_id,
+ expert_id=expert_id,
+ nightly_id=nightly_id,
+ started_at=started_at,
)
# Dream pass submitter. Per-submitter failure stays isolated —
@@ -159,7 +173,7 @@ async def run_nightly_batch_submit(user_id: str) -> NightlyBatchResult:
try:
from .orchestrator import execute_dream_pass
- result.dream = await execute_dream_pass(user_id)
+ result.dream = await execute_dream_pass(user_id, expert_id=expert_id)
if result.dream.error:
logger.warning(
"Nightly batch %s: dream submitter errored for user %s: %s",
@@ -187,7 +201,7 @@ async def run_nightly_batch_submit(user_id: str) -> NightlyBatchResult:
# past their grace period — promotions happen inline at
# retrieval-hit time.
try:
- result.ratification = await run_ratification_pass(user_id)
+ result.ratification = await run_ratification_pass(user_id, expert_id=expert_id)
if result.ratification.error:
logger.warning(
"Nightly batch %s: ratification sweep errored for user %s: %s",
@@ -206,8 +220,12 @@ async def run_nightly_batch_submit(user_id: str) -> NightlyBatchResult:
prev = result.error or ""
result.error = (prev + " | " if prev else "") + f"ratification: {exc}"
+ # Skill learning runs once per account: its pass reviews the account's
+ # skills and its experts', so an expert's nightly batch leaves it out.
try:
- if await is_feature_enabled(Flag.DREAM_SKILL_LEARNING_ENABLED, user_id):
+ if expert_id is None and await is_feature_enabled(
+ Flag.DREAM_SKILL_LEARNING_ENABLED, user_id
+ ):
result.learning = await run_skill_learning_pass(user_id, trigger="nightly")
except Exception as exc:
logger.exception("Nightly skill learning failed for user %s", user_id[:12])
diff --git a/autogpt_platform/backend/backend/copilot/dream/nightly_batch_test.py b/autogpt_platform/backend/backend/copilot/dream/nightly_batch_test.py
index 624f2052ef..249570e8bd 100644
--- a/autogpt_platform/backend/backend/copilot/dream/nightly_batch_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/nightly_batch_test.py
@@ -308,3 +308,103 @@ async def test_skipped_or_errored_batch_path_dream_is_not_in_flight():
result = await run_nightly_batch_submit("u")
assert result.dream_in_flight is False
+
+
+async def test_expert_scope_reaches_both_submitters_and_the_owner_pays():
+ """An expert's nightly cron runs the dream and the sweep on the expert's
+ graph, while the budget check stays on the owner, whose allowance pays
+ for it."""
+ budget = AsyncMock(return_value=(True, None))
+ dream_spy = AsyncMock(return_value=_dream_result())
+ rat_spy = AsyncMock(return_value=_ratification_result())
+ with patch.object(nightly_batch, "check_dream_budget", new=budget), patch(
+ "backend.copilot.dream.orchestrator.execute_dream_pass", new=dream_spy
+ ), patch.object(nightly_batch, "run_ratification_pass", new=rat_spy):
+ result = await run_nightly_batch_submit("u", expert_id="expert-1")
+
+ budget.assert_awaited_once_with("u")
+ dream_spy.assert_awaited_once_with("u", expert_id="expert-1")
+ rat_spy.assert_awaited_once_with("u", expert_id="expert-1")
+ assert result.expert_id == "expert-1"
+
+
+async def test_account_scope_runs_the_account_graph():
+ dream_spy = AsyncMock(return_value=_dream_result())
+ rat_spy = AsyncMock(return_value=_ratification_result())
+ with patch.object(
+ nightly_batch,
+ "check_dream_budget",
+ new=AsyncMock(return_value=(True, None)),
+ ), patch(
+ "backend.copilot.dream.orchestrator.execute_dream_pass", new=dream_spy
+ ), patch.object(
+ nightly_batch, "run_ratification_pass", new=rat_spy
+ ):
+ result = await run_nightly_batch_submit("u")
+
+ dream_spy.assert_awaited_once_with("u", expert_id=None)
+ rat_spy.assert_awaited_once_with("u", expert_id=None)
+ assert result.expert_id is None
+
+
+ "expert_id, runs_learning", [(None, True), ("expert-1", False)]
+)
+async def test_skill_learning_runs_once_per_account_never_per_expert(
+ expert_id: str | None, runs_learning: bool
+):
+ """The learning pass reviews the account's skills and its experts', so
+ with its flag on the account's nightly batch runs it and an expert's
+ leaves it out."""
+ learning = AsyncMock()
+ with patch.object(
+ nightly_batch,
+ "check_dream_budget",
+ new=AsyncMock(return_value=(True, None)),
+ ), patch(
+ "backend.copilot.dream.orchestrator.execute_dream_pass",
+ new=AsyncMock(return_value=_dream_result()),
+ ), patch.object(
+ nightly_batch,
+ "run_ratification_pass",
+ new=AsyncMock(return_value=_ratification_result()),
+ ), patch.object(
+ nightly_batch, "is_feature_enabled", new=AsyncMock(return_value=True)
+ ), patch.object(
+ nightly_batch, "run_skill_learning_pass", new=learning
+ ):
+ await run_nightly_batch_submit("u", expert_id=expert_id)
+
+ if runs_learning:
+ learning.assert_awaited_once_with("u", trigger="nightly")
+ else:
+ learning.assert_not_awaited()
+
+
+async def test_skill_learning_crash_is_reported_not_raised():
+ with patch.object(
+ nightly_batch,
+ "check_dream_budget",
+ new=AsyncMock(return_value=(True, None)),
+ ), patch(
+ "backend.copilot.dream.orchestrator.execute_dream_pass",
+ new=AsyncMock(return_value=_dream_result()),
+ ), patch.object(
+ nightly_batch,
+ "run_ratification_pass",
+ new=AsyncMock(return_value=_ratification_result()),
+ ), patch.object(
+ nightly_batch, "is_feature_enabled", new=AsyncMock(return_value=True)
+ ), patch.object(
+ nightly_batch,
+ "run_skill_learning_pass",
+ new=AsyncMock(side_effect=RuntimeError("boom")),
+ ):
+ result = await run_nightly_batch_submit("u")
+
+ assert result.learning is None
+ assert result.error == "learning: boom"
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry.py b/autogpt_platform/backend/backend/copilot/dream/registry.py
new file mode 100644
index 0000000000..fe55c77f5d
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry.py
@@ -0,0 +1,310 @@
+"""Durable per-scope registry of the dream-system crons.
+
+One ``MemoryScopeSchedule`` row per memory scope (the account, or one hired
+expert) is the source of truth for "is this scope scheduled": its state
+(ACTIVE, PAUSED, WIPED), the timezone its crons were registered in and their
+APScheduler job ids. The crons themselves are the rows of
+``scheduling.DREAM_SYSTEM_JOBS``, keyed by scope: the account's scope key is
+its user id, so its job ids are the historical per-user ones. The work
+against the scheduler and the row lives in ``scope_jobs.py``.
+
+Who calls what:
+
+ * :func:`ensure_scope_scheduled` — ingest, when a memory group's queue is
+ created (``graphiti/ingest.py``), the backfill (``registry_backfill.py``)
+ and the functions below.
+ * :func:`ensure_expert_scheduled` — a new hire or raise, in the background.
+ * :func:`sync_expert_scope` — archive, pause, resume and revive.
+ * :func:`reregister_user` — the owner's timezone changed.
+ * :func:`mark_wiped` — a seam for the scope wipe (workstream 3); nothing
+ calls it yet, and it does not erase the graph itself.
+
+State only changes through pause, resume and wipe. Ensure never overrides a
+PAUSED or WIPED scope, resume only reactivates a PAUSED one, and pause
+leaves a WIPED one WIPED (each of those writes checks the state it moves
+from in the same statement), so neither a stray memory write nor a late
+hire can bring back the crons of an archived expert, and no registry
+operation takes a scope out of WIPED. Every dependency call runs under the
+registry deadline (``deadline.py``), and everything here fails soft (logged,
+never raised), because none of its callers may break or hang over
+scheduling.
+
+These paths act on events, so two of them can interleave and leave a scope's
+crons out of step with its row: a takedown that lands after a later resume,
+or a registration that lands after a timezone change. The scheduler's
+reconcile pass (``scope_reconcile.py``) puts that right when its walk next
+reaches the scope: every cron an ACTIVE row records that the scheduler lacks,
+or runs in another timezone, is registered again in the row's timezone. What
+a row records is decided here, on the owner's flags.
+"""
+
+import logging
+from typing import Any
+
+from prisma.enums import MemoryScopeScheduleState
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.db_accessors import memory_schedule_db
+from backend.data.memory_schedule import MemoryScopeSchedule
+
+from .deadline import within_deadline
+from .scheduling import resolve_user_timezone
+from .scope_jobs import (
+ flag_gate,
+ register_scope_jobs,
+ remove_scope_jobs,
+ scope_label,
+ skip_all,
+ state_reason,
+)
+
+logger = logging.getLogger(__name__)
+
+_WIPED = MemoryScopeScheduleState.WIPED
+# The states a pause moves from: never WIPED, which is stopped for good.
+_PAUSABLE = (MemoryScopeScheduleState.ACTIVE, MemoryScopeScheduleState.PAUSED)
+
+
+async def ensure_scope_scheduled(
+ scope: MemoryScope, *, force_refresh: bool = False
+) -> dict[str, Any]:
+ """Idempotently register every flag-enabled dream-system cron of ``scope``.
+
+ Decides from the scope's registry row, never from Redis: a PAUSED or
+ WIPED scope is left alone, and in an ACTIVE one a cron is (re-)registered
+ only when it has no recorded job or the owner's timezone moved.
+ ``force_refresh`` re-registers every enabled cron regardless.
+
+ Returns one entry per cron, keyed by ``job_id_prefix``: ``None`` when it
+ was already registered (no RPC made), ``{"skipped": True, "reason": ...}``
+ when it was not registered (flag off, ``scope_paused``, ``scope_wiped``,
+ ``scope_changed``, ``timezone_lookup_failed``, ``registry_unavailable``,
+ ``registration_failed``, ``record_failed``), otherwise the scheduler's
+ own result.
+ """
+ results, enabled = await flag_gate(scope)
+ if not enabled:
+ return results
+ try:
+ row = await within_deadline(
+ memory_schedule_db().get_scope_schedule(
+ scope.owner_user_id, scope.scope_key
+ )
+ )
+ except Exception:
+ logger.warning(f"Registry read failed for {scope_label(scope)}", exc_info=True)
+ return results | skip_all(enabled, "registry_unavailable")
+ if row is not None and row.state != MemoryScopeScheduleState.ACTIVE:
+ return results | skip_all(enabled, state_reason(row))
+ user_timezone = await resolve_user_timezone(scope.owner_user_id)
+ if user_timezone is None:
+ # "Unknown" is not "UTC": keep the existing crons until a later call
+ # can read the real timezone.
+ return results | skip_all(enabled, "timezone_lookup_failed")
+ return results | await register_scope_jobs(
+ scope, enabled, row, user_timezone, force_refresh
+ )
+
+
+async def ensure_dream_system_scheduled(
+ user_id: str, *, force_refresh: bool = False
+) -> dict[str, Any]:
+ """The account's crons: :func:`ensure_scope_scheduled` for
+ ``MemoryScope.for_user(user_id)``. ``{}`` for an empty or invalid id."""
+ try:
+ scope = MemoryScope.for_user(user_id)
+ except ValueError:
+ return {}
+ return await ensure_scope_scheduled(scope, force_refresh=force_refresh)
+
+
+async def ensure_expert_scheduled(user_id: str, expert_id: str) -> None:
+ """Register a newly hired or raised expert's crons.
+
+ Runs in the background after the hire returns, so it may land after the
+ expert was already archived or paused: like any ensure, it leaves a
+ PAUSED or WIPED scope alone rather than resuming it.
+ """
+ try:
+ await ensure_scope_scheduled(MemoryScope.for_expert(user_id, expert_id))
+ except Exception:
+ logger.exception(f"Memory schedule registration failed for #{expert_id}")
+
+
+async def sync_expert_scope(user_id: str, expert_id: str, *, active: bool) -> None:
+ """Keep an expert's memory crons in step with an explicit lifecycle change.
+
+ ``active=True`` (schedules resumed, expert revived) resumes a PAUSED
+ scope or registers one with no row; ``active=False`` (archived, schedules
+ paused) pauses it; a WIPED scope stays WIPED either way. Called from API
+ requests and the run-budget gate, so the whole change runs under the
+ registry deadline. A change it cuts off is not retried here: the cron
+ gate still keeps an archived or paused expert's crons from running, and
+ a later lifecycle change or the backfill settles the row. An ordinary
+ registration does not look at the expert's lifecycle, so it will not.
+ """
+ try:
+ scope = MemoryScope.for_expert(user_id, expert_id)
+ change = resume_scope(scope) if active else pause_scope(scope)
+ await within_deadline(change)
+ except TimeoutError:
+ logger.warning(f"Memory schedule sync timed out for expert #{expert_id}")
+ except Exception:
+ # Both already fail soft; this keeps a bug in them from ever
+ # failing the archive, pause or resume that called in.
+ logger.exception(f"Memory schedule sync failed for expert #{expert_id}")
+
+
+async def pause_scope(scope: MemoryScope) -> bool:
+ """Stop the scope's crons until :func:`resume_scope`.
+
+ The row goes PAUSED (created that way if the scope had none, so a later
+ registration cannot activate it) and the jobs are removed. A WIPED row
+ stays WIPED, and its jobs are removed again in case any straggled. True
+ only if both happened. A resume landing between the two gets its crons
+ back from the reconcile pass's next visit to the scope.
+ """
+ return await _leave_active(scope, MemoryScopeScheduleState.PAUSED)
+
+
+async def mark_wiped(scope: MemoryScope) -> bool:
+ """Stop the crons of a scope whose memory was erased. No registry
+ operation brings a wiped scope back (pause keeps it WIPED, resume and
+ ensure skip it); the wipe work decides what does. Erasing the graph is
+ the caller's job."""
+ return await _leave_active(scope, MemoryScopeScheduleState.WIPED)
+
+
+async def resume_scope(
+ scope: MemoryScope, *, force_refresh: bool = False
+) -> dict[str, Any]:
+ """Undo :func:`pause_scope` and register the crons again; a scope with no
+ row is simply registered. A WIPED scope stays WIPED. A resume that cannot
+ write the row reports the crons the owner's flags enable as unavailable,
+ and the others as their flags say, as an ensure would."""
+ try:
+ await within_deadline(
+ memory_schedule_db().set_scope_state(
+ scope.owner_user_id,
+ scope.scope_key,
+ MemoryScopeScheduleState.ACTIVE,
+ only_from=(MemoryScopeScheduleState.PAUSED,),
+ )
+ )
+ except Exception:
+ logger.warning(f"Could not resume {scope_label(scope)}", exc_info=True)
+ results, enabled = await flag_gate(scope)
+ return results | skip_all(enabled, "registry_unavailable")
+ return await ensure_scope_scheduled(scope, force_refresh=force_refresh)
+
+
+async def reregister_user(user_id: str) -> dict[str, dict[str, Any]]:
+ """Re-register every scope of an owner whose timezone changed.
+
+ The account always goes through, row or not: its crons may predate the
+ registry and still run in the old timezone. Each expert with a row
+ follows: paused ones are skipped and take the new timezone when they
+ resume, wiped ones are skipped for good. An owner with every dream flag
+ off costs no database call, as with :func:`ensure_scope_scheduled`.
+ Keyed by scope key.
+ """
+ try:
+ account = MemoryScope.for_user(user_id)
+ except ValueError:
+ return {}
+ skipped, enabled = await flag_gate(account)
+ if not enabled:
+ return {account.scope_key: skipped}
+ scopes = [account, *await _expert_scopes(user_id)]
+ return {
+ scope.scope_key: await ensure_scope_scheduled(scope, force_refresh=True)
+ for scope in scopes
+ }
+
+
+async def _leave_active(scope: MemoryScope, state: MemoryScopeScheduleState) -> bool:
+ """Make the row say ``state`` (a pause leaves a WIPED row WIPED), then
+ remove the crons; True only if both happened. When the state write fails
+ the crons stay: removing them under a row that still says ACTIVE would
+ hide them from the next ensure, and their bodies check the expert's
+ lifecycle themselves. A resume that lands between the write and the
+ removal gets its crons back from the reconcile pass
+ (``scope_reconcile.py``), which restores an ACTIVE row's recorded crons
+ the scheduler lacks."""
+ reached = await _establish_state(scope, state)
+ if reached is None or not await remove_scope_jobs(scope):
+ return False
+ logger.info(f"Dream-system: {scope_label(scope)} is now {reached.value}")
+ return True
+
+
+async def _establish_state(
+ scope: MemoryScope, state: MemoryScopeScheduleState
+) -> MemoryScopeScheduleState | None:
+ """Move the row to ``state``, creating it in that state if it has none;
+ the state the row ends in, or None when that failed. A pause never
+ touches a WIPED row, which is stopped already.
+
+ A registration can claim the row ACTIVE between the first write and the
+ claim, and a claim can fail after the row was created, so unless the
+ claim shows the row already there (or WIPED), the write is made again.
+ """
+ try:
+ if await _move_state(scope, state):
+ return state
+ claimed = await _claim_in_state(scope, state)
+ if claimed is not None and claimed.state in (state, _WIPED):
+ return claimed.state
+ return state if await _move_state(scope, state) else None
+ except Exception:
+ logger.warning(
+ f"Could not mark {scope_label(scope)} {state.value}", exc_info=True
+ )
+ return None
+
+
+async def _move_state(scope: MemoryScope, state: MemoryScopeScheduleState) -> bool:
+ """One conditional write of ``state``: any row can become WIPED, but
+ only an ACTIVE or PAUSED one becomes PAUSED."""
+ only_from = None if state == _WIPED else _PAUSABLE
+ return await within_deadline(
+ memory_schedule_db().set_scope_state(
+ scope.owner_user_id, scope.scope_key, state, only_from=only_from
+ )
+ )
+
+
+async def _claim_in_state(
+ scope: MemoryScope, state: MemoryScopeScheduleState
+) -> MemoryScopeSchedule | None:
+ """Create the scope's row in ``state``; the row as it stands when one
+ already existed, and None when the claim failed."""
+ try:
+ user_timezone = await resolve_user_timezone(scope.owner_user_id)
+ return await within_deadline(
+ memory_schedule_db().claim_scope_schedule(
+ scope.owner_user_id,
+ scope.scope_key,
+ scope.expert_id,
+ user_timezone or "UTC",
+ state,
+ )
+ )
+ except Exception:
+ logger.warning(f"Could not claim {scope_label(scope)}", exc_info=True)
+ return None
+
+
+async def _expert_scopes(user_id: str) -> list[MemoryScope]:
+ try:
+ rows = await within_deadline(
+ memory_schedule_db().list_user_scope_schedules(user_id)
+ )
+ except Exception:
+ logger.warning(f"Could not list scopes of user {user_id[:12]}", exc_info=True)
+ return []
+ return [
+ MemoryScope.for_expert(user_id, row.expert_id)
+ for row in rows
+ if row.expert_id is not None
+ ]
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry_backfill.py b/autogpt_platform/backend/backend/copilot/dream/registry_backfill.py
new file mode 100644
index 0000000000..fe5514e45a
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry_backfill.py
@@ -0,0 +1,282 @@
+"""Backfill the memory-scope schedule registry.
+
+ poetry run memory-schedule-backfill [--dry-run] [--force]
+
+Schedules the dream-system crons of every memory scope that should have them
+and pauses the ones that should not. Scopes come from four places: every
+account with a FalkorDB graph (``GRAPH.LIST``), every account with
+skill-learning sources (an account can learn without a graph), every hired,
+unarchived expert in Postgres, and every ACTIVE registry row. Accounts whose
+user still exists go through ``ensure_scope_scheduled``, which leaves a
+PAUSED or WIPED row as it is; live experts are resumed or, when their
+schedules are paused, paused; an ACTIVE row whose expert is archived or gone
+is paused.
+
+Safe to re-run: an account or live expert already registered in its owner's
+timezone costs a few reads and no scheduler call, but a paused expert is
+paused again (a state write and a remove call) on every run. The report
+lists every scope whose registration, recording or pause failed, with the
+cron and the reason, and the command exits 1 when there is any.
+
+Needs what the backend services need: Postgres (Prisma connects directly),
+FalkorDB, the scheduler service for the registration RPCs, and the feature
+flag backend, since crons are only registered for owners whose flags are
+on.
+"""
+
+import argparse
+import asyncio
+import logging
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from backend.copilot.graphiti.graphs import list_account_graph_owners
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data import db, memory_schedule
+from backend.data.memory_schedule import LiveExpertScope, MemoryScopeSchedule
+from backend.util.feature_flag import initialize_feature_flags, shutdown_feature_flags
+from backend.util.settings import AppEnvironment, Config
+
+from .registry import ensure_scope_scheduled, pause_scope, resume_scope
+
+logger = logging.getLogger(__name__)
+
+Outcome = Literal["registered", "already_scheduled", "skipped", "failed"]
+FAILURE_REASONS = frozenset(
+ {
+ "registration_failed",
+ "record_failed",
+ "registry_unavailable",
+ "timezone_lookup_failed",
+ }
+)
+_PAGE_SIZE = 500
+_CONNECT_TIMEOUT_SECONDS = 60
+
+
+class BackfillFailure(BaseModel):
+ """One cron (or the pause) of one scope that the backfill could not settle."""
+
+ scope_key: str
+ user_id: str
+ expert_id: str | None
+ job: str
+ reason: str
+
+
+class BackfillReport(BaseModel):
+ """What one backfill run found, what it did, and what failed."""
+
+ dry_run: bool
+ accounts: int = 0
+ learning_accounts: int = 0
+ experts: int = 0
+ experts_to_pause: int = 0
+ graphs_without_user: int = 0
+
+ registered: int = 0
+ already_scheduled: int = 0
+ skipped: int = 0
+ paused: int = 0
+ failed: int = 0
+ failures: list[BackfillFailure] = Field(default_factory=list)
+
+
+async def backfill_schedules(
+ *, force: bool = False, dry_run: bool = False
+) -> BackfillReport:
+ """Schedule or pause every memory scope; ``dry_run`` only counts them.
+ ``force`` re-registers every enabled cron of every scope it schedules."""
+ report = BackfillReport(dry_run=dry_run)
+ active_rows = await _active_rows()
+ accounts = await _account_owners(report, active_rows)
+ live = await _live_experts()
+ live_ids = {expert.expert_id for expert in live}
+ stale = [
+ MemoryScope.for_expert(row.user_id, row.expert_id)
+ for row in active_rows
+ if row.expert_id is not None and row.expert_id not in live_ids
+ ]
+ report.accounts, report.experts = len(accounts), len(live)
+ report.experts_to_pause = len(stale) + sum(expert.paused for expert in live)
+ if dry_run:
+ return report
+ for user_id in accounts:
+ scope = MemoryScope.for_user(user_id)
+ _tally(report, scope, await ensure_scope_scheduled(scope, force_refresh=force))
+ for expert in live:
+ await _backfill_expert(report, expert, force)
+ for scope in stale:
+ await _pause(report, scope)
+ return report
+
+
+def main() -> None:
+ """Entry point of ``poetry run memory-schedule-backfill``."""
+ parser = argparse.ArgumentParser(
+ description="Schedule the dream-system crons of every memory scope "
+ "that should have them, and pause the rest. Safe to re-run."
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="count the scopes it would schedule or pause, and change nothing",
+ )
+ parser.add_argument(
+ "--force",
+ action="store_true",
+ help="re-register every enabled cron even when the registry says it "
+ "is current (e.g. after jobs were lost from the scheduler)",
+ )
+ args = parser.parse_args()
+ logging.basicConfig(
+ level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s"
+ )
+ try:
+ report = asyncio.run(_run(force=args.force, dry_run=args.dry_run))
+ except Exception:
+ # Per-scope failures are listed in the report; this is Postgres or
+ # FalkorDB being unreachable, or a listing failing outright.
+ logger.exception("memory-schedule-backfill failed")
+ raise SystemExit(1)
+ print(report.model_dump_json(indent=2))
+ if report.failures:
+ # The report names each one; a re-run retries them.
+ raise SystemExit(1)
+
+
+async def _run(*, force: bool, dry_run: bool) -> BackfillReport:
+ # One bounded attempt, not ``db.connect()``'s retry loop: an operator who
+ # runs this where the database is unreachable wants an error, not an hour
+ # of retries.
+ await asyncio.wait_for(db.prisma.connect(), timeout=_CONNECT_TIMEOUT_SECONDS)
+ # Eager, like the scheduler: a lazily started flag client answers
+ # "off" until it connects, which would skip every registration.
+ flags = not dry_run and Config().app_env != AppEnvironment.LOCAL
+ try:
+ if flags:
+ initialize_feature_flags()
+ return await backfill_schedules(force=force, dry_run=dry_run)
+ finally:
+ if flags:
+ shutdown_feature_flags()
+ await db.disconnect()
+
+
+async def _backfill_expert(
+ report: BackfillReport, expert: LiveExpertScope, force: bool
+) -> None:
+ scope = MemoryScope.for_expert(expert.user_id, expert.expert_id)
+ if expert.paused:
+ await _pause(report, scope)
+ return
+ _tally(report, scope, await resume_scope(scope, force_refresh=force))
+
+
+async def _pause(report: BackfillReport, scope: MemoryScope) -> None:
+ if await pause_scope(scope):
+ report.paused += 1
+ return
+ report.failed += 1
+ report.failures.append(_failure(scope, "pause", "pause_failed"))
+
+
+def _tally(report: BackfillReport, scope: MemoryScope, results: dict[str, Any]) -> None:
+ """Count one scope by its per-cron ensure results, listing each failed
+ cron."""
+ match _classify(results):
+ case "failed":
+ report.failed += 1
+ report.failures.extend(
+ _failure(scope, job, outcome["reason"])
+ for job, outcome in results.items()
+ if _failed(outcome)
+ )
+ case "registered":
+ report.registered += 1
+ case "already_scheduled":
+ report.already_scheduled += 1
+ case "skipped":
+ report.skipped += 1
+
+
+def _classify(results: dict[str, Any]) -> Outcome:
+ """One scope's outcome from its per-cron results. A failed cron makes the
+ scope failed whatever the other crons did."""
+ outcomes = list(results.values())
+ if any(_failed(outcome) for outcome in outcomes):
+ return "failed"
+ if any(outcome and not outcome.get("skipped") for outcome in outcomes):
+ return "registered"
+ if any(outcome is None for outcome in outcomes):
+ return "already_scheduled"
+ return "skipped"
+
+
+def _failed(outcome: Any) -> bool:
+ return bool(outcome) and outcome.get("reason") in FAILURE_REASONS
+
+
+def _failure(scope: MemoryScope, job: str, reason: str) -> BackfillFailure:
+ return BackfillFailure(
+ scope_key=scope.scope_key,
+ user_id=scope.owner_user_id,
+ expert_id=scope.expert_id,
+ job=job,
+ reason=reason,
+ )
+
+
+async def _active_rows() -> list[MemoryScopeSchedule]:
+ rows: list[MemoryScopeSchedule] = []
+ while True:
+ after = rows[-1].scope_key if rows else None
+ page = await memory_schedule.list_active_scope_schedules(
+ after=after, limit=_PAGE_SIZE
+ )
+ rows += page
+ if len(page) < _PAGE_SIZE:
+ return rows
+
+
+async def _live_experts() -> list[LiveExpertScope]:
+ experts: list[LiveExpertScope] = []
+ while True:
+ after = experts[-1].expert_id if experts else None
+ page = await memory_schedule.list_live_expert_scopes(
+ after=after, limit=_PAGE_SIZE
+ )
+ experts += page
+ if len(page) < _PAGE_SIZE:
+ return experts
+
+
+async def _account_owners(
+ report: BackfillReport, active_rows: list[MemoryScopeSchedule]
+) -> list[str]:
+ """Owners of an account graph or of skill-learning sources that still
+ have a user row, plus every account with an ACTIVE registry row."""
+ graph_owners = set(await list_account_graph_owners())
+ learning_owners = set(await _learning_owners())
+ owners = sorted(graph_owners | learning_owners)
+ existing: set[str] = set()
+ for start in range(0, len(owners), _PAGE_SIZE):
+ chunk = owners[start : start + _PAGE_SIZE]
+ existing |= await memory_schedule.existing_user_ids(chunk)
+ report.graphs_without_user = len(graph_owners - existing)
+ report.learning_accounts = len(learning_owners & existing)
+ registered = {row.user_id for row in active_rows if row.expert_id is None}
+ return sorted(existing | registered)
+
+
+async def _learning_owners() -> list[str]:
+ owners: list[str] = []
+ while True:
+ after = owners[-1] if owners else None
+ page = await memory_schedule.list_learning_account_ids(
+ after=after, limit=_PAGE_SIZE
+ )
+ owners += page
+ if len(page) < _PAGE_SIZE:
+ return owners
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry_backfill_test.py b/autogpt_platform/backend/backend/copilot/dream/registry_backfill_test.py
new file mode 100644
index 0000000000..213dd74bb2
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry_backfill_test.py
@@ -0,0 +1,290 @@
+"""Tests for the registry backfill: which scopes it schedules, which it
+pauses, and the counts it reports. The registry functions and the listings
+are mocked; ``registry_test.py`` covers what they do."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from unittest.mock import AsyncMock
+
+import pytest
+from prisma.enums import MemoryScopeScheduleState
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.memory_schedule import LiveExpertScope, MemoryScopeSchedule
+
+from . import registry_backfill
+from .registry_backfill import BackfillFailure, BackfillReport, backfill_schedules
+
+REGISTERED = {"community_rebuild": {"id": "c"}, "dream_nightly_batch": {"id": "n"}}
+ALREADY = {"community_rebuild": None, "dream_nightly_batch": None}
+FLAGS_OFF = {
+ "community_rebuild": {"skipped": True, "reason": "graphiti_communities_disabled"},
+ "dream_nightly_batch": {"skipped": True, "reason": "dream_pass_disabled"},
+}
+FAILED = {
+ "community_rebuild": {"skipped": True, "reason": "registration_failed"},
+ "dream_nightly_batch": None,
+}
+# One cron registered, the other did not: the scope failed.
+PARTLY_FAILED = {
+ "community_rebuild": {"skipped": True, "reason": "registration_failed"},
+ "dream_nightly_batch": {"id": "n"},
+}
+
+
+def _row(user_id: str, expert_id: str | None = None) -> MemoryScopeSchedule:
+ scope = MemoryScope.build(user_id, expert_id)
+ now = datetime.now(timezone.utc)
+ return MemoryScopeSchedule(
+ scope_key=scope.scope_key,
+ user_id=user_id,
+ expert_id=expert_id,
+ timezone="UTC",
+ state=MemoryScopeScheduleState.ACTIVE,
+ community_job_id=None,
+ nightly_job_id=None,
+ learning_job_id=None,
+ last_nightly_run_at=None,
+ last_community_run_at=None,
+ created_at=now,
+ updated_at=now,
+ )
+
+
+class Backfill:
+ """The backfill's inputs and the registry calls it made."""
+
+ def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ self.graph_owners = ["u-graph", "u-gone"]
+ self.learning_owners: list[str] = []
+ self.users = {"u-graph", "u-row"}
+ self.rows = [_row("u-row"), _row("u-row", "e-archived")]
+ self.experts = [
+ LiveExpertScope(expert_id="e-live", user_id="u-graph", paused=False),
+ LiveExpertScope(expert_id="e-paused", user_id="u-graph", paused=True),
+ ]
+ self.ensure = AsyncMock(return_value=REGISTERED)
+ self.resume = AsyncMock(return_value=ALREADY)
+ self.pause = AsyncMock(return_value=True)
+
+ async def owners() -> list[str]:
+ return self.graph_owners
+
+ async def learners(*, after: str | None, limit: int) -> list[str]:
+ later = [
+ user for user in self.learning_owners if after is None or user > after
+ ]
+ return later[:limit]
+
+ async def existing(user_ids: list[str]) -> set[str]:
+ return self.users & set(user_ids)
+
+ async def rows(*, after: str | None, limit: int) -> list[MemoryScopeSchedule]:
+ return [] if after else self.rows
+
+ async def experts(*, after: str | None, limit: int) -> list[LiveExpertScope]:
+ return [] if after else self.experts
+
+ module = registry_backfill.memory_schedule
+ monkeypatch.setattr(registry_backfill, "list_account_graph_owners", owners)
+ monkeypatch.setattr(module, "list_learning_account_ids", learners)
+ monkeypatch.setattr(module, "existing_user_ids", existing)
+ monkeypatch.setattr(module, "list_active_scope_schedules", rows)
+ monkeypatch.setattr(module, "list_live_expert_scopes", experts)
+ monkeypatch.setattr(registry_backfill, "ensure_scope_scheduled", self.ensure)
+ monkeypatch.setattr(registry_backfill, "resume_scope", self.resume)
+ monkeypatch.setattr(registry_backfill, "pause_scope", self.pause)
+
+ def paused_scopes(self) -> list[MemoryScope]:
+ return [call.args[0] for call in self.pause.await_args_list]
+
+
+def backfill(monkeypatch: pytest.MonkeyPatch) -> Backfill:
+ return Backfill(monkeypatch)
+
+
+async def test_backfill_schedules_accounts_and_reconciles_experts(backfill):
+ report = await backfill_schedules()
+
+ assert report == BackfillReport(
+ dry_run=False,
+ accounts=2,
+ experts=2,
+ experts_to_pause=2,
+ graphs_without_user=1,
+ registered=2,
+ already_scheduled=1,
+ paused=2,
+ )
+ # Accounts: every graph owner with a user row, plus every ACTIVE row.
+ accounts = [call.args[0] for call in backfill.ensure.await_args_list]
+ assert accounts == [MemoryScope.for_user("u-graph"), MemoryScope.for_user("u-row")]
+ backfill.resume.assert_awaited_once_with(
+ MemoryScope.for_expert("u-graph", "e-live"), force_refresh=False
+ )
+ # The paused live expert and the ACTIVE row of an archived one.
+ assert backfill.paused_scopes() == [
+ MemoryScope.for_expert("u-graph", "e-paused"),
+ MemoryScope.for_expert("u-row", "e-archived"),
+ ]
+
+
+async def test_an_account_that_learns_without_a_graph_is_scheduled(backfill):
+ """#14551 registers an account's learning cron when its first learning
+ source lands, graph or not: the sources find the account, and only one
+ whose user still exists goes through the ensure."""
+ backfill.learning_owners = ["u-graph", "u-learner", "u-learner-gone"]
+ backfill.users.add("u-learner")
+
+ report = await backfill_schedules()
+
+ assert (report.accounts, report.learning_accounts) == (3, 2)
+ assert report.graphs_without_user == 1
+ accounts = [call.args[0] for call in backfill.ensure.await_args_list]
+ assert accounts == [
+ MemoryScope.for_user("u-graph"),
+ MemoryScope.for_user("u-learner"),
+ MemoryScope.for_user("u-row"),
+ ]
+
+
+async def test_learning_owners_are_read_page_by_page(backfill, monkeypatch):
+ monkeypatch.setattr(registry_backfill, "_PAGE_SIZE", 2)
+ backfill.graph_owners = []
+ backfill.learning_owners = ["u-a", "u-b", "u-c", "u-d", "u-e"]
+ backfill.users = set(backfill.learning_owners)
+ backfill.rows = []
+
+ report = await backfill_schedules(dry_run=True)
+
+ assert (report.accounts, report.learning_accounts) == (5, 5)
+
+
+async def test_dry_run_counts_and_changes_nothing(backfill):
+ report = await backfill_schedules(dry_run=True)
+
+ assert (report.accounts, report.experts, report.experts_to_pause) == (2, 2, 2)
+ assert report.registered == report.paused == 0
+ backfill.ensure.assert_not_awaited()
+ backfill.resume.assert_not_awaited()
+ backfill.pause.assert_not_awaited()
+
+
+async def test_force_re_registers_every_scope(backfill):
+ await backfill_schedules(force=True)
+
+ assert all(
+ call.kwargs["force_refresh"] is True
+ for call in backfill.ensure.await_args_list + backfill.resume.await_args_list
+ )
+
+
+async def test_outcomes_are_counted_per_scope(backfill):
+ backfill.ensure.side_effect = [FLAGS_OFF, FAILED]
+ backfill.pause.return_value = False
+
+ report = await backfill_schedules()
+
+ assert (report.skipped, report.failed, report.already_scheduled) == (1, 3, 1)
+ assert report.registered == report.paused == 0
+
+
+async def test_one_failed_cron_fails_the_scope_and_is_listed(backfill):
+ """Codex's partial-failure case: the community registration failed and
+ the nightly one succeeded. The scope is failed, not registered, and the
+ report names the scope, the cron and the reason."""
+ backfill.ensure.side_effect = [PARTLY_FAILED, ALREADY]
+ backfill.pause.side_effect = [True, False]
+
+ report = await backfill_schedules()
+
+ assert (report.failed, report.registered, report.already_scheduled) == (2, 0, 2)
+ assert report.failures == [
+ BackfillFailure(
+ scope_key="u-graph",
+ user_id="u-graph",
+ expert_id=None,
+ job="community_rebuild",
+ reason="registration_failed",
+ ),
+ BackfillFailure(
+ scope_key=MemoryScope.for_expert("u-row", "e-archived").scope_key,
+ user_id="u-row",
+ expert_id="e-archived",
+ job="pause",
+ reason="pause_failed",
+ ),
+ ]
+
+
+async def test_a_record_failure_counts_as_failed(backfill):
+ backfill.resume.return_value = {
+ "community_rebuild": {"skipped": True, "reason": "record_failed"},
+ "dream_nightly_batch": {"skipped": True, "reason": "record_failed"},
+ }
+
+ report = await backfill_schedules()
+
+ assert [(f.expert_id, f.reason) for f in report.failures] == [
+ ("e-live", "record_failed"),
+ ("e-live", "record_failed"),
+ ]
+
+
+def test_the_command_exits_1_when_anything_failed(monkeypatch):
+ failure = BackfillFailure(
+ scope_key="u", user_id="u", expert_id=None, job="pause", reason="pause_failed"
+ )
+ failed = BackfillReport(dry_run=False, failed=1, failures=[failure])
+
+ async def run(*, force: bool, dry_run: bool) -> BackfillReport:
+ return failed
+
+ monkeypatch.setattr(registry_backfill, "_run", run)
+ monkeypatch.setattr("sys.argv", ["memory-schedule-backfill"])
+ with pytest.raises(SystemExit) as exit_info:
+ registry_backfill.main()
+ assert exit_info.value.code == 1
+
+ async def clean(*, force: bool, dry_run: bool) -> BackfillReport:
+ return BackfillReport(dry_run=False, registered=1)
+
+ monkeypatch.setattr(registry_backfill, "_run", clean)
+ registry_backfill.main() # no exit on success
+
+
+async def test_rerunning_is_safe(backfill):
+ """A second run makes the same calls; what the registry already holds
+ comes back as already scheduled."""
+ first = await backfill_schedules()
+ backfill.ensure.return_value = ALREADY
+
+ second = await backfill_schedules()
+
+ assert first.registered == 2
+ assert second.already_scheduled == 3
+ assert backfill.ensure.await_count == 4
+
+
+def test_classify_puts_failure_before_any_success():
+ assert registry_backfill._classify(PARTLY_FAILED) == "failed"
+ assert registry_backfill._classify(REGISTERED) == "registered"
+ assert registry_backfill._classify(FAILED) == "failed"
+ assert registry_backfill._classify(ALREADY) == "already_scheduled"
+ assert registry_backfill._classify(FLAGS_OFF) == "skipped"
+ partly = {
+ "community_rebuild": None,
+ "dream_nightly_batch": FLAGS_OFF["dream_nightly_batch"],
+ }
+ assert registry_backfill._classify(partly) == "already_scheduled"
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry_deadline_test.py b/autogpt_platform/backend/backend/copilot/dream/registry_deadline_test.py
new file mode 100644
index 0000000000..0cbb51ed19
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry_deadline_test.py
@@ -0,0 +1,245 @@
+"""Every registry entry point returns within the registry deadline when a
+dependency hangs, and cancels the call it gave up on.
+
+The deadline is shortened (``use_short_deadline``). Where the dependency is an
+RPC, the real DatabaseManager or scheduler client is used with a transport
+that never answers, so the client's own retry and timeout layers are part of
+what is bounded. The cron gate and run stamp are covered in
+``scope_crons_test.py``, the API hooks in the experts hook tests.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Any, Awaitable, Callable
+from unittest.mock import AsyncMock
+
+import httpx
+import pytest
+
+from backend.data.db_manager import DatabaseManagerAsyncClient
+from backend.executor.scheduler import SchedulerClient
+from backend.util.service import get_service_client
+
+from . import deadline, registry, scope_crons, scope_jobs
+from .registry_fakes_test import (
+ ACCOUNT,
+ EXPERT,
+ USER,
+ Hang,
+ HangingTransport,
+ RegistryEnv,
+ install_registry_env,
+ use_short_deadline,
+)
+
+# Wall-clock allowance for a call bounded by the short deadline.
+PROMPT = 1.0
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ use_short_deadline(monkeypatch)
+ return install_registry_env(monkeypatch)
+
+
+def _use_registry_rpc(monkeypatch: pytest.MonkeyPatch, rpc: object) -> None:
+ for module in (registry, scope_jobs, scope_crons):
+ monkeypatch.setattr(module, "memory_schedule_db", lambda: rpc)
+
+
+async def test_ensure_is_bounded_when_the_database_manager_hangs(env, monkeypatch):
+ transport = HangingTransport()
+ rpc = transport.client(DatabaseManagerAsyncClient, request_retry=True)
+ _use_registry_rpc(monkeypatch, rpc)
+ started = time.monotonic()
+ try:
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+ finally:
+ await rpc.aclose()
+
+ assert time.monotonic() - started < PROMPT
+ assert {r["reason"] for r in result.values()} == {
+ "registry_unavailable",
+ "skill_learning_disabled",
+ }
+ assert transport.cancelled == transport.paths == ["/get_scope_schedule"]
+
+
+async def test_a_retrying_database_manager_client_is_cut_off(env, monkeypatch):
+ """A refused connection sends the real client into its retry backoff;
+ the deadline cuts the whole retry loop off, not just one attempt."""
+ attempts: list[str] = []
+
+ def refuse(request: httpx.Request) -> httpx.Response:
+ attempts.append(request.url.path)
+ raise httpx.ConnectError("refused", request=request)
+
+ rpc = get_service_client(DatabaseManagerAsyncClient, request_retry=True)
+ rpc._async_clients[asyncio.get_running_loop()] = httpx.AsyncClient(
+ transport=httpx.MockTransport(refuse), base_url="http://refusing"
+ )
+ _use_registry_rpc(monkeypatch, rpc)
+ started = time.monotonic()
+ try:
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+ finally:
+ await rpc.aclose()
+
+ assert time.monotonic() - started < PROMPT
+ assert {r["reason"] for r in result.values()} == {
+ "registry_unavailable",
+ "skill_learning_disabled",
+ }
+ assert attempts == ["/get_scope_schedule"]
+
+
+async def test_a_hanging_scheduler_is_cut_off_and_reported(env, monkeypatch):
+ transport = HangingTransport()
+ scheduler = transport.client(SchedulerClient, request_retry=False)
+ monkeypatch.setattr(scope_jobs, "get_scheduler_client", lambda: scheduler)
+ started = time.monotonic()
+ try:
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+ finally:
+ await scheduler.aclose()
+
+ assert time.monotonic() - started < PROMPT
+ assert {r["reason"] for r in result.values()} == {
+ "registration_failed",
+ "skill_learning_disabled",
+ }
+ assert transport.cancelled == transport.paths
+ assert transport.paths == [
+ "/add_scope_community_rebuild_schedule",
+ "/add_scope_nightly_batch_schedule",
+ ]
+ assert env.db.rows[USER].nightly_job_id is None
+
+
[email protected]("active", [True, False])
+async def test_a_lifecycle_change_is_bounded_as_a_whole(env, monkeypatch, active):
+ """Pause, resume, archive and revive run this inside an API request: the
+ change as a whole returns within the deadline and cancels what hangs."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ await registry.pause_scope(EXPERT)
+ hang = Hang()
+ for rpc in (
+ "remove_scope_memory_jobs",
+ "add_scope_community_rebuild_schedule",
+ "add_scope_nightly_batch_schedule",
+ ):
+ monkeypatch.setattr(env.scheduler, rpc, hang.forever)
+ started = time.monotonic()
+
+ await registry.sync_expert_scope(USER, "expert-1", active=active)
+
+ assert time.monotonic() - started < PROMPT
+ # Whichever deadline fires first, nothing that started is left running.
+ assert hang.started >= 1
+ assert hang.cancelled == hang.started
+
+
+async def test_a_slow_resume_is_cut_off_as_a_whole_not_per_call(env, monkeypatch):
+ """Every call answers well inside the deadline, so no single call times
+ out, but a resume makes about eight of them: the request waits for one
+ deadline, not for their sum. (A longer deadline than the other tests, so
+ Windows' ~16 ms timer granularity cannot push a step past it.)"""
+ monkeypatch.setattr(deadline, "REGISTRY_CALL_TIMEOUT_SECONDS", 0.2)
+ await registry.pause_scope(EXPERT)
+ step = 0.05
+ for owner, name in (
+ (env.db, "set_scope_state"),
+ (env.db, "get_scope_schedule"),
+ (env.db, "claim_scope_schedule"),
+ (env.db, "record_scope_jobs"),
+ (env.scheduler, "add_scope_community_rebuild_schedule"),
+ (env.scheduler, "add_scope_nightly_batch_schedule"),
+ (scope_jobs, "is_feature_enabled"),
+ (registry, "resolve_user_timezone"),
+ ):
+ monkeypatch.setattr(owner, name, _slowed(getattr(owner, name), step))
+ started = time.monotonic()
+
+ await registry.sync_expert_scope(USER, "expert-1", active=True)
+
+ # One deadline is 0.2 s; the eight steps alone would take 0.4 s or more.
+ assert time.monotonic() - started < 0.3
+
+
+def _slowed(call: Callable[..., Awaitable[Any]], seconds: float):
+ async def slow(*args: Any, **kwargs: Any) -> Any:
+ await asyncio.sleep(seconds)
+ return await call(*args, **kwargs)
+
+ return slow
+
+
+async def test_a_new_expert_registration_is_bounded_when_flags_hang(env, monkeypatch):
+ hang = Hang()
+ monkeypatch.setattr(scope_jobs, "is_feature_enabled", hang.forever)
+ started = time.monotonic()
+
+ await registry.ensure_expert_scheduled(USER, "expert-1")
+
+ assert time.monotonic() - started < PROMPT
+ assert hang.started == hang.cancelled == 2
+ assert env.db.rows == {}
+
+
+async def test_a_timezone_change_is_bounded_when_the_listing_hangs(env, monkeypatch):
+ hang = Hang()
+ monkeypatch.setattr(env.db, "list_user_scope_schedules", hang.forever)
+ started = time.monotonic()
+
+ results = await registry.reregister_user(USER)
+
+ assert time.monotonic() - started < PROMPT
+ assert hang.started == hang.cancelled == 1
+ assert list(results) == [USER] # the account still goes through
+
+
+async def test_a_step_at_the_deadline_does_not_extend_a_lifecycle_change(
+ env, monkeypatch
+):
+ """Codex's round-2 reproduction, scaled down: the community cron is
+ registered right at the deadline after a short loop stall, and the
+ nightly registration would then hang. The change must end at the
+ deadline; it used to register the nightly on a fresh deadline and
+ record it, taking twice as long."""
+ budget = 0.3
+ monkeypatch.setattr(deadline, "REGISTRY_CALL_TIMEOUT_SECONDS", budget)
+ loop = asyncio.get_running_loop()
+ started = loop.time()
+ add_community = env.scheduler.add_scope_community_rebuild_schedule
+
+ async def community(**kwargs: Any) -> Any:
+ await asyncio.sleep(started + budget - 0.02 - loop.time())
+ added = await add_community(**kwargs)
+ time.sleep(0.06) # a short loop stall carries it past the deadline
+ return added
+
+ hang = Hang()
+ record = AsyncMock(side_effect=env.db.record_scope_jobs)
+ monkeypatch.setattr(
+ env.scheduler, "add_scope_community_rebuild_schedule", community
+ )
+ monkeypatch.setattr(env.scheduler, "add_scope_nightly_batch_schedule", hang.forever)
+ monkeypatch.setattr(env.db, "record_scope_jobs", record)
+
+ await registry.sync_expert_scope(USER, "expert-1", active=True)
+
+ # Nothing ran on after the deadline: no nightly on a fresh deadline (that
+ # took twice the budget) and no recording of what it registered.
+ assert loop.time() - started < budget + 0.25
+ assert hang.cancelled == hang.started
+ record.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry_fakes_test.py b/autogpt_platform/backend/backend/copilot/dream/registry_fakes_test.py
new file mode 100644
index 0000000000..ed0804d09d
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry_fakes_test.py
@@ -0,0 +1,355 @@
+"""Shared fakes for the schedule registry tests (no tests of its own).
+
+``install_registry_env`` swaps the registry's data module, scheduler client,
+flags and timezone lookup for in-memory fakes with the real ownership rules,
+and ``reconcile`` runs the scheduler's reconcile pass against them (Redis is
+the dream conftest's ``fake_dream_redis``);
+``Hang`` and ``HangingTransport`` stand in for a dependency that never
+answers, recording whether the caller cancelled it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Sequence
+from datetime import datetime, timezone
+from typing import Any
+from unittest.mock import AsyncMock
+
+import httpx
+import pytest
+from prisma.enums import MemoryScopeScheduleState
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.memory_schedule import MemoryScopeSchedule
+from backend.util.exceptions import NotAuthorizedError
+from backend.util.feature_flag import Flag
+from backend.util.service import get_service_client
+
+from . import deadline, registry, scope_crons, scope_jobs, scope_reconcile
+from .scheduling import DREAM_SYSTEM_JOBS, DreamSystemJob
+
+ACTIVE = MemoryScopeScheduleState.ACTIVE
+PAUSED = MemoryScopeScheduleState.PAUSED
+WIPED = MemoryScopeScheduleState.WIPED
+
+USER = "user-1"
+ACCOUNT = MemoryScope.for_user(USER)
+EXPERT = MemoryScope.for_expert(USER, "expert-1")
+OTHER_EXPERT = MemoryScope.for_expert(USER, "expert-2")
+PARIS = "Europe/Paris"
+TOKYO = "Asia/Tokyo"
+
+# Short enough to keep deadline tests quick; long enough for a fake to answer.
+SHORT_DEADLINE = 0.05
+
+
+def job_ids(scope: MemoryScope) -> set[str]:
+ """The crons the default flags give ``scope``: skill learning's is off."""
+ return {
+ f"community_rebuild_{scope.scope_key}",
+ f"dream_nightly_batch_{scope.scope_key}",
+ }
+
+
+def learning_job_id(scope: MemoryScope) -> str:
+ return f"skill_learning_nightly_{scope.scope_key}"
+
+
+class FakeRegistryDB:
+ """``backend.data.memory_schedule`` in memory, with its ownership rules."""
+
+ def __init__(self) -> None:
+ self.rows: dict[str, MemoryScopeSchedule] = {}
+ self.fail = False
+ # Users deleted since their crons were registered; everyone else
+ # exists.
+ self.gone_users: set[str] = set()
+ self.existence_fails = False
+
+ async def get_scope_schedule(
+ self, user_id: str, scope_key: str
+ ) -> MemoryScopeSchedule | None:
+ if self.fail:
+ raise ConnectionError("db down")
+ row = self.rows.get(scope_key)
+ return row if row is not None and row.user_id == user_id else None
+
+ async def claim_scope_schedule(
+ self,
+ user_id: str,
+ scope_key: str,
+ expert_id: str | None,
+ user_timezone: str,
+ state: MemoryScopeScheduleState = ACTIVE,
+ ) -> MemoryScopeSchedule:
+ if self.fail:
+ raise ConnectionError("db down")
+ if scope_key not in self.rows:
+ now = datetime.now(timezone.utc)
+ self.rows[scope_key] = MemoryScopeSchedule(
+ scope_key=scope_key,
+ user_id=user_id,
+ expert_id=expert_id,
+ timezone=user_timezone,
+ state=state,
+ community_job_id=None,
+ nightly_job_id=None,
+ learning_job_id=None,
+ last_nightly_run_at=None,
+ last_community_run_at=None,
+ created_at=now,
+ updated_at=now,
+ )
+ if self.rows[scope_key].user_id != user_id:
+ raise NotAuthorizedError("another user's scope")
+ return self.rows[scope_key]
+
+ async def record_scope_jobs(
+ self,
+ user_id: str,
+ scope_key: str,
+ *,
+ user_timezone: str,
+ community_job_id: str | None,
+ nightly_job_id: str | None,
+ learning_job_id: str | None,
+ ) -> bool:
+ row = await self.get_scope_schedule(user_id, scope_key)
+ if row is None or row.state != ACTIVE:
+ return False
+ self.update(
+ scope_key,
+ timezone=user_timezone,
+ community_job_id=community_job_id,
+ nightly_job_id=nightly_job_id,
+ learning_job_id=learning_job_id,
+ )
+ return True
+
+ async def set_scope_state(
+ self,
+ user_id: str,
+ scope_key: str,
+ state: MemoryScopeScheduleState,
+ *,
+ only_from: Sequence[MemoryScopeScheduleState] | None = None,
+ ) -> bool:
+ row = await self.get_scope_schedule(user_id, scope_key)
+ if row is None or (only_from is not None and row.state not in only_from):
+ return False
+ fields: dict[str, Any] = {"state": state}
+ if state != ACTIVE:
+ fields |= {
+ "community_job_id": None,
+ "nightly_job_id": None,
+ "learning_job_id": None,
+ }
+ self.update(scope_key, **fields)
+ return True
+
+ async def forget_scope_job(self, user_id: str, scope_key: str, job_id: str) -> bool:
+ row = await self.get_scope_schedule(user_id, scope_key)
+ if row is None:
+ return False
+ fields: dict[str, Any] = {}
+ if row.community_job_id == job_id:
+ fields["community_job_id"] = None
+ if row.nightly_job_id == job_id:
+ fields["nightly_job_id"] = None
+ if row.learning_job_id == job_id:
+ fields["learning_job_id"] = None
+ self.update(scope_key, **fields)
+ return bool(fields)
+
+ async def record_scope_run(self, user_id: str, scope_key: str, kind: str) -> bool:
+ if await self.get_scope_schedule(user_id, scope_key) is None:
+ return False
+ field = "last_nightly_run_at" if kind == "nightly" else "last_community_run_at"
+ self.update(scope_key, **{field: datetime.now(timezone.utc)})
+ return True
+
+ async def list_user_scope_schedules(
+ self, user_id: str
+ ) -> list[MemoryScopeSchedule]:
+ return [row for row in self.rows.values() if row.user_id == user_id]
+
+ async def list_scope_schedules(
+ self, *, after: str | None = None, limit: int = 500
+ ) -> list[MemoryScopeSchedule]:
+ if self.fail:
+ raise ConnectionError("db down")
+ keys = sorted(key for key in self.rows if after is None or key > after)
+ return [self.rows[key] for key in keys[:limit]]
+
+ async def existing_user_ids(self, user_ids: list[str]) -> set[str]:
+ if self.fail or self.existence_fails:
+ raise ConnectionError("db down")
+ return set(user_ids) - self.gone_users
+
+ def update(self, scope_key: str, **fields: Any) -> None:
+ self.rows[scope_key] = self.rows[scope_key].model_copy(update=fields)
+
+
+class FakeScheduler:
+ """The memory-cron RPCs of ``SchedulerClient``, in memory, and which
+ crons carry the reconcile's tag (the registry's RPCs register untagged)."""
+
+ def __init__(self) -> None:
+ self.jobs: dict[str, str] = {} # job id -> timezone it runs in
+ self.scopes: dict[str, MemoryScope] = {} # job id -> the scope it is for
+ self.tagged: set[str] = set() # job ids the reconcile registered
+ self.add_scope_community_rebuild_schedule = AsyncMock(
+ side_effect=self._adder("community_rebuild")
+ )
+ self.add_scope_nightly_batch_schedule = AsyncMock(
+ side_effect=self._adder("dream_nightly_batch")
+ )
+ self.add_scope_skill_learning_schedule = AsyncMock(
+ side_effect=self._adder("skill_learning_nightly")
+ )
+ self.remove_scope_memory_jobs = AsyncMock(side_effect=self._remove)
+
+ def _adder(self, prefix: str):
+ async def add(*, scope: MemoryScope, user_timezone: str) -> dict:
+ job_id = f"{prefix}_{scope.scope_key}"
+ self.jobs[job_id] = user_timezone
+ self.scopes[job_id] = scope
+ self.tagged.discard(job_id)
+ return {"id": job_id, "scope_key": scope.scope_key, "next_run_time": None}
+
+ return add
+
+ async def _remove(self, *, scope: MemoryScope) -> list[str]:
+ every = [job.job_id(scope) for job in DREAM_SYSTEM_JOBS]
+ removed = [job_id for job_id in every if job_id in self.jobs]
+ for job_id in removed:
+ del self.jobs[job_id]
+ self.scopes.pop(job_id, None)
+ self.tagged.discard(job_id)
+ return removed
+
+
+class FakeCronStore:
+ """``scope_reconcile.CronStore`` over the fake scheduler, through the
+ same registration calls the registry makes, the reconcile's tagged."""
+
+ def __init__(self, scheduler: FakeScheduler) -> None:
+ self.scheduler = scheduler
+
+ async def listing(self) -> list[scope_reconcile.ListedCron]:
+ return [
+ scope_reconcile.ListedCron(
+ job_id=job_id,
+ scope=scope,
+ timezone=self.scheduler.jobs[job_id],
+ reconciled=job_id in self.scheduler.tagged,
+ )
+ for job_id, scope in self.scheduler.scopes.items()
+ ]
+
+ async def add(
+ self, job: DreamSystemJob, scope: MemoryScope, user_timezone: str
+ ) -> None:
+ await job.register(self.scheduler, scope, user_timezone)
+ self.scheduler.tagged.add(job.job_id(scope))
+
+ async def drop(self, job: DreamSystemJob, scope: MemoryScope) -> None:
+ self.scheduler.jobs.pop(job.job_id(scope), None)
+ self.scheduler.scopes.pop(job.job_id(scope), None)
+ self.scheduler.tagged.discard(job.job_id(scope))
+
+ async def remove(self, scope: MemoryScope) -> None:
+ await self.scheduler.remove_scope_memory_jobs(scope=scope)
+
+
+class RegistryEnv:
+ def __init__(self) -> None:
+ self.db = FakeRegistryDB()
+ self.scheduler = FakeScheduler()
+ self.crons = FakeCronStore(self.scheduler)
+ self.flags = {
+ Flag.GRAPHITI_COMMUNITIES_ENABLED: True,
+ Flag.DREAM_PASS_ENABLED: True,
+ }
+ self.timezone: str | None = PARIS
+
+ def adds(self) -> int:
+ return (
+ self.scheduler.add_scope_community_rebuild_schedule.await_count
+ + self.scheduler.add_scope_nightly_batch_schedule.await_count
+ + self.scheduler.add_scope_skill_learning_schedule.await_count
+ )
+
+ async def reconcile(self) -> scope_reconcile.ReconcileReport:
+ """One reconcile pass, as the scheduler runs it."""
+ return await scope_reconcile.reconcile(self.crons)
+
+
+def install_registry_env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ """Point the registry modules at a fresh set of fakes."""
+ env = RegistryEnv()
+
+ async def flag(flag: Flag, user_id: str) -> bool:
+ return env.flags.get(flag, False)
+
+ async def owner_timezone(user_id: str) -> str | None:
+ return env.timezone
+
+ for module in (registry, scope_jobs, scope_crons, scope_reconcile):
+ monkeypatch.setattr(module, "memory_schedule_db", lambda: env.db)
+ monkeypatch.setattr(scope_jobs, "get_scheduler_client", lambda: env.scheduler)
+ monkeypatch.setattr(scope_jobs, "is_feature_enabled", flag)
+ monkeypatch.setattr(registry, "resolve_user_timezone", owner_timezone)
+ # Every walk starts at the first scope, so a test sees key order.
+ monkeypatch.setattr(scope_reconcile._Place, "last", "")
+ return env
+
+
+def use_short_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(deadline, "REGISTRY_CALL_TIMEOUT_SECONDS", SHORT_DEADLINE)
+ monkeypatch.setattr(deadline, "BRIDGE_MARGIN_SECONDS", SHORT_DEADLINE)
+
+
+class Hang:
+ """A dependency call that never returns, counting the calls that started
+ and the ones their caller cancelled."""
+
+ def __init__(self) -> None:
+ self.started = 0
+ self.cancelled = 0
+
+ async def forever(self, *args: Any, **kwargs: Any) -> Any:
+ self.started += 1
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ self.cancelled += 1
+ raise
+
+
+class HangingTransport:
+ """An HTTP transport for a real RPC client whose requests never answer,
+ recording each request path that reached it and each one cancelled."""
+
+ def __init__(self) -> None:
+ self.paths: list[str] = []
+ self.cancelled: list[str] = []
+
+ async def handle(self, request: httpx.Request) -> httpx.Response:
+ self.paths.append(request.url.path)
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ self.cancelled.append(request.url.path)
+ raise
+ raise AssertionError("unreachable")
+
+ def client(self, client_type: Any, *, request_retry: bool) -> Any:
+ """A real service client of ``client_type`` whose calls on the
+ running loop go through this transport."""
+ rpc = get_service_client(client_type, request_retry=request_retry)
+ rpc._async_clients[asyncio.get_running_loop()] = httpx.AsyncClient(
+ transport=httpx.MockTransport(self.handle), base_url="http://hanging"
+ )
+ return rpc
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry_flag_off_test.py b/autogpt_platform/backend/backend/copilot/dream/registry_flag_off_test.py
new file mode 100644
index 0000000000..c36ea10cc0
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry_flag_off_test.py
@@ -0,0 +1,122 @@
+"""Tests for a dream-system cron whose flag goes off: the registry leaves
+the scope's recorded crons in place, and pause or wipe removes them. Their
+bodies check the flag at run time and do nothing; the scheduler's tests pin
+that (``TestSkillLearningScopeGate::test_the_flag_off_still_short_circuits``,
+``TestExecuteNightlyBatchSyncRuntimeGate``,
+``TestExecuteCommunityRebuildRuntimeGate``). Cleaning them up on flag-off
+needs generation-aware intent, which is left for later.
+
+Fakes come from ``registry_fakes_test.py``.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from backend.util.feature_flag import Flag
+
+from . import registry
+from .registry_fakes_test import (
+ ACCOUNT,
+ EXPERT,
+ PARIS,
+ PAUSED,
+ USER,
+ WIPED,
+ RegistryEnv,
+ install_registry_env,
+ job_ids,
+ learning_job_id,
+)
+
+LEARNING = learning_job_id(ACCOUNT)
+NIGHTLY = f"dream_nightly_batch_{USER}"
+LEARN = Flag.DREAM_SKILL_LEARNING_ENABLED
+DREAM = Flag.DREAM_PASS_ENABLED
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ env = install_registry_env(monkeypatch)
+ env.flags = {LEARN: True}
+ return env
+
+
+async def test_turning_learning_off_leaves_its_cron_recorded(env):
+ """A learning-only account: the ensure after the flag went off returns
+ before it reads the row, and the reconcile keeps the cron the row
+ records."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.flags[LEARN] = False
+
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert result["skill_learning_nightly"] == {
+ "skipped": True,
+ "reason": "skill_learning_disabled",
+ }
+ assert env.db.rows[USER].learning_job_id == LEARNING
+ reports = [await env.reconcile() for _ in range(3)]
+ assert env.scheduler.jobs == {LEARNING: PARIS}
+ assert all(
+ not (report.dropped or report.removed or report.restored) for report in reports
+ )
+
+
+async def test_turning_learning_off_with_the_dream_on_leaves_both_recorded(env):
+ env.flags[DREAM] = True
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.flags[LEARN] = False
+
+ await registry.ensure_scope_scheduled(ACCOUNT)
+
+ row = env.db.rows[USER]
+ assert (row.nightly_job_id, row.learning_job_id) == (NIGHTLY, LEARNING)
+ assert env.scheduler.jobs == {NIGHTLY: PARIS, LEARNING: PARIS}
+
+
+async def test_turning_the_dream_off_leaves_an_experts_crons_recorded(env):
+ env.flags = {DREAM: True, Flag.GRAPHITI_COMMUNITIES_ENABLED: True}
+ await registry.ensure_scope_scheduled(EXPERT)
+ env.flags = {}
+
+ await registry.ensure_scope_scheduled(EXPERT)
+
+ row = env.db.rows[EXPERT.scope_key]
+ assert {row.community_job_id, row.nightly_job_id} == job_ids(EXPERT)
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+
+
[email protected]("state", [PAUSED, WIPED])
+async def test_a_stop_removes_the_crons_of_a_flag_turned_off(env, state):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.flags[LEARN] = False
+ await registry.ensure_scope_scheduled(ACCOUNT)
+
+ stop = registry.pause_scope if state == PAUSED else registry.mark_wiped
+ assert await stop(ACCOUNT)
+
+ row = env.db.rows[USER]
+ assert (row.state, row.learning_job_id) == (state, None)
+ assert env.scheduler.jobs == {}
+
+
+async def test_every_flag_off_costs_no_database_call(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.flags = {}
+ env.db.fail = True
+
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert {r["reason"] for r in result.values()} == {
+ "graphiti_communities_disabled",
+ "dream_pass_disabled",
+ "skill_learning_disabled",
+ }
+ env.db.fail = False
+ assert env.db.rows[USER].learning_job_id == LEARNING
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry_learning_test.py b/autogpt_platform/backend/backend/copilot/dream/registry_learning_test.py
new file mode 100644
index 0000000000..fbcdb27cc8
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry_learning_test.py
@@ -0,0 +1,250 @@
+"""Tests for the account's skill-learning cron in the registry: recorded per
+account like the scope's other crons, on its own flag, never for an expert,
+and paused, resumed, wiped, re-timed and reconciled under the same rules.
+
+Fakes come from ``registry_fakes_test.py``.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from backend.util.feature_flag import Flag
+
+from . import registry, registry_backfill
+from .registry_fakes_test import (
+ ACCOUNT,
+ ACTIVE,
+ EXPERT,
+ PARIS,
+ PAUSED,
+ TOKYO,
+ USER,
+ WIPED,
+ RegistryEnv,
+ install_registry_env,
+ job_ids,
+ learning_job_id,
+)
+
+LEARNING = learning_job_id(ACCOUNT)
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ env = install_registry_env(monkeypatch)
+ env.flags[Flag.DREAM_SKILL_LEARNING_ENABLED] = True
+ return env
+
+
+async def test_learning_alone_registers_and_records_the_accounts_cron(env):
+ """Skill learning runs nightly per account when its flag is on, the
+ dream flags off or not."""
+ env.flags = {Flag.DREAM_SKILL_LEARNING_ENABLED: True}
+
+ results = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert results["skill_learning_nightly"]["id"] == LEARNING
+ assert results["dream_nightly_batch"] == {
+ "skipped": True,
+ "reason": "dream_pass_disabled",
+ }
+ assert env.scheduler.jobs == {LEARNING: PARIS}
+ row = env.db.rows[USER]
+ assert (row.state, row.learning_job_id, row.nightly_job_id) == (
+ ACTIVE,
+ LEARNING,
+ None,
+ )
+
+
+async def test_an_expert_scope_never_gets_a_learning_cron(env):
+ """One pass reviews the account's skills and its experts': an expert's
+ scope neither registers nor even considers a learning cron."""
+ results = await registry.ensure_scope_scheduled(EXPERT)
+
+ assert set(results) == {"community_rebuild", "dream_nightly_batch"}
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+ assert env.db.rows[EXPERT.scope_key].learning_job_id is None
+ env.scheduler.add_scope_skill_learning_schedule.assert_not_awaited()
+
+
+async def test_a_pause_takes_the_learning_cron_down_and_a_resume_brings_it_back(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ assert set(env.scheduler.jobs) == job_ids(ACCOUNT) | {LEARNING}
+
+ assert await registry.pause_scope(ACCOUNT)
+
+ row = env.db.rows[USER]
+ assert (row.state, row.learning_job_id) == (PAUSED, None)
+ assert env.scheduler.jobs == {}
+ skipped = await registry.ensure_scope_scheduled(ACCOUNT)
+ assert skipped["skill_learning_nightly"] == {
+ "skipped": True,
+ "reason": "scope_paused",
+ }
+ assert env.scheduler.jobs == {}
+
+ await registry.resume_scope(ACCOUNT)
+
+ row = env.db.rows[USER]
+ assert (row.state, row.learning_job_id) == (ACTIVE, LEARNING)
+ assert set(env.scheduler.jobs) == job_ids(ACCOUNT) | {LEARNING}
+
+
+async def test_a_wipe_takes_the_learning_cron_down_for_good(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert await registry.mark_wiped(ACCOUNT)
+
+ row = env.db.rows[USER]
+ assert (row.state, row.learning_job_id) == (WIPED, None)
+ assert env.scheduler.jobs == {}
+ await registry.resume_scope(ACCOUNT)
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ await env.reconcile()
+ assert env.db.rows[USER].state == WIPED
+ assert env.scheduler.jobs == {}
+
+
+async def test_the_reconcile_restores_a_lost_learning_cron(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ del env.scheduler.jobs[LEARNING]
+ del env.scheduler.scopes[LEARNING]
+
+ report = await env.reconcile()
+
+ assert report.restored == [LEARNING]
+ assert env.scheduler.jobs[LEARNING] == PARIS
+ assert LEARNING in env.scheduler.tagged
+
+
+async def test_the_reconcile_removes_a_learning_cron_straggling_past_a_pause(env):
+ """A registration that lands after the pause (or a removal that failed)
+ leaves the learning cron behind under a PAUSED row: the next pass takes
+ it down, and its body's gate keeps it from running meanwhile."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ assert await registry.pause_scope(ACCOUNT)
+ await env.scheduler.add_scope_skill_learning_schedule(
+ scope=ACCOUNT, user_timezone=PARIS
+ )
+
+ report = await env.reconcile()
+
+ assert report.removed == [USER]
+ assert env.scheduler.jobs == {}
+ assert env.db.rows[USER].state == PAUSED
+
+
+async def test_the_reconcile_takes_back_a_tagged_learning_cron_no_row_records(env):
+ """The learning flag turned off and an ensure recorded no learning job,
+ while a restore the reconcile made still runs: the next pass takes back
+ its own tagged cron."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ del env.scheduler.jobs[LEARNING]
+ del env.scheduler.scopes[LEARNING]
+ await env.reconcile()
+ assert LEARNING in env.scheduler.tagged
+ env.flags[Flag.DREAM_SKILL_LEARNING_ENABLED] = False
+ await registry.ensure_scope_scheduled(ACCOUNT, force_refresh=True)
+ assert env.db.rows[USER].learning_job_id is None
+
+ report = await env.reconcile()
+
+ assert report.dropped == [LEARNING]
+ assert set(env.scheduler.jobs) == job_ids(ACCOUNT)
+
+
+async def test_a_timezone_change_moves_the_learning_cron(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.timezone = TOKYO
+
+ await registry.reregister_user(USER)
+
+ assert env.scheduler.jobs[LEARNING] == TOKYO
+ assert env.db.rows[USER].learning_job_id == LEARNING
+
+
+async def test_a_timezone_change_reaches_an_owner_with_only_learning_on(env):
+ """An owner whose dream flags are off but whose learning flag is on is
+ not "every dream flag off": the change goes through."""
+ env.flags = {Flag.DREAM_SKILL_LEARNING_ENABLED: True}
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.timezone = TOKYO
+
+ results = await registry.reregister_user(USER)
+
+ assert results[USER]["skill_learning_nightly"]["id"] == LEARNING
+ assert env.scheduler.jobs == {LEARNING: TOKYO}
+
+
+def _backfill_finds(monkeypatch: pytest.MonkeyPatch, learning_owners: list[str]):
+ """The backfill's listings: no graph, no expert, no ACTIVE row; only the
+ learning sources' owners, every one of them a live user."""
+ module = registry_backfill.memory_schedule
+
+ async def learners(*, after: str | None, limit: int) -> list[str]:
+ return [] if after else learning_owners
+
+ async def existing(user_ids: list[str]) -> set[str]:
+ return set(user_ids)
+
+ monkeypatch.setattr(
+ registry_backfill, "list_account_graph_owners", AsyncMock(return_value=[])
+ )
+ monkeypatch.setattr(module, "list_learning_account_ids", learners)
+ monkeypatch.setattr(module, "existing_user_ids", existing)
+ monkeypatch.setattr(
+ module, "list_active_scope_schedules", AsyncMock(return_value=[])
+ )
+ monkeypatch.setattr(module, "list_live_expert_scopes", AsyncMock(return_value=[]))
+
+
+async def test_the_backfill_adopts_a_graphless_accounts_legacy_learning_cron(
+ env, monkeypatch
+):
+ """The review's probe: a legacy learning cron (registered by #14551's
+ capture hook, before the registry) of an account with no graph. The
+ backfill finds it through the account's learning sources and records it,
+ keeping the same job."""
+ env.flags = {Flag.DREAM_SKILL_LEARNING_ENABLED: True}
+ await env.scheduler.add_scope_skill_learning_schedule(
+ scope=ACCOUNT, user_timezone=PARIS
+ )
+ _backfill_finds(monkeypatch, [USER])
+
+ report = await registry_backfill.backfill_schedules()
+
+ assert (report.accounts, report.learning_accounts) == (1, 1)
+ assert env.db.rows[USER].learning_job_id == LEARNING
+ assert env.scheduler.jobs == {LEARNING: PARIS}
+
+
[email protected]("state", [PAUSED, WIPED])
+async def test_the_backfill_leaves_a_stopped_learning_account_stopped(
+ env, monkeypatch, state
+):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ stop = registry.pause_scope if state == PAUSED else registry.mark_wiped
+ assert await stop(ACCOUNT)
+ _backfill_finds(monkeypatch, [USER])
+
+ report = await registry_backfill.backfill_schedules()
+
+ assert report.skipped == 1
+ row = env.db.rows[USER]
+ assert (row.state, row.learning_job_id) == (state, None)
+ assert env.scheduler.jobs == {}
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry_lifecycle_test.py b/autogpt_platform/backend/backend/copilot/dream/registry_lifecycle_test.py
new file mode 100644
index 0000000000..9e7cb23ac0
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry_lifecycle_test.py
@@ -0,0 +1,361 @@
+"""Tests for changing a scope's lifecycle in the registry: pause (archive),
+resume (revive), wipe, a new expert's registration and a timezone change.
+
+Fakes come from ``registry_fakes_test.py``.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from backend.util.feature_flag import Flag
+
+from . import registry
+from .registry_fakes_test import (
+ ACCOUNT,
+ ACTIVE,
+ EXPERT,
+ OTHER_EXPERT,
+ PARIS,
+ PAUSED,
+ TOKYO,
+ USER,
+ WIPED,
+ RegistryEnv,
+ install_registry_env,
+ job_ids,
+)
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ return install_registry_env(monkeypatch)
+
+
+async def test_archive_pauses_and_revive_resumes_the_expert(env):
+ await registry.ensure_expert_scheduled(USER, "expert-1") # hire
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+
+ await registry.sync_expert_scope(USER, "expert-1", active=False) # archive
+ row = env.db.rows[EXPERT.scope_key]
+ assert (row.state, row.community_job_id, row.nightly_job_id) == (PAUSED, None, None)
+ assert env.scheduler.jobs == {}
+
+ await registry.sync_expert_scope(USER, "expert-1", active=True) # re-hire
+ row = env.db.rows[EXPERT.scope_key]
+ assert row.state == ACTIVE
+ assert {row.community_job_id, row.nightly_job_id} == job_ids(EXPERT)
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+ assert list(env.db.rows) == [EXPERT.scope_key]
+
+
+async def test_a_new_expert_registration_never_resumes_a_paused_scope(env):
+ """A hire registers in the background and may land after an archive or a
+ pause: it must leave the scope paused, unlike an explicit resume."""
+ await registry.pause_scope(EXPERT)
+
+ await registry.ensure_expert_scheduled(USER, "expert-1")
+
+ assert env.db.rows[EXPERT.scope_key].state == PAUSED
+ assert env.scheduler.jobs == {}
+ assert env.adds() == 0
+
+
+async def test_the_lifecycle_hook_never_fails_its_caller(env, monkeypatch):
+ """Hire, archive, pause and resume call in; nothing in memory
+ scheduling may fail them — not an invalid id, not a bug."""
+ monkeypatch.setattr(
+ registry, "pause_scope", AsyncMock(side_effect=RuntimeError("bug"))
+ )
+
+ await registry.sync_expert_scope(USER, "expert-1", active=False)
+ await registry.sync_expert_scope(USER, "", active=True)
+ await registry.ensure_expert_scheduled(USER, "")
+
+ assert env.db.rows == {}
+
+
+async def test_pausing_a_scope_without_a_row_records_it_paused(env):
+ assert await registry.pause_scope(EXPERT)
+
+ row = env.db.rows[EXPERT.scope_key]
+ assert (row.state, row.timezone, row.expert_id) == (PAUSED, PARIS, "expert-1")
+ env.scheduler.remove_scope_memory_jobs.assert_awaited_once_with(scope=EXPERT)
+
+
+async def test_a_failed_pause_write_leaves_the_crons_alone(env):
+ await registry.ensure_scope_scheduled(EXPERT)
+ env.db.fail = True
+
+ assert not await registry.pause_scope(EXPERT)
+
+ env.scheduler.remove_scope_memory_jobs.assert_not_awaited()
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+
+
+async def test_a_pause_whose_job_removal_fails_reports_false(env):
+ """The row says PAUSED, so the cron gate already holds the jobs back,
+ but the pause did not finish and must say so."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ env.scheduler.remove_scope_memory_jobs.side_effect = RuntimeError("down")
+
+ assert not await registry.pause_scope(EXPERT)
+
+ assert env.db.rows[EXPERT.scope_key].state == PAUSED
+
+
+async def test_a_pause_racing_a_first_registration_still_ends_paused(env, monkeypatch):
+ """A registration claims the row ACTIVE between the pause's first write
+ (no row yet) and its claim: the pause must set PAUSED explicitly."""
+ claim = env.db.claim_scope_schedule
+
+ async def claimed_active_first(user_id, scope_key, expert_id, tz, state=ACTIVE):
+ await claim(user_id, scope_key, expert_id, tz) # the registration wins
+ return await claim(user_id, scope_key, expert_id, tz, state)
+
+ monkeypatch.setattr(env.db, "claim_scope_schedule", claimed_active_first)
+
+ assert await registry.pause_scope(EXPERT)
+
+ assert env.db.rows[EXPERT.scope_key].state == PAUSED
+ env.scheduler.remove_scope_memory_jobs.assert_awaited_once_with(scope=EXPERT)
+
+
+async def test_a_pause_whose_claim_fails_still_sets_the_state(env, monkeypatch):
+ """A claim that fails after a concurrent registration created the row is
+ not the end: the pause re-reads by writing the state itself."""
+ claim = env.db.claim_scope_schedule
+
+ async def created_elsewhere_then_failed(user_id, scope_key, expert_id, tz, state):
+ await claim(user_id, scope_key, expert_id, tz)
+ raise ConnectionError("lost the response")
+
+ monkeypatch.setattr(env.db, "claim_scope_schedule", created_elsewhere_then_failed)
+
+ assert await registry.pause_scope(EXPERT)
+
+ assert env.db.rows[EXPERT.scope_key].state == PAUSED
+
+
+async def test_wipe_removes_the_jobs_and_resume_does_not_undo_it(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert await registry.mark_wiped(ACCOUNT)
+
+ assert env.db.rows[USER].state == WIPED
+ assert env.scheduler.jobs == {}
+ skipped = await registry.ensure_scope_scheduled(ACCOUNT)
+ assert {r["reason"] for r in skipped.values()} == {
+ "scope_wiped",
+ "skill_learning_disabled",
+ }
+
+ resumed = await registry.resume_scope(ACCOUNT)
+
+ assert {r["reason"] for r in resumed.values()} == {
+ "scope_wiped",
+ "skill_learning_disabled",
+ }
+ assert env.db.rows[USER].state == WIPED
+ assert env.scheduler.jobs == {}
+
+
+async def test_a_resume_landing_inside_a_pause_gets_its_crons_back_next_pass(
+ env, monkeypatch
+):
+ """The pause writes PAUSED; before its removal lands, a resume writes
+ ACTIVE and registers the crons, whose job ids are the scope's own. The
+ removal takes them, leaving the row ACTIVE with its ids recorded and no
+ crons; the pause reports what it did, and the reconcile pass registers
+ them again."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ remove = registry.remove_scope_jobs
+
+ async def resumed_first(scope):
+ monkeypatch.setattr(registry, "remove_scope_jobs", remove)
+ await registry.resume_scope(scope)
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+ return await remove(scope)
+
+ monkeypatch.setattr(registry, "remove_scope_jobs", resumed_first)
+
+ assert await registry.pause_scope(EXPERT)
+ assert env.db.rows[EXPERT.scope_key].state == ACTIVE
+ assert env.scheduler.jobs == {}
+
+ await env.reconcile()
+
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+
+
+async def test_a_resume_after_a_registration_found_the_scope_paused_keeps_its_crons(
+ env, monkeypatch
+):
+ """The round-5 review's stale takedown: a registration records its crons
+ after a pause landed (``scope_changed``), and a resume lands right
+ after. The registration takes nothing down, so the resume's crons stay,
+ recorded on the ACTIVE row, and the next reconcile pass has nothing to
+ change."""
+ record = env.db.record_scope_jobs
+ calls = 0
+
+ async def paused_meanwhile(*args, **kwargs):
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ env.db.update(EXPERT.scope_key, state=PAUSED)
+ result = await record(*args, **kwargs)
+ await registry.resume_scope(EXPERT)
+ return result
+ return await record(*args, **kwargs)
+
+ monkeypatch.setattr(env.db, "record_scope_jobs", paused_meanwhile)
+
+ result = await registry.ensure_scope_scheduled(EXPERT)
+
+ assert {r["reason"] for r in result.values()} == {"scope_changed"}
+ row = env.db.rows[EXPERT.scope_key]
+ assert row.state == ACTIVE
+ assert {row.community_job_id, row.nightly_job_id} == job_ids(EXPERT)
+ assert set(env.scheduler.jobs) == job_ids(EXPERT), "the resume's crons stay"
+
+ report = await env.reconcile()
+
+ assert (report.restored, report.removed, report.dropped) == ([], [], [])
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+
+
+async def test_a_resume_that_cannot_write_the_row_reports_only_enabled_crons_failed(
+ env, monkeypatch
+):
+ """The backfill lists every cron a resume reports unavailable as a
+ failure: a cron the owner's flags turn off is reported as its flag says,
+ as an ensure would, not as a failure."""
+ env.flags[Flag.GRAPHITI_COMMUNITIES_ENABLED] = False
+ monkeypatch.setattr(
+ env.db, "set_scope_state", AsyncMock(side_effect=ConnectionError("db down"))
+ )
+
+ resumed = await registry.resume_scope(EXPERT)
+
+ assert resumed == {
+ "community_rebuild": {
+ "skipped": True,
+ "reason": "graphiti_communities_disabled",
+ },
+ "dream_nightly_batch": {"skipped": True, "reason": "registry_unavailable"},
+ }
+
+
+async def test_a_revive_of_a_wiped_expert_leaves_it_wiped(env):
+ await registry.mark_wiped(EXPERT)
+
+ await registry.sync_expert_scope(USER, "expert-1", active=True)
+
+ assert env.db.rows[EXPERT.scope_key].state == WIPED
+ assert env.adds() == 0
+
+
+async def test_a_wiped_scope_stays_wiped_through_a_pause_and_a_resume(env):
+ """Codex's round-2 reproduction: a pause (or archive) of a wiped expert
+ used to write PAUSED over WIPED, which the next resume (or revive) then
+ turned back into ACTIVE with both crons. The pause now leaves WIPED and
+ only clears any cron that straggled."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ assert await registry.mark_wiped(EXPERT)
+ env.scheduler.jobs = {job_id: PARIS for job_id in job_ids(EXPERT)}
+
+ await registry.sync_expert_scope(USER, "expert-1", active=False)
+
+ assert env.db.rows[EXPERT.scope_key].state == WIPED
+ assert env.scheduler.jobs == {}
+ await registry.sync_expert_scope(USER, "expert-1", active=True)
+ assert env.db.rows[EXPERT.scope_key].state == WIPED
+ assert env.scheduler.jobs == {}
+
+
+async def test_a_pause_that_loses_its_claim_to_a_wipe_keeps_it_wiped(env, monkeypatch):
+ """The pause finds no row, and before its claim lands a wipe creates
+ the row WIPED: the pause's fallback write must not demote it either."""
+ claim = env.db.claim_scope_schedule
+
+ async def wiped_first(user_id, scope_key, expert_id, tz, state):
+ await claim(user_id, scope_key, expert_id, tz, WIPED)
+ return await claim(user_id, scope_key, expert_id, tz, state)
+
+ monkeypatch.setattr(env.db, "claim_scope_schedule", wiped_first)
+
+ assert await registry.pause_scope(EXPERT)
+
+ assert env.db.rows[EXPERT.scope_key].state == WIPED
+
+
+async def test_wiping_an_account_without_a_row_removes_its_legacy_crons(env):
+ env.scheduler.jobs = {job_id: PARIS for job_id in job_ids(ACCOUNT)}
+
+ assert await registry.mark_wiped(ACCOUNT)
+
+ assert env.db.rows[USER].state == WIPED
+ assert env.scheduler.jobs == {}
+
+
+async def test_timezone_change_re_registers_every_active_scope(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ await registry.ensure_scope_scheduled(EXPERT)
+ await registry.pause_scope(OTHER_EXPERT)
+ env.timezone = TOKYO
+
+ results = await registry.reregister_user(USER)
+
+ assert set(results) == {
+ ACCOUNT.scope_key,
+ EXPERT.scope_key,
+ OTHER_EXPERT.scope_key,
+ }
+ paused = results[OTHER_EXPERT.scope_key]
+ assert {r["reason"] for r in paused.values()} == {"scope_paused"}
+ assert set(env.scheduler.jobs) == job_ids(ACCOUNT) | job_ids(EXPERT)
+ assert set(env.scheduler.jobs.values()) == {TOKYO}
+ assert env.db.rows[EXPERT.scope_key].timezone == TOKYO
+
+
+async def test_timezone_change_of_a_flag_off_owner_costs_no_database_call(env):
+ env.flags = {}
+ env.db.fail = True # any read would raise
+
+ results = await registry.reregister_user(USER)
+
+ assert set(results) == {USER}
+ assert {r["reason"] for r in results[USER].values()} == {
+ "graphiti_communities_disabled",
+ "dream_pass_disabled",
+ "skill_learning_disabled",
+ }
+
+
+async def test_timezone_change_re_registers_a_legacy_account_without_row(env):
+ results = await registry.reregister_user(USER)
+
+ assert list(results) == [USER]
+ assert set(env.scheduler.jobs) == job_ids(ACCOUNT)
diff --git a/autogpt_platform/backend/backend/copilot/dream/registry_test.py b/autogpt_platform/backend/backend/copilot/dream/registry_test.py
new file mode 100644
index 0000000000..475d8b95ad
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/registry_test.py
@@ -0,0 +1,343 @@
+"""Tests for registering a scope's crons: ``ensure_scope_scheduled`` and the
+work it hands to ``scope_jobs.py``.
+
+The data module and the scheduler client are in-memory fakes with the real
+ownership rules (``registry_fakes_test.py``). Lifecycle changes are in
+``registry_lifecycle_test.py``, deadlines in ``registry_deadline_test.py``.
+"""
+
+from __future__ import annotations
+
+import logging
+from unittest.mock import AsyncMock
+
+import pytest
+
+from backend.util.feature_flag import Flag
+
+from . import registry, scope_crons, scope_jobs
+from .registry_fakes_test import (
+ ACCOUNT,
+ ACTIVE,
+ EXPERT,
+ PARIS,
+ PAUSED,
+ TOKYO,
+ USER,
+ RegistryEnv,
+ install_registry_env,
+ job_ids,
+)
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ return install_registry_env(monkeypatch)
+
+
+async def test_ensure_registers_both_crons_under_the_legacy_account_ids(env):
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ # The account's scope key is its user id: job ids did not move.
+ assert result["community_rebuild"]["id"] == f"community_rebuild_{USER}"
+ assert result["dream_nightly_batch"]["id"] == f"dream_nightly_batch_{USER}"
+ env.scheduler.add_scope_nightly_batch_schedule.assert_awaited_once_with(
+ scope=ACCOUNT, user_timezone=PARIS
+ )
+ row = env.db.rows[USER]
+ assert (row.state, row.timezone, row.expert_id) == (ACTIVE, PARIS, None)
+ assert {row.community_job_id, row.nightly_job_id} == job_ids(ACCOUNT)
+
+
+async def test_ensure_is_idempotent(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ again = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert again == {
+ "community_rebuild": None,
+ "dream_nightly_batch": None,
+ "skill_learning_nightly": {
+ "skipped": True,
+ "reason": "skill_learning_disabled",
+ },
+ }
+ assert env.adds() == 2
+
+
+async def test_exactly_one_row_per_scope(env):
+ for _ in range(3):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ await registry.ensure_scope_scheduled(EXPERT)
+
+ assert sorted(env.db.rows) == sorted([ACCOUNT.scope_key, EXPERT.scope_key])
+ assert env.db.rows[EXPERT.scope_key].expert_id == "expert-1"
+ assert set(env.scheduler.jobs) == job_ids(ACCOUNT) | job_ids(EXPERT)
+ assert env.adds() == 4
+
+
+async def test_an_expert_write_registers_the_expert_not_the_account(env):
+ result = await registry.ensure_scope_scheduled(EXPERT)
+
+ nightly = f"dream_nightly_batch_{EXPERT.scope_key}"
+ assert result["dream_nightly_batch"]["id"] == nightly
+ assert ACCOUNT.scope_key not in env.db.rows
+
+
+async def test_flags_off_cost_no_read_and_no_rpc(env):
+ env.flags = {}
+ env.db.fail = True # any read would raise
+
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert result == {
+ "community_rebuild": {
+ "skipped": True,
+ "reason": "graphiti_communities_disabled",
+ },
+ "dream_nightly_batch": {"skipped": True, "reason": "dream_pass_disabled"},
+ "skill_learning_nightly": {
+ "skipped": True,
+ "reason": "skill_learning_disabled",
+ },
+ }
+ assert env.adds() == 0
+
+
+async def test_one_flag_off_still_registers_the_other(env):
+ env.flags[Flag.GRAPHITI_COMMUNITIES_ENABLED] = False
+
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert result["community_rebuild"]["reason"] == "graphiti_communities_disabled"
+ assert env.db.rows[USER].nightly_job_id == f"dream_nightly_batch_{USER}"
+ assert env.db.rows[USER].community_job_id is None
+
+
+async def test_a_paused_scope_is_never_registered_by_ensure(env):
+ await registry.pause_scope(EXPERT)
+
+ result = await registry.ensure_scope_scheduled(EXPERT)
+
+ assert all(
+ r == {"skipped": True, "reason": "scope_paused"} for r in result.values()
+ )
+ assert env.adds() == 0
+
+
+async def test_timezone_change_re_registers_in_the_new_zone(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.timezone = TOKYO
+
+ await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert env.adds() == 4
+ assert env.db.rows[USER].timezone == TOKYO
+ assert set(env.scheduler.jobs.values()) == {TOKYO}
+
+
+ "scope,prefix",
+ [
+ (ACCOUNT, "community_rebuild"),
+ (EXPERT, "community_rebuild"),
+ (ACCOUNT, "dream_nightly_batch"),
+ (EXPERT, "dream_nightly_batch"),
+ (ACCOUNT, "skill_learning_nightly"),
+ ],
+)
+async def test_failed_timezone_refresh_keeps_the_cron_repairable(env, scope, prefix):
+ env.flags[Flag.DREAM_SKILL_LEARNING_ENABLED] = True
+ await registry.ensure_scope_scheduled(scope)
+ add = {
+ "community_rebuild": env.scheduler.add_scope_community_rebuild_schedule,
+ "dream_nightly_batch": env.scheduler.add_scope_nightly_batch_schedule,
+ "skill_learning_nightly": env.scheduler.add_scope_skill_learning_schedule,
+ }[prefix]
+ adder = add.side_effect
+ add.side_effect = RuntimeError("scheduler down")
+ env.timezone = TOKYO
+
+ result = await registry.ensure_scope_scheduled(scope)
+
+ job_id = f"{prefix}_{scope.scope_key}"
+ assert result[prefix]["reason"] == "registration_failed"
+ assert env.db.rows[scope.scope_key].timezone == TOKYO
+ assert env.scheduler.jobs[job_id] == PARIS
+ assert job_id not in env.scheduler.tagged
+ add.side_effect = adder
+
+ report = await env.reconcile()
+
+ assert report.restored == [job_id]
+ assert report.failed == []
+ assert env.scheduler.jobs[job_id] == TOKYO
+ assert (await env.reconcile()).restored == []
+
+
+async def test_unknown_timezone_is_not_utc_and_changes_nothing(env):
+ env.timezone = None
+
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert {r["reason"] for r in result.values()} == {
+ "timezone_lookup_failed",
+ "skill_learning_disabled",
+ }
+ assert env.db.rows == {}
+ assert env.adds() == 0
+
+
+async def test_registry_read_failure_skips_the_cycle(env):
+ env.db.fail = True
+
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert {r["reason"] for r in result.values()} == {
+ "registry_unavailable",
+ "skill_learning_disabled",
+ }
+ assert env.adds() == 0
+
+
+async def test_a_failed_registration_is_not_recorded_and_is_retried(env):
+ add = env.scheduler.add_scope_community_rebuild_schedule
+ adder = add.side_effect
+ add.side_effect = RuntimeError("scheduler down")
+
+ first = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert first["community_rebuild"] == {
+ "skipped": True,
+ "reason": "registration_failed",
+ }
+ assert first["dream_nightly_batch"]["id"] == f"dream_nightly_batch_{USER}"
+ assert env.db.rows[USER].community_job_id is None
+
+ add.side_effect = adder
+ second = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert second["community_rebuild"]["id"] == f"community_rebuild_{USER}"
+ assert second["dream_nightly_batch"] is None
+
+
+async def test_a_registration_the_row_cannot_record_is_a_failure(env, monkeypatch):
+ """The jobs exist but the row does not know them: report it, so the
+ backfill counts the scope failed, and the next ensure registers again."""
+
+ async def record_fails(*args, **kwargs):
+ raise ConnectionError("db down")
+
+ monkeypatch.setattr(env.db, "record_scope_jobs", record_fails)
+
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert {r["reason"] for r in result.values()} == {
+ "record_failed",
+ "skill_learning_disabled",
+ }
+ assert env.db.rows[USER].nightly_job_id is None
+
+
+async def test_a_scheduler_side_flag_skip_is_not_recorded(env, caplog):
+ skipped = {"id": None, "skipped": True, "reason": "dream_pass_disabled"}
+ env.scheduler.add_scope_nightly_batch_schedule.side_effect = None
+ env.scheduler.add_scope_nightly_batch_schedule.return_value = skipped
+
+ with caplog.at_level(logging.WARNING, logger=scope_jobs.logger.name):
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert result["dream_nightly_batch"] == skipped
+ assert env.db.rows[USER].nightly_job_id is None
+ assert any("despite the local flag check" in r.getMessage() for r in caplog.records)
+
+
+async def test_a_pause_that_lands_mid_registration_leaves_its_crons_to_the_reconcile(
+ env,
+):
+ """The registration finds the scope paused when it records its crons: it
+ reports ``scope_changed`` and takes nothing down. The fire gate holds
+ the paused scope, and the next reconcile pass removes its crons."""
+ add = env.scheduler.add_scope_nightly_batch_schedule
+ adder = add.side_effect
+
+ async def add_then_archive(*, scope, user_timezone):
+ added = await adder(scope=scope, user_timezone=user_timezone)
+ env.db.update(scope.scope_key, state=PAUSED)
+ return added
+
+ add.side_effect = add_then_archive
+ result = await registry.ensure_scope_scheduled(EXPERT)
+
+ # Not a failure: the scope simply stopped wanting its crons.
+ assert {r["reason"] for r in result.values()} == {"scope_changed"}
+ env.scheduler.remove_scope_memory_jobs.assert_not_awaited()
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+ assert not await scope_crons.memory_scope_may_fire(
+ EXPERT, AsyncMock(return_value="active")
+ )
+
+ report = await env.reconcile()
+
+ assert report.removed == [EXPERT.scope_key]
+ assert env.scheduler.jobs == {}
+
+
+async def test_ensure_dream_system_scheduled_is_the_account_scope(env):
+ await registry.ensure_dream_system_scheduled(USER)
+
+ assert list(env.db.rows) == [USER]
+ assert await registry.ensure_dream_system_scheduled("") == {}
+
+
+async def test_timezone_is_durable_before_scheduler_replacement(env, monkeypatch):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.timezone = TOKYO
+ record = env.db.record_scope_jobs
+ calls = 0
+
+ async def fail_final_record(*args, **kwargs):
+ nonlocal calls
+ calls += 1
+ if calls == 2:
+ raise ConnectionError("final record failed")
+ return await record(*args, **kwargs)
+
+ monkeypatch.setattr(env.db, "record_scope_jobs", fail_final_record)
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+ assert calls == 2
+ assert env.db.rows[USER].timezone == TOKYO
+ assert result["community_rebuild"]["reason"] == "record_failed"
+ assert env.db.rows[USER].community_job_id == f"community_rebuild_{USER}"
+
+
+async def test_failed_timezone_intent_does_not_replace_crons(env, monkeypatch):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.timezone = TOKYO
+ env.scheduler.add_scope_community_rebuild_schedule.reset_mock()
+ env.scheduler.add_scope_nightly_batch_schedule.reset_mock()
+ monkeypatch.setattr(
+ env.db, "record_scope_jobs", AsyncMock(side_effect=ConnectionError("db down"))
+ )
+ result = await registry.ensure_scope_scheduled(ACCOUNT)
+ assert result["community_rebuild"]["reason"] == "record_failed"
+ assert env.db.rows[USER].timezone == PARIS
+ env.scheduler.add_scope_community_rebuild_schedule.assert_not_awaited()
+ env.scheduler.add_scope_nightly_batch_schedule.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/copilot/dream/scheduling.py b/autogpt_platform/backend/backend/copilot/dream/scheduling.py
index 2d3daf11ca..b391790325 100644
--- a/autogpt_platform/backend/backend/copilot/dream/scheduling.py
+++ b/autogpt_platform/backend/backend/copilot/dream/scheduling.py
@@ -1,220 +1,197 @@
-"""Lazy auto-registration of per-user dream-system schedules.
+"""The dream-system cron table and the owner-timezone lookup.
-Two per-user APScheduler cron jobs cover the whole dream system through
-P12 (the cross-scope insight cron for P8 will add a third row when it
-builds):
+Two APScheduler cron jobs serve every memory scope (the account, or one
+hired expert), and a third the account alone:
- * ``community_rebuild_{user_id}`` — Sun 04:00 user-local (P-1.7),
- direct LLM (not batch), activity-gated inside the function.
- * ``dream_nightly_batch_{user_id}`` — daily 03:00 user-local,
+ * ``community_rebuild_{scope_key}`` — weekly, owner-local 04:00 on
+ Mondays (P-1.7): its crontab ``0 4 * * 0`` means Monday to APScheduler,
+ whose weekdays start at 0 = Monday. Direct LLM (not batch),
+ activity-gated inside the function.
+ * ``dream_nightly_batch_{scope_key}`` — daily 03:00 owner-local,
submits all nightly-batch-family work (dream pass, ratification
supersession sweep, plus future P2 / P3 / P4 / P11 stages).
-
-Both crons share the same registration helper. Adding a future cron
-(P8 cross-scope insight, P9 lucid dream queue if it grows beyond
-nightly batch, etc.) is a single row in :data:`DREAM_SYSTEM_JOBS`.
-
-Three layers of flag gating:
-
- 1. **Registration helper (this file)** — cheapest gate; LD flag
- check runs before the Redis CAS-style dedup so a flag-off user
- never burns an RPC or a Redis key.
- 2. **Scheduler ``@expose`` body** — defense-in-depth for direct
- callers (admin endpoint, ad-hoc scripts) that bypass this helper.
- 3. **Execution wrapper (``execute_*_sync``)** — runtime gate; if the
- flag flips off after registration, the scheduled job still fires
- but short-circuits before the body runs.
-
-**Timezone drift handling.** APScheduler binds the cron trigger to
-the timezone at job-creation time — a later ``User.timezone`` change
-silently leaves the cron firing at the old local time. To detect and
-recover from drift, the Redis dedup key stores the timezone the cron
-was registered with (not the literal ``"1"`` it stored historically).
-Every call to :func:`ensure_dream_system_scheduled` compares the
-stored value to the user's current timezone; on mismatch, re-registers
-via ``replace_existing=True``. The eager path (``force_refresh=True``)
-is invoked from the ``User.timezone`` update endpoint; the lazy path
-catches direct-DB / webhook / SSO writes that bypass the API.
-
-Per-job idempotency: each cron has its OWN Redis dedup key. Flipping
-a single flag from off→on after the other cron already registered
-must let the newly-enabled cron in.
-
-Per-job failure isolation: a scheduler RPC failure for one cron
-surfaces in this call's return dict but never blocks the other crons.
-
-Failures are logged at WARN and swallowed; the caller (the graphiti
-ingestion path) must never break because dream-system registration
-failed.
+ * ``skill_learning_nightly_{user_id}`` — daily 03:00 owner-local, the
+ account's skill-learning pass (``copilot/learning/nightly.py``), on its
+ own flag; one pass covers the account's skills and its experts'.
+
+The account's scope key is its user id, so its job ids are unchanged from
+when the crons were keyed per user. Registering, pausing and resuming them
+is ``registry.py``'s job; this module only says what the crons are. Adding a
+future cron (P8 cross-scope insight, ...) is a row in
+:data:`DREAM_SYSTEM_JOBS` plus a job-id column on ``MemoryScopeSchedule``.
+
+Three layers of flag gating, all on the scope owner's flags:
+
+ 1. **Registry** — the cheapest gate; runs before any database read, so
+ a flag-off user costs neither a query nor a scheduler RPC.
+ 2. **Scheduler ``@expose`` method** — defense-in-depth for direct
+ callers (admin endpoint, ad-hoc scripts) that bypass the registry.
+ 3. **Job body** — if the flag flips off after registration, the job
+ still fires but short-circuits before the work runs.
"""
from __future__ import annotations
import logging
-from typing import Any, Awaitable, Callable
+from typing import Any, Awaitable, Callable, Literal
from pydantic.dataclasses import dataclass
from backend.copilot.graphiti.scope import MemoryScope
-from backend.util.feature_flag import Flag, is_feature_enabled
+from backend.data.db_accessors import user_db
+from backend.data.model import USER_TIMEZONE_NOT_SET
+from backend.util.feature_flag import Flag
-logger = logging.getLogger(__name__)
+from .deadline import within_deadline
+logger = logging.getLogger(__name__)
-# Matches the longest cron cadence in the registry (weekly community
-# rebuild) so the lazy drift-detection path re-checks at least once
-# per cron-tick. Going longer leaves a window where Redis says
-# "registered with tz X" but the cron was deleted out-of-band; going
-# shorter wastes scheduler RPC calls when nothing changed.
-REGISTRATION_TTL_SECONDS = 7 * 24 * 3600
-# Redis dedup key prefixes, exported so the scheduler's delete_*
-# @expose methods can clear the matching marker when a cron is removed
-# in-band (see :func:`clear_registration_marker`). Must stay in sync
-# with the registry rows below — pinned by a test.
-COMMUNITY_REBUILD_REGISTRATION_PREFIX = "community_rebuild_registered"
-NIGHTLY_BATCH_REGISTRATION_PREFIX = "dream_nightly_batch_registered"
-SKILL_LEARNING_REGISTRATION_PREFIX = "skill_learning_nightly_registered"
+# Job-id prefixes, shared with the scheduler's ``@expose`` methods so the two
+# can never disagree on a job id.
+COMMUNITY_REBUILD_JOB_PREFIX = "community_rebuild"
+NIGHTLY_BATCH_JOB_PREFIX = "dream_nightly_batch"
+SKILL_LEARNING_JOB_PREFIX = "skill_learning_nightly"
-# A SchedulerClient is the caller's handle to the scheduler service.
-# We don't import the concrete type here to avoid a circular import
-# during the executor's own bootstrap; the helper just calls the
-# named coroutine on whatever the caller hands in.
+# A SchedulerClient is the caller's handle to the scheduler service. The
+# concrete type is not imported here to avoid a circular import during the
+# executor's own bootstrap; the table just calls the named coroutine on
+# whatever the caller hands in.
SchedulerLike = Any
@dataclass(frozen=True)
class DreamSystemJob:
- """One row of the dream-system schedule registry.
-
- The registry is the single source of truth for "what background
- crons does the dream system run per user". Adding a new cron
- (e.g. weekly cross-scope insight for P8) means appending a row
- here; the helper picks it up automatically, the scheduler call
- gets the right job_id, and the Redis dedup key falls out naturally.
- """
+ """One row of the dream-system cron table."""
name: str
"""Human-readable, used only for log messages."""
job_id_prefix: str
- """Job-id naming convention: ``f"{job_id_prefix}_{user_id}"``.
- Must match the scheduler ``@expose`` method's own job_id format."""
-
- registration_key_prefix: str
- """Redis dedup key prefix. Each cron has its OWN key so flipping
- a single flag mid-life lets only that cron re-enter the helper —
- a shared key would block recovery on flag drift."""
+ """Job ids are ``f"{job_id_prefix}_{scope_key}"``."""
flag: Flag
- """LD feature flag gate. Evaluated per-user before any other work."""
+ """LD feature flag gate, evaluated for the scope's owner."""
skip_reason: str
- """The ``reason`` string returned in a flag-off skip result. Keyed
- on the flag value so the helper's return dict is grep-able for
- "why did this user not get a schedule"."""
+ """The ``reason`` recorded when the flag is off, so "why does this scope
+ have no schedule" is grep-able."""
+
+ row_field: Literal["community_job_id", "nightly_job_id", "learning_job_id"]
+ """The ``MemoryScopeSchedule`` field holding this cron's job id."""
+
+ register: Callable[[SchedulerLike, MemoryScope, str], Awaitable[dict]]
+ """``(client, scope, owner_timezone) -> awaitable[result dict]``: the
+ SchedulerClient method that creates the cron job."""
- register: Callable[[SchedulerLike, str, str], Awaitable[dict]]
- """``(client, user_id, user_timezone) -> awaitable[result dict]``.
- The actual SchedulerClient method that creates the cron job."""
+ account_only: bool = False
+ """The cron runs for the account alone: an expert's scope has none."""
+
+ def job_id(self, scope: MemoryScope) -> str:
+ """This cron's APScheduler job id for ``scope``."""
+ return f"{self.job_id_prefix}_{scope.scope_key}"
+
+ def serves(self, scope: MemoryScope) -> bool:
+ """Whether ``scope`` has this cron at all."""
+ return not (self.account_only and scope.is_expert)
def _register_community_rebuild(
- client: SchedulerLike, user_id: str, user_timezone: str
+ client: SchedulerLike, scope: MemoryScope, user_timezone: str
) -> Awaitable[dict]:
- return client.add_community_rebuild_schedule(
- user_id=user_id, user_timezone=user_timezone
+ return client.add_scope_community_rebuild_schedule(
+ scope=scope, user_timezone=user_timezone
)
def _register_nightly_batch(
- client: SchedulerLike, user_id: str, user_timezone: str
+ client: SchedulerLike, scope: MemoryScope, user_timezone: str
) -> Awaitable[dict]:
- return client.add_nightly_batch_schedule(
- user_id=user_id, user_timezone=user_timezone
+ return client.add_scope_nightly_batch_schedule(
+ scope=scope, user_timezone=user_timezone
)
def _register_skill_learning(
- client: SchedulerLike, user_id: str, user_timezone: str
+ client: SchedulerLike, scope: MemoryScope, user_timezone: str
) -> Awaitable[dict]:
- return client.add_skill_learning_schedule(
- user_id=user_id, user_timezone=user_timezone
+ return client.add_scope_skill_learning_schedule(
+ scope=scope, user_timezone=user_timezone
)
-# The registry. Listed in cron-frequency order (rarest first) so the
-# log trail when a new user lands reads "weekly → daily" — the
-# narrative matches how the schedules build up over time. The future
-# P8 cross-scope cron (weekly, batch) will land between these two.
+# Listed in cron-frequency order (rarest first) so a new scope's log trail
+# reads "weekly → daily". The future P8 cross-scope cron (weekly, batch)
+# lands between the first two.
DREAM_SYSTEM_JOBS: list[DreamSystemJob] = [
DreamSystemJob(
name="Community rebuild",
- job_id_prefix="community_rebuild",
- registration_key_prefix=COMMUNITY_REBUILD_REGISTRATION_PREFIX,
+ job_id_prefix=COMMUNITY_REBUILD_JOB_PREFIX,
flag=Flag.GRAPHITI_COMMUNITIES_ENABLED,
skip_reason="graphiti_communities_disabled",
+ row_field="community_job_id",
register=_register_community_rebuild,
),
DreamSystemJob(
name="Dream nightly batch",
- job_id_prefix="dream_nightly_batch",
- # NOT shared with the now-removed individual dream/ratification
- # crons — those keys (``dream_pass_registered``,
- # ``ratification_pass_registered``) are orphaned by the
- # consolidation and naturally expire via their 7-day TTL.
- registration_key_prefix=NIGHTLY_BATCH_REGISTRATION_PREFIX,
- # The nightly batch cron carries dream pass + ratification
- # supersession + future P2/P3/P4/P11 work. All ride the same
- # master gate; finer-grained flags inside individual submitters
- # control whether each stage actually runs within the cron.
+ job_id_prefix=NIGHTLY_BATCH_JOB_PREFIX,
+ # One master gate for the dream pass, the ratification sweep and
+ # the future P2/P3/P4/P11 stages; finer flags inside individual
+ # submitters decide whether each stage runs within the cron.
flag=Flag.DREAM_PASS_ENABLED,
skip_reason="dream_pass_disabled",
+ row_field="nightly_job_id",
register=_register_nightly_batch,
),
DreamSystemJob(
name="Skill learning nightly",
- job_id_prefix="skill_learning_nightly",
- registration_key_prefix=SKILL_LEARNING_REGISTRATION_PREFIX,
+ job_id_prefix=SKILL_LEARNING_JOB_PREFIX,
# Its own gate: skill learning can run with the dream pass off and
# the dream pass never implies skill learning. Same 03:00
- # user-local cadence, separate lease, separate cost rows.
+ # user-local cadence, separate lease, separate cost rows. One pass
+ # per account reviews the account's and its experts' skills, so an
+ # expert's scope has no cron of its own.
flag=Flag.DREAM_SKILL_LEARNING_ENABLED,
skip_reason="skill_learning_disabled",
+ row_field="learning_job_id",
register=_register_skill_learning,
+ account_only=True,
),
]
-async def _resolve_user_timezone(user_id: str) -> str | None:
- """Look up the user's IANA timezone from Postgres.
+def jobs_for(scope: MemoryScope) -> list[DreamSystemJob]:
+ """The crons ``scope`` has: every row for the account, and those not
+ ``account_only`` for an expert."""
+ return [job for job in DREAM_SYSTEM_JOBS if job.serves(scope)]
- Returns ``"UTC"`` only when the answer is authoritative (user
- missing or timezone genuinely unset) and ``None`` when the lookup
- itself failed — a transient DB blip is "unknown", not "UTC", and
- must never silently re-register the user's local-time crons onto
- UTC.
- Single DB call — cached at the helper level for the duration of
- one ``ensure_dream_system_scheduled`` invocation so registering
- multiple crons doesn't take multiple round-trips.
+def dream_system_job(job_id_prefix: str) -> DreamSystemJob:
+ """The table row for ``job_id_prefix``; raises ``KeyError`` if unknown."""
+ for job in DREAM_SYSTEM_JOBS:
+ if job.job_id_prefix == job_id_prefix:
+ return job
+ raise KeyError(job_id_prefix)
+
- Routes through the ``user_db()`` accessor, NOT ``User.prisma()``:
- this runs in the copilot-executor and scheduler processes, which
- never connect a local Prisma client. A direct Prisma call raises
- ``ClientNotConnectedError`` on every invocation there — a
- *permanent* failure the keep-existing-schedules fallback was never
- designed for, which silently prevented dream crons from ever being
- registered. The accessor falls back to the DatabaseManager RPC in
- Prisma-less processes (same pattern as ``dream/apply.py``).
+async def resolve_user_timezone(user_id: str) -> str | None:
+ """The owner's IANA timezone, which every scope of theirs is scheduled in.
+
+ Returns ``"UTC"`` only when the answer is authoritative (user missing or
+ timezone genuinely unset) and ``None`` when the lookup itself failed — a
+ transient DB blip is "unknown", not "UTC", and must never silently
+ re-register the owner's local-time crons onto UTC.
+
+ Routes through the ``user_db()`` accessor, NOT ``User.prisma()``: this
+ runs in the copilot-executor and scheduler processes, which never connect
+ a local Prisma client, and the accessor falls back to the DatabaseManager
+ RPC there. The lookup runs under the registry deadline; a timeout is a
+ failed lookup.
"""
try:
- from backend.data.db_accessors import user_db
- from backend.data.model import USER_TIMEZONE_NOT_SET
-
try:
- user = await user_db().get_user_by_id(user_id)
+ user = await within_deadline(user_db().get_user_by_id(user_id))
except ValueError:
# Authoritative: the user row doesn't exist.
return "UTC"
@@ -230,237 +207,3 @@ async def _resolve_user_timezone(user_id: str) -> str | None:
exc_info=True,
)
return None
-
-
-def _registration_key(user_id: str, key_prefix: str) -> str:
- # Keyed on the owning user whichever memory scope triggered the write:
- # the dream-system crons are registered per user.
- return MemoryScope.for_user(user_id).redis_key(
- "registration", registration_prefix=key_prefix
- )
-
-
-async def _read_registration_tz(user_id: str, key_prefix: str) -> str | None:
- """Read the timezone the cron was last registered with.
-
- Returns:
- * The stored timezone string when the key exists.
- * ``None`` when the key is missing OR Redis is unavailable. The
- caller treats both as "needs registration" — scheduler-side
- ``replace_existing=True`` makes a redundant call a cheap no-op.
- """
- try:
- from backend.data.redis_client import get_redis_async
-
- redis = await get_redis_async()
- key = _registration_key(user_id, key_prefix)
- stored = await redis.get(key)
- if stored is None:
- return None
- if isinstance(stored, bytes):
- return stored.decode("utf-8", errors="replace")
- return str(stored)
- except Exception:
- logger.debug(
- "Redis read failed for %s:%s; treating as not-registered",
- key_prefix,
- user_id[:12],
- exc_info=True,
- )
- return None
-
-
-async def _write_registration_tz(
- user_id: str, key_prefix: str, current_tz: str
-) -> None:
- """Persist the timezone we just registered the cron with.
-
- Best-effort — a Redis write failure means the next call will see
- the key as missing and force a redundant re-register (cheap via
- ``replace_existing=True``).
- """
- try:
- from backend.data.redis_client import get_redis_async
-
- redis = await get_redis_async()
- key = _registration_key(user_id, key_prefix)
- await redis.set(key, current_tz, ex=REGISTRATION_TTL_SECONDS)
- except Exception:
- logger.debug(
- "Redis write failed for %s:%s; lazy path will re-detect later",
- key_prefix,
- user_id[:12],
- exc_info=True,
- )
-
-
-async def clear_registration_marker(user_id: str, key_prefix: str) -> None:
- """Delete the Redis registration marker for one dream-system cron.
-
- Called from the scheduler's ``delete_*_schedule`` @expose methods so
- an in-band cron removal immediately re-opens lazy registration
- instead of leaving the marker to block it for the remainder of its
- 7-day TTL. Single-key DEL so it routes on Redis Cluster.
-
- Best-effort — on Redis failure the marker simply expires via TTL,
- which is the pre-existing out-of-band-deletion recovery window.
- """
- try:
- from backend.data.redis_client import get_redis_async
-
- redis = await get_redis_async()
- await redis.delete(_registration_key(user_id, key_prefix))
- except Exception:
- logger.warning(
- "Redis delete failed for %s:%s; marker will expire via TTL",
- key_prefix,
- user_id[:12],
- exc_info=True,
- )
-
-
-async def ensure_dream_system_scheduled(
- user_id: str, *, force_refresh: bool = False
-) -> dict[str, Any]:
- """Idempotently register every flag-enabled dream-system cron for a user.
-
- Fire-and-forget callable from two trigger points:
-
- * **Lazy path** — called from the graphiti ingestion's
- ``_ensure_worker`` the first time we see a memory write for a
- user in this process. Drift-detects timezone changes via the
- Redis stored value and re-registers when the user's current
- timezone differs from the stored one.
- * **Eager path** — called with ``force_refresh=True`` from the
- ``User.timezone`` update endpoint so a profile change takes
- effect within a single APScheduler tick instead of waiting for
- the dedup key's 7-day TTL to expire.
-
- Walks :data:`DREAM_SYSTEM_JOBS`, gating each entry on its LD flag,
- then on per-job drift detection, then the actual scheduler RPC.
- Each step's failure is isolated; a single bad cron never blocks
- the others.
-
- Returns a dict keyed by ``job_id_prefix`` so callers can audit
- "what happened for this user this call":
-
- * ``None`` — already registered with the current timezone; no
- RPC made. (Lazy path's happy case.)
- * ``{"skipped": True, "reason": "<flag>_disabled"}`` — flag off.
- * ``{"skipped": True, "reason": "timezone_lookup_failed"}`` —
- timezone resolution failed; the existing cron and stored tz
- are left untouched until a later call succeeds.
- * ``{"skipped": True, "reason": "registration_failed"}`` — RPC
- raised; logged.
- * Anything else — the scheduler's own result dict (job id,
- next_run_time, etc.).
-
- Empty ``user_id`` → ``{}`` (no work, no error).
- """
- if not user_id:
- return {}
-
- results: dict[str, Any] = {}
- tz_cached: str | None = None
- tz_lookup_failed = False
- client_cached: SchedulerLike | None = None
-
- for job in DREAM_SYSTEM_JOBS:
- try:
- # Layer 1 of gating: the LD flag check. Cheapest of the
- # three; always run it first.
- if not await is_feature_enabled(job.flag, user_id):
- results[job.job_id_prefix] = {
- "skipped": True,
- "reason": job.skip_reason,
- }
- continue
-
- # Resolve current timezone once per invocation (single DB
- # call shared across enabled crons).
- if tz_cached is None and not tz_lookup_failed:
- tz_cached = await _resolve_user_timezone(user_id)
- tz_lookup_failed = tz_cached is None
-
- if tz_cached is None:
- # Lookup failed — "unknown" is not "UTC". Re-registering
- # would silently rebind the user's 03:00-local crons to
- # UTC; keep the existing cron and stored tz untouched
- # until a later call resolves the real timezone.
- results[job.job_id_prefix] = {
- "skipped": True,
- "reason": "timezone_lookup_failed",
- }
- continue
-
- # Drift detection (unless caller explicitly forced refresh).
- if not force_refresh:
- stored_tz = await _read_registration_tz(
- user_id, job.registration_key_prefix
- )
- if stored_tz == tz_cached:
- # Same tz, still within TTL → no work.
- results[job.job_id_prefix] = None
- continue
- if stored_tz is not None and stored_tz != tz_cached:
- logger.info(
- "Dream-system: timezone drift for user %s job %s "
- "(stored=%s, current=%s) — re-registering",
- user_id[:12],
- job.name,
- stored_tz,
- tz_cached,
- )
- # else: stored_tz is None → first registration OR
- # Redis was unavailable. Either way, register.
-
- # Lazy client handle so a fully-flag-off user never even
- # constructs the scheduler client.
- if client_cached is None:
- from backend.util.clients import get_scheduler_client
-
- client_cached = get_scheduler_client()
-
- result = await job.register(client_cached, user_id, tz_cached)
- if result.get("skipped"):
- # The scheduler's own layer-2 gate refused without
- # creating the job — layer 1 already said ON, so this
- # is always a disagreement (LD cold start in the
- # scheduler pod, transient LD error, targeting-context
- # divergence). Writing the marker here would leave
- # Redis claiming "registered" with no APScheduler job
- # behind it for the full 7-day TTL. Surface it and let
- # the next cycle retry.
- logger.warning(
- "Dream-system: scheduler skipped %s for user %s "
- "(reason=%s) despite the local flag check passing — "
- "not marking registered; next cycle retries",
- job.name,
- user_id[:12],
- result.get("reason"),
- )
- results[job.job_id_prefix] = result
- continue
- await _write_registration_tz(
- user_id, job.registration_key_prefix, tz_cached
- )
- logger.info(
- "Dream-system: registered %s for user %s (tz=%s)",
- job.name,
- user_id[:12],
- tz_cached,
- )
- results[job.job_id_prefix] = result
- except Exception:
- logger.warning(
- "Dream-system: failed to register %s for user %s",
- job.name,
- user_id[:12],
- exc_info=True,
- )
- results[job.job_id_prefix] = {
- "skipped": True,
- "reason": "registration_failed",
- }
-
- return results
diff --git a/autogpt_platform/backend/backend/copilot/dream/scheduling_test.py b/autogpt_platform/backend/backend/copilot/dream/scheduling_test.py
index 566debb080..7ce1cfbc3e 100644
--- a/autogpt_platform/backend/backend/copilot/dream/scheduling_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/scheduling_test.py
@@ -1,97 +1,51 @@
-"""Registry-driven dream-system auto-registration tests.
+"""Tests for the dream-system cron table and the owner-timezone lookup.
Contracts pinned here:
- 1. Registry shape — 2 rows (community + nightly batch), distinct
- dedup keys, distinct flags.
- 2. Per-job flag gating — flag-off → recorded as skipped with the
- right reason; scheduler RPC never called.
- 3. Per-job idempotency — each cron has its own Redis dedup key so
- flipping one flag mid-life only re-enters that cron.
- 4. Per-job failure isolation — one cron's scheduler RPC failure
- doesn't block the other cron.
- 5. Timezone drift detection — Redis dedup key stores the
- registered timezone; on mismatch with current timezone, force
- re-register. ``force_refresh=True`` skips the drift check and
- always re-registers.
- 6. Shared per-call caching — timezone resolution happens at most
- once per invocation.
- 7. Timezone lookup failure is "unknown", not "UTC" — a transient
- DB blip must never re-register the user's local-time crons onto
- UTC; the cycle is skipped and the existing cron left untouched.
+ 1. Table shape — 3 rows (community, nightly batch, skill learning),
+ distinct flags, distinct registry columns; skill learning is the
+ account's alone.
+ 2. Job ids are keyed by scope, and the account's are the per-user ids
+ the crons had before they were per scope.
+ 3. Each row registers through the scheduler's scope-keyed method.
+ 4. Timezone lookup failure is "unknown", not "UTC" — a transient DB blip
+ must never re-register the owner's local-time crons onto UTC.
+
+Registration itself (ensure, pause, resume, wipe) is pinned in
+``registry_test.py``.
"""
from __future__ import annotations
-import logging
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.model import USER_TIMEZONE_NOT_SET
from backend.util.feature_flag import Flag
-from . import scheduling
+from . import deadline, scheduling
+from .registry_fakes_test import Hang
from .scheduling import (
DREAM_SYSTEM_JOBS,
- REGISTRATION_TTL_SECONDS,
- ensure_dream_system_scheduled,
+ dream_system_job,
+ jobs_for,
+ resolve_user_timezone,
)
-# Paths used by the helper — patched at the module level since the
-# helper does lazy imports of the scheduler client + Redis client to
-# avoid bootstrap circular imports.
-_PATH_FLAG = "backend.copilot.dream.scheduling.is_feature_enabled"
-_PATH_TZ = "backend.copilot.dream.scheduling._resolve_user_timezone"
-_PATH_READ_TZ = "backend.copilot.dream.scheduling._read_registration_tz"
-_PATH_WRITE_TZ = "backend.copilot.dream.scheduling._write_registration_tz"
-_PATH_CLIENT = "backend.util.clients.get_scheduler_client"
-
-
-def _mock_scheduler_client(*, fail_jobs: tuple[str, ...] = ()) -> MagicMock:
- """SchedulerClient stub returning a documented dict per call.
-
- ``fail_jobs`` = tuple of job_id_prefix strings that should raise
- instead of returning success — for per-job failure isolation tests.
- """
- client = MagicMock()
-
- def _maker(prefix: str):
- if prefix in fail_jobs:
- return AsyncMock(side_effect=RuntimeError(f"{prefix} scheduler down"))
- return AsyncMock(
- return_value={
- "id": f"{prefix}_abc",
- "user_id": "abc",
- "next_run_time": None,
- }
- )
-
- client.add_community_rebuild_schedule = _maker("community_rebuild")
- client.add_nightly_batch_schedule = _maker("dream_nightly_batch")
- client.add_skill_learning_schedule = _maker("skill_learning_nightly")
- return client
-
-
-def _flag_mock(flag_map: dict[Flag, bool]) -> AsyncMock:
- async def fake(flag: Flag, user_id: str) -> bool:
- return flag_map.get(flag, False)
-
- return AsyncMock(side_effect=fake)
-
-
-def _all_flags_on() -> dict[Flag, bool]:
- return {
- Flag.GRAPHITI_COMMUNITIES_ENABLED: True,
- Flag.DREAM_PASS_ENABLED: True,
- }
+_PATH_USER_DB = "backend.copilot.dream.scheduling.user_db"
+USER = "abc"
+ACCOUNT = MemoryScope.for_user(USER)
+EXPERT = MemoryScope.for_expert(USER, "expert-1")
# ---------------------------------------------------------------------------
-# Registry shape — guard against accidental edits
+# Table shape — guard against accidental edits
# ---------------------------------------------------------------------------
-def test_registry_contains_three_jobs_in_cron_frequency_order():
+def test_table_contains_three_jobs_in_cron_frequency_order():
"""Weekly community first, then the daily crons — the order matches
how schedules build up over time and reads naturally in the log
narrative."""
@@ -103,342 +57,111 @@ def test_registry_contains_three_jobs_in_cron_frequency_order():
]
-def test_each_job_has_distinct_registration_key_prefix():
- """Per-job dedup keys MUST be distinct — a shared key would block
- recovery when a flag flips off→on for one cron but not the other."""
- keys = [j.registration_key_prefix for j in DREAM_SYSTEM_JOBS]
- assert len(set(keys)) == len(keys)
+def test_each_job_has_its_own_registry_column():
+ fields = [j.row_field for j in DREAM_SYSTEM_JOBS]
+ assert fields == ["community_job_id", "nightly_job_id", "learning_job_id"]
def test_community_and_nightly_batch_have_distinct_flags():
"""Community rebuild and nightly batch are independent features
behind separate LD flags. A user can have one enabled without
- the other."""
- community = next(
- j for j in DREAM_SYSTEM_JOBS if j.job_id_prefix == "community_rebuild"
- )
- nightly = next(
- j for j in DREAM_SYSTEM_JOBS if j.job_id_prefix == "dream_nightly_batch"
+ the other. (Skill learning's own flag is pinned below.)"""
+ assert dream_system_job("community_rebuild").flag == (
+ Flag.GRAPHITI_COMMUNITIES_ENABLED
)
- assert community.flag == Flag.GRAPHITI_COMMUNITIES_ENABLED
- assert nightly.flag == Flag.DREAM_PASS_ENABLED
+ assert dream_system_job("dream_nightly_batch").flag == Flag.DREAM_PASS_ENABLED
-def test_skill_learning_is_gated_independently_of_the_dream_pass():
- """Enabling skill learning must not imply the dream pass (or vice
- versa): its own flag, its own cron id, its own dedup key."""
- learning = next(
- j for j in DREAM_SYSTEM_JOBS if j.job_id_prefix == "skill_learning_nightly"
- )
- assert learning.flag == Flag.DREAM_SKILL_LEARNING_ENABLED
- assert learning.skip_reason == "skill_learning_disabled"
- assert (
- learning.registration_key_prefix
- == scheduling.SKILL_LEARNING_REGISTRATION_PREFIX
- )
+def test_unknown_job_prefix_raises():
+ with pytest.raises(KeyError):
+ dream_system_job("morning_briefing")
-async def test_skill_learning_registers_with_dream_pass_off():
- client = _mock_scheduler_client()
- flag_map = {Flag.DREAM_SKILL_LEARNING_ENABLED: True}
- with patch(_PATH_FLAG, new=_flag_mock(flag_map)), patch(
- _PATH_TZ, new=AsyncMock(return_value="Europe/Berlin")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc")
-
- assert result["skill_learning_nightly"]["id"] == "skill_learning_nightly_abc"
- assert result["dream_nightly_batch"] == {
- "skipped": True,
- "reason": "dream_pass_disabled",
- }
- client.add_skill_learning_schedule.assert_awaited_once_with(
- user_id="abc", user_timezone="Europe/Berlin"
- )
- client.add_nightly_batch_schedule.assert_not_called()
-
-
-# ---------------------------------------------------------------------------
-# Empty / null user_id
-# ---------------------------------------------------------------------------
-
-
-async def test_empty_user_id_returns_empty_dict_without_running():
- result = await ensure_dream_system_scheduled("")
- assert result == {}
-
-
-# ---------------------------------------------------------------------------
-# Happy path — both flags on, no existing registration
-# ---------------------------------------------------------------------------
-
-
-async def test_both_crons_register_when_their_flags_are_on():
- client = _mock_scheduler_client()
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value="America/New_York")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc")
-
- assert result["community_rebuild"]["id"] == "community_rebuild_abc"
- assert result["dream_nightly_batch"]["id"] == "dream_nightly_batch_abc"
-
- # Both scheduler calls threaded the same resolved timezone.
- client.add_community_rebuild_schedule.assert_awaited_once_with(
- user_id="abc", user_timezone="America/New_York"
- )
- client.add_nightly_batch_schedule.assert_awaited_once_with(
- user_id="abc", user_timezone="America/New_York"
- )
-
-
-# ---------------------------------------------------------------------------
-# Layer 1 — flag gate
-# ---------------------------------------------------------------------------
-
-
-async def test_community_flag_off_skips_only_community():
- """One flag off → only that cron is skipped; the other registers."""
- client = _mock_scheduler_client()
- flag_map = _all_flags_on()
- flag_map[Flag.GRAPHITI_COMMUNITIES_ENABLED] = False
- with patch(_PATH_FLAG, new=_flag_mock(flag_map)), patch(
- _PATH_TZ, new=AsyncMock(return_value="UTC")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc")
-
- assert result["community_rebuild"] == {
- "skipped": True,
- "reason": "graphiti_communities_disabled",
- }
- assert result["dream_nightly_batch"]["id"] == "dream_nightly_batch_abc"
- client.add_community_rebuild_schedule.assert_not_called()
+def test_exported_prefixes_match_table_rows():
+ """The scheduler imports these constants for its job ids: drift would
+ register or remove the wrong job."""
+ assert [j.job_id_prefix for j in DREAM_SYSTEM_JOBS] == [
+ scheduling.COMMUNITY_REBUILD_JOB_PREFIX,
+ scheduling.NIGHTLY_BATCH_JOB_PREFIX,
+ scheduling.SKILL_LEARNING_JOB_PREFIX,
+ ]
-async def test_dream_flag_off_skips_only_nightly_batch():
- client = _mock_scheduler_client()
- flag_map = _all_flags_on()
- flag_map[Flag.DREAM_PASS_ENABLED] = False
- with patch(_PATH_FLAG, new=_flag_mock(flag_map)), patch(
- _PATH_TZ, new=AsyncMock(return_value="UTC")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc")
-
- assert result["community_rebuild"]["id"] == "community_rebuild_abc"
- assert result["dream_nightly_batch"] == {
- "skipped": True,
- "reason": "dream_pass_disabled",
- }
- client.add_nightly_batch_schedule.assert_not_called()
+def test_skill_learning_is_gated_independently_of_the_dream_pass():
+ """Enabling skill learning must not imply the dream pass (or vice
+ versa): its own flag, its own cron id, its own registry column."""
+ learning = dream_system_job("skill_learning_nightly")
+ assert learning.flag == Flag.DREAM_SKILL_LEARNING_ENABLED
+ assert learning.skip_reason == "skill_learning_disabled"
+ assert learning.row_field == "learning_job_id"
-async def test_all_flags_off_returns_all_skipped_no_rpc_no_tz_lookup():
- """When nothing is enabled, don't waste a DB lookup for tz or an
- RPC to the scheduler service."""
- client = _mock_scheduler_client()
- tz_mock = AsyncMock(return_value="UTC")
- with patch(_PATH_FLAG, new=_flag_mock({})), patch(_PATH_TZ, new=tz_mock), patch(
- _PATH_READ_TZ, new=AsyncMock(return_value=None)
- ), patch(_PATH_WRITE_TZ, new=AsyncMock()), patch(_PATH_CLIENT, return_value=client):
- result = await ensure_dream_system_scheduled("abc")
-
- assert all(r.get("skipped") is True for r in result.values()) # type: ignore[union-attr]
- tz_mock.assert_not_called()
- client.add_community_rebuild_schedule.assert_not_called()
- client.add_nightly_batch_schedule.assert_not_called()
+def test_skill_learning_is_the_accounts_alone():
+ """One pass reviews the account's skills and its experts': an expert's
+ scope has no skill-learning cron, and the account has all three."""
+ assert [j.job_id_prefix for j in jobs_for(ACCOUNT)] == [
+ "community_rebuild",
+ "dream_nightly_batch",
+ "skill_learning_nightly",
+ ]
+ assert [j.job_id_prefix for j in jobs_for(EXPERT)] == [
+ "community_rebuild",
+ "dream_nightly_batch",
+ ]
+ assert [j.job_id_prefix for j in DREAM_SYSTEM_JOBS if j.account_only] == [
+ "skill_learning_nightly"
+ ]
# ---------------------------------------------------------------------------
-# Layer 2 — timezone drift detection (replaces old SETNX-only semantics)
+# Job ids and registration calls
# ---------------------------------------------------------------------------
-async def test_no_drift_returns_none_without_rpc():
- """Cron already registered with the user's current timezone → no
- work, no RPC."""
- client = _mock_scheduler_client()
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value="America/New_York")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value="America/New_York")), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc")
-
- assert result["community_rebuild"] is None
- assert result["dream_nightly_batch"] is None
- client.add_community_rebuild_schedule.assert_not_called()
- client.add_nightly_batch_schedule.assert_not_called()
-
+def test_account_job_ids_are_the_legacy_per_user_ids():
+ """Existing crons were registered as ``{prefix}_{user_id}``; the account
+ scope must land on exactly those ids so nothing is duplicated."""
+ assert [j.job_id(ACCOUNT) for j in DREAM_SYSTEM_JOBS] == [
+ "community_rebuild_abc",
+ "dream_nightly_batch_abc",
+ "skill_learning_nightly_abc",
+ ]
-async def test_timezone_drift_re_registers_via_replace_existing():
- """User moved NY → Paris; existing cron is still bound to NY tz.
- Helper detects the drift via the stored value and re-registers."""
- client = _mock_scheduler_client()
- write_spy = AsyncMock()
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value="Europe/Paris")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value="America/New_York")), patch(
- _PATH_WRITE_TZ, new=write_spy
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc")
- # Both crons re-registered with the new tz.
- client.add_community_rebuild_schedule.assert_awaited_once_with(
- user_id="abc", user_timezone="Europe/Paris"
- )
- client.add_nightly_batch_schedule.assert_awaited_once_with(
- user_id="abc", user_timezone="Europe/Paris"
- )
- # Stored tz updated for both.
- written_tzs = [call.args[2] for call in write_spy.await_args_list]
- assert written_tzs == ["Europe/Paris", "Europe/Paris"]
- assert result["community_rebuild"]["id"] == "community_rebuild_abc"
- assert result["dream_nightly_batch"]["id"] == "dream_nightly_batch_abc"
+def test_expert_job_ids_are_keyed_by_the_expert_scope():
+ assert [j.job_id(EXPERT) for j in jobs_for(EXPERT)] == [
+ f"community_rebuild_{EXPERT.scope_key}",
+ f"dream_nightly_batch_{EXPERT.scope_key}",
+ ]
@pytest.mark.asyncio
-async def test_force_refresh_skips_drift_check_and_always_re_registers():
- """``force_refresh=True`` (from the User.timezone update hook)
- bypasses drift detection and always re-registers."""
- client = _mock_scheduler_client()
- read_spy = AsyncMock(return_value="America/New_York")
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value="America/New_York")
- ), patch(_PATH_READ_TZ, new=read_spy), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc", force_refresh=True)
-
- # Read NEVER consulted when force_refresh=True — saves a Redis
- # round-trip when the eager hook KNOWS something changed.
- read_spy.assert_not_called()
- client.add_community_rebuild_schedule.assert_awaited_once()
- client.add_nightly_batch_schedule.assert_awaited_once()
- assert result["community_rebuild"]["id"] == "community_rebuild_abc"
+async def test_each_row_registers_through_the_scope_keyed_method():
+ client = MagicMock()
+ client.add_scope_community_rebuild_schedule = AsyncMock(return_value={"id": "c"})
+ client.add_scope_nightly_batch_schedule = AsyncMock(return_value={"id": "n"})
+ client.add_scope_skill_learning_schedule = AsyncMock(return_value={"id": "l"})
+ for job in DREAM_SYSTEM_JOBS:
+ await job.register(client, ACCOUNT, "Europe/Paris")
-async def test_redis_read_failure_treats_as_first_registration():
- """Redis unavailable → ``_read_registration_tz`` returns None →
- helper falls through to scheduler RPC. APScheduler's
- ``replace_existing=True`` is the durable backstop."""
- client = _mock_scheduler_client()
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value="UTC")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
+ for method in (
+ client.add_scope_community_rebuild_schedule,
+ client.add_scope_nightly_batch_schedule,
+ client.add_scope_skill_learning_schedule,
):
- result = await ensure_dream_system_scheduled("abc")
-
- assert result["community_rebuild"]["id"] == "community_rebuild_abc"
- assert result["dream_nightly_batch"]["id"] == "dream_nightly_batch_abc"
+ method.assert_awaited_once_with(scope=ACCOUNT, user_timezone="Europe/Paris")
# ---------------------------------------------------------------------------
-# Timezone lookup failure — "unknown" must not become "UTC"
+# Owner timezone
# ---------------------------------------------------------------------------
-async def test_timezone_lookup_failure_leaves_existing_crons_untouched():
- """A transient DB failure during tz resolution must NOT be read as
- 'user is in UTC' — that would re-register the 03:00-local crons
- onto 03:00 UTC and overwrite the stored tz. Skip the cycle: no
- scheduler RPC, no Redis write."""
- client = _mock_scheduler_client()
- write_spy = AsyncMock()
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value=None)
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value="America/Chicago")), patch(
- _PATH_WRITE_TZ, new=write_spy
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc")
-
- assert result["community_rebuild"] == {
- "skipped": True,
- "reason": "timezone_lookup_failed",
- }
- assert result["dream_nightly_batch"] == {
- "skipped": True,
- "reason": "timezone_lookup_failed",
- }
- client.add_community_rebuild_schedule.assert_not_called()
- client.add_nightly_batch_schedule.assert_not_called()
- write_spy.assert_not_called()
-
-
-async def test_timezone_lookup_failure_attempted_once_not_per_job():
- """A failed lookup is cached for the invocation like a successful
- one — two enabled crons must not trigger a second DB round-trip."""
- tz_mock = AsyncMock(return_value=None)
- client = _mock_scheduler_client()
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=tz_mock
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- await ensure_dream_system_scheduled("abc")
-
- tz_mock.assert_awaited_once_with("abc")
-
-
-async def test_force_refresh_with_failed_tz_lookup_still_skips_re_registration():
- """Even the eager path (timezone-update endpoint) must not
- re-register onto UTC when it can't read the new timezone back —
- the lazy drift-detection path recovers once the DB is healthy."""
- client = _mock_scheduler_client()
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value=None)
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc", force_refresh=True)
-
- assert result["community_rebuild"] == {
- "skipped": True,
- "reason": "timezone_lookup_failed",
- }
- client.add_community_rebuild_schedule.assert_not_called()
- client.add_nightly_batch_schedule.assert_not_called()
+def _user_db(**get_user_by_id) -> MagicMock:
+ accessor = MagicMock()
+ accessor.get_user_by_id = AsyncMock(**get_user_by_id)
+ return accessor
@pytest.mark.asyncio
@@ -449,229 +172,45 @@ async def test_resolve_user_timezone_works_without_local_prisma_connection():
invocation, so dream crons were never registered for anyone. The
lookup must route through the ``user_db()`` accessor, which falls
back to the DatabaseManager RPC in Prisma-less processes."""
- accessor = MagicMock()
- accessor.get_user_by_id = AsyncMock(
- return_value=MagicMock(timezone="Europe/Madrid")
- )
- with patch("backend.data.db_accessors.user_db", return_value=accessor):
- assert await scheduling._resolve_user_timezone("abc") == "Europe/Madrid"
- accessor.get_user_by_id.assert_awaited_once_with("abc")
+ accessor = _user_db(return_value=MagicMock(timezone="Europe/Madrid"))
+ with patch(_PATH_USER_DB, return_value=accessor):
+ assert await resolve_user_timezone(USER) == "Europe/Madrid"
+ accessor.get_user_by_id.assert_awaited_once_with(USER)
@pytest.mark.asyncio
async def test_resolve_user_timezone_missing_user_is_authoritative_utc():
"""get_user_by_id raises ValueError for a missing row — that's an
authoritative answer (UTC), not a lookup failure (None)."""
- accessor = MagicMock()
- accessor.get_user_by_id = AsyncMock(side_effect=ValueError("User not found"))
- with patch("backend.data.db_accessors.user_db", return_value=accessor):
- assert await scheduling._resolve_user_timezone("abc") == "UTC"
+ accessor = _user_db(side_effect=ValueError("User not found"))
+ with patch(_PATH_USER_DB, return_value=accessor):
+ assert await resolve_user_timezone(USER) == "UTC"
@pytest.mark.asyncio
async def test_resolve_user_timezone_returns_none_when_db_lookup_fails():
"""The resolver distinguishes 'lookup failed' (None) from
'genuinely unset' (UTC)."""
- accessor = MagicMock()
- accessor.get_user_by_id = AsyncMock(side_effect=ConnectionError("db down"))
- with patch("backend.data.db_accessors.user_db", return_value=accessor):
- assert await scheduling._resolve_user_timezone("abc") is None
+ accessor = _user_db(side_effect=ConnectionError("db down"))
+ with patch(_PATH_USER_DB, return_value=accessor):
+ assert await resolve_user_timezone(USER) is None
@pytest.mark.asyncio
-async def test_resolve_user_timezone_unset_value_falls_back_to_utc():
- """A user whose timezone was never set legitimately defaults to
- UTC — only lookup FAILURES return None."""
- from backend.data.model import USER_TIMEZONE_NOT_SET
-
+async def test_resolve_user_timezone_gives_up_at_the_deadline(monkeypatch):
+ """A hanging lookup is a failed lookup ("unknown", not "UTC"), and the
+ hanging call is cancelled."""
+ monkeypatch.setattr(deadline, "REGISTRY_CALL_TIMEOUT_SECONDS", 0.05)
+ hang = Hang()
accessor = MagicMock()
- accessor.get_user_by_id = AsyncMock(
- return_value=MagicMock(timezone=USER_TIMEZONE_NOT_SET)
- )
- with patch("backend.data.db_accessors.user_db", return_value=accessor):
- assert await scheduling._resolve_user_timezone("abc") == "UTC"
-
-
-# ---------------------------------------------------------------------------
-# Layer 3 — per-job RPC failure isolation
-# ---------------------------------------------------------------------------
-
-
-async def test_one_cron_rpc_failure_does_not_block_the_other():
- """Community rebuild scheduler call raises; nightly batch still
- registers cleanly. Failed cron surfaces as skipped dict, not a
- propagated exception."""
- client = _mock_scheduler_client(fail_jobs=("community_rebuild",))
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value="UTC")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- result = await ensure_dream_system_scheduled("abc")
-
- assert result["community_rebuild"] == {
- "skipped": True,
- "reason": "registration_failed",
- }
- assert result["dream_nightly_batch"]["id"] == "dream_nightly_batch_abc"
-
-
-# ---------------------------------------------------------------------------
-# Layer-1/layer-2 disagreement — a skipped RPC result must NOT stamp the
-# 7-day Redis marker (no APScheduler job exists; the marker would silently
-# block re-registration until the TTL expires)
-# ---------------------------------------------------------------------------
+ accessor.get_user_by_id = hang.forever
+ with patch(_PATH_USER_DB, return_value=accessor):
+ assert await resolve_user_timezone(USER) is None
+ assert hang.started == hang.cancelled == 1
@pytest.mark.asyncio
-async def test_layer2_skipped_rpc_result_writes_no_marker_and_warns(
- caplog: pytest.LogCaptureFixture,
-):
- """Layer-1 said ON but the scheduler's layer-2 gate returned a
- ``{"skipped": True, ...}`` dict without creating the job (e.g. LD
- cold start in the scheduler pod evaluating the flag to its False
- default). Writing the marker would leave Redis saying 'registered'
- with no cron behind it for 7 days. The skip must be recorded in the
- results, surfaced at WARNING (a layer-2 skip after layer-1 said ON
- is always a disagreement), and the marker left unwritten so the
- next cycle retries."""
- client = MagicMock()
- client.add_community_rebuild_schedule = AsyncMock(
- return_value={
- "id": "community_rebuild_abc",
- "user_id": "abc",
- "next_run_time": None,
- }
- )
- skipped_rpc = {
- "id": None,
- "user_id": "abc",
- "user_timezone": "UTC",
- "next_run_time": None,
- "skipped": True,
- "reason": "dream_pass_disabled",
- }
- client.add_nightly_batch_schedule = AsyncMock(return_value=skipped_rpc)
- write_spy = AsyncMock()
-
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=AsyncMock(return_value="UTC")
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=write_spy
- ), patch(
- _PATH_CLIENT, return_value=client
- ), caplog.at_level(
- logging.WARNING, logger="backend.copilot.dream.scheduling"
- ):
- result = await ensure_dream_system_scheduled("abc")
-
- # The scheduler's skip dict is recorded verbatim for auditability.
- assert result["dream_nightly_batch"] == skipped_rpc
- # Marker written ONLY for the cron that actually registered.
- written_prefixes = [call.args[1] for call in write_spy.await_args_list]
- assert written_prefixes == ["community_rebuild_registered"]
- # The disagreement is surfaced at WARNING, not logged as 'registered'.
- assert any(
- record.levelno == logging.WARNING and "skipped" in record.getMessage()
- for record in caplog.records
- )
-
-
-# ---------------------------------------------------------------------------
-# Registration-marker clearing — used by the scheduler's delete_* RPCs so an
-# in-band delete immediately re-opens lazy registration
-# ---------------------------------------------------------------------------
-
-
-def test_exported_key_prefixes_match_registry_rows():
- """The scheduler's delete_* methods import these constants to clear
- markers — drift between them and the registry would clear the wrong
- Redis key and leave the real marker blocking re-registration."""
- by_prefix = {j.job_id_prefix: j.registration_key_prefix for j in DREAM_SYSTEM_JOBS}
- assert (
- by_prefix["community_rebuild"]
- == scheduling.COMMUNITY_REBUILD_REGISTRATION_PREFIX
- )
- assert (
- by_prefix["dream_nightly_batch"] == scheduling.NIGHTLY_BATCH_REGISTRATION_PREFIX
- )
- assert (
- by_prefix["skill_learning_nightly"]
- == scheduling.SKILL_LEARNING_REGISTRATION_PREFIX
- )
-
-
-async def test_clear_registration_marker_deletes_the_single_per_cron_key():
- """Single-key DEL (Redis-cluster-safe) on the per-cron marker."""
- redis = AsyncMock()
- with patch(
- "backend.data.redis_client.get_redis_async",
- new=AsyncMock(return_value=redis),
- ):
- await scheduling.clear_registration_marker(
- "abc", "dream_nightly_batch_registered"
- )
- redis.delete.assert_awaited_once_with("dream_nightly_batch_registered:abc")
-
-
-async def test_clear_registration_marker_swallows_redis_failure(caplog):
- """Best-effort: a Redis outage during clear must never break the
- delete RPC — the marker self-heals via its 7-day TTL. The except block
- must still emit the WARNING (with exc_info) that documents the fallback,
- so a silent-swallow regression is caught."""
- with patch(
- "backend.data.redis_client.get_redis_async",
- new=AsyncMock(side_effect=ConnectionError("redis down")),
- ), caplog.at_level(logging.WARNING, logger=scheduling.logger.name):
- await scheduling.clear_registration_marker("abc", "x_registered")
-
- warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
- assert any("marker will expire via TTL" in r.getMessage() for r in warnings)
- # The fallback warning must carry the traceback for operators.
- assert any(r.exc_info is not None for r in warnings)
-
-
-# ---------------------------------------------------------------------------
-# Shared per-call caching
-# ---------------------------------------------------------------------------
-
-
-async def test_timezone_lookup_runs_at_most_once_per_invocation():
- """Two crons registering must share one Prisma round-trip for the
- user's timezone."""
- client = _mock_scheduler_client()
- tz_mock = AsyncMock(return_value="UTC")
- with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
- _PATH_TZ, new=tz_mock
- ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
- _PATH_WRITE_TZ, new=AsyncMock()
- ), patch(
- _PATH_CLIENT, return_value=client
- ):
- await ensure_dream_system_scheduled("abc")
-
- tz_mock.assert_awaited_once_with("abc")
-
-
-# ---------------------------------------------------------------------------
-# Module constants — defended against TTL drift
-# ---------------------------------------------------------------------------
-
-
-def test_registration_ttl_matches_longest_cron_cadence():
- """TTL must be at least as long as the weekly community rebuild
- cadence so the lazy drift-detection path re-checks at least once
- per cron-tick."""
- one_week_seconds = 7 * 24 * 3600
- assert REGISTRATION_TTL_SECONDS >= one_week_seconds
-
-
-# Suppress "imported but unused" — ``scheduling`` is the module under test;
-# some linters miss the patch() string-target usage.
-_ = scheduling
+async def test_resolve_user_timezone_unset_value_falls_back_to_utc():
+ accessor = _user_db(return_value=MagicMock(timezone=USER_TIMEZONE_NOT_SET))
+ with patch(_PATH_USER_DB, return_value=accessor):
+ assert await resolve_user_timezone(USER) == "UTC"
diff --git a/autogpt_platform/backend/backend/copilot/dream/scope_crons.py b/autogpt_platform/backend/backend/copilot/dream/scope_crons.py
new file mode 100644
index 0000000000..9db9200bca
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/scope_crons.py
@@ -0,0 +1,219 @@
+"""What the dream-system cron bodies in ``executor/scheduler.py`` need: the
+gate they ask before running, the run stamp and in-band-delete bookkeeping
+they write, how they read and log a pass's outcome, and their jobs' args and
+results.
+
+``scheduler.py`` keeps the cron functions and RPC methods that APScheduler
+and clients call by name. Everything here is plain async or pure code, and
+the expert lifecycle lookup is handed in, so imports go scheduler → dream.
+Registry calls run under the registry deadline and fail soft.
+"""
+
+import logging
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Awaitable, Callable
+
+from prisma.enums import MemoryScopeScheduleState
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.db_accessors import memory_schedule_db
+from backend.data.memory_schedule import ScopeRunKind
+
+from .deadline import within_deadline
+from .scheduling import DreamSystemJob
+from .scope_jobs import scope_label
+
+if TYPE_CHECKING:
+ from .nightly_batch import NightlyBatchResult
+
+logger = logging.getLogger(__name__)
+
+# ``(owner_user_id, expert_id) -> "active" | "archived" | ...``: the
+# scheduler's expert lifecycle lookup.
+ExpertStatusLookup = Callable[[str, str], Awaitable[str]]
+
+
+async def memory_scope_may_fire(
+ scope: MemoryScope, expert_status: ExpertStatusLookup
+) -> bool:
+ """Whether a dream-system cron of ``scope`` may run now.
+
+ The registry must let it: an ACTIVE row, or an account with no row
+ (crons from before the registry) whose user still exists. Any other
+ row's crons are stragglers (a pause or archive whose removal failed),
+ and an expert with no row, or an account with no row whose user is
+ gone, was deleted: either way the run is skipped, and the reconcile pass
+ (``scope_reconcile.py``) removes the crons. An expert scope's expert must
+ also be active; archived, paused or unavailable only skips the run, since
+ its crons are the lifecycle hooks' to manage. The whole check runs under
+ the registry deadline, and a slow or failed answer skips the run.
+ """
+ try:
+ return await within_deadline(_may_fire(scope, expert_status))
+ except TimeoutError:
+ logger.warning(f"Memory cron of {scope_label(scope)} skipped: gate timed out")
+ except Exception:
+ logger.warning(
+ f"Memory cron of {scope_label(scope)} skipped: gate failed", exc_info=True
+ )
+ return False
+
+
+async def record_scope_run(scope: MemoryScope, kind: ScopeRunKind) -> bool:
+ """Stamp the time a cron body ran its pass without an error."""
+ try:
+ return await within_deadline(
+ memory_schedule_db().record_scope_run(
+ scope.owner_user_id, scope.scope_key, kind
+ )
+ )
+ except Exception:
+ logger.warning(
+ f"Could not stamp the {kind} run of {scope_label(scope)}", exc_info=True
+ )
+ return False
+
+
+async def forget_registration(scope: MemoryScope, job: DreamSystemJob) -> bool:
+ """Before ``job`` is deleted outside the registry, drop its id from the
+ row, so the reconcile does not restore it and the next ensure registers
+ it again; whether the row's write went through."""
+ try:
+ await within_deadline(
+ memory_schedule_db().forget_scope_job(
+ scope.owner_user_id, scope.scope_key, job.job_id(scope)
+ )
+ )
+ except Exception:
+ logger.warning(
+ f"Could not forget {job.name} of {scope_label(scope)}", exc_info=True
+ )
+ return False
+ return True
+
+
+def nightly_error_parts(result: "NightlyBatchResult") -> list[str]:
+ """Every error a nightly result carries.
+
+ ``run_nightly_batch_submit`` never raises: a submitter CRASH is captured
+ in ``NightlyBatchResult.error``, while a submitter that ran but reported
+ its own failure carries it on its sub-result (``dream.error`` /
+ ``ratification.error`` / ``learning.error``) with the top-level error
+ left unset. A run is clean only when all of these are empty.
+ """
+ dream_error = result.dream.error if result.dream is not None else None
+ ratification_error = (
+ result.ratification.error if result.ratification is not None else None
+ )
+ learning_error = result.learning.error if result.learning is not None else None
+ return [
+ part
+ for part in (
+ result.error,
+ f"dream: {dream_error}" if dream_error else None,
+ f"ratification: {ratification_error}" if ratification_error else None,
+ f"learning: {learning_error}" if learning_error else None,
+ )
+ if part
+ ]
+
+
+def log_nightly_outcome(scope: MemoryScope, result: "NightlyBatchResult") -> None:
+ label = f"{scope_label(scope)} (nightly {result.nightly_id})"
+ if result.error:
+ logger.warning(f"Nightly batch errored for {label}: {result.error}")
+ return
+ if result.skipped:
+ logger.info(f"Nightly batch skipped for {label}: {result.skip_reason}")
+ return
+ dream, ratification = result.dream, result.ratification
+ logger.info(
+ f"Nightly batch completed for {label} in "
+ f"{result.elapsed_seconds or 0.0:.1f}s: "
+ f"dream_writes={dream.consolidated_count if dream else 0} "
+ f"dream_proposals={dream.proposal_count if dream else 0} "
+ f"ratified={ratification.ratified_count if ratification else 0} "
+ f"superseded={ratification.superseded_count if ratification else 0}"
+ )
+
+
+def log_community_outcome(scope: MemoryScope, result: dict[str, Any]) -> None:
+ if result.get("error"):
+ logger.warning(
+ f"Community rebuild errored for {scope_label(scope)}: {result['error']}"
+ )
+ return
+ logger.info(
+ f"Community rebuild completed for {scope_label(scope)} in "
+ f"{result.get('elapsed_seconds') or 0.0:.1f}s: "
+ f"{result.get('communities_built')}"
+ )
+
+
+def memory_job_kwargs(scope: MemoryScope) -> dict[str, str]:
+ """Job kwargs of a dream-system cron. The account's keep the
+ ``{"user_id": ...}`` they had when the crons were per user, so an
+ existing job, a re-registered one and a rolled-back scheduler all call
+ the body the same way."""
+ if scope.expert_id is None:
+ return {"user_id": scope.owner_user_id}
+ return {"user_id": scope.owner_user_id, "expert_id": scope.expert_id}
+
+
+def memory_job_result(
+ scope: MemoryScope,
+ job_id: str,
+ next_run_time: datetime | None,
+ user_timezone: str,
+) -> dict[str, Any]:
+ return {
+ "id": job_id,
+ "user_id": scope.owner_user_id,
+ "scope_key": scope.scope_key,
+ "user_timezone": user_timezone,
+ "next_run_time": next_run_time.isoformat() if next_run_time else None,
+ }
+
+
+def memory_job_skipped(
+ scope: MemoryScope, user_timezone: str, reason: str
+) -> dict[str, Any]:
+ return {
+ "id": None,
+ "user_id": scope.owner_user_id,
+ "scope_key": scope.scope_key,
+ "user_timezone": user_timezone,
+ "next_run_time": None,
+ "skipped": True,
+ "reason": reason,
+ }
+
+
+async def _may_fire(scope: MemoryScope, expert_status: ExpertStatusLookup) -> bool:
+ if not await _schedule_active(scope):
+ return False
+ if scope.expert_id is None:
+ return True
+ status = await expert_status(scope.owner_user_id, scope.expert_id)
+ if status != "active":
+ logger.info(f"Memory cron of {scope_label(scope)} skipped: expert is {status}")
+ return status == "active"
+
+
+async def _schedule_active(scope: MemoryScope) -> bool:
+ """Whether the scope's row lets its crons run; an account with no row
+ runs while its user exists. A cron that fires under a row that says
+ otherwise is a straggler, for the reconcile pass to remove."""
+ db = memory_schedule_db()
+ row = await db.get_scope_schedule(scope.owner_user_id, scope.scope_key)
+ if row is not None and row.state == MemoryScopeScheduleState.ACTIVE:
+ return True
+ if row is None and not scope.is_expert:
+ owner = scope.owner_user_id
+ if owner in await db.existing_user_ids([owner]):
+ return True
+ logger.info(f"Memory cron of {scope_label(scope)} skipped: its user is gone")
+ return False
+ state = row.state.value.lower() if row is not None else "without a registry row"
+ logger.info(f"Memory cron of {scope_label(scope)} skipped: fired {state}")
+ return False
diff --git a/autogpt_platform/backend/backend/copilot/dream/scope_crons_test.py b/autogpt_platform/backend/backend/copilot/dream/scope_crons_test.py
new file mode 100644
index 0000000000..16e19aeaa7
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/scope_crons_test.py
@@ -0,0 +1,239 @@
+"""Tests for what the dream-system cron bodies ask of the registry
+(``scope_crons.py``): the gate, the run stamp, in-band-delete bookkeeping,
+reading a pass's outcome, and the jobs' args and results."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime, timezone
+from unittest.mock import AsyncMock
+
+import pytest
+
+from backend.data.db_manager import DatabaseManagerAsyncClient
+
+from . import registry, scope_crons
+from .registry_fakes_test import (
+ ACCOUNT,
+ EXPERT,
+ USER,
+ Hang,
+ HangingTransport,
+ RegistryEnv,
+ install_registry_env,
+ use_short_deadline,
+)
+from .scheduling import dream_system_job
+
+PROMPT = 1.0
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ return install_registry_env(monkeypatch)
+
+
+def _status(value: str) -> AsyncMock:
+ return AsyncMock(return_value=value)
+
+
+async def test_the_gate_follows_the_row_state(env):
+ await registry.ensure_scope_scheduled(EXPERT)
+ active = _status("active")
+ assert await scope_crons.memory_scope_may_fire(EXPERT, active)
+
+ await registry.pause_scope(EXPERT)
+ assert not await scope_crons.memory_scope_may_fire(EXPERT, active)
+
+ await registry.mark_wiped(EXPERT)
+ assert not await scope_crons.memory_scope_may_fire(EXPERT, active)
+
+
+async def test_a_cron_that_outlived_its_scopes_pause_is_skipped_then_reconciled(
+ env, monkeypatch
+):
+ """A pause whose removal failed left the crons behind a PAUSED row: one
+ that fires is skipped, removes nothing (no takedown to race a resume),
+ and the reconcile pass removes them."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ monkeypatch.setattr(registry, "remove_scope_jobs", AsyncMock(return_value=False))
+ assert not await registry.pause_scope(EXPERT)
+ assert env.scheduler.jobs # the pause's removal failed
+
+ assert not await scope_crons.memory_scope_may_fire(EXPERT, _status("active"))
+ assert env.scheduler.jobs, "the gate only skips"
+
+ await env.reconcile()
+
+ assert env.scheduler.jobs == {}
+ assert env.db.rows[EXPERT.scope_key].state.value == "PAUSED"
+
+
+async def test_a_legacy_account_without_row_still_fires(env):
+ status = _status("active")
+
+ assert await scope_crons.memory_scope_may_fire(ACCOUNT, status)
+
+ status.assert_not_awaited() # the account has no expert to check
+ env.scheduler.remove_scope_memory_jobs.assert_not_awaited()
+
+
+async def test_an_expert_without_row_is_skipped_and_reconciled_away(env):
+ """A deleted expert's crons fire with no row: skipped, and the reconcile
+ pass removes them."""
+ await env.scheduler.add_scope_nightly_batch_schedule(
+ scope=EXPERT, user_timezone="UTC"
+ )
+
+ assert not await scope_crons.memory_scope_may_fire(EXPERT, _status("active"))
+ env.scheduler.remove_scope_memory_jobs.assert_not_awaited()
+
+ await env.reconcile()
+
+ assert env.scheduler.jobs == {}
+
+
+ "status, fires",
+ [
+ ("active", True),
+ ("paused", False),
+ ("archived", False),
+ ("unavailable", False),
+ ("missing", False),
+ ],
+)
+async def test_an_expert_scope_fires_only_while_the_expert_is_active(
+ env, status: str, fires: bool
+):
+ await registry.ensure_scope_scheduled(EXPERT)
+ lookup = _status(status)
+
+ assert await scope_crons.memory_scope_may_fire(EXPERT, lookup) is fires
+
+ lookup.assert_awaited_once_with(USER, "expert-1")
+
+
+async def test_a_failed_registry_read_blocks_the_fire(env):
+ env.db.fail = True
+ assert not await scope_crons.memory_scope_may_fire(ACCOUNT, _status("active"))
+
+
+async def test_the_gate_skips_promptly_when_the_registry_hangs(env, monkeypatch):
+ use_short_deadline(monkeypatch)
+ transport = HangingTransport()
+ rpc = transport.client(DatabaseManagerAsyncClient, request_retry=True)
+ monkeypatch.setattr(scope_crons, "memory_schedule_db", lambda: rpc)
+ started = time.monotonic()
+ try:
+ fires = await scope_crons.memory_scope_may_fire(ACCOUNT, _status("active"))
+ finally:
+ await rpc.aclose()
+
+ assert not fires
+ assert time.monotonic() - started < PROMPT
+ assert transport.cancelled == transport.paths == ["/get_scope_schedule"]
+
+
+async def test_the_gate_skips_promptly_when_the_expert_lookup_hangs(env, monkeypatch):
+ await registry.ensure_scope_scheduled(EXPERT)
+ use_short_deadline(monkeypatch)
+ hang = Hang()
+ started = time.monotonic()
+
+ fires = await scope_crons.memory_scope_may_fire(EXPERT, hang.forever)
+
+ assert not fires
+ assert time.monotonic() - started < PROMPT
+ assert hang.started == hang.cancelled == 1
+
+
+async def test_record_scope_run_stamps_the_row_and_fails_soft(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert await scope_crons.record_scope_run(ACCOUNT, "nightly")
+
+ assert env.db.rows[USER].last_nightly_run_at is not None
+ assert env.db.rows[USER].last_community_run_at is None
+ env.db.fail = True
+ assert not await scope_crons.record_scope_run(ACCOUNT, "community")
+
+
+async def test_record_scope_run_gives_up_at_the_deadline(env, monkeypatch):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ use_short_deadline(monkeypatch)
+ hang = Hang()
+ monkeypatch.setattr(env.db, "record_scope_run", hang.forever)
+ started = time.monotonic()
+
+ assert not await scope_crons.record_scope_run(ACCOUNT, "nightly")
+
+ assert time.monotonic() - started < PROMPT
+ assert hang.started == hang.cancelled == 1
+
+
+async def test_forgetting_a_deleted_cron_lets_ensure_register_it_again(
+ env, fake_dream_redis
+):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+
+ assert await scope_crons.forget_registration(
+ ACCOUNT, dream_system_job("dream_nightly_batch")
+ )
+
+ assert env.db.rows[USER].nightly_job_id is None
+ assert env.db.rows[USER].community_job_id == f"community_rebuild_{USER}"
+ assert f"dream_nightly_batch_registered:{USER}" not in fake_dream_redis.store
+ again = await registry.ensure_scope_scheduled(ACCOUNT)
+ assert again["community_rebuild"] is None
+ assert again["dream_nightly_batch"]["id"] == f"dream_nightly_batch_{USER}"
+
+
+async def test_a_forget_that_cannot_write_the_row_says_so(env, monkeypatch):
+ """The delete it serves then keeps its cron (``executor/scheduler.py``):
+ a row still recording it would have the reconcile restore it."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ monkeypatch.setattr(
+ env.db, "forget_scope_job", AsyncMock(side_effect=ConnectionError("down"))
+ )
+
+ assert not await scope_crons.forget_registration(
+ ACCOUNT, dream_system_job("dream_nightly_batch")
+ )
+ assert env.db.rows[USER].nightly_job_id == f"dream_nightly_batch_{USER}"
+
+
+def test_job_kwargs_keep_the_accounts_legacy_shape():
+ assert scope_crons.memory_job_kwargs(ACCOUNT) == {"user_id": USER}
+ assert scope_crons.memory_job_kwargs(EXPERT) == {
+ "user_id": USER,
+ "expert_id": "expert-1",
+ }
+
+
+def test_job_results_carry_the_scope_key():
+ when = datetime(2026, 9, 28, 4, tzinfo=timezone.utc)
+
+ added = scope_crons.memory_job_result(EXPERT, "job-1", when, "Asia/Tokyo")
+ skipped = scope_crons.memory_job_skipped(ACCOUNT, "UTC", "dream_pass_disabled")
+
+ assert added == {
+ "id": "job-1",
+ "user_id": USER,
+ "scope_key": EXPERT.scope_key,
+ "user_timezone": "Asia/Tokyo",
+ "next_run_time": when.isoformat(),
+ }
+ assert skipped["skipped"] is True and skipped["scope_key"] == USER
diff --git a/autogpt_platform/backend/backend/copilot/dream/scope_jobs.py b/autogpt_platform/backend/backend/copilot/dream/scope_jobs.py
new file mode 100644
index 0000000000..465d0278e0
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/scope_jobs.py
@@ -0,0 +1,260 @@
+"""The dream-system crons of one memory scope, against the scheduler and the
+``MemoryScopeSchedule`` row: registering and recording them, and removing
+them.
+
+``registry.py`` decides when a scope's crons should exist; this module does
+the work, and ``scope_crons.py`` holds what the cron bodies ask of it. Every
+call to a dependency runs under the registry deadline (``deadline.py``).
+Everything fails soft: logged, never raised.
+
+Turning a flag off leaves the scope's recorded crons in place. Their bodies
+check the flag at run time and do nothing. Pause or wipe removes them.
+Cleaning them up on flag-off needs generation-aware intent, which is left for
+later. (A pass that re-registers, on a timezone change or a forced refresh,
+stops recording a cron whose flag is off and leaves the cron itself, unless
+the reconcile had registered it, when its next pass takes it back.)
+"""
+
+import logging
+from typing import Any, Iterable
+
+from prisma.enums import MemoryScopeScheduleState
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.db_accessors import memory_schedule_db
+from backend.data.memory_schedule import MemoryScopeSchedule
+from backend.util.clients import get_scheduler_client
+from backend.util.feature_flag import is_feature_enabled
+
+from .deadline import within_deadline
+from .scheduling import DREAM_SYSTEM_JOBS, DreamSystemJob, jobs_for
+
+logger = logging.getLogger(__name__)
+
+
+async def flag_gate(scope: MemoryScope) -> tuple[dict[str, Any], list[DreamSystemJob]]:
+ """Split the scope's crons (``jobs_for``) on the owner's flags: skip
+ results for the disabled ones, and the enabled ones still to consider."""
+ results: dict[str, Any] = {}
+ enabled: list[DreamSystemJob] = []
+ for job in jobs_for(scope):
+ try:
+ on = await within_deadline(
+ is_feature_enabled(job.flag, scope.owner_user_id)
+ )
+ except Exception:
+ logger.warning(f"Flag check failed for {job.name}", exc_info=True)
+ results[job.job_id_prefix] = skipped("registration_failed")
+ continue
+ if on:
+ enabled.append(job)
+ else:
+ results[job.job_id_prefix] = skipped(job.skip_reason)
+ return results, enabled
+
+
+async def register_scope_jobs(
+ scope: MemoryScope,
+ enabled: list[DreamSystemJob],
+ row: MemoryScopeSchedule | None,
+ user_timezone: str,
+ force_refresh: bool,
+) -> dict[str, Any]:
+ """Claim the scope's row if it has none, register every enabled cron
+ that is missing or was registered in another timezone, and record the
+ job ids on the row. One result per enabled cron, keyed by prefix."""
+ if row is None:
+ row = await _claim(scope, user_timezone)
+ if row is None:
+ return skip_all(enabled, "registry_unavailable")
+ if row.state != MemoryScopeScheduleState.ACTIVE:
+ return skip_all(enabled, state_reason(row))
+ stale = force_refresh or row.timezone != user_timezone
+ if row.timezone != user_timezone:
+ failure = await _record_timezone(scope, row, user_timezone)
+ if failure is not None:
+ return skip_all(enabled, failure)
+ outcomes: dict[str, Any] = {}
+ for job in enabled:
+ if not stale and recorded_job_id(row, job) is not None:
+ outcomes[job.job_id_prefix] = None
+ continue
+ outcomes[job.job_id_prefix] = await _register_job(scope, job, user_timezone)
+ return await _record(scope, row, user_timezone, outcomes, stale)
+
+
+async def _record_timezone(
+ scope: MemoryScope, row: MemoryScopeSchedule, user_timezone: str
+) -> str | None:
+ """Persist the new timezone before replacing crons, so reconciliation
+ retains it even if the final job-id write fails."""
+ try:
+ recorded = await within_deadline(
+ memory_schedule_db().record_scope_jobs(
+ scope.owner_user_id,
+ scope.scope_key,
+ user_timezone=user_timezone,
+ community_job_id=row.community_job_id,
+ nightly_job_id=row.nightly_job_id,
+ learning_job_id=row.learning_job_id,
+ )
+ )
+ except Exception:
+ logger.warning(
+ f"Could not record timezone of {scope_label(scope)}", exc_info=True
+ )
+ return "record_failed"
+ return None if recorded else "scope_changed"
+
+
+async def remove_scope_jobs(scope: MemoryScope) -> bool:
+ """Remove every dream-system cron of the scope from the scheduler;
+ whether the scheduler confirmed it."""
+ try:
+ await within_deadline(
+ get_scheduler_client().remove_scope_memory_jobs(scope=scope)
+ )
+ except Exception:
+ logger.warning(
+ f"Could not remove the crons of {scope_label(scope)}", exc_info=True
+ )
+ return False
+ return True
+
+
+def skipped(reason: str) -> dict[str, Any]:
+ return {"skipped": True, "reason": reason}
+
+
+def skip_all(jobs: Iterable[DreamSystemJob], reason: str) -> dict[str, Any]:
+ return {job.job_id_prefix: skipped(reason) for job in jobs}
+
+
+def state_reason(row: MemoryScopeSchedule) -> str:
+ return f"scope_{row.state.value.lower()}"
+
+
+def scope_label(scope: MemoryScope) -> str:
+ owner = f"user {scope.owner_user_id[:12]}"
+ return f"{owner} expert {scope.expert_id[:12]}" if scope.expert_id else owner
+
+
+async def _claim(scope: MemoryScope, user_timezone: str) -> MemoryScopeSchedule | None:
+ try:
+ return await within_deadline(
+ memory_schedule_db().claim_scope_schedule(
+ scope.owner_user_id, scope.scope_key, scope.expert_id, user_timezone
+ )
+ )
+ except Exception:
+ logger.warning(f"Could not claim {scope_label(scope)}", exc_info=True)
+ return None
+
+
+async def _register_job(
+ scope: MemoryScope, job: DreamSystemJob, user_timezone: str
+) -> dict[str, Any]:
+ try:
+ result = await within_deadline(
+ job.register(get_scheduler_client(), scope, user_timezone)
+ )
+ except Exception:
+ logger.warning(
+ f"Dream-system: failed to register {job.name} for {scope_label(scope)}",
+ exc_info=True,
+ )
+ return skipped("registration_failed")
+ if result.get("skipped"):
+ # The scheduler's own flag gate refused although ours said on (LD
+ # cold start, targeting divergence): record nothing, retry next time.
+ logger.warning(
+ f"Dream-system: scheduler skipped {job.name} for {scope_label(scope)} "
+ f"(reason={result.get('reason')}) despite the local flag check"
+ )
+ return result
+ logger.info(
+ f"Dream-system: registered {job.name} for {scope_label(scope)} "
+ f"(tz={user_timezone})"
+ )
+ return result
+
+
+async def _record(
+ scope: MemoryScope,
+ row: MemoryScopeSchedule,
+ user_timezone: str,
+ outcomes: dict[str, Any],
+ stale: bool,
+) -> dict[str, Any]:
+ """Write the job ids this pass settled on and return the outcomes as
+ they stand afterwards: a registration the row could not record is a
+ failure, and one the scope left ACTIVE under is ``scope_changed``. Its
+ crons are left where they are: the fire gate holds a scope that is not
+ ACTIVE, the reconcile removes them (``scope_reconcile.py``), and taking
+ them down here could take a concurrent resume's crons with them."""
+ ids = {
+ job.row_field: _next_job_id(row, job, outcomes.get(job.job_id_prefix), stale)
+ for job in DREAM_SYSTEM_JOBS
+ }
+ if not stale and all(
+ ids[job.row_field] == recorded_job_id(row, job) for job in DREAM_SYSTEM_JOBS
+ ):
+ return outcomes
+ try:
+ recorded = await within_deadline(
+ memory_schedule_db().record_scope_jobs(
+ scope.owner_user_id,
+ scope.scope_key,
+ user_timezone=user_timezone,
+ community_job_id=ids["community_job_id"],
+ nightly_job_id=ids["nightly_job_id"],
+ learning_job_id=ids["learning_job_id"],
+ )
+ )
+ except Exception:
+ logger.warning(
+ f"Could not record the jobs of {scope_label(scope)}", exc_info=True
+ )
+ return _replace_registered(outcomes, "record_failed")
+ if recorded:
+ return outcomes
+ logger.info(
+ f"{scope_label(scope)} left ACTIVE while registering; the reconcile "
+ "removes its crons"
+ )
+ return _replace_registered(outcomes, "scope_changed")
+
+
+def _replace_registered(outcomes: dict[str, Any], reason: str) -> dict[str, Any]:
+ """The outcomes with every registration made this pass turned into a
+ skip with ``reason``."""
+ return {
+ prefix: skipped(reason) if _registered(outcome) else outcome
+ for prefix, outcome in outcomes.items()
+ }
+
+
+def _registered(outcome: Any) -> bool:
+ return bool(outcome) and not outcome.get("skipped") and bool(outcome.get("id"))
+
+
+def _next_job_id(
+ row: MemoryScopeSchedule, job: DreamSystemJob, outcome: Any, stale: bool
+) -> str | None:
+ """Keep failed registration intent so reconciliation can retry it."""
+ if _registered(outcome):
+ return outcome["id"]
+ if outcome and outcome.get("reason") == "registration_failed":
+ return recorded_job_id(row, job)
+ return None if stale else recorded_job_id(row, job)
+
+
+def recorded_job_id(row: MemoryScopeSchedule, job: DreamSystemJob) -> str | None:
+ """The job id ``row`` records for ``job``, None when it records none."""
+ match job.row_field:
+ case "community_job_id":
+ return row.community_job_id
+ case "nightly_job_id":
+ return row.nightly_job_id
+ case "learning_job_id":
+ return row.learning_job_id
diff --git a/autogpt_platform/backend/backend/copilot/dream/scope_reconcile.py b/autogpt_platform/backend/backend/copilot/dream/scope_reconcile.py
new file mode 100644
index 0000000000..fffa09a736
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/scope_reconcile.py
@@ -0,0 +1,350 @@
+"""Keep the scheduler's dream-system crons in step with the registry rows.
+
+The registry acts on events, and events interleave (a stale takedown after a
+resume, a failed re-read, a registration landing after a timezone change, a
+restore landing after a delete), leaving the scheduler's crons out of step
+with a row.
+
+:func:`reconcile` is the level-triggered backstop the scheduler runs every
+``RECONCILE_INTERVAL_MINUTES``. It lists the dream-system crons and every row,
+then walks the scopes as a ring, from just after the last scope a pass in this
+process took (a random one after the process starts), and settles each the
+listing shows out of step, on the row as read again just before:
+
+ * an ACTIVE row's crons are the ones it records, in its timezone: what the
+ scheduler lacks or runs elsewhere is registered, tagged as the
+ reconcile's (``RECONCILED_SUFFIX`` on the job's name);
+ * a tagged cron its row does not record, or whose row is not ACTIVE or
+ gone, is removed, before the scope's crons are repaired and whatever
+ that repair does. So a restore that raced a delete or a pause, failed
+ partway, was cancelled or finished after its pass, leaves at worst a
+ tagged cron that the next pass reaching the scope takes away, even
+ while the scope's repair keeps failing;
+ * every cron of a PAUSED or WIPED row, of an expert with no row, or of an
+ account with no row whose user is gone, is removed. Which of those
+ accounts are gone is asked once per pass, in pages, each under the
+ registry deadline; when that cannot be answered, their crons wait for a
+ later pass. An untagged cron no row records (a live account's from
+ before the registry, a registration whose record failed) is the
+ registry's, and stays.
+
+No flag is read here. Every change is idempotent, so two schedulers (a
+rollout's overlap) each walk on their own and converge. A pass takes a scope
+as its place before it works it, so a pass cut off at its ceiling hands the
+rest of the ring to the next: a lost, stale or stray cron is put right once
+the walk next reaches its scope. A pass that cannot list the crons or the rows
+changes nothing.
+
+Known limit: a pass's changes are not fenced against the registry's. An
+ensure or a resume that registers and records a cron after the pass's
+re-read can lose it to the pass's take-back or removal, and the next pass
+that reaches the scope restores it from the row. For the nightly learning
+cron that can cost one night's run; its sources persist, so the next
+night's run reviews them.
+"""
+
+import bisect
+import logging
+import random
+from collections.abc import Awaitable, Sequence
+from typing import ClassVar, Protocol
+
+from prisma.enums import MemoryScopeScheduleState
+from pydantic import BaseModel, Field
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.db_accessors import memory_schedule_db
+from backend.data.memory_schedule import MemoryScopeSchedule
+
+from .deadline import within_deadline
+from .scheduling import DREAM_SYSTEM_JOBS, DreamSystemJob
+from .scope_jobs import recorded_job_id, scope_label
+
+logger = logging.getLogger(__name__)
+
+RECONCILE_INTERVAL_MINUTES = 30
+# Rows read from the registry per call while a pass lists them.
+PAGE_SIZE = 500
+# Ends the name of every cron the reconcile registers: a later pass takes
+# such a cron back once its row stops recording it.
+RECONCILED_SUFFIX = " (reconciled)"
+_ACTIVE = MemoryScopeScheduleState.ACTIVE
+
+
+class ListedCron(BaseModel):
+ """One dream-system cron the scheduler holds: its job id, the scope its
+ kwargs name, the timezone its trigger runs in, and whether the reconcile
+ registered it."""
+
+ job_id: str
+ scope: MemoryScope
+ timezone: str | None
+ reconciled: bool = False
+
+
+class CronStore(Protocol):
+ """The scheduler side of a pass: its dream-system crons, and adding one
+ cron of a scope (tagged as the reconcile's), dropping one, or removing
+ them all."""
+
+ async def listing(self) -> Sequence[ListedCron]: ...
+
+ async def add(
+ self, job: DreamSystemJob, scope: MemoryScope, user_timezone: str
+ ) -> None: ...
+
+ async def drop(self, job: DreamSystemJob, scope: MemoryScope) -> None: ...
+
+ async def remove(self, scope: MemoryScope) -> None: ...
+
+
+class ReconcileReport(BaseModel):
+ """What one pass changed: the job ids it registered and the tagged ones
+ it took back, the scopes whose crons it removed, and the scopes a change
+ failed for."""
+
+ restored: list[str] = Field(default_factory=list)
+ dropped: list[str] = Field(default_factory=list)
+ removed: list[str] = Field(default_factory=list)
+ failed: list[str] = Field(default_factory=list)
+
+
+async def reconcile(store: CronStore) -> ReconcileReport:
+ """One pass (above). Never raises; the scheduler cancels it at its
+ ceiling, and the place it took carries the next pass on."""
+ report = ReconcileReport()
+ try:
+ listed = await store.listing()
+ rows = {row.scope_key: row for row in await _all_rows()}
+ except Exception:
+ logger.warning(
+ "Memory cron reconcile skipped: the crons or the registry could "
+ "not be listed",
+ exc_info=True,
+ )
+ return report
+ held: dict[str, list[ListedCron]] = {}
+ for cron in listed:
+ held.setdefault(cron.scope.scope_key, []).append(cron)
+ gone = await _gone_accounts(rows, held)
+ for key in _ring(sorted(rows.keys() | held.keys())):
+ row, crons = rows.get(key), held.get(key, [])
+ if not _out_of_step(row, crons, gone):
+ continue
+ _Place.last = key
+ scope = _scope_of(row) if row else crons[0].scope
+ await _settle(scope, crons, store, report, gone)
+ return report
+
+
+async def _gone_accounts(
+ rows: dict[str, MemoryScopeSchedule], held: dict[str, list[ListedCron]]
+) -> set[str]:
+ """The owners of account crons with no row whose user is gone, asked in
+ pages under the registry deadline. Empty when the answer cannot be had:
+ those crons wait for a later pass rather than go on a guess."""
+ owners = sorted(
+ {
+ crons[0].scope.owner_user_id
+ for key, crons in held.items()
+ if key not in rows and not crons[0].scope.is_expert
+ }
+ )
+ live: set[str] = set()
+ try:
+ for start in range(0, len(owners), PAGE_SIZE):
+ page = owners[start : start + PAGE_SIZE]
+ live |= await within_deadline(memory_schedule_db().existing_user_ids(page))
+ except Exception:
+ logger.warning(
+ "Memory cron reconcile could not tell which accounts are gone; "
+ "their crons wait for a later pass",
+ exc_info=True,
+ )
+ return set()
+ return set(owners) - live
+
+
+def _ring(keys: list[str]) -> list[str]:
+ """``keys`` from just after the last scope a pass in this process took,
+ wrapping round; from a random one when it has taken none."""
+ if not keys:
+ return []
+ if _Place.last is None:
+ start = random.randrange(len(keys))
+ else:
+ start = bisect.bisect_right(keys, _Place.last)
+ return keys[start:] + keys[:start]
+
+
+class _Place:
+ """The last scope a pass in this process began to settle."""
+
+ last: ClassVar[str | None] = None
+
+
+def _out_of_step(
+ row: MemoryScopeSchedule | None, crons: list[ListedCron], gone: set[str]
+) -> bool:
+ """Whether the listing shows the scope out of step: an ACTIVE row's
+ recorded crons missing or elsewhere, a tagged cron it does not record,
+ or crons that must go."""
+ if row is not None and row.state == _ACTIVE:
+ scope = _scope_of(row)
+ return bool(_off(row, scope, _zones(crons))) or bool(_stray(row, crons))
+ if crons and _straggling(crons[0].scope, row, gone):
+ return True
+ return bool(_stray(row, crons))
+
+
+async def _settle(
+ scope: MemoryScope,
+ crons: list[ListedCron],
+ store: CronStore,
+ report: ReconcileReport,
+ gone: set[str],
+) -> None:
+ """Settle the scope on its row as read now: remove the crons of a scope
+ that must have none; else take back the tagged crons the row does not
+ record, then restore an ACTIVE row's crons. Each step stands on its own,
+ the take-back first, so a restore that keeps failing never keeps a stray
+ running."""
+ try:
+ row = await _read(scope)
+ except Exception:
+ logger.warning(f"Could not read {scope_label(scope)} again", exc_info=True)
+ return
+ if _straggling(scope, row, gone):
+ await _step(scope, report, _remove(scope, store, report))
+ return
+ await _step(scope, report, _drop(scope, _stray(row, crons), store, report))
+ if row is not None and row.state == _ACTIVE:
+ await _step(scope, report, _restore(scope, row, crons, store, report))
+
+
+async def _step(
+ scope: MemoryScope, report: ReconcileReport, change: Awaitable[None]
+) -> None:
+ """Make one of the scope's changes; a failure is logged and the scope
+ reported failed once, and the pass goes on."""
+ try:
+ await change
+ except Exception:
+ logger.warning(
+ f"Could not settle the crons of {scope_label(scope)}", exc_info=True
+ )
+ if scope.scope_key not in report.failed:
+ report.failed.append(scope.scope_key)
+
+
+async def _remove(
+ scope: MemoryScope, store: CronStore, report: ReconcileReport
+) -> None:
+ await store.remove(scope)
+ report.removed.append(scope.scope_key)
+ logger.info(f"Memory cron reconcile removed the crons of {scope_label(scope)}")
+
+
+async def _restore(
+ scope: MemoryScope,
+ row: MemoryScopeSchedule,
+ crons: list[ListedCron],
+ store: CronStore,
+ report: ReconcileReport,
+) -> None:
+ """Register, tagged, each cron the row records that ``crons`` lacks or
+ runs in another timezone."""
+ for job in _off(row, scope, _zones(crons)):
+ await store.add(job, scope, row.timezone)
+ report.restored.append(job.job_id(scope))
+ logger.info(
+ f"Memory cron reconcile registered {job.name} for "
+ f"{scope_label(scope)} in {row.timezone}"
+ )
+
+
+async def _drop(
+ scope: MemoryScope,
+ jobs: list[DreamSystemJob],
+ store: CronStore,
+ report: ReconcileReport,
+) -> None:
+ """Take back the tagged crons ``jobs`` of ``scope``."""
+ for job in jobs:
+ await store.drop(job, scope)
+ report.dropped.append(job.job_id(scope))
+ logger.info(
+ f"Memory cron reconcile took back {job.name} of {scope_label(scope)}: "
+ "its row does not record it"
+ )
+
+
+def _off(
+ row: MemoryScopeSchedule, scope: MemoryScope, zones: dict[str, str | None]
+) -> list[DreamSystemJob]:
+ """The crons ``row`` records that ``zones`` (each listed job id's
+ timezone) lacks, or has in a timezone other than the row's."""
+ return [
+ job
+ for job in DREAM_SYSTEM_JOBS
+ if recorded_job_id(row, job) is not None
+ and zones.get(job.job_id(scope), "") != row.timezone
+ ]
+
+
+def _stray(
+ row: MemoryScopeSchedule | None, crons: list[ListedCron]
+) -> list[DreamSystemJob]:
+ """The tagged crons among ``crons`` that ``row`` does not keep: it does
+ not record them, is not ACTIVE, or is gone."""
+ tagged = {cron.job_id for cron in crons if cron.reconciled}
+ if not tagged:
+ return []
+ scope = crons[0].scope
+ return [
+ job
+ for job in DREAM_SYSTEM_JOBS
+ if job.job_id(scope) in tagged
+ and (row is None or row.state != _ACTIVE or recorded_job_id(row, job) is None)
+ ]
+
+
+def _straggling(
+ scope: MemoryScope, row: MemoryScopeSchedule | None, gone: set[str]
+) -> bool:
+ """Whether crons of ``scope`` must go: its row says PAUSED or WIPED, or it
+ has no row and is an expert's or a gone account's. A live account with
+ no row keeps its crons until the registry adopts them."""
+ if row is None:
+ return scope.is_expert or scope.owner_user_id in gone
+ return row.state != _ACTIVE
+
+
+def _scope_of(row: MemoryScopeSchedule) -> MemoryScope:
+ return MemoryScope.build(row.user_id, row.expert_id)
+
+
+def _zones(crons: list[ListedCron]) -> dict[str, str | None]:
+ return {cron.job_id: cron.timezone for cron in crons}
+
+
+async def _read(scope: MemoryScope) -> MemoryScopeSchedule | None:
+ """The scope's row as it stands now, None when it has none, under the
+ registry deadline. Raises when it cannot be read."""
+ return await within_deadline(
+ memory_schedule_db().get_scope_schedule(scope.owner_user_id, scope.scope_key)
+ )
+
+
+async def _all_rows() -> list[MemoryScopeSchedule]:
+ """Every registry row, paged by scope key, each page under the registry
+ deadline."""
+ rows: list[MemoryScopeSchedule] = []
+ while True:
+ after = rows[-1].scope_key if rows else None
+ page = await within_deadline(
+ memory_schedule_db().list_scope_schedules(after=after, limit=PAGE_SIZE)
+ )
+ rows += page
+ if len(page) < PAGE_SIZE:
+ return rows
diff --git a/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_gone_test.py b/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_gone_test.py
new file mode 100644
index 0000000000..c0cc55fcf6
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_gone_test.py
@@ -0,0 +1,136 @@
+"""Tests for an account's dream-system crons with no registry row once its
+user is gone: the reconcile removes them, keeps a live account's for the
+registry to adopt, and leaves both when it cannot tell; the cron gate skips
+the gone account's. No flag is read.
+
+The probe is an independent review's: a deleted user's untagged legacy
+learning cron was kept for good, and its gate let it run. Fakes come from
+``registry_fakes_test.py``; ``backend/data/memory_schedule_db_test.py``
+runs the probe with a real user, scheduler and jobstore.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from backend.copilot.graphiti.scope import MemoryScope
+
+from . import registry, scope_crons, scope_reconcile
+from .registry_fakes_test import (
+ ACCOUNT,
+ EXPERT,
+ PARIS,
+ RegistryEnv,
+ install_registry_env,
+)
+from .scheduling import DREAM_SYSTEM_JOBS
+
+LIVE = MemoryScope.for_user("live-user")
+GONE = MemoryScope.for_user("gone-user")
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ return install_registry_env(monkeypatch)
+
+
+async def _legacy_crons(env: RegistryEnv, *scopes: MemoryScope) -> None:
+ """Crons from before the registry: registered untagged, no row."""
+ for scope in scopes:
+ for job in DREAM_SYSTEM_JOBS:
+ await job.register(env.scheduler, scope, PARIS)
+
+
+def _ids(scope: MemoryScope) -> set[str]:
+ return {job.job_id(scope) for job in DREAM_SYSTEM_JOBS}
+
+
+async def test_a_gone_accounts_crons_go_and_a_live_ones_stay(env):
+ await _legacy_crons(env, LIVE, GONE)
+ env.db.gone_users = {GONE.owner_user_id}
+
+ report = await env.reconcile()
+
+ assert report.removed == [GONE.scope_key]
+ assert set(env.scheduler.jobs) == _ids(LIVE)
+ assert env.db.rows == {}
+
+
+async def test_an_unanswered_existence_check_leaves_them_for_a_later_pass(env):
+ await _legacy_crons(env, LIVE, GONE)
+ env.db.gone_users = {GONE.owner_user_id}
+ env.db.existence_fails = True
+
+ report = await env.reconcile()
+
+ assert (report.removed, report.failed) == ([], [])
+ assert set(env.scheduler.jobs) == _ids(LIVE) | _ids(GONE)
+
+ env.db.existence_fails = False
+ report = await env.reconcile()
+
+ assert report.removed == [GONE.scope_key]
+ assert set(env.scheduler.jobs) == _ids(LIVE)
+
+
+async def test_the_check_asks_about_rowless_accounts_only_a_page_at_a_time(
+ env, monkeypatch
+):
+ """One query per page of owners, none about an account with a row or an
+ expert, whose crons with no row go whatever its owner."""
+ monkeypatch.setattr(scope_reconcile, "PAGE_SIZE", 2)
+ rowless = [MemoryScope.for_user(f"legacy-{n}") for n in range(5)]
+ await _legacy_crons(env, *rowless)
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ await env.scheduler.add_scope_nightly_batch_schedule(
+ scope=EXPERT, user_timezone=PARIS
+ )
+ asked: list[list[str]] = []
+ existing = env.db.existing_user_ids
+
+ async def counted(user_ids: list[str]) -> set[str]:
+ asked.append(user_ids)
+ return await existing(user_ids)
+
+ monkeypatch.setattr(env.db, "existing_user_ids", counted)
+
+ report = await env.reconcile()
+
+ owners = sorted(scope.owner_user_id for scope in rowless)
+ assert asked == [owners[0:2], owners[2:4], owners[4:5]]
+ assert report.removed == [EXPERT.scope_key]
+
+
+async def test_the_gate_skips_a_gone_accounts_cron_and_lets_a_live_ones_run(env):
+ env.db.gone_users = {GONE.owner_user_id}
+ status = AsyncMock(return_value="active")
+
+ assert not await scope_crons.memory_scope_may_fire(GONE, status)
+ assert await scope_crons.memory_scope_may_fire(LIVE, status)
+
+
+async def test_a_gate_that_cannot_tell_skips_the_run(env):
+ env.db.existence_fails = True
+
+ assert not await scope_crons.memory_scope_may_fire(
+ LIVE, AsyncMock(return_value="active")
+ )
+
+
+async def test_an_account_with_a_row_is_not_asked_about(env):
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ env.db.existence_fails = True
+
+ assert await scope_crons.memory_scope_may_fire(
+ ACCOUNT, AsyncMock(return_value="active")
+ )
+ report = await env.reconcile()
+ assert (report.removed, report.failed) == ([], [])
diff --git a/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_jobstore_test.py b/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_jobstore_test.py
new file mode 100644
index 0000000000..9edf5f6959
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_jobstore_test.py
@@ -0,0 +1,649 @@
+"""The memory cron reconcile against the real scheduler and a real jobstore
+(SQLite, or PostgreSQL when ``DIRECT_URL`` names one), with the registry's
+fakes: an account cron's delete racing a restore, whole or cut short, and a
+write that finds its row locked.
+
+The probes are an independent review's. Round 3's: a restore that read the
+row before a successful delete re-created the deleted cron for good, and a
+write blocked on a locked row held its pass, and every pass after it, at
+the same scope. Round 4's: that restore cut short after its stale add, by
+a failed or locked write or a cancellation. A stale add is tagged as the
+reconcile's, and the next pass takes it back. Round 5's: a sibling's repair
+that keeps failing on every pass kept that stray running; the stray now
+goes first, whatever the repair does. The learning fit's review: a
+take-back racing a new registration of the same cron removes it, a known
+limit the next pass repairs.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import random
+import threading
+import time
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from datetime import timezone
+from pathlib import Path
+from unittest.mock import AsyncMock
+
+import pytest
+from apscheduler.schedulers.background import BackgroundScheduler
+from sqlalchemy import create_engine, text
+
+from backend.executor import scheduler as sm
+from backend.executor.jobstore import ResilientSQLAlchemyJobStore
+from backend.util.feature_flag import Flag
+
+from . import registry, scope_reconcile
+from .registry_fakes_test import (
+ ACCOUNT,
+ PARIS,
+ PAUSED,
+ TOKYO,
+ USER,
+ RegistryEnv,
+ install_registry_env,
+ job_ids,
+ learning_job_id,
+)
+from .scheduling import SKILL_LEARNING_JOB_PREFIX, dream_system_job
+
+_EXECUTION = sm.Jobstores.EXECUTION.value
+TIMEOUT = 5
+_NIGHTLY = f"dream_nightly_batch_{ACCOUNT.scope_key}"
+_COMMUNITY = f"community_rebuild_{ACCOUNT.scope_key}"
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ return install_registry_env(monkeypatch)
+
+
+@contextmanager
+def _scheduler(store: ResilientSQLAlchemyJobStore) -> Iterator[sm.Scheduler]:
+ """A ``Scheduler`` whose APScheduler, started paused, keeps its EXECUTION
+ jobs in ``store``."""
+ apscheduler = BackgroundScheduler(
+ jobstores={_EXECUTION: store}, timezone=timezone.utc
+ )
+ apscheduler.start(paused=True)
+ service = sm.Scheduler(register_system_tasks=False)
+ service.scheduler, service._execution_jobstore = apscheduler, store
+ try:
+ yield service
+ finally:
+ apscheduler.remove_jobstore(_EXECUTION)
+ apscheduler.shutdown(wait=True)
+ store.engine.dispose()
+
+
+def _sqlite(tmp_path: Path) -> ResilientSQLAlchemyJobStore:
+ return ResilientSQLAlchemyJobStore(
+ url=f"sqlite:///{(tmp_path / 'jobs.sqlite').as_posix()}"
+ )
+
+
+def _add_crons(service: sm.Scheduler, scope, timezone_name: str) -> None:
+ for job in sm.DREAM_SYSTEM_JOBS:
+ service._add_memory_job(
+ *sm._MEMORY_CRONS[job.job_id_prefix], job, scope, timezone_name
+ )
+
+
+def _job(service: sm.Scheduler, job_id: str):
+ return service.scheduler.get_job(job_id, jobstore=_EXECUTION)
+
+
+async def _pass(service: sm.Scheduler) -> scope_reconcile.ReconcileReport:
+ return await scope_reconcile.reconcile(sm._SchedulerCrons(service))
+
+
+async def _set(event: threading.Event) -> None:
+ while not event.is_set():
+ await asyncio.sleep(0.01)
+
+
+async def test_a_delete_after_a_restores_read_is_not_undone(env, monkeypatch, tmp_path):
+ """The review's probe: a restore re-timing the account's crons parks
+ after its read, just before writing the nightly cron; the delete RPC
+ forgets the cron and removes it; the restore then writes it back,
+ tagged as the reconcile's. The next pass takes it back, and later
+ passes leave it deleted."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ monkeypatch.setattr(sm, "_event_loop", asyncio.get_running_loop())
+ with _scheduler(_sqlite(tmp_path)) as service:
+ _add_crons(service, ACCOUNT, TOKYO)
+ reached, release = threading.Event(), threading.Event()
+ add = service._add_memory_job
+
+ def parked_add(*args):
+ if args[2].job_id_prefix == "dream_nightly_batch" and not reached.is_set():
+ reached.set()
+ assert release.wait(5)
+ return add(*args)
+
+ monkeypatch.setattr(service, "_add_memory_job", parked_add)
+ restore = asyncio.create_task(_pass(service))
+ await asyncio.wait_for(_set(reached), TIMEOUT)
+ assert await asyncio.to_thread(service.delete_nightly_batch_schedule, USER)
+ assert env.db.rows[ACCOUNT.scope_key].nightly_job_id is None
+ release.set()
+ await asyncio.wait_for(restore, 5)
+ for _ in range(3):
+ await _pass(service)
+
+ assert _job(service, _NIGHTLY) is None
+ assert str(_job(service, _COMMUNITY).trigger.timezone) == PARIS
+
+
+async def test_a_delete_forgets_the_cron_before_it_removes_it(
+ env, monkeypatch, tmp_path
+):
+ """A pass that runs between the delete's two steps finds the row no
+ longer recording the cron and leaves it, so the delete's removal is the
+ last word. Removing first, the pass would restore the cron the row
+ still records, and the forget after it would leave it running for
+ good."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ monkeypatch.setattr(sm, "_event_loop", asyncio.get_running_loop())
+ with _scheduler(_sqlite(tmp_path)) as service:
+ _add_crons(service, ACCOUNT, PARIS)
+ first, release = threading.Event(), threading.Event()
+ forget, removed = sm._forget_dream_registration, sm._removed
+
+ def parked(step):
+ def run(*args):
+ done = step(*args)
+ if not first.is_set():
+ first.set()
+ assert release.wait(5)
+ return done
+
+ return run
+
+ monkeypatch.setattr(sm, "_forget_dream_registration", parked(forget))
+ monkeypatch.setattr(sm, "_removed", parked(removed))
+ delete = asyncio.create_task(
+ asyncio.to_thread(service.delete_nightly_batch_schedule, USER)
+ )
+ await asyncio.wait_for(_set(first), TIMEOUT)
+ await asyncio.wait_for(_pass(service), TIMEOUT)
+ release.set()
+
+ assert await asyncio.wait_for(delete, 5)
+ await _pass(service)
+ assert _job(service, _NIGHTLY) is None
+ assert env.db.rows[ACCOUNT.scope_key].nightly_job_id is None
+
+
+async def test_a_delete_whose_forget_fails_keeps_its_cron(env, monkeypatch, tmp_path):
+ """The row still records the cron, so removing it would only have the
+ reconcile restore it: the delete says it failed and leaves it."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ monkeypatch.setattr(sm, "_event_loop", asyncio.get_running_loop())
+ monkeypatch.setattr(
+ env.db, "forget_scope_job", AsyncMock(side_effect=ConnectionError("down"))
+ )
+ with _scheduler(_sqlite(tmp_path)) as service:
+ _add_crons(service, ACCOUNT, PARIS)
+
+ assert not await asyncio.to_thread(service.delete_nightly_batch_schedule, USER)
+
+ assert _job(service, _NIGHTLY) is not None
+ assert env.db.rows[ACCOUNT.scope_key].nightly_job_id == _NIGHTLY
+
+
+# The flow a step belongs to: set where each flow starts, carried into its
+# threads and tasks with the context.
+_FLOW: ContextVar[str] = ContextVar("reconcile_test_flow")
+
+
+class _Turns:
+ """Runs flows one step at a time: each wrapped call waits its turn, and
+ once every running flow is waiting, one of them, picked at random
+ (seeded) by the flow's name, takes its step. A seed fixes the order of
+ the flows' steps, whatever their timing."""
+
+ def __init__(self, rng: random.Random, flows: list[str]) -> None:
+ self._rng = rng
+ self._lock = threading.Lock()
+ self._waiting: dict[str, threading.Event] = {}
+ self._running = set(flows)
+ self._on = True
+
+ def step(self, call):
+ def run(*args, **kwargs):
+ assert self._turn().wait(TIMEOUT), "no turn came"
+ return call(*args, **kwargs)
+
+ return run
+
+ def astep(self, call):
+ async def run(*args, **kwargs):
+ await asyncio.wait_for(_set(self._turn()), TIMEOUT)
+ return await call(*args, **kwargs)
+
+ return run
+
+ def finished(self, flow: str) -> None:
+ with self._lock:
+ self._running.discard(flow)
+ self._next()
+
+ def off(self) -> None:
+ with self._lock:
+ self._on = False
+ for turn in self._waiting.values():
+ turn.set()
+ self._waiting.clear()
+
+ def _turn(self) -> threading.Event:
+ turn = threading.Event()
+ with self._lock:
+ if not self._on:
+ turn.set()
+ return turn
+ self._waiting[_FLOW.get()] = turn
+ self._next()
+ return turn
+
+ def _next(self) -> None:
+ if self._waiting and self._waiting.keys() >= self._running:
+ flow = self._rng.choice(sorted(self._waiting))
+ self._waiting.pop(flow).set()
+
+
[email protected]("ending", ["healthy", "add_fails", "cancelled"])
[email protected]("seed", range(20))
+async def test_no_interleaving_of_a_restore_and_a_delete_leaves_a_cron_unrecorded(
+ env, monkeypatch, tmp_path, seed: int, ending: str
+):
+ """A restore re-timing the account's crons races the nightly cron's
+ delete, their registry reads and writes and jobstore writes taken one at
+ a time in a seeded order. The restore runs to its end, its second add
+ fails, or it is cancelled there. Once both are done, and two more
+ passes, the scheduler holds exactly the crons the row records: after the
+ delete, never the nightly one."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ monkeypatch.setattr(sm, "_event_loop", asyncio.get_running_loop())
+ turns = _Turns(random.Random(seed), ["restore", "delete"])
+ monkeypatch.setattr(
+ env.db, "get_scope_schedule", turns.astep(env.db.get_scope_schedule)
+ )
+ monkeypatch.setattr(
+ env.db, "forget_scope_job", turns.astep(env.db.forget_scope_job)
+ )
+ with _scheduler(_sqlite(tmp_path)) as service:
+ _add_crons(service, ACCOUNT, TOKYO)
+ add = turns.step(service._add_memory_job)
+ adds = {"made": 0}
+
+ def ending_add(*args):
+ if adds["made"] == 1 and ending == "add_fails":
+ raise ConnectionError("the second add failed")
+ if adds["made"] == 1 and ending == "cancelled":
+ raise asyncio.CancelledError("the pass's ceiling")
+ adds["made"] += 1
+ return add(*args)
+
+ monkeypatch.setattr(service, "_add_memory_job", ending_add)
+ monkeypatch.setattr(
+ service.scheduler, "remove_job", turns.step(service.scheduler.remove_job)
+ )
+
+ async def restoring() -> None:
+ _FLOW.set("restore")
+ try:
+ await _pass(service)
+ except asyncio.CancelledError:
+ pass
+ finally:
+ turns.finished("restore")
+
+ def deleting() -> bool:
+ _FLOW.set("delete")
+ try:
+ return service.delete_nightly_batch_schedule(USER)
+ finally:
+ turns.finished("delete")
+
+ await asyncio.wait_for(
+ asyncio.gather(restoring(), asyncio.to_thread(deleting)), 4 * TIMEOUT
+ )
+ turns.off()
+ monkeypatch.setattr(service, "_add_memory_job", add)
+ for _ in range(2):
+ await _pass(service)
+
+ row = env.db.rows[ACCOUNT.scope_key]
+ assert row.nightly_job_id is None, "the delete forgot it"
+ assert _job(service, _NIGHTLY) is None
+ assert _job(service, _COMMUNITY) is not None
+ assert str(_job(service, _COMMUNITY).trigger.timezone) == PARIS
+
+
+def _postgres_url() -> str | None:
+ direct = os.getenv("DIRECT_URL") or ""
+ return direct.split("?")[0] if direct.startswith("postgresql") else None
+
+
[email protected](_postgres_url() is None, reason="needs the PostgreSQL test DB")
+async def test_a_write_held_by_a_locked_row_fails_its_scope_and_the_pass_goes_on(
+ env, monkeypatch
+):
+ """The review's blocked write on PostgreSQL: another transaction holds
+ the first scope's community cron row while every scope needs repair.
+ That scope's write gives up after the write timeout and fails it; the
+ same pass re-times the next scope and removes a paused one's crons, and
+ leaves no write waiting on the row. The next cycle repairs the first."""
+ monkeypatch.setattr(sm, "MEMORY_CRON_WRITE_TIMEOUT_SECONDS", 0.5)
+ locked, later, paused = (
+ registry.MemoryScope.for_user(f"pg-cut-{n}") for n in (0, 1, 9)
+ )
+ for scope in (locked, later, paused):
+ await registry.ensure_scope_scheduled(scope)
+ await env.db.set_scope_state(paused.owner_user_id, paused.scope_key, PAUSED)
+ name = f"memcore_reconcile_{uuid.uuid4().hex[:8]}"
+ url = _postgres_url()
+ assert url is not None
+ engine = create_engine(
+ url, pool_size=4, max_overflow=0, connect_args={"application_name": name}
+ )
+ locker = create_engine(url).connect()
+ try:
+ with _scheduler(
+ ResilientSQLAlchemyJobStore(engine=engine, tablename=name)
+ ) as service:
+ for scope in (locked, later, paused):
+ _add_crons(service, scope, TOKYO)
+ hold = locker.begin()
+ try:
+ locker.execute(
+ text(f"SELECT id FROM {name} WHERE id = :id FOR UPDATE"),
+ {"id": f"community_rebuild_{locked.scope_key}"},
+ ).all()
+ started = time.monotonic()
+
+ report = await asyncio.wait_for(_pass(service), 20)
+
+ assert time.monotonic() - started < 10
+ assert report.failed == [locked.scope_key]
+ assert report.removed == [paused.scope_key]
+ zones = {
+ job.id: str(job.trigger.timezone) for job in service._memory_jobs()
+ }
+ assert {zones[job_id] for job_id in job_ids(later)} == {PARIS}
+ assert {zones[job_id] for job_id in job_ids(locked)} == {TOKYO}
+ assert not job_ids(paused) & zones.keys()
+ waiting = locker.execute(
+ text(
+ "SELECT count(*) FROM pg_stat_activity "
+ "WHERE application_name = :name "
+ "AND wait_event_type = 'Lock'"
+ ),
+ {"name": name},
+ ).scalar()
+ assert waiting == 0, "no write is left waiting on the row"
+ finally:
+ # Before the scheduler shuts down: a write still waiting on
+ # the row holds the jobstores lock its shutdown needs.
+ hold.rollback()
+
+ await _pass(service)
+
+ zones = {
+ job.id: str(job.trigger.timezone) for job in service._memory_jobs()
+ }
+ assert {zones[job_id] for job_id in job_ids(locked)} == {PARIS}
+ with engine.begin() as connection:
+ connection.execute(text(f"DROP TABLE IF EXISTS {name}"))
+ finally:
+ locker.close()
+ locker.engine.dispose()
+
+
[email protected]("ending", ["error", "cancel", "pg_timeout"])
+async def test_a_restore_cut_short_after_a_stale_add_leaves_no_cron_behind(
+ env, monkeypatch, tmp_path, ending: str
+):
+ """The round-4 review's partial restore, with the production delete and
+ jobstore: the restore re-timing the account's crons parks before its
+ community add, the delete RPC forgets that cron and removes it, and the
+ add then puts it back. The nightly add after it fails, is cancelled, or
+ waits on a row another transaction holds (on PostgreSQL) until the write
+ timeout. The community cron came back tagged as the reconcile's, so the
+ next pass takes it back, the nightly's repair failing, cut short or held
+ again there (the round-5 review's production delete race), and three
+ more after the nightly recovers leave it gone."""
+ if ending == "pg_timeout" and _postgres_url() is None:
+ pytest.skip("needs the PostgreSQL test DB")
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ monkeypatch.setattr(sm, "_event_loop", asyncio.get_running_loop())
+ name = f"memcore_partial_{uuid.uuid4().hex[:8]}"
+ if ending == "pg_timeout":
+ monkeypatch.setattr(sm, "MEMORY_CRON_WRITE_TIMEOUT_SECONDS", 0.2)
+ url = _postgres_url()
+ assert url is not None
+ store = ResilientSQLAlchemyJobStore(url=url, tablename=name)
+ else:
+ store = _sqlite(tmp_path)
+ locker = store.engine.connect() if ending == "pg_timeout" else None
+ try:
+ with _scheduler(store) as service:
+ _add_crons(service, ACCOUNT, TOKYO)
+ reached, release = threading.Event(), threading.Event()
+ add = service._add_memory_job
+
+ def parked_community(*args):
+ if args[2].job_id_prefix == "community_rebuild":
+ reached.set()
+ assert release.wait(TIMEOUT)
+ elif ending == "error":
+ raise ConnectionError("the nightly add failed")
+ elif ending == "cancel":
+ raise asyncio.CancelledError("the pass's ceiling")
+ return add(*args)
+
+ monkeypatch.setattr(service, "_add_memory_job", parked_community)
+ hold = locker.begin() if locker is not None else None
+ try:
+ if locker is not None:
+ locker.execute(
+ text(f"SELECT id FROM {name} WHERE id = :id FOR UPDATE"),
+ {"id": _NIGHTLY},
+ ).all()
+ restore = asyncio.create_task(_pass(service))
+ await asyncio.wait_for(_set(reached), TIMEOUT)
+ assert await asyncio.to_thread(
+ service.delete_community_rebuild_schedule, USER
+ )
+ release.set()
+ if ending == "cancel":
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(restore, 10)
+ else:
+ report = await asyncio.wait_for(restore, 10)
+ assert report.failed == [ACCOUNT.scope_key]
+ assert _job(service, _COMMUNITY) is not None, "the stale add landed"
+
+ if ending == "cancel":
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(_pass(service), 10)
+ else:
+ report = await asyncio.wait_for(_pass(service), 10)
+ assert report.dropped == [_COMMUNITY]
+ assert report.failed == [ACCOUNT.scope_key]
+ assert _job(service, _COMMUNITY) is None, "gone, the repair failing"
+ finally:
+ release.set()
+ if hold is not None:
+ hold.rollback()
+ monkeypatch.setattr(service, "_add_memory_job", add)
+
+ for _ in range(3):
+ await _pass(service)
+
+ assert env.db.rows[ACCOUNT.scope_key].community_job_id is None
+ assert _job(service, _COMMUNITY) is None
+ assert str(_job(service, _NIGHTLY).trigger.timezone) == PARIS
+ if ending == "pg_timeout":
+ with store.engine.begin() as connection:
+ connection.execute(text(f"DROP TABLE IF EXISTS {name}"))
+ finally:
+ if locker is not None:
+ locker.close()
+
+
[email protected]("failure", ["error", "pg_lock"])
+async def test_a_tagged_stray_goes_while_its_siblings_repair_keeps_failing(
+ env, monkeypatch, tmp_path, failure: str
+):
+ """The round-5 review's sibling failure: the ACTIVE account scope holds a
+ tagged community cron its row no longer records and a nightly cron in
+ the wrong timezone, whose repair fails on every pass (its add raises, or
+ another transaction holds its row on PostgreSQL until the write
+ timeout). The first pass takes the stray back all the same, and the
+ nightly is repaired once its write goes through."""
+ if failure == "pg_lock" and _postgres_url() is None:
+ pytest.skip("needs the PostgreSQL test DB")
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ monkeypatch.setattr(sm, "_event_loop", asyncio.get_running_loop())
+ community, nightly = sm.DREAM_SYSTEM_JOBS[:2]
+ name = f"memcore_sibling_{uuid.uuid4().hex[:8]}"
+ if failure == "pg_lock":
+ monkeypatch.setattr(sm, "MEMORY_CRON_WRITE_TIMEOUT_SECONDS", 0.2)
+ url = _postgres_url()
+ assert url is not None
+ store = ResilientSQLAlchemyJobStore(url=url, tablename=name)
+ else:
+ store = _sqlite(tmp_path)
+ locker = store.engine.connect() if failure == "pg_lock" else None
+ try:
+ with _scheduler(store) as service:
+ service._add_memory_job(
+ *sm._MEMORY_CRONS[community.job_id_prefix],
+ community,
+ ACCOUNT,
+ TOKYO,
+ True,
+ )
+ service._add_memory_job(
+ *sm._MEMORY_CRONS[nightly.job_id_prefix], nightly, ACCOUNT, TOKYO
+ )
+ await env.db.forget_scope_job(USER, ACCOUNT.scope_key, _COMMUNITY)
+ add = service._add_memory_job
+
+ def failing(*args):
+ raise ConnectionError("the nightly's repair is unavailable")
+
+ hold = locker.begin() if locker is not None else None
+ try:
+ if locker is not None:
+ locker.execute(
+ text(f"SELECT id FROM {name} WHERE id = :id FOR UPDATE"),
+ {"id": _NIGHTLY},
+ ).all()
+ else:
+ monkeypatch.setattr(service, "_add_memory_job", failing)
+ reports = [await asyncio.wait_for(_pass(service), 10) for _ in range(3)]
+ finally:
+ if hold is not None:
+ hold.rollback()
+
+ assert reports[0].dropped == [_COMMUNITY]
+ assert [report.failed for report in reports] == [[ACCOUNT.scope_key]] * 3
+ assert _job(service, _COMMUNITY) is None
+ assert str(_job(service, _NIGHTLY).trigger.timezone) == TOKYO
+ monkeypatch.setattr(service, "_add_memory_job", add)
+
+ report = await _pass(service)
+
+ assert report.restored == [_NIGHTLY] and report.failed == []
+ assert str(_job(service, _NIGHTLY).trigger.timezone) == PARIS
+ assert _job(service, _COMMUNITY) is None
+ if failure == "pg_lock":
+ with store.engine.begin() as connection:
+ connection.execute(text(f"DROP TABLE IF EXISTS {name}"))
+ finally:
+ if locker is not None:
+ locker.close()
+
+
[email protected]("backend", ["sqlite", "postgres"])
+async def test_a_take_back_racing_a_new_registration_is_repaired_next_pass(
+ env, monkeypatch, tmp_path, backend: str
+):
+ """The learning fit review's probe, kept as a test of eventual repair
+ (the known limit in ``scope_reconcile.py``): a pass takes back a tagged
+ learning cron its row stopped recording, and after the pass's re-read a
+ real ensure registers it again, untagged, in a new timezone, and records
+ it. The take-back removes the new cron; the next pass restores it from
+ the row, in the row's timezone."""
+ if backend == "postgres" and _postgres_url() is None:
+ pytest.skip("needs the PostgreSQL test DB")
+ env.flags = {Flag.DREAM_SKILL_LEARNING_ENABLED: True}
+ learning = learning_job_id(ACCOUNT)
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ await env.db.forget_scope_job(USER, ACCOUNT.scope_key, learning)
+ monkeypatch.setattr(sm, "_event_loop", asyncio.get_running_loop())
+ monkeypatch.setattr(
+ "backend.util.feature_flag.is_feature_enabled", AsyncMock(return_value=True)
+ )
+ name = f"memcore_learning_{uuid.uuid4().hex[:8]}"
+ url = _postgres_url()
+ store = (
+ ResilientSQLAlchemyJobStore(url=url, tablename=name)
+ if backend == "postgres" and url is not None
+ else _sqlite(tmp_path)
+ )
+ job = dream_system_job(SKILL_LEARNING_JOB_PREFIX)
+ with _scheduler(store) as service:
+ service._add_memory_job(
+ *sm._MEMORY_CRONS[SKILL_LEARNING_JOB_PREFIX], job, ACCOUNT, PARIS, True
+ )
+
+ async def register(**kwargs) -> dict:
+ return await asyncio.to_thread(
+ service.add_scope_skill_learning_schedule, **kwargs
+ )
+
+ env.scheduler.add_scope_skill_learning_schedule.side_effect = register
+ crons = sm._SchedulerCrons(service)
+ take_back = crons.drop
+
+ async def raced(dropped, scope) -> None:
+ env.timezone = TOKYO
+ await registry.ensure_scope_scheduled(ACCOUNT, force_refresh=True)
+ assert env.db.rows[USER].learning_job_id == learning
+ replaced = _job(service, learning)
+ assert not replaced.name.endswith(scope_reconcile.RECONCILED_SUFFIX)
+ await take_back(dropped, scope)
+
+ monkeypatch.setattr(crons, "drop", raced)
+ report = await asyncio.wait_for(scope_reconcile.reconcile(crons), TIMEOUT)
+
+ assert report.dropped == [learning]
+ assert _job(service, learning) is None, "the known limit"
+ monkeypatch.setattr(crons, "drop", take_back)
+
+ report = await asyncio.wait_for(scope_reconcile.reconcile(crons), TIMEOUT)
+
+ assert report.restored == [learning]
+ restored = _job(service, learning)
+ assert str(restored.trigger.timezone) == TOKYO
+ assert env.db.rows[USER].learning_job_id == learning
+ if backend == "postgres":
+ with store.engine.begin() as connection:
+ connection.execute(text(f"DROP TABLE IF EXISTS {name}"))
diff --git a/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_test.py b/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_test.py
new file mode 100644
index 0000000000..9df05bbfe6
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/scope_reconcile_test.py
@@ -0,0 +1,724 @@
+"""Tests for the scheduler's reconcile pass over the memory-scope crons
+(``scope_reconcile.py``), against the registry fakes: it restores what the
+event-driven paths lose, removes what they leave behind, and converges.
+
+The race probes are an independent review's. Round 1's left an ACTIVE row
+with its job ids recorded and no crons; each now ends with the crons back
+after one pass. Round 2's: a restore that lands after a timezone change,
+set right by the next pass, and passes cut off at their ceiling, each now
+walking on round the ring. Round 3's: a pass cut off inside a scope's work,
+now moving on, and a restore whose cron the row stopped recording
+meanwhile. Round 4's: that restore cut short by a failure or a
+cancellation, and a stale place. A stale restore leaves a cron tagged as
+the reconcile's, which the next pass takes back.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import AsyncMock
+
+import pytest
+
+from . import registry, scope_crons, scope_jobs, scope_reconcile
+from .registry_fakes_test import (
+ ACCOUNT,
+ ACTIVE,
+ EXPERT,
+ OTHER_EXPERT,
+ PARIS,
+ PAUSED,
+ TOKYO,
+ USER,
+ WIPED,
+ RegistryEnv,
+ install_registry_env,
+ job_ids,
+)
+
+TIMEOUT = 2
+
+
+def env(monkeypatch: pytest.MonkeyPatch) -> RegistryEnv:
+ return install_registry_env(monkeypatch)
+
+
+def _lose_every_cron(env: RegistryEnv) -> None:
+ env.scheduler.jobs.clear()
+ env.scheduler.scopes.clear()
+
+
+async def _refused_then_resumed(
+ env: RegistryEnv, monkeypatch: pytest.MonkeyPatch
+) -> dict[str, Any]:
+ """The interleaving from the round-2 review: an ensure parks before
+ recording while a pause lands, so its record is refused, and a resume
+ registers and records the crons before the ensure goes on. Returns the
+ ensure's outcomes."""
+ record = env.db.record_scope_jobs
+ reached, go = asyncio.Event(), asyncio.Event()
+ ensure: asyncio.Task | None = None
+
+ async def parked_record(*args: Any, **kwargs: Any) -> bool:
+ if asyncio.current_task() is not ensure or reached.is_set():
+ return await record(*args, **kwargs)
+ reached.set()
+ await go.wait()
+ refused = await record(*args, **kwargs)
+ await registry.resume_scope(EXPERT)
+ return refused
+
+ monkeypatch.setattr(env.db, "record_scope_jobs", parked_record)
+ ensure = asyncio.create_task(
+ registry.ensure_scope_scheduled(EXPERT, force_refresh=True)
+ )
+ await asyncio.wait_for(reached.wait(), TIMEOUT)
+ assert await registry.pause_scope(EXPERT)
+ go.set()
+ outcomes = await asyncio.wait_for(ensure, TIMEOUT)
+ monkeypatch.setattr(env.db, "record_scope_jobs", record)
+ return outcomes
+
+
[email protected]("cycles", [1, 2])
+async def test_a_registration_a_pause_refused_takes_no_later_resumes_crons(
+ env, monkeypatch, cycles
+):
+ """After ``cycles`` of the interleaving the refused registrations report
+ ``scope_changed`` and take nothing down: the row is ACTIVE with both ids
+ recorded and the resume's crons in place, and a pass has nothing to
+ change. (A takedown there once took the resume's crons, for the pass
+ to restore.)"""
+ await registry.ensure_scope_scheduled(EXPERT)
+ for _ in range(cycles):
+ outcomes = await _refused_then_resumed(env, monkeypatch)
+ assert {r["reason"] for r in outcomes.values()} == {"scope_changed"}
+ row = env.db.rows[EXPERT.scope_key]
+ assert row.state == ACTIVE
+ assert {row.community_job_id, row.nightly_job_id} == job_ids(EXPERT)
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+
+ report = await env.reconcile()
+
+ assert (report.restored, report.removed, report.dropped) == ([], [], [])
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+
+
+async def test_a_gate_fire_under_a_paused_row_never_takes_crons_down(env):
+ """The review's probe through the gate needed the gate to remove a
+ straggler's crons, racing a resume. It no longer does: a fire under a
+ PAUSED row only skips, so a resume after it keeps the crons, and the
+ pass after leaves them."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ await env.db.set_scope_state(EXPERT.owner_user_id, EXPERT.scope_key, PAUSED)
+
+ assert not await scope_crons.memory_scope_may_fire(
+ EXPERT, AsyncMock(return_value="active")
+ )
+ env.scheduler.remove_scope_memory_jobs.assert_not_awaited()
+ await registry.resume_scope(EXPERT)
+ await env.reconcile()
+
+ assert env.db.rows[EXPERT.scope_key].state == ACTIVE
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+
+
[email protected]("state", [PAUSED, WIPED])
+async def test_a_pass_removes_the_crons_of_a_scope_that_left_active(env, state):
+ await registry.ensure_scope_scheduled(EXPERT)
+ await registry.ensure_scope_scheduled(OTHER_EXPERT)
+ await env.db.set_scope_state(EXPERT.owner_user_id, EXPERT.scope_key, state)
+
+ report = await env.reconcile()
+
+ assert set(env.scheduler.jobs) == job_ids(OTHER_EXPERT)
+ assert report.removed == [EXPERT.scope_key]
+
+
+async def test_a_pass_removes_a_deleted_experts_crons_and_keeps_a_legacy_accounts(
+ env,
+):
+ """An expert's crons with no row are an expert deleted; an account's are
+ crons from before the registry, which the cron gate lets run."""
+ for scope in (EXPERT, ACCOUNT):
+ await env.scheduler.add_scope_nightly_batch_schedule(
+ scope=scope, user_timezone="UTC"
+ )
+
+ report = await env.reconcile()
+
+ assert set(env.scheduler.jobs) == {f"dream_nightly_batch_{ACCOUNT.scope_key}"}
+ assert report.removed == [EXPERT.scope_key]
+
+
+async def test_a_pass_registers_what_the_row_records_and_reads_no_flag(
+ env, monkeypatch
+):
+ """An ACTIVE row's crons are the ones it records: the owner's flags, off
+ by now, are not read, a cron the row no longer records stays missing,
+ and a row recording none is left to the next ensure."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ await registry.ensure_scope_scheduled(OTHER_EXPERT)
+ env.db.update(EXPERT.scope_key, community_job_id=None)
+ env.db.update(OTHER_EXPERT.scope_key, community_job_id=None, nightly_job_id=None)
+ _lose_every_cron(env)
+ flags = AsyncMock(return_value=False)
+ monkeypatch.setattr(scope_jobs, "is_feature_enabled", flags)
+
+ report = await env.reconcile()
+
+ assert set(env.scheduler.jobs) == {f"dream_nightly_batch_{EXPERT.scope_key}"}
+ assert report.restored == [f"dream_nightly_batch_{EXPERT.scope_key}"]
+ flags.assert_not_awaited()
+
+
+async def test_a_cron_the_row_does_not_record_is_left_where_it_is(env):
+ """A cron an ACTIVE row does not record (its record was refused, or it
+ predates the registry) is the registry's to decide: the pass neither
+ removes it nor moves it to the row's timezone."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ env.db.update(EXPERT.scope_key, community_job_id=None, timezone=TOKYO)
+ community = f"community_rebuild_{EXPERT.scope_key}"
+ env.scheduler.jobs[f"dream_nightly_batch_{EXPERT.scope_key}"] = TOKYO
+
+ report = await env.reconcile()
+
+ assert env.scheduler.jobs[community] == PARIS
+ assert report.restored == report.removed == []
+
+
+async def test_a_pass_restores_a_cron_in_the_rows_timezone(env):
+ env.timezone = TOKYO
+ await registry.ensure_scope_scheduled(EXPERT)
+ env.timezone = PARIS # the owner moved; the row still says Tokyo
+ _lose_every_cron(env)
+
+ await env.reconcile()
+
+ assert set(env.scheduler.jobs.values()) == {TOKYO}
+
+
+async def test_a_cron_in_another_timezone_is_registered_again_in_the_rows(env):
+ await registry.ensure_scope_scheduled(EXPERT)
+ for job_id in job_ids(EXPERT):
+ env.scheduler.jobs[job_id] = TOKYO
+
+ report = await env.reconcile()
+
+ assert set(env.scheduler.jobs.values()) == {PARIS}
+ assert sorted(report.restored) == sorted(job_ids(EXPERT))
+ assert (await env.reconcile()).restored == []
+
+
+async def test_a_restore_landing_after_a_timezone_change_is_undone_by_the_next_pass(
+ env, monkeypatch
+):
+ """The review's probe: a restore reads the row (Tokyo) and parks while
+ the owner moves to Paris and ``reregister_user`` registers and records
+ the crons there; released, it registers them in Tokyo again. The row
+ says Paris with both ids recorded, so the next pass, comparing
+ timezones, puts them back in Paris."""
+ env.timezone = TOKYO
+ await registry.ensure_scope_scheduled(EXPERT)
+ _lose_every_cron(env)
+ add = env.crons.add
+ reached, go = asyncio.Event(), asyncio.Event()
+
+ async def parked_add(job, scope, user_timezone):
+ if not reached.is_set():
+ reached.set()
+ await go.wait()
+ await add(job, scope, user_timezone)
+
+ monkeypatch.setattr(env.crons, "add", parked_add)
+ stale = asyncio.create_task(env.reconcile())
+ await asyncio.wait_for(reached.wait(), TIMEOUT)
+ env.timezone = PARIS
+ await registry.reregister_user(USER)
+ assert _zones(env, EXPERT) == {PARIS}
+ go.set()
+ await asyncio.wait_for(stale, TIMEOUT)
+ assert _zones(env, EXPERT) == {TOKYO}, "the stale restore"
+ assert env.db.rows[EXPERT.scope_key].timezone == PARIS
+
+ report = await env.reconcile()
+
+ assert _zones(env, EXPERT) == {PARIS}
+ assert sorted(report.restored) == sorted(job_ids(EXPERT))
+
+
+def _zones(env: RegistryEnv, scope) -> set[str]:
+ """The timezones the scheduler runs ``scope``'s two crons in."""
+ return {env.scheduler.jobs[job_id] for job_id in job_ids(scope)}
+
+
+async def test_a_pause_landing_during_a_pass_wins(env, monkeypatch):
+ """The listing saw the row ACTIVE and its crons gone; a pause lands before
+ the pass registers them: the pass reads the row again and registers
+ nothing."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ _lose_every_cron(env)
+ listing = env.db.list_scope_schedules
+
+ async def paused_after_listing(**kwargs: Any):
+ page = await listing(**kwargs)
+ await env.db.set_scope_state(EXPERT.owner_user_id, EXPERT.scope_key, PAUSED)
+ return page
+
+ monkeypatch.setattr(env.db, "list_scope_schedules", paused_after_listing)
+
+ report = await env.reconcile()
+
+ assert env.scheduler.jobs == {}
+ assert report.restored == []
+
+
+async def test_a_resume_landing_during_a_pass_wins(env, monkeypatch):
+ """The listing saw the row PAUSED with crons behind it; a resume lands
+ before the pass removes them: the pass reads the row again and removes
+ nothing."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ await env.db.set_scope_state(EXPERT.owner_user_id, EXPERT.scope_key, PAUSED)
+ listing = env.db.list_scope_schedules
+
+ async def resumed_after_listing(**kwargs: Any):
+ page = await listing(**kwargs)
+ await env.db.set_scope_state(EXPERT.owner_user_id, EXPERT.scope_key, ACTIVE)
+ return page
+
+ monkeypatch.setattr(env.db, "list_scope_schedules", resumed_after_listing)
+
+ report = await env.reconcile()
+
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+ assert report.removed == []
+
+
+async def test_two_passes_at_once_converge(env, monkeypatch):
+ """Two schedulers reconciling at once, as in a rolling deploy, each
+ listing before the other changes anything: the same end state, and
+ nothing failed."""
+ await registry.ensure_scope_scheduled(EXPERT)
+ await registry.ensure_scope_scheduled(OTHER_EXPERT)
+ _lose_every_cron(env)
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ await env.db.set_scope_state(ACCOUNT.owner_user_id, ACCOUNT.scope_key, PAUSED)
+ listing = env.crons.listing
+ both_listed = asyncio.Barrier(2)
+
+ async def list_together():
+ listed = await listing()
+ await both_listed.wait()
+ return listed
+
+ monkeypatch.setattr(env.crons, "listing", list_together)
+
+ reports = await asyncio.gather(env.reconcile(), env.reconcile())
+
+ assert set(env.scheduler.jobs) == job_ids(EXPERT) | job_ids(OTHER_EXPERT)
+ assert all(report.failed == [] for report in reports)
+ monkeypatch.setattr(env.crons, "listing", listing)
+ again = await env.reconcile()
+ assert (again.restored, again.removed) == ([], [])
+
+
[email protected]("down", ["registry", "scheduler"])
+async def test_a_pass_that_cannot_list_changes_nothing(env, monkeypatch, down):
+ await registry.ensure_scope_scheduled(EXPERT)
+ await env.db.set_scope_state(EXPERT.owner_user_id, EXPERT.scope_key, PAUSED)
+ if down == "registry":
+ env.db.fail = True
+ else:
+ monkeypatch.setattr(
+ env.crons, "listing", AsyncMock(side_effect=TimeoutError("jobstore"))
+ )
+
+ report = await env.reconcile()
+
+ assert set(env.scheduler.jobs) == job_ids(EXPERT)
+ assert report == scope_reconcile.ReconcileReport()
+
+
+async def test_a_failed_registration_is_reported_and_the_pass_goes_on(env):
+ await registry.ensure_scope_scheduled(EXPERT)
+ await registry.ensure_scope_scheduled(OTHER_EXPERT)
+ _lose_every_cron(env)
+ add = env.scheduler.add_scope_nightly_batch_schedule.side_effect
+
+ async def refuse_expert_one(*, scope, user_timezone):
+ if scope == EXPERT:
+ raise ConnectionError("jobstore down")
+ return await add(scope=scope, user_timezone=user_timezone)
+
+ env.scheduler.add_scope_nightly_batch_schedule.side_effect = refuse_expert_one
+
+ report = await env.reconcile()
+
+ assert report.failed == [EXPERT.scope_key]
+ assert job_ids(OTHER_EXPERT) <= set(env.scheduler.jobs)
+
+
+async def test_a_pass_reads_every_page_of_the_registry(env, monkeypatch):
+ monkeypatch.setattr(scope_reconcile, "PAGE_SIZE", 2)
+ scopes = [registry.MemoryScope.for_expert(USER, f"e-{i}") for i in range(5)]
+ for scope in scopes:
+ await registry.ensure_scope_scheduled(scope)
+ _lose_every_cron(env)
+
+ await env.reconcile()
+
+ assert set(env.scheduler.jobs) == set().union(*(job_ids(s) for s in scopes))
+
+
+async def test_a_pass_settles_each_scope_in_key_order(env, monkeypatch):
+ """Removal and restore go scope by scope, in key order: a straggler's
+ crons go before a later scope's are restored, not after every restore."""
+ scopes = sorted(
+ (registry.MemoryScope.for_expert(USER, f"e-{i}") for i in range(4)),
+ key=lambda scope: scope.scope_key,
+ )
+ for scope in scopes:
+ await registry.ensure_scope_scheduled(scope)
+ for scope in scopes[1:]:
+ for job_id in job_ids(scope):
+ env.scheduler.jobs.pop(job_id)
+ env.scheduler.scopes.pop(job_id)
+ await env.db.set_scope_state(USER, scopes[0].scope_key, PAUSED)
+ done: list[str] = []
+ remove, add = env.crons.remove, env.crons.add
+
+ async def noted_remove(scope):
+ done.append(f"remove {scope.scope_key}")
+ await remove(scope)
+
+ async def noted_add(job, scope, user_timezone):
+ done.append(f"add {scope.scope_key}")
+ await add(job, scope, user_timezone)
+
+ monkeypatch.setattr(env.crons, "remove", noted_remove)
+ monkeypatch.setattr(env.crons, "add", noted_add)
+
+ await env.reconcile()
+
+ assert done == [f"remove {scopes[0].scope_key}"] + [
+ f"add {scope.scope_key}" for scope in scopes[1:] for _ in range(2)
+ ]
+
+
+async def test_passes_cut_off_at_their_ceiling_walk_on_round_the_ring(env, monkeypatch):
+ """The review's starvation probe, in one process. Eight scopes lost their
+ crons, the first three of them on a jobstore that fails every
+ registration, and the last scope's row was paused with its crons left
+ behind; every pass is cut off at its third registration attempt. Each
+ pass takes a scope as its place before it works it and the next walks on
+ after it, round the ring, so the passes get through every scope, the
+ straggler's removal included, however often the failing ones come
+ round."""
+ scopes = sorted(
+ (registry.MemoryScope.for_expert(USER, f"e-{i}") for i in range(9)),
+ key=lambda scope: scope.scope_key,
+ )
+ for scope in scopes:
+ await registry.ensure_scope_scheduled(scope)
+ for scope in scopes[:-1]:
+ for job_id in job_ids(scope):
+ env.scheduler.jobs.pop(job_id)
+ env.scheduler.scopes.pop(job_id)
+ straggler = scopes[-1]
+ await env.db.set_scope_state(USER, straggler.scope_key, PAUSED)
+ stalled = {scope.scope_key for scope in scopes[:3]}
+ add = env.crons.add
+ budget = {"left": 0}
+
+ async def ceiling_add(job, scope, user_timezone):
+ if budget["left"] == 0:
+ raise asyncio.CancelledError("the pass's ceiling")
+ budget["left"] -= 1
+ if scope.scope_key in stalled:
+ raise ConnectionError("the jobstore stalled")
+ await add(job, scope, user_timezone)
+
+ monkeypatch.setattr(env.crons, "add", ceiling_add)
+ healthy = set().union(*(job_ids(s) for s in scopes[3:-1]))
+ for _ in range(12):
+ budget["left"] = 2
+ try:
+ await env.reconcile()
+ except asyncio.CancelledError:
+ pass
+ if set(env.scheduler.jobs) == healthy:
+ break
+ else:
+ pytest.fail(f"the passes never got through; jobs {sorted(env.scheduler.jobs)}")
+
+ assert not job_ids(straggler) & env.scheduler.jobs.keys()
+
+
+async def test_no_scope_starves_whatever_place_a_pass_starts_from(env, monkeypatch):
+ """The review's stale-place probe without Redis: a place left near the
+ start, every scope failing, and every pass cut off at its third attempt
+ (the scope it was cut in taken, its work not done). Started over, six
+ passes visited the same two scopes; each pass walks on from the last
+ scope taken, so the six reach the last scope and come round again."""
+ scopes = sorted(
+ (registry.MemoryScope.for_user(f"stale-{i}") for i in range(7)),
+ key=lambda scope: scope.scope_key,
+ )
+ for scope in scopes:
+ await registry.ensure_scope_scheduled(scope)
+ _lose_every_cron(env)
+ monkeypatch.setattr(scope_reconcile._Place, "last", scopes[0].scope_key)
+ tried: list[str] = []
+ budget = {"left": 0}
+
+ async def failing_add(job, scope, user_timezone):
+ if budget["left"] == 0:
+ raise asyncio.CancelledError("the pass's ceiling")
+ budget["left"] -= 1
+ tried.append(scope.scope_key)
+ raise ConnectionError("this scope fails")
+
+ monkeypatch.setattr(env.crons, "add", failing_add)
+ for _ in range(6):
+ budget["left"] = 2
+ with pytest.raises(asyncio.CancelledError):
+ await env.reconcile()
+
+ assert scopes[-1].scope_key in tried, tried
+ assert scopes[0].scope_key in tried, "and round again"
+
+
+async def test_a_pass_cut_off_in_a_scope_still_moves_the_next_one_on(env, monkeypatch):
+ """The review's hung write: a registration that never returns until the
+ ceiling cuts its pass. The pass took the scope as its place before it
+ began it, so the next one walks on after it and repairs the rest; the
+ scope it was cut in comes round again at the end of the ring."""
+ scopes = [registry.MemoryScope.for_user(f"cut-{i}") for i in range(3)]
+ for scope in scopes:
+ await registry.ensure_scope_scheduled(scope)
+ _lose_every_cron(env)
+ entered = asyncio.Event()
+ tried: list[str] = []
+ add = env.crons.add
+
+ async def hung_add(job, scope, user_timezone):
+ tried.append(scope.scope_key)
+ if scope == scopes[0] and not entered.is_set():
+ entered.set()
+ await asyncio.Event().wait()
+ await add(job, scope, user_timezone)
+
+ monkeypatch.setattr(env.crons, "add", hung_add)
+ cut = asyncio.create_task(env.reconcile())
+ await asyncio.wait_for(entered.wait(), TIMEOUT)
+ cut.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await cut
+
+ await asyncio.wait_for(env.reconcile(), TIMEOUT)
+
+ assert set(env.scheduler.jobs) == set().union(*(job_ids(s) for s in scopes))
+ assert tried.index(scopes[1].scope_key) < len(tried) - 2, "the rest went first"
+ assert tried[-2:] == [scopes[0].scope_key] * 2, "the cut scope last"
+
+
+async def test_a_process_starts_its_walk_at_a_random_scope(env, monkeypatch):
+ """So a scheduler that restarts often can't keep repairing the same
+ first scopes."""
+ scopes = sorted(
+ (registry.MemoryScope.for_user(f"start-{i}") for i in range(4)),
+ key=lambda scope: scope.scope_key,
+ )
+ for scope in scopes:
+ await registry.ensure_scope_scheduled(scope)
+ _lose_every_cron(env)
+ monkeypatch.setattr(scope_reconcile._Place, "last", None)
+ monkeypatch.setattr(scope_reconcile.random, "randrange", lambda n: 2)
+ order: list[str] = []
+ add = env.crons.add
+
+ async def noted_add(job, scope, user_timezone):
+ if scope.scope_key not in order:
+ order.append(scope.scope_key)
+ await add(job, scope, user_timezone)
+
+ monkeypatch.setattr(env.crons, "add", noted_add)
+
+ await env.reconcile()
+
+ keys = [scope.scope_key for scope in scopes]
+ assert order == keys[2:] + keys[:2]
+ assert scope_reconcile._Place.last == keys[1], "the last scope it took"
+
+
[email protected]("change", ["delete", "pause"])
+async def test_a_restore_the_row_stopped_recording_meanwhile_is_taken_back_next_pass(
+ env, monkeypatch, change: str
+):
+ """The review's stale restore: a cron is deleted (its record forgotten
+ first, then the job), or the scope paused, between the restore's read
+ and its write, which puts the cron back, tagged as the reconcile's. The
+ next pass takes back the tagged cron the row no longer records (the
+ pause's: every cron of the scope)."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ _lose_every_cron(env)
+ add = env.crons.add
+ nightly = f"dream_nightly_batch_{ACCOUNT.scope_key}"
+ community = f"community_rebuild_{ACCOUNT.scope_key}"
+
+ async def interleaved_add(job, scope, user_timezone):
+ if job.job_id(scope) == nightly:
+ if change == "delete":
+ await env.db.forget_scope_job(USER, ACCOUNT.scope_key, nightly)
+ env.scheduler.jobs.pop(nightly, None)
+ else:
+ await env.db.set_scope_state(USER, ACCOUNT.scope_key, PAUSED)
+ await add(job, scope, user_timezone)
+
+ monkeypatch.setattr(env.crons, "add", interleaved_add)
+ await env.reconcile()
+ assert nightly in env.scheduler.tagged, "the stale restore landed"
+ monkeypatch.setattr(env.crons, "add", add)
+
+ report = await env.reconcile()
+
+ if change == "delete":
+ assert set(env.scheduler.jobs) == {community}
+ assert report.dropped == [nightly]
+ else:
+ assert env.scheduler.jobs == {}
+ assert report.removed == [ACCOUNT.scope_key]
+
+
[email protected]("ending", ["error", "cancel"])
+async def test_a_restore_cut_short_after_a_stale_add_leaves_nothing_next_pass(
+ env, monkeypatch, ending: str
+):
+ """The round-4 review's partial restore: the community cron is deleted
+ between the restore's read and its add, which puts it back, then the
+ nightly add fails or the pass is cancelled. The tagged community cron
+ the row no longer records goes on the next pass."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ _lose_every_cron(env)
+ add = env.crons.add
+ community = f"community_rebuild_{ACCOUNT.scope_key}"
+
+ async def partial_add(job, scope, user_timezone):
+ if job.job_id(scope) == community:
+ await env.db.forget_scope_job(USER, ACCOUNT.scope_key, community)
+ await add(job, scope, user_timezone)
+ return
+ if ending == "cancel":
+ raise asyncio.CancelledError("the pass's ceiling")
+ raise ConnectionError("the second add failed")
+
+ monkeypatch.setattr(env.crons, "add", partial_add)
+ if ending == "cancel":
+ with pytest.raises(asyncio.CancelledError):
+ await env.reconcile()
+ else:
+ assert (await env.reconcile()).failed == [ACCOUNT.scope_key]
+ assert community in env.scheduler.jobs, "the stale add landed"
+ monkeypatch.setattr(env.crons, "add", add)
+
+ await env.reconcile()
+
+ assert set(env.scheduler.jobs) == {f"dream_nightly_batch_{ACCOUNT.scope_key}"}
+ assert env.db.rows[ACCOUNT.scope_key].community_job_id is None
+
+
[email protected]("ending", ["error", "cancel"])
+async def test_a_stray_goes_before_a_restore_that_fails_in_its_scope(
+ env, monkeypatch, ending: str
+):
+ """The round-5 review's ordering: the scope's restore fails (or the pass
+ is cancelled there) on every pass; its tagged stray goes first all the
+ same, so a repair that keeps failing never keeps one running."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ community, nightly = scope_reconcile.DREAM_SYSTEM_JOBS[:2]
+ env.scheduler.tagged.add(community.job_id(ACCOUNT))
+ await env.db.forget_scope_job(USER, ACCOUNT.scope_key, community.job_id(ACCOUNT))
+ env.scheduler.jobs[nightly.job_id(ACCOUNT)] = TOKYO
+
+ async def failing(job, scope, user_timezone):
+ if ending == "cancel":
+ raise asyncio.CancelledError
+ raise ConnectionError("the nightly's repair is unavailable")
+
+ monkeypatch.setattr(env.crons, "add", failing)
+
+ if ending == "cancel":
+ with pytest.raises(asyncio.CancelledError):
+ await env.reconcile()
+ else:
+ report = await env.reconcile()
+ assert report.dropped == [community.job_id(ACCOUNT)]
+ assert report.failed == [ACCOUNT.scope_key]
+
+ assert community.job_id(ACCOUNT) not in env.scheduler.jobs
+ assert env.scheduler.jobs[nightly.job_id(ACCOUNT)] == TOKYO
+
+
+async def test_a_take_back_that_fails_does_not_hold_up_the_restore(env, monkeypatch):
+ """Each step stands on its own: the scope's stray cannot be taken back
+ (its drop fails), and the row's cron running in the wrong timezone is
+ repaired all the same; the scope is reported failed once."""
+ await registry.ensure_scope_scheduled(ACCOUNT)
+ community, nightly = scope_reconcile.DREAM_SYSTEM_JOBS[:2]
+ env.scheduler.tagged.add(community.job_id(ACCOUNT))
+ await env.db.forget_scope_job(USER, ACCOUNT.scope_key, community.job_id(ACCOUNT))
+ env.scheduler.jobs[nightly.job_id(ACCOUNT)] = TOKYO
+
+ async def failing(job, scope):
+ raise ConnectionError("the take-back is unavailable")
+
+ monkeypatch.setattr(env.crons, "drop", failing)
+
+ report = await env.reconcile()
+
+ assert report.failed == [ACCOUNT.scope_key]
+ assert report.restored == [nightly.job_id(ACCOUNT)]
+ assert env.scheduler.jobs[nightly.job_id(ACCOUNT)] == PARIS
+
+
+async def test_a_tagged_cron_with_no_row_goes_and_an_untagged_one_stays(env):
+ """An account's crons with no row predate the registry and stay; one the
+ reconcile tagged had a row once, and goes with it."""
+ legacy = registry.MemoryScope.for_user("legacy")
+ gone = registry.MemoryScope.for_user("gone")
+ for job in scope_reconcile.DREAM_SYSTEM_JOBS:
+ await job.register(env.scheduler, legacy, PARIS)
+ await env.crons.add(job, gone, PARIS)
+
+ report = await env.reconcile()
+
+ every = scope_reconcile.DREAM_SYSTEM_JOBS
+ assert set(env.scheduler.jobs) == {job.job_id(legacy) for job in every}
+ assert sorted(report.dropped) == sorted(job.job_id(gone) for job in every)
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/AGENTS.md b/autogpt_platform/backend/backend/copilot/graphiti/AGENTS.md
index 6a0b3ef1cc..9e3436d806 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/AGENTS.md
+++ b/autogpt_platform/backend/backend/copilot/graphiti/AGENTS.md
@@ -21,6 +21,20 @@ Runtime prompt instructions live in `prompting.py:get_graphiti_supplement()`.
- Be careful about memory pollution from assistant/tool phrasing; extraction quality matters as much as ingestion success.
- Keep warm-context and tool-driven recall resilient: failures should degrade gracefully rather than break chat execution.
+## Dream Schedules
+
+A memory scope (the account, or one hired expert) that has been registered has one `MemoryScopeSchedule` row, and while that row is ACTIVE, the crons whose flag is on: `dream_nightly_batch_{scope_key}` (daily, 03:00 owner-local) and `community_rebuild_{scope_key}` (weekly, 04:00 owner-local on Mondays: APScheduler reads the `0` in `0 4 * * 0` as Monday). The account's scope key is its user id, so its job ids predate the table. The table decides whether a scope is scheduled; `copilot/dream/registry.py` keeps it in step:
+
+- **Registration.** Ingest registers a scope when its group's queue is created: the group's first write in a process, and again whenever the queue comes back after retiring (60 s idle), with at most one registration in flight per group. A hire or raise registers the expert in the background. Registration never reactivates a PAUSED or WIPED scope, and writes nothing while all applicable flags are off (dream, community, and account-only skill learning), so a scope that was never registered or paused has no row.
+- **Lifecycle.** Archive and a schedule pause (including a budget breach) pause the expert's scope and remove its crons; the resume route and a revive resume a PAUSED scope (or register one that has no row). A lifecycle change the deadline cuts off is not retried: the cron gate still keeps an archived or paused expert's crons from running, and a later lifecycle change or the backfill settles the row (an ordinary registration does not look at the expert's lifecycle). `mark_wiped` is a seam for the wipe work: nothing calls it yet and it does not erase the graph. No registry operation takes a scope out of WIPED: registration and resume skip it, and pause or archive leave it WIPED (a pause only moves an ACTIVE or PAUSED row, checked in the same statement), so the wipe work decides what brings one back.
+- **Reconcile.** Registration, pause and resume act on events, and two of them can interleave: a pause's takedown that lands after a later resume registered the same deterministic job ids, or a registration that read the row before a timezone change and lands after it. Each can leave an ACTIVE row whose recorded crons the scheduler lacks or runs in the wrong timezone, which an ensure, trusting the recorded ids, never notices. On each scheduler start and then every 30 minutes, the scheduler runs `reconcile_memory_scope_crons` (`copilot/dream/scope_reconcile.py`). It lists its dream-system crons (inside the pass's 20-minute ceiling, the database stopping the query after 60 s) and every registry row, paged, then settles each scope out of step, on the row as read again right before. Every cron an ACTIVE row records that the scheduler lacks, or runs in another timezone, is registered in the row's timezone and tagged as the reconcile's (` (reconciled)` on the job's name). A tagged cron its row does not record, or whose row is not ACTIVE or gone, is removed, before the scope's crons are repaired and whatever that repair does: so a restore that raced a delete or a pause (the account's delete RPCs have the registry forget the cron before they remove it, and keep it, failing, when the forget fails), failed partway, was cancelled or finished after its pass, leaves at worst a tagged cron the next pass that reaches the scope takes away, even while the scope's repair keeps failing. An untagged cron the row does not record is the registry's (an account's from before the registry, a registration whose record failed) and is left alone, and no flag is read. Every cron of a PAUSED or WIPED row, or of an expert with no row, is removed; an account's untagged crons with no row are left, as the cron gate lets them run. Every change is idempotent (a removal that finds the job already gone is no error). Each of the pass's own jobstore writes gives up after 10 s on a row another transaction holds (`jobstore.WRITE_TIMEOUT`: a transaction-local `lock_timeout` and `statement_timeout`; the scheduler's other writes are not limited), failing only that scope. A pass walks the scopes as a ring from just after the last scope a pass in this process took (a random one after the process starts), taking each as its place before it works it, so a pass cut off at its ceiling, even inside a scope's work, hands the rest of the ring to the next, and a restart can't keep repairing the same first scopes. The place is the process's own: APScheduler 3 can't share a jobstore between processes, so production runs one scheduler, and during a rollout two each walk on their own. So a lost, stale or stray cron is put right once the walk next reaches its scope, not within a fixed 30 minutes. The job lives in a process-local `MemoryJobStore`, registered after start with an immediate next run; it never enters the shared SQL job table. During a rolling deploy each new scheduler runs its own pass; overlapping passes replay bounded, idempotent work. Expert crons use a separate callable reference so old binaries park them rather than execute incompatible kwargs. A new-code listing that includes a parked row restores its SQL next-run timestamp from the readable payload, guarded by unchanged id, payload and NULL column. The memory-prefix listing, schedule-index reconciliation at startup/daily, and all-jobs cache can recover rows this way. Explicitly paused payloads stay paused. An overdue recovered cron coalesces missed runs subject to its misfire grace; an old pod can park it again until the overlap ends. A registration whose record finds the row no longer ACTIVE reports `scope_changed` and takes nothing down (a takedown there could take a concurrent resume's crons): the gate holds the scope, and the reconcile removes its crons. The cron gate itself only skips a straggler's run; removing it is the reconcile's job, and so is restoring the crons a resume landing inside a pause's takedown lost.
+- **Timezone.** A timezone change re-registers every scope of the user in the new zone. The new timezone is recorded before cron replacement. A failed replacement retains the existing recorded job id so reconciliation can detect the old timezone and retry; a first registration that fails still has no id. A failure to record timezone intent is retried by a later registration or backfill.
+- **Deadlines.** Every registry call to the DatabaseManager, the scheduler, the flag backend or the user lookup runs under a 10 s deadline (`copilot/dream/deadline.py`), built on `asyncio.timeout` so a deadline nested in another still holds on Python 3.11. On timeout the call's cancellation is requested; a timeout does not prove the call had no effect, since the server may already have acted on it. The API hooks bound the whole change, so the memory part of a request ends when that deadline requests cancellation, plus the cancelled call's cleanup and any time the event loop itself is blocked, and a cron whose registry gate cannot answer in time skips that run. Background registration (ingest, hire, raise) and the backfill pay the deadline per call, so during an outage the backfill's run time grows with the number of scopes.
+
+Existing deployments fill the table with `poetry run memory-schedule-backfill` (`--dry-run` only counts; `--force` re-registers every cron, e.g. after jobs were lost from the scheduler). It discovers accounts with a `user_*` FalkorDB graph and a user row, accounts with skill-learning sources, and existing ACTIVE account rows; it ensures those accounts, resumes or registers every live hired expert, pauses paused and archived ones, and prints a JSON report that lists each failed scope with its cron and reason; it exits 1 if anything failed. Re-running is safe; it registers nothing twice, but pauses the paused experts again (a state write and a remove call each). It needs Postgres, FalkorDB, the scheduler service and the flag backend, so run it where the backend services' environment is set; registration uses `dream-pass-enabled`, `graphiti-communities-enabled`, and the independent account-only `dream-skill-learning-enabled` flag. Force refresh does not override WIPED or paused account state. Flag-off cron bodies skip work while recorded intent remains available to reconciliation.
+
+**Deploy order.** Apply both registry migrations and deploy the DatabaseManager and scheduler services first, then the writers (REST API, copilot-executor), then run the backfill: newer writers get a 404 from an older scheduler's scope RPCs, and a newer scheduler's cron gate skips every memory run, the account's included, while an older DatabaseManager cannot answer the registry. Rolling scheduler overlap can temporarily park expert crons; the new scheduler recovers them on a later listing, including its startup/periodic reconciliation. Account cron references and kwargs remain backward-compatible. For rollback, stop or remove expert crons before reverting all schedulers: old binaries leave them parked, and no new reader remains to recover them. The process-local reconcile needs no persistent system-job repair. Backfill follows service deployment and is needed for dormant-scope adoption, not as a prerequisite for the additive schema deployment.
+
## Query Cookbook
Run everything from `autogpt_platform/backend` and use `poetry run ...`.
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/dream_ratification_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/dream_ratification_integration_test.py
index f69c7ffd7f..6aab3c3553 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/dream_ratification_integration_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/dream_ratification_integration_test.py
@@ -207,9 +207,9 @@ def stub_boundaries(
* ``get_graphiti_client`` — patched on ``ingest`` (where it is used) to
return a Graphiti wired to the test's real driver with the LLM stubbed.
- * ``ensure_dream_system_scheduled`` — ``_ensure_worker`` imports it lazily
- from ``dream.scheduling``, so that module IS the use site. Left live it
- would fire LaunchDarkly + scheduler RPCs from a background task.
+ * ``ensure_scope_scheduled`` — patched on ``ingest`` (where the first
+ write per group calls it). Left live it would fire LaunchDarkly,
+ registry and scheduler calls from a background task.
* ``get_redis_async`` — likewise imported lazily by the hit tracker.
"""
driver, _user_id = dream_graph
@@ -217,9 +217,8 @@ def stub_boundaries(
mocker.patch.object(
ingest_mod, "get_graphiti_client", mocker.AsyncMock(return_value=client)
)
- mocker.patch(
- "backend.copilot.dream.scheduling.ensure_dream_system_scheduled",
- mocker.AsyncMock(return_value={}),
+ mocker.patch.object(
+ ingest_mod, "ensure_scope_scheduled", mocker.AsyncMock(return_value={})
)
mocker.patch(
"backend.data.redis_client.get_redis_async",
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/graphs.py b/autogpt_platform/backend/backend/copilot/graphiti/graphs.py
new file mode 100644
index 0000000000..c3965e5cc5
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/graphiti/graphs.py
@@ -0,0 +1,45 @@
+"""Enumerate the memory graphs on the FalkorDB instance.
+
+Nothing records which memory scopes exist — drivers never create a graph
+until the first write — so ``GRAPH.LIST`` is the only inventory of accounts
+that have memory. Expert graphs are named by a digest of the expert id and
+cannot be mapped back; list experts from Postgres instead.
+"""
+
+from falkordb.asyncio import FalkorDB
+
+from .client import derive_group_id
+from .config import graphiti_config
+
+ACCOUNT_GRAPH_PREFIX = "user_"
+
+
+async def list_graph_names() -> list[str]:
+ """Every graph on the FalkorDB instance, via ``GRAPH.LIST``."""
+ client = FalkorDB(
+ host=graphiti_config.falkordb_host,
+ port=graphiti_config.falkordb_port,
+ password=graphiti_config.falkordb_password or None,
+ )
+ try:
+ return list(await client.list_graphs())
+ finally:
+ await client.aclose()
+
+
+async def list_account_graph_owners() -> list[str]:
+ """The user ids that own an account graph (``user_<id>``), sorted."""
+ names = await list_graph_names()
+ return sorted({owner for name in names if (owner := account_graph_owner(name))})
+
+
+def account_graph_owner(graph_name: str) -> str | None:
+ """The user id whose account graph ``graph_name`` is, or None for any
+ other graph (an expert's, or one this code did not derive)."""
+ if not graph_name.startswith(ACCOUNT_GRAPH_PREFIX):
+ return None
+ user_id = graph_name.removeprefix(ACCOUNT_GRAPH_PREFIX)
+ try:
+ return user_id if derive_group_id(user_id) == graph_name else None
+ except ValueError:
+ return None
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/ingest.py b/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
index 5d636b83c9..89cb4a110c 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
@@ -12,6 +12,9 @@ from datetime import datetime, timezone
from graphiti_core.nodes import EpisodeType
+from backend.copilot.dream.registry import ensure_scope_scheduled
+from backend.util.background import spawn_background_task
+
from .client import ensure_indices_once, get_graphiti_client
from .memory_model import MemoryEnvelope, MemoryKind, MemoryStatus, SourceKind
from .scope import MemoryScope
@@ -27,11 +30,14 @@ logger = logging.getLogger(__name__)
# different loop". Scope the registry per running loop so each loop has its
# own queues, workers, and lock. Entries auto-clean when the loop is GC'd.
class _LoopIngestState:
- __slots__ = ("group_queues", "group_workers", "workers_lock")
+ __slots__ = ("group_queues", "group_workers", "registrations", "workers_lock")
def __init__(self) -> None:
self.group_queues: dict[str, asyncio.Queue] = {}
self.group_workers: dict[str, asyncio.Task] = {}
+ # The in-flight dream-schedule registration of each memory group,
+ # dropped when it finishes; see ``_register_dream_schedules``.
+ self.registrations: dict[str, asyncio.Task] = {}
self.workers_lock = asyncio.Lock()
@@ -386,10 +392,11 @@ async def enqueue_conversation_turn(
return
try:
- group_id = MemoryScope.build(user_id, expert_id).group_id
+ scope = MemoryScope.build(user_id, expert_id)
except ValueError:
logger.warning("Invalid memory scope for ingestion: %s", user_id[:12])
return
+ group_id = scope.group_id
user_display_name = await resolve_user_name(user_id)
@@ -403,8 +410,7 @@ async def enqueue_conversation_turn(
source_description = f"User message in session {session_id}"
queued = await _enqueue_payload(
- user_id,
- group_id,
+ scope,
{
"name": episode_name,
"episode_body": episode_body_for_graphiti,
@@ -437,8 +443,7 @@ async def enqueue_conversation_turn(
provenance=f"session:{session_id}",
)
await _enqueue_payload(
- user_id,
- group_id,
+ scope,
{
"name": f"finding_{session_id}",
"episode_body": envelope.model_dump_json(),
@@ -504,8 +509,7 @@ async def enqueue_episode(
source = EpisodeType.json if is_json else EpisodeType.text
queued = await _enqueue_payload(
- user_id,
- group_id,
+ scope,
{
"name": name,
"episode_body": episode_body,
@@ -555,21 +559,22 @@ async def wait_for_ingestion(
return await completion.wait(timeout_seconds)
-async def _enqueue_payload(user_id: str, group_id: str, payload: dict) -> bool:
+async def _enqueue_payload(scope: MemoryScope, payload: dict) -> bool:
"""Atomically select a group worker and enqueue one payload.
Queue selection, worker creation, and ``put_nowait`` share
``workers_lock`` with idle retirement. A caller can therefore never put
work onto a queue after its worker has unregistered it.
- Also fires the auto-registration of the user's dream-system
- schedules (community rebuild + dream pass + ratification pass) the
- first time we see them in this process — lazy on first memory write,
- per-job flag-gated, per-job idempotent. See
- ``copilot/dream/scheduling.py:ensure_dream_system_scheduled``.
- Fire-and-forget; failures are swallowed inside the helper so
- ingestion is never affected.
+ Creating a group's queue also registers the scope's own dream-system
+ crons (community rebuild + nightly batch): an expert group registers the
+ expert's scope, not the account's. That happens on the group's first
+ write in this process and again each time its queue comes back after
+ retiring (``_WORKER_IDLE_TIMEOUT`` idle). Fire-and-forget, flag-gated
+ and idempotent; see ``_register_dream_schedules``.
"""
+ user_id = scope.owner_user_id
+ group_id = scope.group_id
state = _get_loop_state()
is_new_group_for_this_process = False
async with state.workers_lock:
@@ -591,20 +596,36 @@ async def _enqueue_payload(user_id: str, group_id: str, payload: dict) -> bool:
return False
if is_new_group_for_this_process:
- # Fire-and-forget; per-job Redis SETNX inside the helper
- # provides cross-process / cross-restart idempotency. Done
- # outside the workers_lock so the scheduler RPC can't
- # deadlock ingestion.
- from backend.copilot.dream.scheduling import ensure_dream_system_scheduled
-
- asyncio.create_task(
- ensure_dream_system_scheduled(user_id),
- name=f"dream-system-register-{user_id[:12]}",
- )
+ # Outside the workers_lock so the registry's RPCs can't hold up
+ # ingestion.
+ _register_dream_schedules(state, scope)
return True
+def _register_dream_schedules(state: _LoopIngestState, scope: MemoryScope) -> None:
+ """Register the scope's dream-system crons in the background, at most
+ one registration in flight per group: a queue that retires and comes
+ back while the last one is still running does not start another. The
+ scope's registry row makes a repeat registration a no-op; every call it
+ makes is bounded (``copilot/dream/deadline.py``)."""
+ group_id = scope.group_id
+ running = state.registrations.get(group_id)
+ if running is not None and not running.done():
+ return
+ task = spawn_background_task(
+ ensure_scope_scheduled(scope),
+ name=f"dream-system-register-{scope.scope_key[:12]}",
+ )
+ state.registrations[group_id] = task
+
+ def _drop(done: asyncio.Task) -> None:
+ if state.registrations.get(group_id) is done:
+ del state.registrations[group_id]
+
+ task.add_done_callback(_drop)
+
+
async def resolve_user_name(user_id: str) -> str:
"""Get the user's display name from BusinessUnderstanding, or fall back to 'User'."""
try:
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/ingest_test.py b/autogpt_platform/backend/backend/copilot/graphiti/ingest_test.py
index 28ad4e9e38..3c0fa7026d 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/ingest_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/ingest_test.py
@@ -15,20 +15,19 @@ from .scope import MemoryScope
@pytest.fixture(autouse=True)
-def _stub_dream_registration(mocker):
- """_enqueue_payload fires ensure_dream_system_scheduled fire-and-forget for
- every first-seen user. Unmocked it runs a REAL Prisma timezone lookup on
- this test's function-scoped event loop whenever an earlier test already
- connected Prisma, leaving a pool connection bound to a dead loop that
- later kills a session-loop test with "Event loop is closed"."""
- mocker.patch(
- "backend.copilot.dream.scheduling.ensure_dream_system_scheduled",
- AsyncMock(return_value=None),
+def stub_dream_registration(mocker) -> AsyncMock:
+ """_enqueue_payload fires ensure_scope_scheduled fire-and-forget for
+ every first-seen memory group. Unmocked it runs a REAL Prisma timezone
+ lookup on this test's function-scoped event loop whenever an earlier test
+ already connected Prisma, leaving a pool connection bound to a dead loop
+ that later kills a session-loop test with "Event loop is closed"."""
+ return mocker.patch.object(
+ ingest, "ensure_scope_scheduled", AsyncMock(return_value={})
)
def _enqueue_mock(queue: asyncio.Queue) -> AsyncMock:
- async def enqueue(_user_id: str, _group_id: str, payload: dict) -> bool:
+ async def enqueue(_scope: MemoryScope, payload: dict) -> bool:
try:
queue.put_nowait(payload)
except asyncio.QueueFull:
@@ -341,24 +340,21 @@ class TestEnqueueConversationTurn:
expert_id="expert-1",
)
- expert_group = MemoryScope.for_expert("user-1", "expert-1").group_id
+ expert_scope = MemoryScope.for_expert("user-1", "expert-1")
enqueue_mock.assert_awaited_once()
- assert enqueue_mock.await_args.args[:2] == ("user-1", expert_group)
- assert queue.get_nowait()["group_id"] == expert_group
+ assert enqueue_mock.await_args.args[0] == expert_scope
+ assert queue.get_nowait()["group_id"] == expert_scope.group_id
class TestMemoryGroupQueueIsolation:
@pytest.mark.asyncio
async def test_same_user_experts_get_distinct_workers(self) -> None:
- first_group = "expert_first"
- second_group = "expert_second"
+ first = MemoryScope.for_expert("user-1", "first")
+ second = MemoryScope.for_expert("user-1", "second")
+ first_group, second_group = first.group_id, second.group_id
with patch.object(ingest, "_ingestion_worker", new_callable=AsyncMock):
- await ingest._enqueue_payload(
- "user-1", first_group, {"group_id": first_group}
- )
- await ingest._enqueue_payload(
- "user-1", second_group, {"group_id": second_group}
- )
+ await ingest._enqueue_payload(first, {"group_id": first_group})
+ await ingest._enqueue_payload(second, {"group_id": second_group})
state = ingest._get_loop_state()
assert (
@@ -372,16 +368,15 @@ class TestMemoryGroupQueueIsolation:
@pytest.mark.asyncio
async def test_existing_queue_without_worker_is_restarted(self) -> None:
- group_id = "expert_stale_worker"
+ scope = MemoryScope.for_expert("user-1", "stale-worker")
+ group_id = scope.group_id
queue: asyncio.Queue = asyncio.Queue(maxsize=10)
state = ingest._get_loop_state()
state.group_queues[group_id] = queue
worker_mock = AsyncMock()
with patch.object(ingest, "_ingestion_worker", new=worker_mock):
- assert await ingest._enqueue_payload(
- "user-1", group_id, {"group_id": group_id}
- )
+ assert await ingest._enqueue_payload(scope, {"group_id": group_id})
worker = state.group_workers[group_id]
await worker
@@ -389,6 +384,66 @@ class TestMemoryGroupQueueIsolation:
assert queue.get_nowait()["group_id"] == group_id
+class TestDreamScheduleRegistration:
+ @pytest.mark.asyncio
+ async def test_first_write_per_group_registers_that_groups_own_scope(
+ self, stub_dream_registration: AsyncMock
+ ) -> None:
+ """An expert group registers the expert's crons, not the account's,
+ and only on its first write in this process."""
+ expert = MemoryScope.for_expert("user-1", "expert-1")
+ account = MemoryScope.for_user("user-1")
+ with patch.object(ingest, "_ingestion_worker", new_callable=AsyncMock):
+ await ingest._enqueue_payload(expert, {"group_id": expert.group_id})
+ await ingest._enqueue_payload(expert, {"group_id": expert.group_id})
+ await ingest._enqueue_payload(account, {"group_id": account.group_id})
+ state = ingest._get_loop_state()
+ await asyncio.gather(*state.group_workers.values(), return_exceptions=True)
+
+ registered = [c.args[0] for c in stub_dream_registration.call_args_list]
+ assert registered == [expert, account]
+
+ @pytest.mark.asyncio
+ async def test_a_returning_queue_registers_again_but_never_twice_at_once(
+ self, monkeypatch
+ ) -> None:
+ """A group's queue retires after idling and comes back on the next
+ write, which registers the scope again; while a registration is
+ still in flight, the returning queue does not start another."""
+ release = asyncio.Event()
+ calls: list[MemoryScope] = []
+
+ async def ensure(scope: MemoryScope) -> dict:
+ calls.append(scope)
+ await release.wait()
+ return {}
+
+ monkeypatch.setattr(ingest, "ensure_scope_scheduled", ensure)
+ scope = MemoryScope.for_user("user-1")
+ state = ingest._get_loop_state()
+
+ async def write_after_retirement() -> None:
+ state.group_queues.pop(scope.group_id, None)
+ state.group_workers.pop(scope.group_id, None)
+ assert await ingest._enqueue_payload(scope, {"group_id": scope.group_id})
+ await asyncio.sleep(0)
+
+ with patch.object(ingest, "_ingestion_worker", new_callable=AsyncMock):
+ await write_after_retirement()
+ in_flight = state.registrations[scope.group_id]
+ await write_after_retirement()
+ assert calls == [scope]
+
+ release.set()
+ await in_flight
+ await asyncio.sleep(0)
+ assert scope.group_id not in state.registrations
+
+ await write_after_retirement()
+ assert calls == [scope, scope]
+ await state.registrations[scope.group_id]
+
+
class TestQueueFullScenario:
@pytest.mark.asyncio
async def test_queue_full_logs_warning_no_crash(self) -> None:
@@ -495,7 +550,7 @@ class TestEnqueueEpisode:
assert await ingest.enqueue_episode(
scope, "sess1", name="test_ep", episode_body="hello"
)
- assert enqueue_mock.await_args.args[:2] == ("abc", scope.group_id)
+ assert enqueue_mock.await_args.args[0] == scope
assert q.get_nowait()["group_id"] == scope.group_id
@pytest.mark.asyncio
@@ -690,8 +745,7 @@ class TestWorkerIdleTimeout:
try:
enqueue_task = asyncio.create_task(
ingest._enqueue_payload(
- user_id,
- group_id,
+ MemoryScope.for_user(user_id),
{
"name": "racing-episode",
"episode_body": "hello",
@@ -748,8 +802,7 @@ class TestWorkerIdleTimeout:
await asyncio.sleep(0)
enqueue_task = asyncio.create_task(
ingest._enqueue_payload(
- user_id,
- group_id,
+ MemoryScope.for_user(user_id),
{
"name": "cancellation-race-episode",
"episode_body": "hello",
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/scope.py b/autogpt_platform/backend/backend/copilot/graphiti/scope.py
index 4c47de1ed0..796ec5b0e0 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/scope.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/scope.py
@@ -16,9 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
from .client import derive_memory_group_id, derive_memory_scope_key
-RedisKeyFamily = Literal[
- "dream_lock", "last_completed", "hits", "rebuild_lock", "registration"
-]
+RedisKeyFamily = Literal["dream_lock", "last_completed", "hits", "rebuild_lock"]
# One in-flight dream per scope (dream/locks.py).
DREAM_LOCK_KEY_PREFIX = "dream:inflight:"
@@ -78,23 +76,13 @@ class MemoryScope(BaseModel):
def is_expert(self) -> bool:
return self.expert_id is not None
- def redis_key(
- self,
- family: RedisKeyFamily,
- *,
- edge_uuid: str | None = None,
- registration_prefix: str | None = None,
- ) -> str:
+ def redis_key(self, family: RedisKeyFamily, *, edge_uuid: str | None = None) -> str:
"""This scope's Redis key for ``family``, exactly as it was keyed
before this class existed.
- Most families key on ``scope_key`` (the raw user id for the account,
- the group id for an expert). Two do not, and keep their layouts:
- ``rebuild_lock`` keys on ``group_id``, and ``registration`` keys on
- the owner's user id even for an expert scope, because the dream
- crons are registered per user. ``hits`` needs the counted
- ``edge_uuid``; ``registration`` needs the cron's marker prefix (see
- ``dream/scheduling.py``).
+ Every family keys on ``scope_key`` (the raw user id for the account,
+ the group id for an expert) except ``rebuild_lock``, which keeps its
+ ``group_id`` layout. ``hits`` needs the counted ``edge_uuid``.
"""
match family:
case "dream_lock":
@@ -107,10 +95,6 @@ class MemoryScope(BaseModel):
return f"{HIT_TRACKER_KEY_PREFIX}:{self.scope_key}:{edge_uuid}"
case "rebuild_lock":
return f"{REBUILD_LOCK_KEY_PREFIX}{self.group_id}"
- case "registration":
- if registration_prefix is None:
- raise ValueError("the registration key needs a prefix")
- return f"{registration_prefix}:{self.owner_user_id}"
@model_validator(mode="after")
def _ids_match_derivation(self) -> "MemoryScope":
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/scope_test.py b/autogpt_platform/backend/backend/copilot/graphiti/scope_test.py
index f0744ede7c..a3c101c3cc 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/scope_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/scope_test.py
@@ -3,11 +3,6 @@
import pytest
from pydantic import ValidationError
-from backend.copilot.dream.scheduling import (
- COMMUNITY_REBUILD_REGISTRATION_PREFIX,
- NIGHTLY_BATCH_REGISTRATION_PREFIX,
-)
-
from .client import derive_group_id, derive_memory_group_id, derive_memory_scope_key
from .scope import (
DREAM_LOCK_KEY_PREFIX,
@@ -100,26 +95,10 @@ class TestRedisKeys:
legacy = f"{REBUILD_LOCK_KEY_PREFIX}{group_id}"
assert scope.redis_key("rebuild_lock") == legacy
- @BOTH_SCOPES
- @pytest.mark.parametrize(
- "prefix",
- [COMMUNITY_REBUILD_REGISTRATION_PREFIX, NIGHTLY_BATCH_REGISTRATION_PREFIX],
- )
- def test_registration_keys_on_owner_user(
- self, expert_id: str | None, prefix: str
- ) -> None:
- scope = MemoryScope.build(USER_ID, expert_id)
- legacy = f"{prefix}:{USER_ID}"
- assert scope.redis_key("registration", registration_prefix=prefix) == legacy
-
def test_hits_key_requires_edge_uuid(self) -> None:
with pytest.raises(ValueError, match="edge_uuid"):
MemoryScope.for_user(USER_ID).redis_key("hits")
- def test_registration_key_requires_prefix(self) -> None:
- with pytest.raises(ValueError, match="prefix"):
- MemoryScope.for_user(USER_ID).redis_key("registration")
-
class TestValidation:
def test_empty_user_id_is_rejected(self) -> None:
diff --git a/autogpt_platform/backend/backend/copilot/learning/capture.py b/autogpt_platform/backend/backend/copilot/learning/capture.py
index 3a8d77dd4c..dcbeeae60c 100644
--- a/autogpt_platform/backend/backend/copilot/learning/capture.py
+++ b/autogpt_platform/backend/backend/copilot/learning/capture.py
@@ -10,7 +10,7 @@ from __future__ import annotations
import asyncio
import logging
-from backend.copilot.dream.scheduling import ensure_dream_system_scheduled
+from backend.copilot.dream.registry import ensure_dream_system_scheduled
from backend.copilot.model import ChatMessage, ChatSession
from backend.data.skill_learning import LearningSourceRecord
from backend.util.feature_flag import Flag, is_feature_enabled
diff --git a/autogpt_platform/backend/backend/copilot/learning/dreaming_test.py b/autogpt_platform/backend/backend/copilot/learning/dreaming_test.py
index 213f8e88a8..0663901b84 100644
--- a/autogpt_platform/backend/backend/copilot/learning/dreaming_test.py
+++ b/autogpt_platform/backend/backend/copilot/learning/dreaming_test.py
@@ -49,3 +49,41 @@ async def test_dreaming_reviews_private_skills_even_when_memory_dreaming_fails(
run.assert_awaited_once_with(user, trigger="nightly")
assert result.learning == learned
assert result.dream.error == "memory unavailable"
+
+
+async def test_an_experts_nightly_batch_leaves_skill_learning_to_the_account(
+ monkeypatch,
+):
+ """One pass reviews the account's skills and its experts': each expert's
+ nightly batch running it again would only queue up behind the lease."""
+ now = datetime.now(timezone.utc)
+ user = "account-a"
+ run = AsyncMock()
+ monkeypatch.setattr(
+ nightly_batch, "is_feature_enabled", AsyncMock(return_value=True), raising=False
+ )
+ monkeypatch.setattr(nightly_batch, "run_skill_learning_pass", run, raising=False)
+ monkeypatch.setattr(
+ nightly_batch, "check_dream_budget", AsyncMock(return_value=(True, None))
+ )
+ monkeypatch.setattr(
+ orchestrator,
+ "execute_dream_pass",
+ AsyncMock(
+ return_value=DreamPassResult(
+ user_id=user,
+ pass_id="dream-1",
+ execution_path="sync_baseline",
+ started_at=now,
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ nightly_batch,
+ "run_ratification_pass",
+ AsyncMock(return_value=RatificationResult(user_id=user, started_at=now)),
+ )
+ result = await nightly_batch.run_nightly_batch_submit(user, expert_id="expert-1")
+ run.assert_not_awaited()
+ assert result.learning is None
diff --git a/autogpt_platform/backend/backend/data/db_accessors.py b/autogpt_platform/backend/backend/data/db_accessors.py
index 60e504bf8b..46fd103519 100644
--- a/autogpt_platform/backend/backend/data/db_accessors.py
+++ b/autogpt_platform/backend/backend/data/db_accessors.py
@@ -144,6 +144,19 @@ def user_db():
return user_db
+def memory_schedule_db():
+ if db.is_connected():
+ from backend.data import memory_schedule as _memory_schedule_db
+
+ memory_schedule_db = _memory_schedule_db
+ else:
+ from backend.util.clients import get_database_manager_async_client
+
+ memory_schedule_db = get_database_manager_async_client()
+
+ return memory_schedule_db
+
+
def understanding_db():
if db.is_connected():
from backend.data import understanding as _understanding_db
diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.py
index 43f758ad45..a3ea42ba95 100644
--- a/autogpt_platform/backend/backend/data/db_manager.py
+++ b/autogpt_platform/backend/backend/data/db_manager.py
@@ -63,6 +63,7 @@ from backend.copilot.sharing.db import link_new_execution_to_chat_share
from backend.data import bot_analytics as bot_analytics_db
from backend.data import bot_installs as bot_installs_db
from backend.data import db
+from backend.data import memory_schedule as memory_schedule_db
from backend.data import skill_deletion as skill_deletion_data
from backend.data import skill_learning as skill_learning_data
from backend.data import skill_publication as skill_publication_data
@@ -593,6 +594,20 @@ class DatabaseManager(AppService):
upsert_bot_install = _(bot_installs_db.upsert_bot_install)
revoke_bot_install = _(bot_installs_db.revoke_bot_install)
+ # ============ Memory Scope Schedules ============ #
+ # Exposed so the Prisma-less scheduler (cron bodies) and copilot-executor
+ # (lazy registration on the first memory write) reach the registry via
+ # db_accessors.memory_schedule_db().
+ get_scope_schedule = _(memory_schedule_db.get_scope_schedule)
+ claim_scope_schedule = _(memory_schedule_db.claim_scope_schedule)
+ record_scope_jobs = _(memory_schedule_db.record_scope_jobs)
+ set_scope_state = _(memory_schedule_db.set_scope_state)
+ forget_scope_job = _(memory_schedule_db.forget_scope_job)
+ record_scope_run = _(memory_schedule_db.record_scope_run)
+ list_user_scope_schedules = _(memory_schedule_db.list_user_scope_schedules)
+ list_scope_schedules = _(memory_schedule_db.list_scope_schedules)
+ existing_user_ids = _(memory_schedule_db.existing_user_ids)
+
# ============ Bot Analytics ============ #
record_bot_event = _(bot_analytics_db.record_bot_event)
record_guild_joined = _(bot_analytics_db.record_guild_joined)
@@ -1100,6 +1115,17 @@ class DatabaseManagerAsyncClient(AppServiceClient):
upsert_bot_install = d.upsert_bot_install
revoke_bot_install = d.revoke_bot_install
+ # ============ Memory Scope Schedules ============ #
+ get_scope_schedule = d.get_scope_schedule
+ claim_scope_schedule = d.claim_scope_schedule
+ record_scope_jobs = d.record_scope_jobs
+ set_scope_state = d.set_scope_state
+ forget_scope_job = d.forget_scope_job
+ record_scope_run = d.record_scope_run
+ list_user_scope_schedules = d.list_user_scope_schedules
+ list_scope_schedules = d.list_scope_schedules
+ existing_user_ids = d.existing_user_ids
+
# ============ Bot Analytics ============ #
record_bot_event = d.record_bot_event
record_guild_joined = d.record_guild_joined
diff --git a/autogpt_platform/backend/backend/data/db_manager_test.py b/autogpt_platform/backend/backend/data/db_manager_test.py
index 98badbf370..4a1a8a6ea1 100644
--- a/autogpt_platform/backend/backend/data/db_manager_test.py
+++ b/autogpt_platform/backend/backend/data/db_manager_test.py
@@ -24,6 +24,31 @@ def test_add_store_agent_rpc_request_schema_is_constructible() -> None:
manager._create_fastapi_endpoint(manager.add_store_agent_to_library)
+def test_memory_schedule_methods_registered() -> None:
+ """The scheduler and copilot-executor have no Prisma client; the
+ schedule registry reaches its rows through these."""
+ for method in (
+ "get_scope_schedule",
+ "claim_scope_schedule",
+ "record_scope_jobs",
+ "set_scope_state",
+ "forget_scope_job",
+ "record_scope_run",
+ "list_user_scope_schedules",
+ "list_scope_schedules",
+ "existing_user_ids",
+ ):
+ assert hasattr(DatabaseManager, method)
+ assert hasattr(DatabaseManagerAsyncClient, method)
+
+
+def test_memory_schedule_rpc_request_schemas_are_constructible() -> None:
+ manager = DatabaseManager()
+ manager._create_fastapi_endpoint(manager.claim_scope_schedule)
+ manager._create_fastapi_endpoint(manager.record_scope_run)
+ manager._create_fastapi_endpoint(manager.existing_user_ids)
+
+
def test_the_audience_consumer_can_read_the_opt_out() -> None:
assert hasattr(DatabaseManager, "is_marketing_opted_out")
assert hasattr(DatabaseManagerAsyncClient, "is_marketing_opted_out")
diff --git a/autogpt_platform/backend/backend/data/memory_schedule.py b/autogpt_platform/backend/backend/data/memory_schedule.py
new file mode 100644
index 0000000000..b095090ff1
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/memory_schedule.py
@@ -0,0 +1,316 @@
+"""Data access for the memory-scope schedule registry.
+
+One ``MemoryScopeSchedule`` row per memory scope (the account, or one hired
+expert; see ``copilot/graphiti/scope.py``) records whether the scope's dream
+crons may run, the timezone they were registered in, their APScheduler job ids
+and when each last ran. ``copilot/dream/registry.py`` owns the logic; this
+module is only its storage. The scheduler and copilot-executor processes have
+no Prisma client and reach it through ``db_accessors.memory_schedule_db()``.
+
+Ownership: every function that names a scope also takes its owner's user id
+and matches on both, so a caller only ever reaches rows of the user it names.
+The listings at the bottom are unscoped on purpose: the scheduler's cron
+reconcile pages through every row over the DatabaseManager RPC
+(``list_scope_schedules``) and asks which owners of an account cron with no
+row still exist (``existing_user_ids``), and the backfill's own listings are
+not exposed there.
+"""
+
+from collections.abc import Sequence
+from datetime import datetime, timezone
+from typing import Literal
+
+import prisma.models
+import prisma.types
+from prisma.enums import MemoryScopeScheduleState, ResourceVisibility
+from prisma.errors import UniqueViolationError
+from pydantic import BaseModel
+
+from backend.util.exceptions import NotAuthorizedError
+
+ScopeRunKind = Literal["nightly", "community"]
+
+
+class MemoryScopeSchedule(BaseModel):
+ """One memory scope's registry row."""
+
+ scope_key: str
+ user_id: str
+ expert_id: str | None
+ timezone: str
+ state: MemoryScopeScheduleState
+ community_job_id: str | None
+ nightly_job_id: str | None
+ learning_job_id: str | None
+ last_nightly_run_at: datetime | None
+ last_community_run_at: datetime | None
+ created_at: datetime
+ updated_at: datetime
+
+ @staticmethod
+ def from_db(row: prisma.models.MemoryScopeSchedule) -> "MemoryScopeSchedule":
+ return MemoryScopeSchedule(
+ scope_key=row.scopeKey,
+ user_id=row.userId,
+ expert_id=row.expertId,
+ timezone=row.timezone,
+ state=row.state,
+ community_job_id=row.communityJobId,
+ nightly_job_id=row.nightlyJobId,
+ learning_job_id=row.learningJobId,
+ last_nightly_run_at=row.lastNightlyRunAt,
+ last_community_run_at=row.lastCommunityRunAt,
+ created_at=row.createdAt,
+ updated_at=row.updatedAt,
+ )
+
+
+class LiveExpertScope(BaseModel):
+ """A hired, unarchived expert: a scope the backfill schedules or pauses."""
+
+ expert_id: str
+ user_id: str
+ paused: bool
+
+
+async def get_scope_schedule(
+ user_id: str, scope_key: str
+) -> MemoryScopeSchedule | None:
+ """The scope's row, or None when it has none or it is another user's."""
+ row = await prisma.models.MemoryScopeSchedule.prisma().find_first(
+ where={"scopeKey": scope_key, "userId": user_id}
+ )
+ return MemoryScopeSchedule.from_db(row) if row else None
+
+
+async def claim_scope_schedule(
+ user_id: str,
+ scope_key: str,
+ expert_id: str | None,
+ user_timezone: str,
+ state: MemoryScopeScheduleState = MemoryScopeScheduleState.ACTIVE,
+) -> MemoryScopeSchedule:
+ """Create the scope's row in ``state`` unless one exists, and return it.
+
+ An existing row comes back unchanged: whoever created it set its state.
+ Concurrent first claims all return the one row that won: the create is
+ tried, and on the unique violation the winner is read back. Raises
+ ``NotAuthorizedError`` when the key is another user's.
+ """
+ data: prisma.types.MemoryScopeScheduleCreateInput = {
+ "scopeKey": scope_key,
+ "userId": user_id,
+ "expertId": expert_id,
+ "timezone": user_timezone,
+ "state": state,
+ }
+ try:
+ row = await prisma.models.MemoryScopeSchedule.prisma().create(data=data)
+ except UniqueViolationError:
+ row = await _claim_winner(data)
+ if row.userId != user_id:
+ raise NotAuthorizedError(f"Memory scope {scope_key[:12]} is another user's")
+ return MemoryScopeSchedule.from_db(row)
+
+
+async def _claim_winner(
+ data: prisma.types.MemoryScopeScheduleCreateInput,
+) -> prisma.models.MemoryScopeSchedule:
+ """The row a concurrent claim created. If it was deleted before this
+ read, the create is tried once more, and a second violation (or one on
+ another key, such as a second row for the same expert) propagates."""
+ table = prisma.models.MemoryScopeSchedule.prisma()
+ row = await table.find_unique(where={"scopeKey": data["scopeKey"]})
+ return row if row is not None else await table.create(data=data)
+
+
+async def record_scope_jobs(
+ user_id: str,
+ scope_key: str,
+ *,
+ user_timezone: str,
+ community_job_id: str | None,
+ nightly_job_id: str | None,
+ learning_job_id: str | None,
+) -> bool:
+ """Record the scope's job ids and the timezone they run in.
+
+ Writes only an ACTIVE row: False means the write was refused, the row
+ being paused, wiped or deleted meanwhile. The caller reports
+ ``scope_changed`` and takes nothing down
+ (``copilot/dream/scope_jobs._record``): the fire gate skips a run under a
+ PAUSED or WIPED row or an expert with no row, and the scheduler's
+ reconcile removes those crons (``copilot/dream/scope_reconcile.py``). An
+ account with no row is the exception: it is not gated, and the reconcile
+ leaves the crons its registration made (untagged) where they are.
+ """
+ updated = await prisma.models.MemoryScopeSchedule.prisma().update_many(
+ where={
+ "scopeKey": scope_key,
+ "userId": user_id,
+ "state": MemoryScopeScheduleState.ACTIVE,
+ },
+ data={
+ "timezone": user_timezone,
+ "communityJobId": community_job_id,
+ "nightlyJobId": nightly_job_id,
+ "learningJobId": learning_job_id,
+ },
+ )
+ return updated > 0
+
+
+async def set_scope_state(
+ user_id: str,
+ scope_key: str,
+ state: MemoryScopeScheduleState,
+ *,
+ only_from: Sequence[MemoryScopeScheduleState] | None = None,
+) -> bool:
+ """Move the scope to ``state``; False when it has no row, or, with
+ ``only_from``, when the row is in none of those states. The check and
+ the write are one statement, so a concurrent change cannot slip between
+ them.
+
+ Leaving ACTIVE forgets the job ids too: the registry removes the jobs
+ together with the state change.
+ """
+ where: prisma.types.MemoryScopeScheduleWhereInput = {
+ "scopeKey": scope_key,
+ "userId": user_id,
+ }
+ if only_from is not None:
+ where["state"] = {"in": list(only_from)}
+ data: prisma.types.MemoryScopeScheduleUpdateManyMutationInput = {"state": state}
+ if state != MemoryScopeScheduleState.ACTIVE:
+ data["communityJobId"] = None
+ data["nightlyJobId"] = None
+ data["learningJobId"] = None
+ updated = await prisma.models.MemoryScopeSchedule.prisma().update_many(
+ where=where, data=data
+ )
+ return updated > 0
+
+
+async def forget_scope_job(user_id: str, scope_key: str, job_id: str) -> bool:
+ """Drop ``job_id`` from the scope's row once the job itself is gone, so
+ the next ``ensure_scope_scheduled`` registers it again."""
+ table = prisma.models.MemoryScopeSchedule.prisma()
+ community = await table.update_many(
+ where={"scopeKey": scope_key, "userId": user_id, "communityJobId": job_id},
+ data={"communityJobId": None},
+ )
+ nightly = await table.update_many(
+ where={"scopeKey": scope_key, "userId": user_id, "nightlyJobId": job_id},
+ data={"nightlyJobId": None},
+ )
+ learning = await table.update_many(
+ where={"scopeKey": scope_key, "userId": user_id, "learningJobId": job_id},
+ data={"learningJobId": None},
+ )
+ return community + nightly + learning > 0
+
+
+async def record_scope_run(user_id: str, scope_key: str, kind: ScopeRunKind) -> bool:
+ """Stamp the time the scope's nightly or community job last ran cleanly."""
+ now = datetime.now(timezone.utc)
+ data: prisma.types.MemoryScopeScheduleUpdateManyMutationInput = (
+ {"lastNightlyRunAt": now} if kind == "nightly" else {"lastCommunityRunAt": now}
+ )
+ updated = await prisma.models.MemoryScopeSchedule.prisma().update_many(
+ where={"scopeKey": scope_key, "userId": user_id}, data=data
+ )
+ return updated > 0
+
+
+async def list_user_scope_schedules(user_id: str) -> list[MemoryScopeSchedule]:
+ """Every scope row the user owns: the account's and each expert's."""
+ rows = await prisma.models.MemoryScopeSchedule.prisma().find_many(
+ where={"userId": user_id}, order={"scopeKey": "asc"}
+ )
+ return [MemoryScopeSchedule.from_db(row) for row in rows]
+
+
+async def list_scope_schedules(
+ *, after: str | None = None, limit: int = 500
+) -> list[MemoryScopeSchedule]:
+ """One page of every row, whatever its state, across all users, by scope
+ key; pass the last key of a page as ``after`` for the next. For the
+ scheduler's cron reconcile (``copilot/dream/scope_reconcile.py``)."""
+ where: prisma.types.MemoryScopeScheduleWhereInput = {}
+ if after is not None:
+ where["scopeKey"] = {"gt": after}
+ rows = await prisma.models.MemoryScopeSchedule.prisma().find_many(
+ where=where, order={"scopeKey": "asc"}, take=limit
+ )
+ return [MemoryScopeSchedule.from_db(row) for row in rows]
+
+
+async def list_active_scope_schedules(
+ *, after: str | None = None, limit: int = 500
+) -> list[MemoryScopeSchedule]:
+ """One page of ACTIVE rows across all users, by scope key; pass the last
+ key of a page as ``after`` for the next. Backfill only."""
+ where: prisma.types.MemoryScopeScheduleWhereInput = {
+ "state": MemoryScopeScheduleState.ACTIVE
+ }
+ if after is not None:
+ where["scopeKey"] = {"gt": after}
+ rows = await prisma.models.MemoryScopeSchedule.prisma().find_many(
+ where=where, order={"scopeKey": "asc"}, take=limit
+ )
+ return [MemoryScopeSchedule.from_db(row) for row in rows]
+
+
+async def list_live_expert_scopes(
+ *, after: str | None = None, limit: int = 500
+) -> list[LiveExpertScope]:
+ """One page of hired, unarchived, owner-only experts, by expert id; pass
+ the last id of a page as ``after`` for the next. Backfill only."""
+ where: prisma.types.ExpertWhereInput = {
+ "isTemplate": False,
+ "isArchived": False,
+ "ownerUserId": {"not": None},
+ "visibility": ResourceVisibility.PRIVATE,
+ }
+ if after is not None:
+ where["id"] = {"gt": after}
+ rows = await prisma.models.Expert.prisma().find_many(
+ where=where, order={"id": "asc"}, take=limit
+ )
+ return [
+ LiveExpertScope(
+ expert_id=row.id,
+ user_id=row.ownerUserId,
+ paused=row.schedulesPausedAt is not None,
+ )
+ for row in rows
+ if row.ownerUserId is not None
+ ]
+
+
+async def list_learning_account_ids(
+ *, after: str | None = None, limit: int = 500
+) -> list[str]:
+ """One page of the users with skill-learning sources, by user id; pass
+ the last id of a page as ``after`` for the next. Backfill only: an
+ account can learn without a memory graph, and its learning cron was
+ registered when its first source landed."""
+ where: prisma.types.SkillLearningSourceWhereInput = {}
+ if after is not None:
+ where["userId"] = {"gt": after}
+ rows = await prisma.models.SkillLearningSource.prisma().group_by(
+ by=["userId"], where=where, order={"userId": "asc"}, take=limit
+ )
+ return [row["userId"] for row in rows]
+
+
+async def existing_user_ids(user_ids: list[str]) -> set[str]:
+ """The subset of ``user_ids`` that still have a User row: the backfill's
+ check of the owners it found, and the reconcile's and the cron gate's of
+ an account cron with no row (``copilot/dream/scope_reconcile.py``,
+ ``scope_crons.py``). One query; callers page their ids."""
+ if not user_ids:
+ return set()
+ rows = await prisma.models.User.prisma().find_many(where={"id": {"in": user_ids}})
+ return {row.id for row in rows}
diff --git a/autogpt_platform/backend/backend/data/memory_schedule_db_test.py b/autogpt_platform/backend/backend/data/memory_schedule_db_test.py
new file mode 100644
index 0000000000..17779c51ac
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/memory_schedule_db_test.py
@@ -0,0 +1,306 @@
+"""The memory-scope schedule registry against a real Postgres: concurrent
+first claims of one scope, a pause racing them, a wiped expert's scope
+going through the real pause, resume, archive and revive functions, a
+deleted user's legacy cron (with a real scheduler and jobstore), and the
+backfill's listing of the accounts that learn.
+
+Needs the database (``server`` starts the test stack). The unit tests in
+``memory_schedule_test.py`` cover the same code with Prisma mocked.
+"""
+
+import asyncio
+import uuid
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import prisma.models
+import pytest
+from prisma.enums import MemoryScopeScheduleState
+
+from backend.api.features.experts import experts_db
+from backend.api.features.experts import scheduling as expert_scheduling
+from backend.copilot.dream import registry, scope_crons, scope_jobs, scope_reconcile
+from backend.copilot.dream.registry_fakes_test import FakeScheduler
+from backend.copilot.dream.scheduling import SKILL_LEARNING_JOB_PREFIX, dream_system_job
+from backend.copilot.dream.scope_reconcile_jobstore_test import (
+ _job,
+ _scheduler,
+ _sqlite,
+)
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data import memory_schedule
+from backend.data.memory_schedule import MemoryScopeSchedule
+from backend.executor import scheduler as scheduler_module
+from backend.util.test import SpinTestServer
+
+SCOPES = 10
+CONTENDERS = 20
+WIPED = MemoryScopeScheduleState.WIPED
+
+
+async def _make_users(prefix: str) -> list[str]:
+ user_ids = [f"{prefix}-{uuid.uuid4().hex[:16]}" for _ in range(SCOPES)]
+ await prisma.models.User.prisma().create_many(
+ data=[{"id": uid, "email": f"{uid}@example.invalid"} for uid in user_ids]
+ )
+ return user_ids
+
+
+async def _drop_users(user_ids: list[str]) -> None:
+ await prisma.models.User.prisma().delete_many(where={"id": {"in": user_ids}})
+
+
[email protected](loop_scope="session")
+async def test_concurrent_first_claims_all_return_the_one_row(server: SpinTestServer):
+ user_ids = await _make_users("claim-race")
+ try:
+ for user_id in user_ids:
+ results = await asyncio.gather(
+ *(
+ memory_schedule.claim_scope_schedule(user_id, user_id, None, "UTC")
+ for _ in range(CONTENDERS)
+ ),
+ return_exceptions=True,
+ )
+ failures = [r for r in results if not isinstance(r, MemoryScopeSchedule)]
+ assert failures == []
+ rows = await prisma.models.MemoryScopeSchedule.prisma().count(
+ where={"userId": user_id}
+ )
+ assert rows == 1
+ finally:
+ await _drop_users(user_ids)
+
+
[email protected](loop_scope="session")
+async def test_a_pause_racing_first_registrations_always_ends_paused(
+ server: SpinTestServer,
+):
+ """A pause of a scope with no row claims it PAUSED while registrations
+ claim it ACTIVE. In every other round a registration's claim lands
+ first, so the pause's own claim loses and gets the ACTIVE row back;
+ either way the pause ends PAUSED with the crons removed."""
+ user_ids = await _make_users("pause-race")
+ try:
+ for round_number, user_id in enumerate(user_ids):
+ await _race_pause_against_claims(
+ user_id, registration_first=round_number % 2 == 1
+ )
+ finally:
+ await _drop_users(user_ids)
+
+
+async def _race_pause_against_claims(user_id: str, registration_first: bool) -> None:
+ barrier = asyncio.Barrier(CONTENDERS + 1)
+ registered = asyncio.Event()
+
+ async def register(*args: Any) -> MemoryScopeSchedule:
+ await barrier.wait()
+ row = await memory_schedule.claim_scope_schedule(*args)
+ registered.set()
+ return row
+
+ async def pause_claim(*args: Any) -> MemoryScopeSchedule:
+ await barrier.wait()
+ if registration_first:
+ await registered.wait()
+ return await memory_schedule.claim_scope_schedule(*args)
+
+ db = SimpleNamespace(
+ set_scope_state=memory_schedule.set_scope_state,
+ claim_scope_schedule=pause_claim,
+ )
+ remove = AsyncMock(return_value=True)
+ with (
+ patch.object(registry, "memory_schedule_db", return_value=db),
+ patch.object(registry, "resolve_user_timezone", AsyncMock(return_value="UTC")),
+ patch.object(registry, "remove_scope_jobs", remove),
+ ):
+ paused, *_ = await asyncio.gather(
+ registry.pause_scope(MemoryScope.for_user(user_id)),
+ *(register(user_id, user_id, None, "UTC") for _ in range(CONTENDERS)),
+ )
+ row = await memory_schedule.get_scope_schedule(user_id, user_id)
+ assert paused is True
+ assert row is not None and row.state == MemoryScopeScheduleState.PAUSED
+ remove.assert_awaited_once()
+
+
[email protected](loop_scope="session")
+async def test_a_wiped_expert_stays_wiped_through_its_lifecycle(
+ server: SpinTestServer, monkeypatch: pytest.MonkeyPatch
+):
+ """Codex's round-2 reproduction: pause then resume, and archive then
+ revive, used to bring a wiped expert's scope back ACTIVE with both
+ crons, because the pause wrote PAUSED over WIPED. A direct resume of a
+ wiped scope was already refused."""
+ scheduler = _lifecycle_boundaries(monkeypatch)
+ user_id, scope = await _make_expert("wiped-lifecycle")
+ try:
+ await registry.ensure_scope_scheduled(scope)
+ assert await registry.mark_wiped(scope)
+ await registry.resume_scope(scope)
+ assert await _stopped(scope, scheduler) == WIPED
+
+ assert await expert_scheduling.pause_expert_schedules(
+ user_id, scope.expert_id or "", "test"
+ )
+ assert await expert_scheduling.resume_expert_schedules(
+ user_id, scope.expert_id or ""
+ )
+ assert await _stopped(scope, scheduler) == WIPED
+
+ await experts_db.archive_expert(user_id, scope.expert_id or "")
+ revived = await prisma.models.Expert.prisma().update(
+ where={"id": scope.expert_id or ""}, data={"isArchived": False}
+ )
+ assert revived is not None
+ await experts_db._resume_revived_hire(revived)
+ assert await _stopped(scope, scheduler) == WIPED
+ finally:
+ await _drop_users([user_id])
+
+
[email protected](loop_scope="session")
+async def test_a_paused_expert_still_resumes(
+ server: SpinTestServer, monkeypatch: pytest.MonkeyPatch
+):
+ scheduler = _lifecycle_boundaries(monkeypatch)
+ user_id, scope = await _make_expert("paused-lifecycle")
+ try:
+ await registry.ensure_scope_scheduled(scope)
+ await expert_scheduling.pause_expert_schedules(
+ user_id, scope.expert_id or "", "test"
+ )
+ assert await _stopped(scope, scheduler) == MemoryScopeScheduleState.PAUSED
+
+ await expert_scheduling.resume_expert_schedules(user_id, scope.expert_id or "")
+
+ row = await memory_schedule.get_scope_schedule(user_id, scope.scope_key)
+ assert row is not None and row.state == MemoryScopeScheduleState.ACTIVE
+ assert len(scheduler.jobs) == 2
+ finally:
+ await _drop_users([user_id])
+
+
+def _lifecycle_boundaries(monkeypatch: pytest.MonkeyPatch) -> FakeScheduler:
+ """Everything but Postgres: the scheduler, flags, the timezone lookup
+ and the triggers the lifecycle functions touch."""
+ scheduler = FakeScheduler()
+ monkeypatch.setattr(scope_jobs, "is_feature_enabled", AsyncMock(return_value=True))
+ monkeypatch.setattr(scope_jobs, "get_scheduler_client", lambda: scheduler)
+ monkeypatch.setattr(
+ registry, "resolve_user_timezone", AsyncMock(return_value="UTC")
+ )
+ for name in (
+ "reset_weekly_spend",
+ "detach_expert_triggers",
+ "reattach_expert_triggers",
+ ):
+ monkeypatch.setattr(expert_scheduling, name, AsyncMock())
+ monkeypatch.setattr(experts_db, "emit_funnel_event", MagicMock())
+ monkeypatch.setattr(
+ experts_db, "get_user_default_team", AsyncMock(return_value=("org", "team"))
+ )
+ return scheduler
+
+
+async def _make_expert(prefix: str) -> tuple[str, MemoryScope]:
+ user_id = f"{prefix}-{uuid.uuid4().hex[:16]}"
+ await prisma.models.User.prisma().create(
+ data={"id": user_id, "email": f"{user_id}@example.invalid", "timezone": "UTC"}
+ )
+ expert = await prisma.models.Expert.prisma().create(
+ data={
+ "ownerUserId": user_id,
+ "name": "Lifecycle",
+ "role": "test",
+ "identity": "test",
+ "weeklyBudget": 10,
+ }
+ )
+ return user_id, MemoryScope.for_expert(user_id, expert.id)
+
+
+async def _stopped(
+ scope: MemoryScope, scheduler: FakeScheduler
+) -> MemoryScopeScheduleState | None:
+ """The scope's state, asserting it has no crons left."""
+ assert scheduler.jobs == {}
+ row = await memory_schedule.get_scope_schedule(scope.owner_user_id, scope.scope_key)
+ return row.state if row is not None else None
+
+
[email protected](loop_scope="session")
+async def test_a_deleted_accounts_legacy_cron_goes_and_a_live_ones_stays(
+ server: SpinTestServer, tmp_path
+):
+ """The learning fit review's probe: an untagged learning cron from
+ before the registry, of an account with no row, in a real scheduler and
+ jobstore. Once the user is deleted, the reconcile removes it and its
+ gate skips it; a live account's cron stays and its gate lets it run."""
+ user_ids = [f"legacy-cron-{uuid.uuid4().hex[:16]}" for _ in range(2)]
+ await prisma.models.User.prisma().create_many(
+ data=[{"id": uid, "email": f"{uid}@example.invalid"} for uid in user_ids]
+ )
+ live, gone = (MemoryScope.for_user(uid) for uid in user_ids)
+ job = dream_system_job(SKILL_LEARNING_JOB_PREFIX)
+ body = scheduler_module._MEMORY_CRONS[SKILL_LEARNING_JOB_PREFIX]
+ try:
+ with _scheduler(_sqlite(tmp_path)) as service:
+ for scope in (live, gone):
+ service._add_memory_job(*body, job, scope, "UTC")
+ await prisma.models.User.prisma().delete(where={"id": gone.owner_user_id})
+
+ crons = scheduler_module._SchedulerCrons(service)
+ report = await asyncio.wait_for(scope_reconcile.reconcile(crons), 30)
+
+ assert gone.scope_key in report.removed
+ assert live.scope_key not in report.removed
+ assert _job(service, job.job_id(gone)) is None
+ assert _job(service, job.job_id(live)) is not None
+ assert not await scope_crons._schedule_active(gone)
+ assert await scope_crons._schedule_active(live)
+ finally:
+ await _drop_users(user_ids)
+
+
[email protected](loop_scope="session")
+async def test_the_accounts_that_learn_are_listed_once_each_page_by_page(
+ server: SpinTestServer,
+):
+ """One entry per user with learning sources, however many it has, in
+ id order; a user without any is not listed."""
+ stem = f"learning-list-{uuid.uuid4().hex[:16]}"
+ user_ids = [f"{stem}-a", f"{stem}-b", f"{stem}-c"]
+ await prisma.models.User.prisma().create_many(
+ data=[{"id": uid, "email": f"{uid}@example.invalid"} for uid in user_ids]
+ )
+ try:
+ for user_id, sources in zip(user_ids, (2, 1, 0)):
+ for n in range(sources):
+ await prisma.models.SkillLearningSource.prisma().create(
+ data={
+ "userId": user_id,
+ "ownerKey": "personal",
+ "sourceKind": "chat_session",
+ "sourceId": f"session-{n}",
+ "revision": "1",
+ }
+ )
+
+ first = await memory_schedule.list_learning_account_ids(after=stem, limit=1)
+ second = await memory_schedule.list_learning_account_ids(
+ after=first[-1], limit=1
+ )
+ third = await memory_schedule.list_learning_account_ids(
+ after=second[-1], limit=1
+ )
+ together = await memory_schedule.list_learning_account_ids(after=stem, limit=10)
+
+ assert (first, second) == ([user_ids[0]], [user_ids[1]])
+ assert user_ids[2] not in third
+ assert together[:2] == user_ids[:2] and user_ids[2] not in together
+ finally:
+ await _drop_users(user_ids)
diff --git a/autogpt_platform/backend/backend/data/memory_schedule_test.py b/autogpt_platform/backend/backend/data/memory_schedule_test.py
new file mode 100644
index 0000000000..0619ca6ab6
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/memory_schedule_test.py
@@ -0,0 +1,306 @@
+"""Unit tests for the memory-scope schedule data module: the ownership rules
+and the writes the registry relies on, against a mocked Prisma table."""
+
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import prisma.models
+import pytest
+from prisma.enums import MemoryScopeScheduleState, ResourceVisibility
+from prisma.errors import UniqueViolationError
+
+from backend.data import memory_schedule
+from backend.util.exceptions import NotAuthorizedError
+
+ACTIVE = MemoryScopeScheduleState.ACTIVE
+PAUSED = MemoryScopeScheduleState.PAUSED
+
+
+def _row(**overrides) -> SimpleNamespace:
+ now = datetime.now(timezone.utc)
+ fields = {
+ "scopeKey": "user-1",
+ "userId": "user-1",
+ "expertId": None,
+ "timezone": "Europe/Paris",
+ "state": ACTIVE,
+ "communityJobId": "community_rebuild_user-1",
+ "nightlyJobId": None,
+ "learningJobId": None,
+ "lastNightlyRunAt": None,
+ "lastCommunityRunAt": None,
+ "createdAt": now,
+ "updatedAt": now,
+ }
+ return SimpleNamespace(**(fields | overrides))
+
+
+def table(mocker) -> MagicMock:
+ client = MagicMock()
+ client.find_first = AsyncMock(return_value=None)
+ client.find_unique = AsyncMock(return_value=None)
+ client.find_many = AsyncMock(return_value=[])
+ client.create = AsyncMock(return_value=_row())
+ client.update_many = AsyncMock(return_value=1)
+ mocker.patch.object(
+ prisma.models.MemoryScopeSchedule, "prisma", return_value=client
+ )
+ return client
+
+
+async def test_get_matches_on_scope_and_owner(table):
+ table.find_first.return_value = _row()
+
+ row = await memory_schedule.get_scope_schedule("user-1", "user-1")
+
+ table.find_first.assert_awaited_once_with(
+ where={"scopeKey": "user-1", "userId": "user-1"}
+ )
+ assert row is not None
+ assert (row.scope_key, row.community_job_id) == (
+ "user-1",
+ "community_rebuild_user-1",
+ )
+
+
+def _unique_violation() -> UniqueViolationError:
+ return UniqueViolationError(
+ {"user_facing_error": {"message": "Unique constraint failed: scopeKey"}}
+ )
+
+
+async def test_claim_creates_in_the_given_state(table):
+ await memory_schedule.claim_scope_schedule(
+ "user-1", "expert_abc", "expert-1", "Asia/Tokyo", PAUSED
+ )
+
+ table.create.assert_awaited_once_with(
+ data={
+ "scopeKey": "expert_abc",
+ "userId": "user-1",
+ "expertId": "expert-1",
+ "timezone": "Asia/Tokyo",
+ "state": PAUSED,
+ }
+ )
+ table.find_unique.assert_not_awaited()
+
+
+async def test_a_claim_that_loses_the_race_returns_the_winner_unchanged(table):
+ """Concurrent first claims: the create hits the unique key, and the row
+ the other claim created comes back as it is, never overwritten."""
+ table.create.side_effect = _unique_violation()
+ table.find_unique.return_value = _row(state=PAUSED)
+
+ row = await memory_schedule.claim_scope_schedule("user-1", "user-1", None, "UTC")
+
+ assert row.state == PAUSED
+ table.find_unique.assert_awaited_once_with(where={"scopeKey": "user-1"})
+
+
+async def test_a_winner_deleted_before_the_re_read_is_claimed_again(table):
+ table.create.side_effect = [_unique_violation(), _row()]
+ table.find_unique.return_value = None
+
+ row = await memory_schedule.claim_scope_schedule("user-1", "user-1", None, "UTC")
+
+ assert row.scope_key == "user-1"
+ assert table.create.await_count == 2
+
+
+async def test_a_claim_gives_up_after_two_tries(table):
+ """A violation with no row to read back (the winner deleted twice, or a
+ clash on the expert's own unique key) propagates on the second try."""
+ table.create.side_effect = _unique_violation()
+ table.find_unique.return_value = None
+
+ with pytest.raises(UniqueViolationError):
+ await memory_schedule.claim_scope_schedule("user-1", "user-1", None, "UTC")
+ assert table.create.await_count == 2
+
+
[email protected]("raced", [False, True])
+async def test_claim_refuses_another_users_scope(table, raced: bool):
+ theirs = _row(userId="someone-else")
+ if raced:
+ table.create.side_effect = _unique_violation()
+ table.find_unique.return_value = theirs
+ else:
+ table.create.return_value = theirs
+
+ with pytest.raises(NotAuthorizedError):
+ await memory_schedule.claim_scope_schedule("user-1", "user-1", None, "UTC")
+
+
+async def test_record_jobs_only_writes_an_active_row(table):
+ table.update_many.return_value = 0
+
+ recorded = await memory_schedule.record_scope_jobs(
+ "user-1",
+ "user-1",
+ user_timezone="UTC",
+ community_job_id="c",
+ nightly_job_id="n",
+ learning_job_id="l",
+ )
+
+ assert recorded is False
+ where = table.update_many.await_args.kwargs["where"]
+ assert where == {"scopeKey": "user-1", "userId": "user-1", "state": ACTIVE}
+ assert table.update_many.await_args.kwargs["data"] == {
+ "timezone": "UTC",
+ "communityJobId": "c",
+ "nightlyJobId": "n",
+ "learningJobId": "l",
+ }
+
+
+async def test_leaving_active_forgets_the_job_ids(table):
+ assert await memory_schedule.set_scope_state("user-1", "user-1", PAUSED)
+
+ assert table.update_many.await_args.kwargs == {
+ "where": {"scopeKey": "user-1", "userId": "user-1"},
+ "data": {
+ "state": PAUSED,
+ "communityJobId": None,
+ "nightlyJobId": None,
+ "learningJobId": None,
+ },
+ }
+
+
+async def test_resuming_leaves_the_job_ids_to_the_next_registration(table):
+ await memory_schedule.set_scope_state("user-1", "user-1", ACTIVE)
+
+ assert table.update_many.await_args.kwargs["data"] == {"state": ACTIVE}
+
+
+async def test_a_guarded_state_change_only_moves_those_states(table):
+ """A pause moves only an ACTIVE or PAUSED row, never a WIPED one, and
+ the check is part of the UPDATE itself rather than a read before it."""
+ table.update_many.return_value = 0
+
+ moved = await memory_schedule.set_scope_state(
+ "user-1", "user-1", PAUSED, only_from=(ACTIVE, PAUSED)
+ )
+
+ assert moved is False
+ assert table.update_many.await_args.kwargs["where"] == {
+ "scopeKey": "user-1",
+ "userId": "user-1",
+ "state": {"in": [ACTIVE, PAUSED]},
+ }
+
+
+async def test_forget_clears_whichever_column_holds_the_job(table):
+ table.update_many.side_effect = [0, 1, 0]
+
+ assert await memory_schedule.forget_scope_job("user-1", "user-1", "job-1")
+
+ community, nightly, learning = table.update_many.await_args_list
+ assert community.kwargs["where"]["communityJobId"] == "job-1"
+ assert nightly.kwargs["data"] == {"nightlyJobId": None}
+ assert learning.kwargs["where"]["learningJobId"] == "job-1"
+ assert learning.kwargs["data"] == {"learningJobId": None}
+ assert all(
+ c.kwargs["where"]["userId"] == "user-1" for c in (community, nightly, learning)
+ )
+
+
+async def test_forget_finds_the_learning_job(table):
+ table.update_many.side_effect = [0, 0, 1]
+
+ assert await memory_schedule.forget_scope_job(
+ "user-1", "user-1", "skill_learning_nightly_user-1"
+ )
+
+
+ "kind, column",
+ [("nightly", "lastNightlyRunAt"), ("community", "lastCommunityRunAt")],
+)
+async def test_record_run_stamps_the_kinds_column(
+ table, kind: memory_schedule.ScopeRunKind, column: str
+):
+ await memory_schedule.record_scope_run("user-1", "user-1", kind)
+
+ assert list(table.update_many.await_args.kwargs["data"]) == [column]
+
+
+async def test_active_rows_page_by_scope_key(table):
+ await memory_schedule.list_active_scope_schedules(after="k", limit=10)
+
+ assert table.find_many.await_args.kwargs == {
+ "where": {"state": ACTIVE, "scopeKey": {"gt": "k"}},
+ "order": {"scopeKey": "asc"},
+ "take": 10,
+ }
+
+
+async def test_every_row_pages_by_scope_key_whatever_its_state(table):
+ table.find_many.return_value = [_row(), _row(scopeKey="user-2", state=PAUSED)]
+
+ first = await memory_schedule.list_scope_schedules(limit=10)
+ await memory_schedule.list_scope_schedules(after="user-2", limit=10)
+
+ assert [row.state for row in first] == [ACTIVE, PAUSED]
+ assert [call.kwargs for call in table.find_many.await_args_list] == [
+ {"where": {}, "order": {"scopeKey": "asc"}, "take": 10},
+ {
+ "where": {"scopeKey": {"gt": "user-2"}},
+ "order": {"scopeKey": "asc"},
+ "take": 10,
+ },
+ ]
+
+
+async def test_live_experts_are_hired_unarchived_and_owner_only(mocker):
+ experts = MagicMock()
+ experts.find_many = AsyncMock(
+ return_value=[
+ SimpleNamespace(id="e1", ownerUserId="u1", schedulesPausedAt=None),
+ SimpleNamespace(
+ id="e2", ownerUserId="u1", schedulesPausedAt=datetime.now(timezone.utc)
+ ),
+ ]
+ )
+ mocker.patch.object(prisma.models.Expert, "prisma", return_value=experts)
+
+ scopes = await memory_schedule.list_live_expert_scopes()
+
+ where = experts.find_many.await_args.kwargs["where"]
+ assert where == {
+ "isTemplate": False,
+ "isArchived": False,
+ "ownerUserId": {"not": None},
+ "visibility": ResourceVisibility.PRIVATE,
+ }
+ assert [(s.expert_id, s.paused) for s in scopes] == [("e1", False), ("e2", True)]
+
+
+async def test_existing_user_ids_skips_the_query_for_nothing(mocker):
+ users = MagicMock()
+ users.find_many = AsyncMock()
+ mocker.patch.object(prisma.models.User, "prisma", return_value=users)
+
+ assert await memory_schedule.existing_user_ids([]) == set()
+ users.find_many.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.py
index 0b154d8845..b5209af1f9 100644
--- a/autogpt_platform/backend/backend/data/user.py
+++ b/autogpt_platform/backend/backend/data/user.py
@@ -891,16 +891,17 @@ async def update_user_timezone(user_id: str, timezone: str) -> User:
# Dream-system schedules are bound to the timezone at job-creation
# time; without an eager re-register they'd keep firing at the old
- # local time. Fire-and-forget so this profile update returns
- # immediately — the helper's lazy drift-detection path (via the
- # Redis dedup key value) is the durable backstop if this
- # fails or the user doesn't trigger a memory write within the
- # 7-day key TTL.
+ # local time. Every memory scope of the user (the account and each
+ # scheduled expert) is re-registered. Fire-and-forget so this
+ # profile update returns immediately. If it fails, the registry's
+ # timezone check on the scope's next registration (a returning
+ # ingest queue, a lifecycle change, the backfill) is the backstop,
+ # not necessarily the next memory write.
try:
- from backend.copilot.dream.scheduling import ensure_dream_system_scheduled
+ from backend.copilot.dream.registry import reregister_user
task = asyncio.create_task(
- ensure_dream_system_scheduled(user_id, force_refresh=True),
+ reregister_user(user_id),
name=f"tz-reregister-{user_id[:12]}",
)
_background_tasks.add(task)
diff --git a/autogpt_platform/backend/backend/data/user_test.py b/autogpt_platform/backend/backend/data/user_test.py
index c140061965..4de3aae7d1 100644
--- a/autogpt_platform/backend/backend/data/user_test.py
+++ b/autogpt_platform/backend/backend/data/user_test.py
@@ -35,6 +35,25 @@ def _application_user(user_id: str, email: str) -> user_module.User:
class TestUpdateUserTimezone:
+ @pytest.fixture(autouse=True)
+ def _stub_background_reregistration(self, mocker):
+ """The update spawns the dream and briefing re-registrations as
+ background tasks. Left live, they reach the flag service, the
+ registry and Redis from the test's loop and can outlive the test;
+ the tests that pin those tasks patch them again themselves."""
+ mocker.patch(
+ "backend.copilot.dream.registry.reregister_user",
+ new=AsyncMock(return_value={}),
+ )
+ mocker.patch(
+ "backend.copilot.briefing.scheduling.clear_briefing_registration_marker",
+ new=AsyncMock(),
+ )
+ mocker.patch(
+ "backend.copilot.briefing.scheduling.ensure_morning_briefing_scheduled",
+ new=AsyncMock(),
+ )
+
@pytest.mark.asyncio
async def test_invalidates_all_three_user_caches(self):
prisma_user = MagicMock(id="user-1", email="[email protected]")
@@ -91,19 +110,19 @@ class TestUpdateUserTimezone:
assert "connection lost" in str(exc.value)
@pytest.mark.asyncio
- async def test_eagerly_re_registers_dream_schedules_with_force_refresh(self):
+ async def test_eagerly_re_registers_every_memory_scope_of_the_user(self):
"""APScheduler cron triggers bind to the timezone at registration
- time. A profile-page timezone change MUST eagerly re-register
- the dream-system crons so they fire at the right local time
- without waiting for the 7-day Redis dedup-key TTL to expire."""
+ time. A profile-page timezone change MUST eagerly re-register the
+ dream-system crons of every memory scope the user has (the account
+ and each expert) so they fire at the right local time."""
from backend.copilot.briefing import scheduling as briefing_scheduling
- from backend.copilot.dream import scheduling as dream_scheduling
+ from backend.copilot.dream import registry as dream_registry
prisma_user = MagicMock(id="user-tz", email="[email protected]")
- captured: list[tuple[str, bool]] = []
+ captured: list[str] = []
- async def fake_ensure(user_id: str, *, force_refresh: bool = False):
- captured.append((user_id, force_refresh))
+ async def fake_reregister(user_id: str):
+ captured.append(user_id)
return {}
with (
@@ -112,9 +131,7 @@ class TestUpdateUserTimezone:
patch.object(user_module.get_user_by_id, "cache_delete"),
patch.object(user_module.get_user_by_email, "cache_delete"),
patch.object(user_module.get_or_create_user, "cache_clear"),
- patch.object(
- dream_scheduling, "ensure_dream_system_scheduled", new=fake_ensure
- ),
+ patch.object(dream_registry, "reregister_user", new=fake_reregister),
# This test isolates the dream-system re-registration contract;
# the sibling morning-briefing re-registration (also wired here)
# is covered separately by scheduling_test.py.
@@ -137,7 +154,7 @@ class TestUpdateUserTimezone:
# assert it was called.
await asyncio.sleep(0)
- assert captured == [("user-tz", True)]
+ assert captured == ["user-tz"]
@pytest.mark.asyncio
async def test_re_register_task_is_retained_and_its_failure_logged(self):
@@ -147,11 +164,11 @@ class TestUpdateUserTimezone:
``_background_tasks`` until done and log failures via the
done-callback instead of dropping them."""
from backend.copilot.briefing import scheduling as briefing_scheduling
- from backend.copilot.dream import scheduling as dream_scheduling
+ from backend.copilot.dream import registry as dream_registry
prisma_user = MagicMock(id="user-tz", email="[email protected]")
- async def failing_ensure(user_id: str, *, force_refresh: bool = False):
+ async def failing_reregister(user_id: str):
raise RuntimeError("scheduler unreachable")
with (
@@ -160,9 +177,7 @@ class TestUpdateUserTimezone:
patch.object(user_module.get_user_by_id, "cache_delete"),
patch.object(user_module.get_user_by_email, "cache_delete"),
patch.object(user_module.get_or_create_user, "cache_clear"),
- patch.object(
- dream_scheduling, "ensure_dream_system_scheduled", new=failing_ensure
- ),
+ patch.object(dream_registry, "reregister_user", new=failing_reregister),
# Isolate the dream-system task-retention contract under test
# from the sibling morning-briefing re-registration (also wired
# here) — real Redis/flag I/O in that path would otherwise give
@@ -208,7 +223,7 @@ class TestUpdateUserTimezone:
stored marker before re-ensuring, or the drift check would read
the just-superseded timezone and skip the re-register."""
from backend.copilot.briefing import scheduling as briefing_scheduling
- from backend.copilot.dream import scheduling as dream_scheduling
+ from backend.copilot.dream import registry as dream_registry
prisma_user = MagicMock(id="user-tz", email="[email protected]")
calls: list[str] = []
@@ -225,9 +240,7 @@ class TestUpdateUserTimezone:
patch.object(user_module.get_user_by_id, "cache_delete"),
patch.object(user_module.get_user_by_email, "cache_delete"),
patch.object(user_module.get_or_create_user, "cache_clear"),
- patch.object(
- dream_scheduling, "ensure_dream_system_scheduled", new=AsyncMock()
- ),
+ patch.object(dream_registry, "reregister_user", new=AsyncMock()),
patch.object(
briefing_scheduling,
"clear_briefing_registration_marker",
diff --git a/autogpt_platform/backend/backend/executor/jobstore.py b/autogpt_platform/backend/backend/executor/jobstore.py
index 1866c8c027..d6d902316f 100644
--- a/autogpt_platform/backend/backend/executor/jobstore.py
+++ b/autogpt_platform/backend/backend/executor/jobstore.py
@@ -1,11 +1,25 @@
import logging
import pickle
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from apscheduler.jobstores.base import ConflictingIdError, JobLookupError
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
-from sqlalchemy import and_, select
+from apscheduler.util import datetime_to_utc_timestamp
+from sqlalchemy import Connection, and_, select, text
+from sqlalchemy.exc import IntegrityError
logger = logging.getLogger(__name__)
+# Seconds a job write in this context may wait for a row another transaction
+# holds, or run, on PostgreSQL: set around its own writes by a caller that
+# must not be held by one (the memory cron reconcile,
+# ``executor/scheduler.py``). The scheduler's other writes leave it unset.
+WRITE_TIMEOUT: ContextVar[float | None] = ContextVar(
+ "jobstore_write_timeout", default=None
+)
+
class ResilientSQLAlchemyJobStore(SQLAlchemyJobStore):
"""Parks jobs it cannot restore instead of deleting them.
@@ -18,31 +32,47 @@ class ResilientSQLAlchemyJobStore(SQLAlchemyJobStore):
``get_due_jobs`` returning the row, so one bad job cannot spin the
scheduler.
- Parked jobs do not run. ``get_parked_job_ids`` is the operator surface
- that keeps that from being silent, and ``reconcile_repaired_jobs``
- finishes the job once the payload is repaired.
+ A listing that can restore a parked job resumes its serialized schedule.
+ Explicitly paused jobs have no serialized next run and stay paused.
+ ``get_parked_job_ids`` reports jobs that still cannot be restored.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._parked_ids: set[str] = set()
- def _get_jobs(self, *conditions):
+ def _get_jobs(self, *conditions, statement_timeout: float | None = None):
+ """Upstream's read, parking what it cannot restore. On PostgreSQL a
+ ``statement_timeout`` (seconds) bounds this read alone, in its own
+ transaction, for a caller that must not wait on a stalled query: the
+ engine's other queries keep their limits."""
jobs = []
selectable = select(
self.jobs_t.c.id, self.jobs_t.c.job_state, self.jobs_t.c.next_run_time
).order_by(self.jobs_t.c.next_run_time)
selectable = selectable.where(and_(*conditions)) if conditions else selectable
unrestorable: list = []
+ restored: list = []
with self.engine.begin() as connection:
+ if statement_timeout is not None:
+ _limit(connection, statement_timeout, "statement_timeout")
for row in connection.execute(selectable):
try:
- jobs.append(self._reconstitute_job(row.job_state))
+ job = self._reconstitute_job(row.job_state)
except (KeyboardInterrupt, SystemExit):
raise
except BaseException as e:
unrestorable.append((row, e))
+ else:
+ jobs.append(job)
+ if row.next_run_time is None and job.next_run_time is not None:
+ restored.append((row, job.next_run_time))
+
+ for row, next_run_time in restored:
+ if self._unpark(connection, row, next_run_time):
+ self._parked_ids.discard(row.id)
+ self._logger.info('Resuming restored job "%s"', row.id)
for row, error in unrestorable:
if not self._park(connection, row):
@@ -60,6 +90,52 @@ class ResilientSQLAlchemyJobStore(SQLAlchemyJobStore):
return jobs
+ def add_job(self, job):
+ """Upstream's insert, bounded by ``WRITE_TIMEOUT`` when set."""
+ insert = self.jobs_t.insert().values(
+ id=job.id,
+ next_run_time=datetime_to_utc_timestamp(job.next_run_time),
+ job_state=pickle.dumps(job.__getstate__(), self.pickle_protocol),
+ )
+ with self._write() as connection:
+ try:
+ connection.execute(insert)
+ except IntegrityError:
+ raise ConflictingIdError(job.id) from None
+
+ def update_job(self, job):
+ """Upstream's update, bounded by ``WRITE_TIMEOUT`` when set."""
+ update = (
+ self.jobs_t.update()
+ .values(
+ next_run_time=datetime_to_utc_timestamp(job.next_run_time),
+ job_state=pickle.dumps(job.__getstate__(), self.pickle_protocol),
+ )
+ .where(self.jobs_t.c.id == job.id)
+ )
+ with self._write() as connection:
+ if connection.execute(update).rowcount == 0:
+ raise JobLookupError(job.id)
+
+ def remove_job(self, job_id):
+ """Upstream's delete, bounded by ``WRITE_TIMEOUT`` when set."""
+ delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id)
+ with self._write() as connection:
+ if connection.execute(delete).rowcount == 0:
+ raise JobLookupError(job_id)
+
+ @contextmanager
+ def _write(self) -> Iterator[Connection]:
+ """A write's transaction, its lock waits and statements limited to
+ ``WRITE_TIMEOUT`` when this context sets it: the write then fails
+ instead of holding its thread, and APScheduler's lock, on a row
+ another transaction keeps."""
+ with self.engine.begin() as connection:
+ timeout = WRITE_TIMEOUT.get()
+ if timeout is not None:
+ _limit(connection, timeout, "lock_timeout", "statement_timeout")
+ yield connection
+
def _park(self, connection, row) -> bool:
"""Clear ``next_run_time`` for the row we read, and say whether we did.
@@ -79,6 +155,20 @@ class ResilientSQLAlchemyJobStore(SQLAlchemyJobStore):
)
return result.rowcount == 1
+ def _unpark(self, connection, row, next_run_time) -> bool:
+ result = connection.execute(
+ self.jobs_t.update()
+ .where(
+ and_(
+ self.jobs_t.c.id == row.id,
+ self.jobs_t.c.job_state == row.job_state,
+ self.jobs_t.c.next_run_time.is_(None),
+ )
+ )
+ .values(next_run_time=datetime_to_utc_timestamp(next_run_time))
+ )
+ return result.rowcount == 1
+
def get_parked_job_ids(self, limit: int | None = None) -> list[str]:
"""Ids of rows that are paused *and* still unrestorable.
@@ -115,8 +205,9 @@ class ResilientSQLAlchemyJobStore(SQLAlchemyJobStore):
copy and refuses. Writing that copy back to None makes it an
ordinary paused job, which resume revives.
- ``jobstore_backfill`` already repairs the rows it rewrites, so this is
- the path for a repair made some other way, and it is not run at
+ Listings automatically resume restorable jobs. This explicit operator
+ path instead normalizes them to paused before a listing resumes them;
+ ``jobstore_backfill`` also pauses rows it rewrites. It is not run at
startup. It walks the whole paused set in id order, a batch per
transaction: a cap alone would keep re-reading the first page and
never reach a repaired row behind a backlog nothing deletes, while one
@@ -174,3 +265,15 @@ class ResilientSQLAlchemyJobStore(SQLAlchemyJobStore):
self._parked_ids.discard(row.id)
healed.append(row.id)
return seen
+
+
+def _limit(connection: Connection, seconds: float, *settings: str) -> None:
+ """Limit ``settings`` for the rest of ``connection``'s transaction, on
+ PostgreSQL; other databases have no such settings."""
+ if connection.dialect.name != "postgresql":
+ return
+ for setting in settings:
+ connection.execute(
+ text("SELECT set_config(:setting, :ms, true)"),
+ {"setting": setting, "ms": str(int(seconds * 1000))},
+ )
diff --git a/autogpt_platform/backend/backend/executor/jobstore_test.py b/autogpt_platform/backend/backend/executor/jobstore_test.py
index 0c60eb4ddc..e0e41fdb17 100644
--- a/autogpt_platform/backend/backend/executor/jobstore_test.py
+++ b/autogpt_platform/backend/backend/executor/jobstore_test.py
@@ -1,12 +1,15 @@
+import logging
import os
import pickle
import tempfile
from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
from enum import Enum
from unittest.mock import MagicMock, patch
import pytest
from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.util import datetime_to_utc_timestamp
from sqlalchemy import select
from backend.executor.jobstore import ResilientSQLAlchemyJobStore
@@ -107,10 +110,9 @@ def test_scheduler_listing_path_parks_an_unrestorable_schedule():
assert sched.get_parked_jobs()[Jobstores.EXECUTION.value] == ["poisoned"]
-def test_repaired_job_can_be_resumed_and_then_fires():
- """Park, repair, resume, run. Parking can only clear the COLUMN, so without
- reconciliation the repaired row is stranded: it reports healthy, never comes
- due, and resume refuses it because the pickled copy is still set."""
+def test_manual_repair_can_pause_before_a_listing_resumes_the_job():
+ """The explicit repair path can normalize both copies to paused before
+ any automatic listing restores the serialized next run."""
with _store() as (store, scheduler):
scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")
healthy_state = _job_state(store, "poisoned")
@@ -122,7 +124,6 @@ def test_repaired_job_can_be_resumed_and_then_fires():
# which is exactly what jobstore_backfill does.
_set_job_state(store, "poisoned", healthy_state)
assert store.get_parked_job_ids() == []
- assert store._get_jobs()[0].next_run_time is not None # stranded
assert _next_run_time(store, "poisoned") is None
assert store.reconcile_repaired_jobs() == ["poisoned"]
@@ -143,6 +144,90 @@ def test_reconcile_leaves_a_user_paused_job_alone():
assert _next_run_time(store, "paused") is None
[email protected]("prefix", [None, "ordinary_"])
+def test_listing_unparks_a_restorable_job_once(caplog, prefix):
+ with _store() as (store, scheduler):
+ job_id = "ordinary_job"
+ job = scheduler.add_job(
+ noop,
+ "interval",
+ seconds=60,
+ id=job_id,
+ next_run_time=datetime.now(timezone.utc) - timedelta(minutes=2),
+ )
+ healthy = _job_state(store, job_id)
+ _poison(store, job_id)
+ store.get_all_jobs()
+ _set_job_state(store, job_id, healthy)
+ caplog.clear()
+ conditions = (store.jobs_t.c.id.startswith(prefix),) if prefix else ()
+
+ with caplog.at_level(logging.INFO):
+ assert [j.id for j in store._get_jobs(*conditions)] == [job_id]
+ store._get_jobs(*conditions)
+
+ assert _next_run_time(store, job_id) == datetime_to_utc_timestamp(
+ job.next_run_time
+ )
+ assert _job_state(store, job_id) == healthy
+ assert store._parked_ids == set()
+ assert caplog.text.count('Resuming restored job "ordinary_job"') == 1
+ assert [j.id for j in store.get_due_jobs(datetime.now(timezone.utc))] == [
+ job_id
+ ]
+
+
+def test_listing_does_not_unpark_a_user_paused_job():
+ with _store() as (store, scheduler):
+ scheduler.add_job(noop, "interval", seconds=60, id="paused")
+ scheduler.pause_job("paused")
+ healthy = _job_state(store, "paused")
+
+ [job] = store.get_all_jobs()
+
+ assert job.next_run_time is None
+ assert _next_run_time(store, "paused") is None
+ assert _job_state(store, "paused") == healthy
+
+
[email protected]("changed", ["job_state", "next_run_time"])
+def test_listing_does_not_unpark_a_row_changed_since_the_scan(changed):
+ with _store() as (store, scheduler):
+ scheduler.add_job(noop, "interval", seconds=60, id="parked")
+ healthy = _job_state(store, "parked")
+ _poison(store, "parked")
+ store.get_all_jobs()
+ _set_job_state(store, "parked", healthy)
+ restore = store._reconstitute_job
+ state = pickle.loads(healthy)
+ state["next_run_time"] = None
+ paused_state = pickle.dumps(state, store.pickle_protocol)
+
+ def change_midway(job_state):
+ values = (
+ {"job_state": paused_state}
+ if changed == "job_state"
+ else {"next_run_time": 1.0}
+ )
+ with store.engine.begin() as connection:
+ connection.execute(
+ store.jobs_t.update()
+ .where(store.jobs_t.c.id == "parked")
+ .values(**values)
+ )
+ return restore(job_state)
+
+ with patch.object(store, "_reconstitute_job", side_effect=change_midway):
+ store.get_all_jobs()
+
+ assert _next_run_time(store, "parked") == (
+ None if changed == "job_state" else 1.0
+ )
+ assert _job_state(store, "parked") == (
+ paused_state if changed == "job_state" else healthy
+ )
+
+
def test_reconcile_leaves_a_still_broken_job_parked():
with _store() as (store, scheduler):
scheduler.add_job(noop, "interval", seconds=3600, id="poisoned")
@@ -157,12 +242,14 @@ def test_reconcile_walks_past_its_batch_size():
"""Batching bounds the transaction, not the work: every stranded row must
still be reached, which a plain cap would not do."""
with _store() as (store, scheduler):
+ healthy = {}
for i in range(7):
scheduler.add_job(noop, "interval", seconds=3600, id=f"j{i}")
- healthy = _job_state(store, f"j{i}")
+ healthy[f"j{i}"] = _job_state(store, f"j{i}")
_poison(store, f"j{i}")
- store._get_jobs()
- _set_job_state(store, f"j{i}", healthy)
+ store._get_jobs()
+ for job_id, state in healthy.items():
+ _set_job_state(store, job_id, state)
healed = store.reconcile_repaired_jobs(batch_size=2)
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index ef1aa7506c..e506edcfc9 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -1,4 +1,5 @@
import asyncio
+import concurrent.futures
import hashlib
import logging
import os
@@ -11,7 +12,7 @@ from concurrent.futures import Future
from contextlib import nullcontext
from datetime import datetime, timedelta, timezone
from enum import Enum
-from typing import Annotated, Any, Literal, Optional, Union
+from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Union
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
from zoneinfo import ZoneInfoNotFoundError
@@ -22,14 +23,16 @@ from apscheduler.events import (
EVENT_JOB_MISSED,
)
from apscheduler.job import Job as JobObj
+from apscheduler.jobstores.base import JobLookupError
from apscheduler.jobstores.memory import MemoryJobStore
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.date import DateTrigger
+from apscheduler.triggers.interval import IntervalTrigger
from apscheduler.util import ZoneInfo
from dotenv import load_dotenv
from pydantic import BaseModel, Field, ValidationError
-from sqlalchemy import MetaData, create_engine
+from sqlalchemy import MetaData, create_engine, or_
from backend.api.features.experts.models import ExpertRoutine
from backend.copilot.active_turns import ConcurrentTurnLimitError
@@ -45,14 +48,37 @@ from backend.copilot.config import CopilotLlmAuthProvider
from backend.copilot.continuation import RESUMED_AT_RESET_NOTE
from backend.copilot.credential_selection import CredentialPins
from backend.copilot.db import PaginatedMessages
+from backend.copilot.dream.deadline import bridge_timeout
from backend.copilot.dream.scheduling import (
- COMMUNITY_REBUILD_REGISTRATION_PREFIX,
- NIGHTLY_BATCH_REGISTRATION_PREFIX,
- SKILL_LEARNING_REGISTRATION_PREFIX,
- clear_registration_marker,
+ COMMUNITY_REBUILD_JOB_PREFIX,
+ DREAM_SYSTEM_JOBS,
+ NIGHTLY_BATCH_JOB_PREFIX,
+ SKILL_LEARNING_JOB_PREFIX,
+ DreamSystemJob,
+ dream_system_job,
+ jobs_for,
+)
+from backend.copilot.dream.scope_crons import (
+ forget_registration,
+ log_community_outcome,
+ log_nightly_outcome,
+ memory_job_kwargs,
+ memory_job_result,
+ memory_job_skipped,
+ memory_scope_may_fire,
+ nightly_error_parts,
+ record_scope_run,
+)
+from backend.copilot.dream.scope_jobs import scope_label
+from backend.copilot.dream.scope_reconcile import (
+ RECONCILE_INTERVAL_MINUTES,
+ RECONCILED_SUFFIX,
+ ListedCron,
+ reconcile,
)
from backend.copilot.executor.utils import ScheduledTurnOrigin, schedule_turn
from backend.copilot.graphiti.communities import rebuild_communities_for_user
+from backend.copilot.graphiti.scope import MemoryScope
from backend.copilot.model import ChatSessionInfo, create_chat_session, get_chat_session
from backend.copilot.optimize_blocks import optimize_block_descriptions
from backend.copilot.permissions import CopilotPermissions, routine_disabled_tools
@@ -61,12 +87,13 @@ from backend.copilot.transports import resolve_default_chat_route
from backend.copilot.turn_queue import TurnRefusal, turn_refusal
from backend.data.db_accessors import experts_db
from backend.data.execution import ExecutionTrigger, GraphExecutionWithNodes
+from backend.data.memory_schedule import ScopeRunKind
from backend.data.model import CredentialsMetaInput, GraphInput
from backend.data.redis_client import get_redis_async
from backend.data.schedule import normalize_schedule_name
from backend.executor import schedule_events
from backend.executor import utils as execution_utils
-from backend.executor.jobstore import ResilientSQLAlchemyJobStore
+from backend.executor.jobstore import WRITE_TIMEOUT, ResilientSQLAlchemyJobStore
from backend.executor.schedule_index import ScheduleIndex, ScheduleIndexEntry
from backend.integrations.creds_manager import IntegrationCredentialsManager
from backend.monitoring import (
@@ -107,6 +134,9 @@ from backend.util.service import (
)
from backend.util.settings import Config
+if TYPE_CHECKING:
+ from backend.copilot.dream.nightly_batch import NightlyBatchResult
+
def _extract_schema_from_url(database_url) -> tuple[str, str]:
"""
@@ -147,6 +177,18 @@ SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS = 1800
# generous ceiling relative to the per-op default.
STRIPE_RECONCILE_TIMEOUT_SECONDS = 1800 # 30 minutes
+# The memory cron reconcile reads every registry row; it must finish well
+# inside its own interval (``scope_reconcile.RECONCILE_INTERVAL_MINUTES``).
+# Its listing of the dream-system crons runs inside that ceiling, and the
+# database stops that query at its own limit: cancelling the wait leaves
+# the thread running.
+MEMORY_CRON_RECONCILE_TIMEOUT_SECONDS = 20 * 60
+MEMORY_CRON_LISTING_TIMEOUT_SECONDS = 60
+# Each of the reconcile's jobstore writes gives up after this on a row another
+# transaction holds (``jobstore.WRITE_TIMEOUT``), failing only its scope.
+MEMORY_CRON_WRITE_TIMEOUT_SECONDS = 10
+_MEMORY_CRON_RECONCILE_JOB_ID = "reconcile_memory_scope_crons"
+
def job_listener(event):
"""Logs job execution outcomes for better monitoring."""
@@ -204,12 +246,37 @@ def get_event_loop():
return _event_loop
-def run_async(coro, timeout: float | None = SCHEDULER_OPERATION_TIMEOUT_SECONDS):
- """Run a coroutine in the shared event loop and wait for completion."""
+def run_async(
+ coro,
+ timeout: float | None = SCHEDULER_OPERATION_TIMEOUT_SECONDS,
+ *,
+ cancel_on_timeout: bool = False,
+):
+ """Run a coroutine in the shared event loop and wait for completion.
+
+ A ``TimeoutError`` does not prove the operation had no effect: it may
+ have finished just as the wait ended, or be past a write it cannot take
+ back. By default the coroutine is left running after a timeout.
+ ``cancel_on_timeout`` requests its cancellation (requested, not awaited:
+ it stops at its next await, possibly after this returns); only callers
+ whose coroutine is safe to stop anywhere pass it, today the memory
+ registry bridges. Graph dispatch is not: ``_add_graph_execution``
+ creates the execution row before its remaining lookups, and cancelled
+ there it leaves that row INCOMPLETE, never published or failed. Making
+ it cancellation-safe is the follow-up that would let the default flip.
+ """
loop = get_event_loop()
future = asyncio.run_coroutine_threadsafe(coro, loop)
try:
return future.result(timeout=timeout)
+ except concurrent.futures.TimeoutError:
+ if cancel_on_timeout:
+ future.cancel()
+ logger.warning(
+ f"Async operation timed out after {timeout}s; "
+ + ("cancellation requested" if cancel_on_timeout else "not cancelled")
+ )
+ raise
except Exception as e:
logger.warning(f"Async operation failed: {type(e).__name__}: {e}")
raise
@@ -1325,45 +1392,77 @@ def cleanup_expired_files():
run_async(cleanup_expired_files_async())
-def execute_community_rebuild(user_id: str):
- """Per-user Graphiti community rebuild (P-1.7).
+def execute_community_rebuild(user_id: str, expert_id: str | None = None):
+ """Weekly community-rebuild cron body of one memory scope (P-1.7).
+
+ The job is ``community_rebuild_{scope_key}``. The account's jobs pass
+ only ``user_id``, as they did when the crons were per user. Runs only
+ while ``_memory_scope_may_fire`` lets it, and stamps
+ ``lastCommunityRunAt`` when the rebuild finishes without an error.
+ """
+ scope = MemoryScope.build(user_id, expert_id)
+ if not _memory_scope_may_fire(scope):
+ return
+ result = _rebuild_scope_communities(scope)
+ if result is not None and not result.get("error"):
+ _stamp_scope_run(scope, "community")
+
+
+def _rebuild_scope_communities(scope: MemoryScope) -> dict | None:
+ """Flag-gated community rebuild of one scope, with its outcome logged.
Sync wrapper around the async ``rebuild_communities_for_user`` so it
can run on the APScheduler thread pool. Failures are caught inside
the coroutine; this wrapper logs the outcome.
Runtime flag gate: if ``GRAPHITI_COMMUNITIES_ENABLED`` flipped from
- on→off after the schedule was registered, this body short-circuits
- instead of running. Registration-time gating is in
- ``add_community_rebuild_schedule``; this is the third layer of
+ on→off after the schedule was registered, this returns None instead
+ of running. Registration-time gating is in
+ ``add_scope_community_rebuild_schedule``; this is the third layer of
defense (see ``copilot/dream/scheduling.py`` module docstring).
"""
from backend.copilot.graphiti.config import is_communities_enabled_for_user
- if not run_async(is_communities_enabled_for_user(user_id)):
+ if not run_async(is_communities_enabled_for_user(scope.owner_user_id)):
logger.info(
- "Community rebuild skipped for user %s — flag flipped off post-registration",
- user_id[:12],
+ "Community rebuild skipped for %s — flag flipped off post-registration",
+ scope_label(scope),
)
- return
+ return None
result = run_async(
- rebuild_communities_for_user(user_id),
+ rebuild_communities_for_user(scope.owner_user_id, expert_id=scope.expert_id),
timeout=SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS,
)
- if result.get("error"):
- logger.warning(
- "Community rebuild errored for user %s: %s",
- user_id[:12],
- result["error"],
+ log_community_outcome(scope, result)
+ return result
+
+
+def _memory_scope_may_fire(scope: MemoryScope) -> bool:
+ """The registry gate of a dream-system cron (``memory_scope_may_fire``),
+ run on the scheduler loop. The gate bounds itself; if the bridge still
+ gives up, the gate's cancellation is requested and the run skipped."""
+ try:
+ return run_async(
+ memory_scope_may_fire(scope, _expert_scope_status),
+ timeout=bridge_timeout(),
+ cancel_on_timeout=True,
)
- else:
- logger.info(
- "Community rebuild completed for user %s in %.1fs: %s",
- user_id[:12],
- result.get("elapsed_seconds") or 0.0,
- result.get("communities_built"),
+ except concurrent.futures.TimeoutError:
+ logger.warning(f"Memory cron of {scope_label(scope)} skipped: gate timed out")
+ return False
+
+
+def _stamp_scope_run(scope: MemoryScope, kind: ScopeRunKind) -> None:
+ """Record a clean run on the scope's registry row; best-effort."""
+ try:
+ run_async(
+ record_scope_run(scope, kind),
+ timeout=bridge_timeout(),
+ cancel_on_timeout=True,
)
+ except concurrent.futures.TimeoutError:
+ logger.warning(f"Could not stamp the {kind} run of {scope_label(scope)}")
def _morning_briefing_crontab(user_id: str) -> str:
@@ -1435,8 +1534,30 @@ async def _self_delete_morning_briefing_schedule(user_id: str) -> None:
)
-def execute_nightly_batch_sync(user_id: str):
- """Per-user nightly batch-family fan-out cron body.
+def execute_nightly_batch_sync(
+ user_id: str, expert_id: str | None = None
+) -> "NightlyBatchResult | None":
+ """Nightly batch-family cron body of one memory scope.
+
+ The job is ``dream_nightly_batch_{scope_key}``. The account's jobs pass
+ only ``user_id``, as they did when the crons were per user. Runs only
+ while ``_memory_scope_may_fire`` lets it, and stamps ``lastNightlyRunAt``
+ when the fan-out reports no error. Returns the ``NightlyBatchResult``,
+ or ``None`` when a gate short-circuits.
+ """
+ scope = MemoryScope.build(user_id, expert_id)
+ if not _memory_scope_may_fire(scope):
+ return None
+ result = _submit_nightly_batch(user_id, expert_id)
+ if result is not None and not nightly_error_parts(result):
+ _stamp_scope_run(scope, "nightly")
+ return result
+
+
+def _submit_nightly_batch(
+ user_id: str, expert_id: str | None = None
+) -> "NightlyBatchResult | None":
+ """Flag-gated nightly batch-family fan-out of one scope, outcome logged.
Sync wrapper for APScheduler. The body inside
``run_nightly_batch_submit`` sequentially invokes each enabled
@@ -1449,9 +1570,8 @@ def execute_nightly_batch_sync(user_id: str):
Runtime flag gate (layer 3 of the 3-layer design): if
``DREAM_PASS_ENABLED`` flipped off after the cron was registered,
- this body short-circuits before any submitter runs. The
- consolidation removes the separate dream / ratification crons —
- both are now submitters inside this single nightly cron.
+ this short-circuits before any submitter runs. The budget check
+ inside the fan-out stays on the owner, whichever scope runs.
Returns the typed ``NightlyBatchResult`` so the admin
``*_with_status`` wrapper can persist it on the JobStatus row.
@@ -1464,63 +1584,39 @@ def execute_nightly_batch_sync(user_id: str):
)
from backend.util.feature_flag import Flag, is_feature_enabled
+ scope = MemoryScope.build(user_id, expert_id)
if not run_async(is_feature_enabled(Flag.DREAM_PASS_ENABLED, user_id)):
logger.info(
- "Nightly batch skipped for user %s — DREAM_PASS_ENABLED flipped off",
- user_id[:12],
+ "Nightly batch skipped for %s — DREAM_PASS_ENABLED flipped off",
+ scope_label(scope),
)
return None
result: NightlyBatchResult = run_async(
- run_nightly_batch_submit(user_id),
+ run_nightly_batch_submit(user_id, expert_id=expert_id),
timeout=SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS,
)
- if result.error:
- logger.warning(
- "Nightly batch errored for user %s (nightly %s): %s",
- user_id[:12],
- result.nightly_id,
- result.error,
- )
- elif result.skipped:
- logger.info(
- "Nightly batch skipped for user %s (nightly %s): %s",
- user_id[:12],
- result.nightly_id,
- result.skip_reason,
- )
- else:
- dream_writes = result.dream.consolidated_count if result.dream else 0
- dream_proposals = result.dream.proposal_count if result.dream else 0
- rat_ratified = result.ratification.ratified_count if result.ratification else 0
- rat_superseded = (
- result.ratification.superseded_count if result.ratification else 0
- )
- logger.info(
- "Nightly batch completed for user %s in %.1fs (nightly %s): "
- "dream_writes=%d dream_proposals=%d ratified=%d superseded=%d",
- user_id[:12],
- result.elapsed_seconds or 0.0,
- result.nightly_id,
- dream_writes,
- dream_proposals,
- rat_ratified,
- rat_superseded,
- )
+ log_nightly_outcome(scope, result)
return result
def execute_skill_learning_sync(user_id: str):
- """Per-user nightly skill-learning cron body (independent of the dream pass).
-
- Runtime flag gate (layer 3): if ``DREAM_SKILL_LEARNING_ENABLED`` flipped
- off after the cron was registered, the body returns before any work.
- The pass itself never raises; its outcome is logged here and kept in
- the per-source review ledger.
+ """The account's nightly skill-learning cron body (independent of the
+ dream pass); the job is ``skill_learning_nightly_{user_id}``.
+
+ Runs only while ``_memory_scope_may_fire`` lets the account's scope, as
+ the other dream-system crons do: a cron that straggles past a pause or a
+ wipe of the account's memory does nothing. Runtime flag gate (layer 3):
+ if ``DREAM_SKILL_LEARNING_ENABLED`` flipped off after the cron was
+ registered, the body returns before any work. The pass itself never
+ raises; its outcome is logged here and kept in the per-source review
+ ledger.
"""
from backend.copilot.learning.nightly import run_skill_learning_pass
from backend.util.feature_flag import Flag, is_feature_enabled
+ if not _memory_scope_may_fire(MemoryScope.for_user(user_id)):
+ return None
if not run_async(is_feature_enabled(Flag.DREAM_SKILL_LEARNING_ENABLED, user_id)):
logger.info(
"Skill learning skipped for user %s — flag flipped off", user_id[:12]
@@ -1610,7 +1706,9 @@ def execute_nightly_batch_with_status(user_id: str, job_id: str):
)
try:
- result = execute_nightly_batch_sync(user_id)
+ # The admin trigger bypasses the registry gate of the cron body:
+ # it runs the account's pass whatever state its scope row is in.
+ result = _submit_nightly_batch(user_id)
except Exception as exc:
logger.exception(
"Admin-triggered nightly batch crashed for user %s job %s",
@@ -1648,36 +1746,11 @@ def execute_nightly_batch_with_status(user_id: str, job_id: str):
skip_reason="dream_pass_disabled_runtime",
)
- # ``run_nightly_batch_submit`` never raises — a submitter CRASH is
- # captured in ``NightlyBatchResult.error``, while a submitter that
- # ran but reported its own failure carries it on its sub-result
- # (``dream.error`` / ``ratification.error``) with the top-level
- # error left unset. The admin row must read errored for all of
- # these — otherwise the Memory Visualizer renders a failed dream
- # as a successful nightly run (same contract as the dream-pass
- # wrapper below).
- error_parts = [
- part
- for part in (
- result.error,
- (
- f"dream: {result.dream.error}"
- if result.dream is not None and result.dream.error
- else None
- ),
- (
- f"ratification: {result.ratification.error}"
- if result.ratification is not None and result.ratification.error
- else None
- ),
- (
- f"learning: {result.learning.error}"
- if result.learning is not None and result.learning.error
- else None
- ),
- )
- if part
- ]
+ # The admin row must read errored for every error the result carries
+ # (see ``nightly_error_parts``) — otherwise the Memory Visualizer
+ # renders a failed dream as a successful nightly run (same contract as
+ # the dream-pass wrapper below).
+ error_parts = nightly_error_parts(result)
if error_parts:
try:
run_async(
@@ -1912,22 +1985,124 @@ def execute_community_rebuild_with_status(user_id: str, job_id: str):
)
-def _clear_dream_registration_marker(user_id: str, key_prefix: str) -> None:
- """Bridge ``clear_registration_marker`` onto the scheduler's event loop.
+# Each dream-system cron's body and crontab, by job-id prefix: weekly
+# community rebuild at owner-local 04:00 on Mondays (APScheduler counts
+# weekdays from 0 = Monday), nightly batch at owner-local 03:00.
+_MEMORY_CRONS: dict[str, tuple[Callable[..., object], str]] = {
+ COMMUNITY_REBUILD_JOB_PREFIX: (execute_community_rebuild, "0 4 * * 0"),
+ NIGHTLY_BATCH_JOB_PREFIX: (execute_nightly_batch_sync, "0 3 * * *"),
+ SKILL_LEARNING_JOB_PREFIX: (execute_skill_learning_sync, "0 3 * * *"),
+}
+# A missed run is caught up only within this many seconds; None (the
+# scheduler's default) catches it up however late. Skill learning's catch-up
+# after downtime is capped to the same morning: at most one run (coalesce),
+# and the pass re-checks every source's eligibility anyway.
+_MEMORY_CRON_MISFIRE_GRACE: dict[str, int | None] = {
+ SKILL_LEARNING_JOB_PREFIX: 6 * 3600,
+}
- Best-effort: a failed clear only delays lazy re-registration until
- the marker's 7-day TTL expires, so it must never break the delete
- RPC that called it.
- """
+
+def execute_expert_memory_cron(
+ func: Callable[..., object], user_id: str, expert_id: str
+) -> object:
+ """A distinct reference so old binaries park expert-only job payloads."""
+ return func(user_id=user_id, expert_id=expert_id)
+
+
+def reconcile_memory_scope_crons() -> None:
+ """System task, every ``RECONCILE_INTERVAL_MINUTES``: the dream-system
+ crons back in step with the registry rows (``copilot/dream/
+ scope_reconcile.py``). Safe on two schedulers at once."""
+ if _scheduler_instance is None:
+ logger.warning("Memory cron reconcile skipped: scheduler not running")
+ return
+ _scheduler_instance._reconcile_memory_crons()
+
+
+def listed_cron(job: JobObj) -> ListedCron | None:
+ """The scope a dream-system job runs for, from its kwargs, and the
+ timezone its trigger runs in; None when the kwargs name no scope."""
+ try:
+ scope = MemoryScope.build(job.kwargs["user_id"], job.kwargs.get("expert_id"))
+ except (KeyError, ValueError):
+ logger.warning("Memory cron %s names no memory scope; left alone", job.id)
+ return None
+ return ListedCron(
+ job_id=job.id,
+ scope=scope,
+ timezone=_job_timezone_name(job),
+ reconciled=job.name.endswith(RECONCILED_SUFFIX),
+ )
+
+
+class _SchedulerCrons:
+ """``scope_reconcile.CronStore`` over this scheduler's own jobstore, off
+ the event loop. Each write gives up after
+ ``MEMORY_CRON_WRITE_TIMEOUT_SECONDS`` on a row another transaction holds,
+ failing only its scope: cancelling the wait for it would leave the
+ thread, and APScheduler's lock, held."""
+
+ def __init__(self, scheduler: "Scheduler") -> None:
+ self._scheduler = scheduler
+
+ async def listing(self) -> list[ListedCron]:
+ return await asyncio.to_thread(self._scheduler._memory_crons)
+
+ async def add(
+ self, job: DreamSystemJob, scope: MemoryScope, user_timezone: str
+ ) -> None:
+ """Register ``job`` of ``scope``, tagged as the reconcile's."""
+ await _bounded_write(
+ self._scheduler._add_memory_job,
+ *_MEMORY_CRONS[job.job_id_prefix],
+ job,
+ scope,
+ user_timezone,
+ True,
+ )
+
+ async def drop(self, job: DreamSystemJob, scope: MemoryScope) -> None:
+ await _bounded_write(self._scheduler._drop_memory_job, job, scope)
+
+ async def remove(self, scope: MemoryScope) -> None:
+ await _bounded_write(self._scheduler.remove_scope_memory_jobs, scope)
+
+
+async def _bounded_write(call: Callable[..., object], *args: object) -> None:
+ """``call`` off the event loop, its jobstore writes limited to
+ ``MEMORY_CRON_WRITE_TIMEOUT_SECONDS`` (``asyncio.to_thread`` carries
+ ``WRITE_TIMEOUT`` into the thread)."""
+ token = WRITE_TIMEOUT.set(MEMORY_CRON_WRITE_TIMEOUT_SECONDS)
+ try:
+ await asyncio.to_thread(call, *args)
+ finally:
+ WRITE_TIMEOUT.reset(token)
+
+
+def _removed(job: JobObj) -> bool:
+ """Remove ``job``; False when another scheduler removed it first."""
try:
- run_async(clear_registration_marker(user_id, key_prefix), timeout=10)
+ job.remove()
+ except JobLookupError:
+ return False
+ return True
+
+
+def _forget_dream_registration(scope: MemoryScope, job: DreamSystemJob) -> bool:
+ """Bridge ``forget_registration`` onto the scheduler's event loop: whether
+ the scope's row no longer records the cron. Never raises: a delete that
+ cannot forget keeps its cron."""
+ try:
+ return run_async(
+ forget_registration(scope, job),
+ timeout=bridge_timeout(),
+ cancel_on_timeout=True,
+ )
except Exception:
logger.warning(
- "Failed to clear registration marker %s for user %s",
- key_prefix,
- user_id[:12],
- exc_info=True,
+ "Failed to forget %s of %s", job.name, scope_label(scope), exc_info=True
)
+ return False
def refresh_near_expiry_credentials() -> None:
@@ -2119,6 +2294,7 @@ class Jobstores(Enum):
EXECUTION = "execution"
BATCHED_NOTIFICATIONS = "batched_notifications"
WEEKLY_NOTIFICATIONS = "weekly_notifications"
+ MEMORY_CRON_RECONCILE = "memory_cron_reconcile"
class GraphExecutionJobArgs(BaseModel):
@@ -2579,6 +2755,7 @@ class Scheduler(AppService):
**self._persistent_jobstores,
# These don't really need persistence
Jobstores.WEEKLY_NOTIFICATIONS.value: MemoryJobStore(),
+ Jobstores.MEMORY_CRON_RECONCILE.value: MemoryJobStore(),
},
logger=apscheduler_logger,
timezone=ZoneInfo("UTC"),
@@ -2764,16 +2941,19 @@ class Scheduler(AppService):
self.scheduler.add_listener(job_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
self.scheduler.add_listener(job_missed_listener, EVENT_JOB_MISSED)
self.scheduler.add_listener(job_max_instances_listener, EVENT_JOB_MAX_INSTANCES)
+ # Published before start: a system job due at start (such as the
+ # schedule index reconcile) runs at once, on the scheduler's thread,
+ # and reaches this instance only through _scheduler_instance.
+ global _scheduler_instance
+ _scheduler_instance = self
self.scheduler.start()
if self.register_system_tasks:
# After start: until then get_job only sees pending jobs, not the
# jobstore, so the "leave an unchanged job alone" check can't work.
self._register_idle_credential_sweep()
self._register_posthog_lifecycle_sweep()
+ self._register_memory_cron_reconcile()
self._report_parked_jobs()
-
- global _scheduler_instance
- _scheduler_instance = self
self._start_schedule_index_backfill()
# Keep the service running since BackgroundScheduler doesn't block
@@ -3752,11 +3932,12 @@ class Scheduler(AppService):
# --- Graphiti community detection (P-1.7) ---
#
- # Communities are off-by-default behind LD flag ``GRAPHITI_COMMUNITIES_ENABLED``
- # at the call sites. The scheduler unconditionally accepts the
- # registration call; callers gate on the flag. Rebuilds run weekly at
- # user-local 04:00 Sunday to avoid the Leiden cost spike during active
- # hours (and to stagger from a future per-user dream pass at 03:00).
+ # Communities are off-by-default behind LD flag ``GRAPHITI_COMMUNITIES_ENABLED``.
+ # Registration checks the flag here too (defense in depth: the registry
+ # gates first). Rebuilds run weekly at owner-local 04:00 on Mondays
+ # (``0 4 * * 0``: APScheduler counts weekdays from 0 = Monday) to avoid
+ # the Leiden cost spike during active hours and to stagger from the
+ # nightly dream at 03:00.
@expose
def add_community_rebuild_schedule(
@@ -3764,70 +3945,188 @@ class Scheduler(AppService):
user_id: str,
user_timezone: str = "UTC",
) -> dict:
- """Register a weekly community rebuild for one user.
+ """The account's weekly community rebuild: the scope-keyed method
+ for ``MemoryScope.for_user(user_id)``, whose job id is unchanged."""
+ return self.add_scope_community_rebuild_schedule(
+ MemoryScope.for_user(user_id), user_timezone
+ )
- Gated by ``Flag.GRAPHITI_COMMUNITIES_ENABLED`` per-user. When the
- flag is off the call is a no-op — returns a structured "skipped"
- dict so callers see the same shape as a successful registration.
+ @expose
+ def add_scope_community_rebuild_schedule(
+ self,
+ scope: MemoryScope,
+ user_timezone: str = "UTC",
+ ) -> dict:
+ """Register the weekly community rebuild of one memory scope.
+
+ Gated by ``Flag.GRAPHITI_COMMUNITIES_ENABLED`` for the scope's
+ owner. When the flag is off the call is a no-op — returns a
+ structured "skipped" dict so callers see the same shape as a
+ successful registration. Only adds the job: the registry row that
+ lets it fire is ``copilot/dream/registry.py``'s to write.
"""
from backend.copilot.graphiti.config import is_communities_enabled_for_user
- if not run_async(is_communities_enabled_for_user(user_id)):
+ user_timezone = user_timezone or "UTC"
+ if not run_async(is_communities_enabled_for_user(scope.owner_user_id)):
logger.info(
- "Community rebuild registration skipped for user %s — "
+ "Community rebuild registration skipped for %s — "
"GRAPHITI_COMMUNITIES_ENABLED flag is off.",
- user_id[:12],
+ scope_label(scope),
)
- return {
- "id": None,
- "user_id": user_id,
- "user_timezone": user_timezone,
- "next_run_time": None,
- "skipped": True,
- "reason": "graphiti_communities_disabled",
- }
+ return memory_job_skipped(
+ scope, user_timezone, "graphiti_communities_disabled"
+ )
+ job = self._add_memory_job(
+ *_MEMORY_CRONS[COMMUNITY_REBUILD_JOB_PREFIX],
+ dream_system_job(COMMUNITY_REBUILD_JOB_PREFIX),
+ scope,
+ user_timezone,
+ )
+ return memory_job_result(scope, job.id, job.next_run_time, user_timezone)
- if not user_timezone:
- user_timezone = "UTC"
+ @expose
+ def delete_community_rebuild_schedule(self, user_id: str) -> bool:
+ """Remove the account's weekly community rebuild."""
+ return self._delete_account_memory_job(
+ user_id, dream_system_job(COMMUNITY_REBUILD_JOB_PREFIX)
+ )
- job_id = f"community_rebuild_{user_id}"
- job = self.scheduler.add_job(
- execute_community_rebuild,
- kwargs={"user_id": user_id},
- trigger=CronTrigger.from_crontab("0 4 * * 0", timezone=user_timezone),
- id=job_id,
- name=f"Graphiti community rebuild for {user_id[:12]}",
+ @expose
+ def remove_scope_memory_jobs(self, scope: MemoryScope) -> list[str]:
+ """Remove every dream-system cron of one memory scope (``jobs_for``:
+ an expert's has no skill-learning cron); returns the job ids removed.
+ The ids derive from the scope, so a caller can only ever reach that
+ scope's crons. The registry calls this when it pauses or wipes a
+ scope."""
+ jobs = [
+ self.scheduler.get_job(
+ job.job_id(scope), jobstore=Jobstores.EXECUTION.value
+ )
+ for job in jobs_for(scope)
+ ]
+ removed = [job.id for job in jobs if job is not None and _removed(job)]
+ if removed:
+ logger.info("Removed the memory crons of %s", scope_label(scope))
+ return removed
+
+ def _reconcile_memory_crons(self) -> None:
+ """One reconcile pass (``reconcile_memory_scope_crons``), its listing
+ of the dream-system crons included, under
+ ``MEMORY_CRON_RECONCILE_TIMEOUT_SECONDS``."""
+ try:
+ report = run_async(
+ reconcile(_SchedulerCrons(self)),
+ timeout=MEMORY_CRON_RECONCILE_TIMEOUT_SECONDS,
+ cancel_on_timeout=True,
+ )
+ except concurrent.futures.TimeoutError:
+ logger.warning(
+ "Memory cron reconcile cut off; the next pass carries on after "
+ "the last scope it took"
+ )
+ return
+ if report.restored or report.dropped or report.removed or report.failed:
+ logger.info(
+ "Memory cron reconcile: restored %d, took back %d, removed the "
+ "crons of %d scopes, %d failed",
+ len(report.restored),
+ len(report.dropped),
+ len(report.removed),
+ len(report.failed),
+ )
+
+ def _memory_crons(self) -> list[ListedCron]:
+ """Every dream-system cron that names a scope, with its timezone."""
+ return [
+ cron
+ for job in self._memory_jobs()
+ if (cron := listed_cron(job)) is not None
+ ]
+
+ def _memory_jobs(self) -> list[JobObj]:
+ """Every dream-system cron in the EXECUTION jobstore, in one read
+ filtered by job-id prefix and stopped by the database after
+ ``MEMORY_CRON_LISTING_TIMEOUT_SECONDS``."""
+ ids = self._execution_jobstore.jobs_t.c.id
+ return self._execution_jobstore._get_jobs(
+ or_(
+ *(
+ ids.startswith(f"{job.job_id_prefix}_", autoescape=True)
+ for job in DREAM_SYSTEM_JOBS
+ )
+ ),
+ statement_timeout=MEMORY_CRON_LISTING_TIMEOUT_SECONDS,
+ )
+
+ def _add_memory_job(
+ self,
+ func: Callable[..., object],
+ crontab: str,
+ job: DreamSystemJob,
+ scope: MemoryScope,
+ user_timezone: str,
+ reconciled: bool = False,
+ ) -> JobObj:
+ """Add, or replace, one dream-system cron of ``scope``; tagged as the
+ reconcile's when it registers it (``RECONCILED_SUFFIX``)."""
+ tag = RECONCILED_SUFFIX if reconciled else ""
+ kwargs: dict[str, object] = {**memory_job_kwargs(scope)}
+ if scope.expert_id is not None:
+ kwargs["func"] = func
+ func = execute_expert_memory_cron
+ added = self.scheduler.add_job(
+ func,
+ kwargs=kwargs,
+ trigger=CronTrigger.from_crontab(crontab, timezone=user_timezone),
+ id=job.job_id(scope),
+ name=f"{job.name} for {scope_label(scope)}{tag}",
jobstore=Jobstores.EXECUTION.value,
replace_existing=True,
max_instances=1,
+ misfire_grace_time=_MEMORY_CRON_MISFIRE_GRACE.get(job.job_id_prefix),
)
logger.info(
- "Registered community rebuild job %s for user %s in tz %s",
- job.id,
- user_id[:12],
+ "Registered %s job %s for %s in tz %s",
+ job.name,
+ added.id,
+ scope_label(scope),
user_timezone,
)
- return {
- "id": job.id,
- "user_id": user_id,
- "user_timezone": user_timezone,
- "next_run_time": (
- job.next_run_time.isoformat() if job.next_run_time else None
- ),
- }
-
- @expose
- def delete_community_rebuild_schedule(self, user_id: str) -> bool:
- """Remove the weekly community rebuild for one user."""
- job_id = f"community_rebuild_{user_id}"
- job = self.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)
- if not job:
+ return added
+
+ def _delete_account_memory_job(self, user_id: str, job: DreamSystemJob) -> bool:
+ """Remove one of the account's crons, the registry forgetting it
+ first: the reconcile restores a cron a row records, and takes back
+ only one it registered itself once the row stops recording it, so the
+ record goes before the job. A forget that fails keeps the job, and
+ says so. The next ``ensure_scope_scheduled`` registers it again."""
+ scope = MemoryScope.for_user(user_id)
+ existing = self.scheduler.get_job(
+ job.job_id(scope), jobstore=Jobstores.EXECUTION.value
+ )
+ if not existing:
return False
- job.remove()
- _clear_dream_registration_marker(user_id, COMMUNITY_REBUILD_REGISTRATION_PREFIX)
- logger.info("Removed community rebuild job for user %s", user_id[:12])
+ if not _forget_dream_registration(scope, job):
+ logger.warning(
+ "Kept %s job for user %s: the registry could not forget it",
+ job.name,
+ user_id[:12],
+ )
+ return False
+ _removed(existing)
+ logger.info("Removed %s job for user %s", job.name, user_id[:12])
return True
+ def _drop_memory_job(self, job: DreamSystemJob, scope: MemoryScope) -> None:
+ """Remove one dream-system cron of ``scope`` if it is there."""
+ try:
+ self.scheduler.remove_job(
+ job.job_id(scope), jobstore=Jobstores.EXECUTION.value
+ )
+ except JobLookupError:
+ pass
+
@expose
def execute_community_rebuild_pass(self, user_id: str, force: bool = False) -> dict:
"""Manually trigger a community rebuild for one user (bypasses cron).
@@ -3867,6 +4166,28 @@ class Scheduler(AppService):
jobstore=Jobstores.EXECUTION.value,
)
+ def _register_memory_cron_reconcile(self) -> None:
+ """The memory cron reconcile (``reconcile_memory_scope_crons``), the
+ level-triggered backstop of the registry's pause, resume and ensure
+ (``copilot/dream/scope_reconcile.py``), every
+ ``RECONCILE_INTERVAL_MINUTES``.
+
+ Each process starts with an immediate pass. Keep the job in memory
+ so an old scheduler cannot park it during a rolling deploy.
+ Overlapping passes replay the same bounded, idempotent work.
+ """
+ trigger = IntervalTrigger(minutes=RECONCILE_INTERVAL_MINUTES)
+ self.scheduler.add_job(
+ reconcile_memory_scope_crons,
+ trigger,
+ id=_MEMORY_CRON_RECONCILE_JOB_ID,
+ replace_existing=True,
+ max_instances=1,
+ coalesce=True,
+ next_run_time=datetime.now(ZoneInfo("UTC")),
+ jobstore=Jobstores.MEMORY_CRON_RECONCILE.value,
+ )
+
# --- Morning briefing ---
#
# Daily per-user cron at user-local 09:00. The job body's flag gate and
@@ -3979,72 +4300,54 @@ class Scheduler(AppService):
user_id: str,
user_timezone: str = "UTC",
) -> dict:
- """Register the nightly batch-family fan-out cron for one user.
-
- Gated by ``Flag.DREAM_PASS_ENABLED`` per-user. When the flag
- is off the call is a no-op — returns a structured "skipped"
- dict matching the shape of a successful registration.
- Defense-in-depth: the auto-registration helper in
- ``copilot/dream/scheduling.py`` already gates this, but direct
- callers (admin endpoint, ad-hoc scripts) bypass that helper.
+ """The account's nightly batch cron: the scope-keyed method for
+ ``MemoryScope.for_user(user_id)``, whose job id is unchanged."""
+ return self.add_scope_nightly_batch_schedule(
+ MemoryScope.for_user(user_id), user_timezone
+ )
+
+ @expose
+ def add_scope_nightly_batch_schedule(
+ self,
+ scope: MemoryScope,
+ user_timezone: str = "UTC",
+ ) -> dict:
+ """Register the nightly batch-family fan-out cron of one memory scope.
+
+ Gated by ``Flag.DREAM_PASS_ENABLED`` for the scope's owner. When
+ the flag is off the call is a no-op — returns a structured
+ "skipped" dict matching the shape of a successful registration.
+ Defense-in-depth: the registry in ``copilot/dream/registry.py``
+ already gates this, but direct callers (admin endpoint, ad-hoc
+ scripts) bypass it. Only adds the job: the registry row that lets
+ it fire is the registry's to write.
"""
from backend.util.feature_flag import Flag, is_feature_enabled
- if not run_async(is_feature_enabled(Flag.DREAM_PASS_ENABLED, user_id)):
+ user_timezone = user_timezone or "UTC"
+ if not run_async(
+ is_feature_enabled(Flag.DREAM_PASS_ENABLED, scope.owner_user_id)
+ ):
logger.info(
- "Nightly batch registration skipped for user %s — "
+ "Nightly batch registration skipped for %s — "
"DREAM_PASS_ENABLED flag is off.",
- user_id[:12],
+ scope_label(scope),
)
- return {
- "id": None,
- "user_id": user_id,
- "user_timezone": user_timezone,
- "next_run_time": None,
- "skipped": True,
- "reason": "dream_pass_disabled",
- }
-
- if not user_timezone:
- user_timezone = "UTC"
-
- job_id = f"dream_nightly_batch_{user_id}"
- job = self.scheduler.add_job(
- execute_nightly_batch_sync,
- kwargs={"user_id": user_id},
- trigger=CronTrigger.from_crontab("0 3 * * *", timezone=user_timezone),
- id=job_id,
- name=f"Dream nightly batch for {user_id[:12]}",
- jobstore=Jobstores.EXECUTION.value,
- replace_existing=True,
- max_instances=1,
- )
- logger.info(
- "Registered nightly batch job %s for user %s in tz %s",
- job.id,
- user_id[:12],
+ return memory_job_skipped(scope, user_timezone, "dream_pass_disabled")
+ job = self._add_memory_job(
+ *_MEMORY_CRONS[NIGHTLY_BATCH_JOB_PREFIX],
+ dream_system_job(NIGHTLY_BATCH_JOB_PREFIX),
+ scope,
user_timezone,
)
- return {
- "id": job.id,
- "user_id": user_id,
- "user_timezone": user_timezone,
- "next_run_time": (
- job.next_run_time.isoformat() if job.next_run_time else None
- ),
- }
+ return memory_job_result(scope, job.id, job.next_run_time, user_timezone)
@expose
def delete_nightly_batch_schedule(self, user_id: str) -> bool:
- """Remove the nightly batch cron for one user."""
- job_id = f"dream_nightly_batch_{user_id}"
- job = self.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)
- if not job:
- return False
- job.remove()
- _clear_dream_registration_marker(user_id, NIGHTLY_BATCH_REGISTRATION_PREFIX)
- logger.info("Removed nightly batch job for user %s", user_id[:12])
- return True
+ """Remove the account's nightly batch cron."""
+ return self._delete_account_memory_job(
+ user_id, dream_system_job(NIGHTLY_BATCH_JOB_PREFIX)
+ )
@expose
def add_skill_learning_schedule(
@@ -4052,89 +4355,65 @@ class Scheduler(AppService):
user_id: str,
user_timezone: str = "UTC",
) -> dict:
- """Register the nightly skill-learning cron for one user.
-
- Gated by ``Flag.DREAM_SKILL_LEARNING_ENABLED`` — independent of the
- dream pass gate. Fires at 03:00 user-local. ``coalesce`` plus a
- six-hour misfire grace give a capped catch-up after downtime: at
- most one run, and only if the scheduler is back the same morning;
- the pass itself re-checks every source's eligibility, so a stale
- proposal is never applied by a catch-up run.
+ """The account's nightly skill-learning cron: the scope-keyed method
+ for ``MemoryScope.for_user(user_id)``, whose job id is unchanged."""
+ return self.add_scope_skill_learning_schedule(
+ MemoryScope.for_user(user_id), user_timezone
+ )
+
+ @expose
+ def add_scope_skill_learning_schedule(
+ self,
+ scope: MemoryScope,
+ user_timezone: str = "UTC",
+ ) -> dict:
+ """Register the nightly skill-learning cron of an account's scope.
+
+ Gated by ``Flag.DREAM_SKILL_LEARNING_ENABLED`` for the owner,
+ independent of the dream pass gate. An expert's scope has no such
+ cron (one pass reviews the account's skills and its experts'), so it
+ is refused. Fires at 03:00 owner-local. A cron already registered in
+ that timezone is kept as it is, its next run included, so an overdue
+ run is not pushed back a day, and reported as registered. A missed
+ run is caught up only the same morning
+ (``_MEMORY_CRON_MISFIRE_GRACE``): at most one run, and the pass
+ re-checks every source's eligibility, so a stale proposal is never
+ applied by a catch-up run. Only adds the job: the registry row that
+ lets it fire is the registry's to write.
"""
from backend.util.feature_flag import Flag, is_feature_enabled
+ user_timezone = user_timezone or "UTC"
+ if scope.is_expert:
+ return memory_job_skipped(scope, user_timezone, "account_only")
if not run_async(
- is_feature_enabled(Flag.DREAM_SKILL_LEARNING_ENABLED, user_id)
+ is_feature_enabled(Flag.DREAM_SKILL_LEARNING_ENABLED, scope.owner_user_id)
):
logger.info(
- "Skill learning registration skipped for user %s — flag is off.",
- user_id[:12],
+ "Skill learning registration skipped for %s — "
+ "DREAM_SKILL_LEARNING_ENABLED flag is off.",
+ scope_label(scope),
)
- return {
- "id": None,
- "user_id": user_id,
- "user_timezone": user_timezone,
- "next_run_time": None,
- "skipped": True,
- "reason": "skill_learning_disabled",
- }
-
- if not user_timezone:
- user_timezone = "UTC"
-
- job_id = f"skill_learning_nightly_{user_id}"
- existing = self.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)
- if existing is not None and _job_timezone_name(existing) == user_timezone:
- return {
- "id": existing.id,
- "user_id": user_id,
- "user_timezone": user_timezone,
- "next_run_time": (
- existing.next_run_time.isoformat()
- if existing.next_run_time
- else None
- ),
- "skipped": True,
- "reason": "already_registered",
- }
- job = self.scheduler.add_job(
- execute_skill_learning_sync,
- kwargs={"user_id": user_id},
- trigger=CronTrigger.from_crontab("0 3 * * *", timezone=user_timezone),
- id=job_id,
- name=f"Skill learning nightly for {user_id[:12]}",
- jobstore=Jobstores.EXECUTION.value,
- replace_existing=True,
- max_instances=1,
- coalesce=True,
- misfire_grace_time=6 * 3600,
+ return memory_job_skipped(scope, user_timezone, "skill_learning_disabled")
+ job = dream_system_job(SKILL_LEARNING_JOB_PREFIX)
+ existing = self.scheduler.get_job(
+ job.job_id(scope), jobstore=Jobstores.EXECUTION.value
)
- logger.info(
- "Registered skill learning job %s for user %s in tz %s",
- job.id,
- user_id[:12],
- user_timezone,
+ if existing is not None and _job_timezone_name(existing) == user_timezone:
+ return memory_job_result(
+ scope, existing.id, existing.next_run_time, user_timezone
+ )
+ added = self._add_memory_job(
+ *_MEMORY_CRONS[SKILL_LEARNING_JOB_PREFIX], job, scope, user_timezone
)
- return {
- "id": job.id,
- "user_id": user_id,
- "user_timezone": user_timezone,
- "next_run_time": (
- job.next_run_time.isoformat() if job.next_run_time else None
- ),
- }
+ return memory_job_result(scope, added.id, added.next_run_time, user_timezone)
@expose
def delete_skill_learning_schedule(self, user_id: str) -> bool:
- """Remove the nightly skill-learning cron for one user."""
- job_id = f"skill_learning_nightly_{user_id}"
- job = self.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)
- if not job:
- return False
- job.remove()
- _clear_dream_registration_marker(user_id, SKILL_LEARNING_REGISTRATION_PREFIX)
- logger.info("Removed skill learning job for user %s", user_id[:12])
- return True
+ """Remove the account's nightly skill-learning cron."""
+ return self._delete_account_memory_job(
+ user_id, dream_system_job(SKILL_LEARNING_JOB_PREFIX)
+ )
@expose
def execute_skill_learning_now(self, user_id: str) -> dict:
@@ -4290,9 +4569,13 @@ class SchedulerClient(AppServiceClient):
add_community_rebuild_schedule = endpoint_to_async(
Scheduler.add_community_rebuild_schedule
)
+ add_scope_community_rebuild_schedule = endpoint_to_async(
+ Scheduler.add_scope_community_rebuild_schedule
+ )
delete_community_rebuild_schedule = endpoint_to_async(
Scheduler.delete_community_rebuild_schedule
)
+ remove_scope_memory_jobs = endpoint_to_async(Scheduler.remove_scope_memory_jobs)
execute_community_rebuild_pass = endpoint_to_async(
Scheduler.execute_community_rebuild_pass
)
@@ -4305,12 +4588,18 @@ class SchedulerClient(AppServiceClient):
)
add_nightly_batch_schedule = endpoint_to_async(Scheduler.add_nightly_batch_schedule)
+ add_scope_nightly_batch_schedule = endpoint_to_async(
+ Scheduler.add_scope_nightly_batch_schedule
+ )
delete_nightly_batch_schedule = endpoint_to_async(
Scheduler.delete_nightly_batch_schedule
)
add_skill_learning_schedule = endpoint_to_async(
Scheduler.add_skill_learning_schedule
)
+ add_scope_skill_learning_schedule = endpoint_to_async(
+ Scheduler.add_scope_skill_learning_schedule
+ )
delete_skill_learning_schedule = endpoint_to_async(
Scheduler.delete_skill_learning_schedule
)
diff --git a/autogpt_platform/backend/backend/executor/scheduler_memory_reconcile_test.py b/autogpt_platform/backend/backend/executor/scheduler_memory_reconcile_test.py
new file mode 100644
index 0000000000..7c7f307e01
--- /dev/null
+++ b/autogpt_platform/backend/backend/executor/scheduler_memory_reconcile_test.py
@@ -0,0 +1,565 @@
+"""The scheduler's side of the memory cron reconcile
+(``reconcile_memory_scope_crons``): its filtered listing of the dream-system
+crons (inside the pass's ceiling, and stopped by the database), the scope
+and timezone each names, the cron store it hands the pass, its registration
+as a system job, and removal that another scheduler got to first. The
+pass's logic is ``copilot/dream/scope_reconcile_test.py``.
+
+The listing, registration and removal probes are an independent review's
+(PR 9 round 2), on real SQLite jobstores as there.
+"""
+
+import asyncio
+import concurrent.futures
+import os
+import re
+import tempfile
+import threading
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import pytest
+from apscheduler.jobstores.memory import MemoryJobStore
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.cron import CronTrigger
+from apscheduler.triggers.interval import IntervalTrigger
+from sqlalchemy import text
+from sqlalchemy.exc import OperationalError
+
+from backend.copilot.dream.scheduling import (
+ COMMUNITY_REBUILD_JOB_PREFIX,
+ NIGHTLY_BATCH_JOB_PREFIX,
+ dream_system_job,
+)
+from backend.copilot.dream.scope_reconcile import (
+ RECONCILE_INTERVAL_MINUTES,
+ CronStore,
+ ListedCron,
+ ReconcileReport,
+)
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.executor import scheduler as scheduler_module
+from backend.executor.jobstore import WRITE_TIMEOUT, ResilientSQLAlchemyJobStore
+from backend.executor.scheduler import (
+ MEMORY_CRON_LISTING_TIMEOUT_SECONDS,
+ MEMORY_CRON_WRITE_TIMEOUT_SECONDS,
+ Jobstores,
+ Scheduler,
+ _extract_schema_from_url,
+ _SchedulerCrons,
+ listed_cron,
+ reconcile_memory_scope_crons,
+)
+
+EXPERT = MemoryScope.for_expert("user-1", "expert-1")
+_EXECUTION = Jobstores.EXECUTION.value
+_RECONCILE_JOB = "reconcile_memory_scope_crons"
+
+
+def noop(**kwargs: Any) -> None:
+ """A job body the SQLite jobstore can pickle by reference."""
+
+
+@contextmanager
+def _sqlite_file() -> Iterator[str]:
+ fd, path = tempfile.mkstemp(suffix=".sqlite")
+ os.close(fd)
+ try:
+ yield f"sqlite:///{path}"
+ finally:
+ os.unlink(path)
+
+
+@contextmanager
+def _scheduler_on(url: str) -> Iterator[Scheduler]:
+ """A ``Scheduler`` whose APScheduler, started paused, keeps its EXECUTION
+ jobs in the SQLite database at ``url``, as two schedulers in a rolling
+ deploy share one jobstore."""
+ store = ResilientSQLAlchemyJobStore(url=url)
+ apscheduler = BackgroundScheduler()
+ apscheduler.add_jobstore(store, _EXECUTION)
+ apscheduler.add_jobstore(MemoryJobStore(), "memory_cron_reconcile")
+ apscheduler.start(paused=True)
+ scheduler = Scheduler(register_system_tasks=False)
+ scheduler.scheduler = apscheduler
+ scheduler._execution_jobstore = store
+ try:
+ yield scheduler
+ finally:
+ # The store goes first: a scheduler shutting down processes the jobs
+ # due on it, and would move an overdue one on.
+ apscheduler.remove_jobstore(_EXECUTION, shutdown=True)
+ apscheduler.shutdown(wait=False)
+ store.engine.dispose() # Windows will not unlink an open file
+
+
+def scheduler_loop():
+ """A real event loop on its own thread, installed as the scheduler's
+ shared loop so ``run_async`` bridges to it as in production."""
+ loop = asyncio.new_event_loop()
+ thread = threading.Thread(target=loop.run_forever, daemon=True)
+ thread.start()
+ try:
+ with patch.object(scheduler_module, "_event_loop", loop):
+ yield loop
+ finally:
+ loop.call_soon_threadsafe(loop.stop)
+ thread.join(timeout=2)
+ loop.close()
+
+
+def test_the_listing_reads_only_the_dream_system_crons() -> None:
+ """One read, filtered by prefix, with ``_`` matched literally: a job id
+ that only a LIKE wildcard would match is left out."""
+ with _sqlite_file() as url, _scheduler_on(url) as scheduler:
+ for job_id in (
+ "community_rebuild_user-1",
+ "dream_nightly_batch_user-2",
+ "communityXrebuild_user-3",
+ "dream_nightly_batchy",
+ "graph-execution-1",
+ ):
+ scheduler.scheduler.add_job(
+ noop,
+ "interval",
+ hours=1,
+ id=job_id,
+ kwargs={"user_id": "user-1"},
+ jobstore=_EXECUTION,
+ )
+
+ listed = sorted(job.id for job in scheduler._memory_jobs())
+
+ assert listed == ["community_rebuild_user-1", "dream_nightly_batch_user-2"]
+
+
+def test_the_listing_names_each_crons_scope_and_timezone() -> None:
+ """Registered as the scope RPCs register them, each cron lists with the
+ scope its kwargs name and the timezone its trigger runs in, the form
+ the registry row keeps."""
+ account = MemoryScope.for_user("user-1")
+ with _sqlite_file() as url, _scheduler_on(url) as scheduler:
+ community = dream_system_job(COMMUNITY_REBUILD_JOB_PREFIX)
+ nightly = dream_system_job(NIGHTLY_BATCH_JOB_PREFIX)
+ scheduler._add_memory_job(noop, "0 4 * * 0", community, EXPERT, "Asia/Tokyo")
+ scheduler._add_memory_job(noop, "0 3 * * *", nightly, account, "UTC")
+
+ listed = sorted(scheduler._memory_crons(), key=lambda cron: cron.job_id)
+
+ assert [(cron.job_id, cron.scope, cron.timezone) for cron in listed] == [
+ (community.job_id(EXPERT), EXPERT, "Asia/Tokyo"),
+ (nightly.job_id(account), account, "UTC"),
+ ]
+ assert not any(cron.reconciled for cron in listed)
+
+
+def test_a_cron_the_reconcile_registers_lists_as_its():
+ """Its tag is on the job's name, which the jobstore keeps; registering
+ the same job again as the registry does takes the tag off."""
+ nightly = dream_system_job(NIGHTLY_BATCH_JOB_PREFIX)
+ with _sqlite_file() as url, _scheduler_on(url) as scheduler:
+ scheduler._add_memory_job(noop, "0 3 * * *", nightly, EXPERT, "UTC", True)
+ [tagged] = scheduler._memory_crons()
+ scheduler._add_memory_job(noop, "0 3 * * *", nightly, EXPERT, "UTC")
+ [untagged] = scheduler._memory_crons()
+
+ assert (tagged.job_id, tagged.reconciled) == (nightly.job_id(EXPERT), True)
+ assert untagged.reconciled is False
+
+
+ "prefix", [COMMUNITY_REBUILD_JOB_PREFIX, NIGHTLY_BATCH_JOB_PREFIX]
+)
+def test_expert_crons_use_a_separate_callable_reference(prefix):
+ job = dream_system_job(prefix)
+ body, crontab = scheduler_module._MEMORY_CRONS[prefix]
+ with _sqlite_file() as url, _scheduler_on(url) as scheduler:
+ expert = scheduler._add_memory_job(body, crontab, job, EXPERT, "UTC")
+ account = scheduler._add_memory_job(
+ body, crontab, job, MemoryScope.for_user("user-1"), "UTC"
+ )
+
+ assert (
+ expert.func_ref == "backend.executor.scheduler:execute_expert_memory_cron"
+ )
+ assert expert.kwargs == {
+ "func": body,
+ "user_id": "user-1",
+ "expert_id": "expert-1",
+ }
+ assert account.func is body
+ assert account.kwargs == {"user_id": "user-1"}
+
+
+def test_expert_cron_dispatch_preserves_the_body_arguments_and_result():
+ body = MagicMock(return_value="completed")
+
+ result = scheduler_module.execute_expert_memory_cron(body, "user-1", "expert-1")
+
+ body.assert_called_once_with(user_id="user-1", expert_id="expert-1")
+ assert result == "completed"
+
+
+ "kwargs, scope",
+ [
+ ({"user_id": "user-1"}, MemoryScope.for_user("user-1")),
+ ({"user_id": "user-1", "expert_id": "expert-1"}, EXPERT),
+ ({"user_id": "user-1", "expert_id": None}, MemoryScope.for_user("user-1")),
+ ({}, None),
+ ({"user_id": ""}, None),
+ ],
+)
+def test_a_listed_cron_names_the_scope_its_kwargs_do(
+ kwargs: dict[str, Any], scope: MemoryScope | None
+) -> None:
+ job = MagicMock(
+ id="dream_nightly_batch_x",
+ kwargs=kwargs,
+ trigger=CronTrigger.from_crontab("0 3 * * *", timezone="Europe/Paris"),
+ )
+ job.name = "Dream nightly batch for user user-1"
+
+ cron = listed_cron(job)
+
+ assert (cron.scope if cron is not None else None) == scope
+ assert cron is None or cron.timezone == "Europe/Paris"
+
+
+def test_the_listing_asks_the_database_to_stop_it() -> None:
+ """The jobstore read carries ``MEMORY_CRON_LISTING_TIMEOUT_SECONDS``:
+ cancelling the wait for it leaves its thread running, so the database
+ is what stops a stalled read."""
+ scheduler = Scheduler(register_system_tasks=False)
+ scheduler._execution_jobstore = ResilientSQLAlchemyJobStore(url="sqlite://")
+
+ with patch.object(
+ scheduler._execution_jobstore, "_get_jobs", return_value=[]
+ ) as read:
+ scheduler._memory_jobs()
+
+ assert read.call_args.kwargs == {
+ "statement_timeout": MEMORY_CRON_LISTING_TIMEOUT_SECONDS
+ }
+
+
+def test_a_statement_timeout_leaves_a_sqlite_read_as_it_was() -> None:
+ """SQLite has no statement timeout; the read goes through unbounded."""
+ with _sqlite_file() as url, _scheduler_on(url) as scheduler:
+ scheduler.scheduler.add_job(
+ noop,
+ "interval",
+ hours=1,
+ id="community_rebuild_user-1",
+ jobstore=_EXECUTION,
+ )
+
+ jobs = scheduler._execution_jobstore._get_jobs(statement_timeout=0.001)
+
+ assert [job.id for job in jobs] == ["community_rebuild_user-1"]
+
+
+def _postgres_url() -> str | None:
+ """The test database's SQLAlchemy URL, as the scheduler builds it, when
+ the run has a PostgreSQL one."""
+ direct = os.getenv("DIRECT_URL") or ""
+ if not direct.startswith("postgresql"):
+ return None
+ return _extract_schema_from_url(direct)[1]
+
+
[email protected](_postgres_url() is None, reason="needs the PostgreSQL test DB")
+def test_postgresql_stops_a_listing_that_runs_past_its_statement_timeout() -> None:
+ """On PostgreSQL the bounded read is cancelled by the server, and the
+ same read without the bound runs to its end."""
+ store = ResilientSQLAlchemyJobStore(
+ url=_postgres_url(), tablename=f"apscheduler_jobs_probe_{os.getpid()}"
+ )
+ apscheduler = BackgroundScheduler()
+ apscheduler.add_jobstore(store, _EXECUTION)
+ apscheduler.start(paused=True)
+ try:
+ apscheduler.add_job(noop, "interval", hours=1, id="a", jobstore=_EXECUTION)
+ slow = text("pg_sleep(1) IS NOT NULL")
+
+ with pytest.raises(OperationalError, match="statement timeout"):
+ store._get_jobs(slow, statement_timeout=0.1)
+ assert [job.id for job in store._get_jobs(slow)] == ["a"]
+ finally:
+ apscheduler.shutdown(wait=False)
+ store.jobs_t.drop(store.engine, checkfirst=True)
+ store.engine.dispose()
+
+
+async def test_the_cron_store_lists_and_registers_as_the_scope_rpcs_do() -> None:
+ """The pass lists through ``_memory_crons``, registers through the same
+ body and crontab as the scope RPCs, tagged as the reconcile's, and
+ removes through ``remove_scope_memory_jobs``."""
+ scheduler = Scheduler(register_system_tasks=False)
+ crons: CronStore = _SchedulerCrons(scheduler)
+ listed = [ListedCron(job_id="x", scope=EXPERT, timezone="UTC")]
+ nightly = dream_system_job(NIGHTLY_BATCH_JOB_PREFIX)
+ with (
+ patch.object(scheduler, "_memory_crons", return_value=listed),
+ patch.object(scheduler, "_add_memory_job") as add,
+ patch.object(scheduler, "_drop_memory_job") as drop,
+ patch.object(scheduler, "remove_scope_memory_jobs") as remove,
+ ):
+ assert await crons.listing() == listed
+ await crons.add(dream_system_job(COMMUNITY_REBUILD_JOB_PREFIX), EXPERT, "UTC")
+ await crons.add(nightly, EXPERT, "UTC")
+ await crons.drop(nightly, EXPERT)
+ await crons.remove(EXPERT)
+
+ drop.assert_called_once_with(nightly, EXPERT)
+
+ assert [call.args[:2] for call in add.call_args_list] == [
+ (scheduler_module.execute_community_rebuild, "0 4 * * 0"),
+ (scheduler_module.execute_nightly_batch_sync, "0 3 * * *"),
+ ]
+ assert [call.args[3:] for call in add.call_args_list] == [(EXPERT, "UTC", True)] * 2
+ remove.assert_called_once_with(EXPERT)
+
+
+async def test_the_cron_stores_writes_carry_the_write_timeout() -> None:
+ """Each write runs in a thread that sees ``WRITE_TIMEOUT``, so the
+ jobstore limits it; the caller's context is left as it was."""
+ scheduler = Scheduler(register_system_tasks=False)
+ crons: CronStore = _SchedulerCrons(scheduler)
+ seen: list[float | None] = []
+
+ def note(*args) -> None:
+ seen.append(WRITE_TIMEOUT.get())
+
+ with (
+ patch.object(scheduler, "_add_memory_job", note),
+ patch.object(scheduler, "_drop_memory_job", note),
+ patch.object(scheduler, "remove_scope_memory_jobs", note),
+ ):
+ await crons.add(dream_system_job(NIGHTLY_BATCH_JOB_PREFIX), EXPERT, "UTC")
+ await crons.drop(dream_system_job(NIGHTLY_BATCH_JOB_PREFIX), EXPERT)
+ await crons.remove(EXPERT)
+
+ assert seen == [MEMORY_CRON_WRITE_TIMEOUT_SECONDS] * 3
+ assert WRITE_TIMEOUT.get() is None
+
+
+def test_dropping_a_cron_that_is_gone_is_no_error() -> None:
+ """Another scheduler, or the delete RPC, may have removed it first."""
+ with _sqlite_file() as url, _scheduler_on(url) as scheduler:
+ nightly = dream_system_job(NIGHTLY_BATCH_JOB_PREFIX)
+ scheduler._add_memory_job(noop, "0 3 * * *", nightly, EXPERT, "UTC")
+
+ scheduler._drop_memory_job(nightly, EXPERT)
+ scheduler._drop_memory_job(nightly, EXPERT)
+
+ assert scheduler.scheduler.get_jobs(jobstore=_EXECUTION) == []
+
+
[email protected](_postgres_url() is None, reason="needs the PostgreSQL test DB")
+def test_a_write_under_the_write_timeout_gives_up_on_a_locked_row() -> None:
+ """On PostgreSQL a write in a context that sets ``WRITE_TIMEOUT`` fails
+ once it has waited that long for a row another transaction holds,
+ leaving no thread behind; one in a context that does not, as every
+ other scheduler write, waits for the row."""
+ name = f"apscheduler_jobs_lock_{os.getpid()}"
+ store = ResilientSQLAlchemyJobStore(url=_postgres_url(), tablename=name)
+ apscheduler = BackgroundScheduler()
+ apscheduler.add_jobstore(store, _EXECUTION)
+ apscheduler.start(paused=True)
+ locker = store.engine.connect()
+ try:
+ apscheduler.add_job(noop, "interval", hours=1, id="a", jobstore=_EXECUTION)
+ hold = locker.begin()
+ locker.execute(text(f"SELECT id FROM {name} WHERE id = 'a' FOR UPDATE")).all()
+ raised: list[BaseException] = []
+
+ def bounded() -> None:
+ # A new thread starts with an empty context: set it there.
+ WRITE_TIMEOUT.set(0.2)
+ try:
+ apscheduler.modify_job("a", jobstore=_EXECUTION, name="bounded")
+ except BaseException as exc:
+ raised.append(exc)
+
+ writer = threading.Thread(target=bounded)
+ writer.start()
+ writer.join(timeout=5)
+ assert not writer.is_alive(), "the bounded write gave up on the row"
+ [error] = raised
+ assert isinstance(error, OperationalError)
+ assert re.search("(lock|statement) timeout", str(error))
+ unbounded = threading.Thread(
+ target=apscheduler.modify_job,
+ args=("a",),
+ kwargs={"jobstore": _EXECUTION, "name": "waited"},
+ )
+ unbounded.start()
+ unbounded.join(timeout=1)
+ assert unbounded.is_alive(), "an unbounded write waits for the row"
+ hold.rollback()
+ unbounded.join(timeout=5)
+ assert not unbounded.is_alive()
+ assert apscheduler.get_job("a", jobstore=_EXECUTION).name == "waited"
+ finally:
+ locker.close()
+ apscheduler.remove_jobstore(_EXECUTION, shutdown=False)
+ apscheduler.shutdown(wait=False)
+ store.jobs_t.drop(store.engine, checkfirst=True)
+ store.engine.dispose()
+
+
+def test_a_pass_hands_the_reconcile_its_cron_store() -> None:
+ scheduler = Scheduler(register_system_tasks=False)
+ seen: dict[str, Any] = {}
+
+ async def fake_reconcile(store: CronStore):
+ seen["store"] = store
+ return ReconcileReport(restored=["dream_nightly_batch_user-1"])
+
+ with (
+ patch.object(scheduler_module, "reconcile", fake_reconcile),
+ patch.object(
+ scheduler_module,
+ "run_async",
+ side_effect=lambda coro, **kwargs: asyncio.run(coro),
+ ),
+ ):
+ scheduler._reconcile_memory_crons()
+
+ assert isinstance(seen["store"], _SchedulerCrons)
+
+
+def test_a_stalled_listing_is_cut_off_at_the_passs_ceiling(
+ scheduler_loop, monkeypatch, caplog
+) -> None:
+ """The review's probe: the listing ran before the ceiling started, so a
+ stalled one held the pass (and, one at a time, every later one). It now
+ runs inside the ceiling: the pass returns at the ceiling while the
+ listing still hangs."""
+ monkeypatch.setattr(scheduler_module, "MEMORY_CRON_RECONCILE_TIMEOUT_SECONDS", 0.2)
+ scheduler = Scheduler(register_system_tasks=False)
+ release = threading.Event()
+ started = threading.Event()
+
+ def stalled_listing() -> list:
+ started.set()
+ release.wait(timeout=5)
+ return []
+
+ monkeypatch.setattr(scheduler, "_memory_jobs", stalled_listing)
+ began = time.monotonic()
+ try:
+ scheduler._reconcile_memory_crons()
+ took = time.monotonic() - began
+ finally:
+ release.set()
+
+ assert started.is_set(), "the listing ran inside the pass"
+ assert took < 2
+ assert "Memory cron reconcile cut off" in caplog.text
+
+
+def test_a_pass_cut_off_at_its_timeout_waits_for_the_next(caplog) -> None:
+ scheduler = Scheduler(register_system_tasks=False)
+
+ def cut_off(coro, **kwargs):
+ coro.close()
+ assert kwargs["cancel_on_timeout"] is True
+ raise concurrent.futures.TimeoutError
+
+ with patch.object(scheduler_module, "run_async", side_effect=cut_off):
+ scheduler._reconcile_memory_crons()
+
+ assert "Memory cron reconcile cut off" in caplog.text
+
+
+def test_the_system_task_waits_for_a_running_scheduler(caplog) -> None:
+ with patch.object(scheduler_module, "_scheduler_instance", None):
+ reconcile_memory_scope_crons()
+
+ assert "scheduler not running" in caplog.text
+
+
+class TestRegistration:
+ """Each scheduler runs an immediate pass without persisting the system job."""
+
+ def _register(self, existing=None) -> MagicMock:
+ scheduler = Scheduler.__new__(Scheduler)
+ scheduler.scheduler = MagicMock()
+ scheduler.scheduler.get_job.return_value = existing
+ Scheduler._register_memory_cron_reconcile(scheduler)
+ return scheduler.scheduler.add_job
+
+ def test_registers_an_immediate_interval_job(self) -> None:
+ before = datetime.now(timezone.utc)
+ add_job = self._register()
+
+ add_job.assert_called_once()
+ assert add_job.call_args.args[0] is reconcile_memory_scope_crons
+ assert str(add_job.call_args.args[1]) == str(
+ IntervalTrigger(minutes=RECONCILE_INTERVAL_MINUTES)
+ )
+ kwargs = dict(add_job.call_args.kwargs)
+ assert before <= kwargs.pop("next_run_time") <= datetime.now(timezone.utc)
+ assert kwargs == {
+ "id": _RECONCILE_JOB,
+ "replace_existing": True,
+ "max_instances": 1,
+ "coalesce": True,
+ "jobstore": "memory_cron_reconcile",
+ }
+
+ def test_overlapping_schedulers_and_a_restart_each_get_an_immediate_pass(
+ self,
+ ) -> None:
+ with _sqlite_file() as url:
+ with _scheduler_on(url) as first, _scheduler_on(url) as second:
+ before = datetime.now(timezone.utc)
+ first._register_memory_cron_reconcile()
+ second._register_memory_cron_reconcile()
+ for service in (first, second):
+ [job] = service.scheduler.get_jobs(jobstore="memory_cron_reconcile")
+ assert job.id == _RECONCILE_JOB
+ assert before <= job.next_run_time <= datetime.now(timezone.utc)
+ assert service._execution_jobstore.get_all_jobs() == []
+ with _scheduler_on(url) as restarted:
+ assert restarted.scheduler.get_jobs() == []
+ before = datetime.now(timezone.utc)
+ restarted._register_memory_cron_reconcile()
+ [job] = restarted.scheduler.get_jobs(jobstore="memory_cron_reconcile")
+ assert before <= job.next_run_time <= datetime.now(timezone.utc)
+ assert restarted._execution_jobstore.get_all_jobs() == []
+
+
+def test_a_removal_another_scheduler_made_first_is_no_error() -> None:
+ """The review's probe: two schedulers on one jobstore both read a
+ scope's crons, the other removes them first, and this one's removal
+ finds them gone. It reports only what it removed, and raises nothing."""
+ with _sqlite_file() as url, _scheduler_on(url) as one, _scheduler_on(url) as other:
+ for prefix in (COMMUNITY_REBUILD_JOB_PREFIX, NIGHTLY_BATCH_JOB_PREFIX):
+ one._add_memory_job(
+ noop, "0 3 * * *", dream_system_job(prefix), EXPERT, "UTC"
+ )
+ get_job = one.scheduler.get_job
+
+ def read_then_lose(job_id: str, jobstore: str):
+ job = get_job(job_id, jobstore=jobstore)
+ if job_id.startswith(NIGHTLY_BATCH_JOB_PREFIX):
+ other.scheduler.remove_job(job_id, jobstore=jobstore)
+ return job
+
+ with patch.object(one.scheduler, "get_job", side_effect=read_then_lose):
+ removed = one.remove_scope_memory_jobs(EXPERT)
+
+ left = one.scheduler.get_jobs(jobstore=_EXECUTION)
+
+ assert removed == [dream_system_job(COMMUNITY_REBUILD_JOB_PREFIX).job_id(EXPERT)]
+ assert left == []
diff --git a/autogpt_platform/backend/backend/executor/scheduler_run_async_test.py b/autogpt_platform/backend/backend/executor/scheduler_run_async_test.py
new file mode 100644
index 0000000000..8b78d7ea99
--- /dev/null
+++ b/autogpt_platform/backend/backend/executor/scheduler_run_async_test.py
@@ -0,0 +1,124 @@
+"""``run_async``, the scheduler's bridge from its sync job bodies onto the
+shared event loop: what a timeout does to the coroutine it gave up on.
+
+Only the memory-registry bridges ask for cancellation (see
+``TestMemoryScopeGateBridge`` in ``scheduler_test.py``); every other
+caller's coroutine runs on, because graph dispatch is not safe to stop
+midway.
+"""
+
+import asyncio
+import threading
+import time
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from backend.executor.scheduler import GraphExecutionJobArgs, execute_graph, run_async
+
+_SCHEDULER_PATH = "backend.executor.scheduler"
+_JOB = GraphExecutionJobArgs(
+ schedule_id="sched-1",
+ user_id="user-1",
+ graph_id="graph-1",
+ graph_version=1,
+ cron="* * * * *",
+ input_data={},
+ input_credentials={},
+)
+
+
+def scheduler_loop():
+ """A real event loop on its own thread, installed as the scheduler's
+ shared loop so ``run_async`` bridges to it as in production."""
+ loop = asyncio.new_event_loop()
+ thread = threading.Thread(target=loop.run_forever, daemon=True)
+ thread.start()
+ try:
+ with patch(f"{_SCHEDULER_PATH}._event_loop", loop):
+ yield loop
+ finally:
+ loop.call_soon_threadsafe(loop.stop)
+ thread.join(timeout=2)
+ loop.close()
+
+
+def test_run_async_cancels_the_coroutine_it_gives_up_on_when_asked(scheduler_loop):
+ """The memory bridges ask for it: a timed-out gate must not stay on the
+ shared loop with nobody waiting for it."""
+ cancelled = threading.Event()
+
+ async def hang() -> None:
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ cancelled.set()
+ raise
+
+ with pytest.raises(TimeoutError):
+ run_async(hang(), timeout=0.05, cancel_on_timeout=True)
+ assert cancelled.wait(timeout=2)
+
+
+def test_run_async_leaves_the_coroutine_running_by_default(scheduler_loop):
+ """Every other caller keeps the old behaviour: the coroutine runs to its
+ end after the timeout, since not every one is safe to stop midway."""
+ finished = threading.Event()
+
+ async def slow() -> None:
+ await asyncio.sleep(0.2)
+ finished.set()
+
+ with pytest.raises(TimeoutError):
+ run_async(slow(), timeout=0.05)
+ assert finished.wait(timeout=2)
+
+
+def test_a_timed_out_graph_dispatch_still_publishes_its_execution(scheduler_loop):
+ """Codex's round-2 reproduction at the durable boundary: graph dispatch
+ creates its execution before its remaining lookups, so the bridge timing
+ out there must not cancel it, or the execution stays INCOMPLETE and is
+ never published. Once the slow lookup answers, dispatch completes."""
+ dispatch = _SlowDispatch()
+
+ def scaled(coro, timeout=None, **kwargs):
+ return run_async(coro, timeout=0.05, **kwargs)
+
+ db = MagicMock(increment_onboarding_runs=AsyncMock())
+ with (
+ patch(f"{_SCHEDULER_PATH}.run_async", new=scaled),
+ patch(
+ f"{_SCHEDULER_PATH}.execution_utils.add_graph_execution",
+ new=dispatch.add,
+ ),
+ patch(f"{_SCHEDULER_PATH}.get_database_manager_async_client", return_value=db),
+ patch(f"{_SCHEDULER_PATH}.product_analytics.track_schedule_fired"),
+ ):
+ with pytest.raises(TimeoutError):
+ execute_graph(**_JOB.model_dump(mode="json"))
+ dispatch.lookup_answered.set()
+ dispatch.wait_until_published()
+
+ assert dispatch.events == ["created", "published"]
+
+
+class _SlowDispatch:
+ """An ``add_graph_execution`` that creates its execution at once and
+ publishes it only after a slow lookup answers."""
+
+ def __init__(self) -> None:
+ self.events: list[str] = []
+ self.lookup_answered = threading.Event()
+
+ async def add(self, **kwargs) -> MagicMock:
+ self.events.append("created")
+ while not self.lookup_answered.is_set():
+ await asyncio.sleep(0.01)
+ self.events.append("published")
+ return MagicMock(id="exec-1")
+
+ def wait_until_published(self, seconds: float = 2.0) -> None:
+ until = time.monotonic() + seconds
+ while "published" not in self.events and time.monotonic() < until:
+ time.sleep(0.01)
diff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.py
index 15c05812f2..4f7b3a094d 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py
@@ -1,21 +1,32 @@
+import asyncio
+import threading
+import time
import uuid
from datetime import datetime, timedelta, timezone
-from unittest.mock import MagicMock, patch
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import pytz
from apscheduler.triggers.cron import CronTrigger
from backend.api.model import CreateGraph
+from backend.copilot.dream import deadline, scope_crons
+from backend.copilot.dream.scheduling import DREAM_SYSTEM_JOBS
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.copilot.learning.dispositions import SkillLearningResult
from backend.data import db
from backend.data.execution import get_graph_executions
+from backend.executor import scheduler as scheduler_module
from backend.executor.scheduler import (
GraphExecutionJobArgs,
Jobstores,
Scheduler,
_build_trigger,
_execute_graph,
+ _memory_scope_may_fire,
_normalize_cron_day_of_week,
+ execute_skill_learning_sync,
)
from backend.usecases.sample import create_test_graph, create_test_user
from backend.util.clients import get_scheduler_client
@@ -405,7 +416,7 @@ class TestAddCommunityRebuildSchedule:
assert kwargs["replace_existing"] is True
# Lands in the EXECUTION jobstore (Postgres-backed).
assert kwargs["jobstore"] == Jobstores.EXECUTION.value
- # Cron is 04:00 Sunday per spec (P-1.7). Trigger repr is the
+ # Cron is 04:00 on Mondays (P-1.7; see the next test). Trigger repr is the
# most stable surface to assert against without depending on
# APScheduler's internal CronTrigger.field accessors.
trigger_repr = repr(kwargs["trigger"])
@@ -419,6 +430,21 @@ class TestAddCommunityRebuildSchedule:
assert result["user_id"] == "abc"
assert result["user_timezone"] == "America/New_York"
+ def test_the_weekly_rebuild_fires_on_mondays(self) -> None:
+ """``0 4 * * 0`` is Monday to APScheduler, whose weekdays count from
+ 0 = Monday (the docs used to say Sunday; the expression is kept)."""
+ s = _stub_scheduler()
+ s.scheduler.add_job.return_value = MagicMock(id="j", next_run_time=None)
+ with patch("backend.executor.scheduler.run_async", return_value=True):
+ s.add_community_rebuild_schedule(user_id="abc", user_timezone="UTC")
+ trigger = s.scheduler.add_job.call_args.kwargs["trigger"]
+ sunday = datetime(2026, 9, 27, tzinfo=timezone.utc)
+
+ fires = trigger.get_next_fire_time(None, sunday)
+
+ assert fires == datetime(2026, 9, 28, 4, tzinfo=timezone.utc)
+ assert fires.strftime("%A") == "Monday"
+
def test_empty_timezone_falls_back_to_utc(self) -> None:
s = _stub_scheduler()
s.scheduler.add_job.return_value = MagicMock(
@@ -446,7 +472,7 @@ class TestDeleteCommunityRebuildSchedule:
s.scheduler.get_job.return_value = fake_job
with (
patch("backend.executor.scheduler.run_async"),
- patch("backend.executor.scheduler.clear_registration_marker"),
+ patch("backend.executor.scheduler.forget_registration"),
):
assert s.delete_community_rebuild_schedule("abc") is True
# Look up by the canonical job id
@@ -460,21 +486,23 @@ class TestDeleteCommunityRebuildSchedule:
s.scheduler.get_job.return_value = None
assert s.delete_community_rebuild_schedule("abc") is False
- def test_delete_clears_registration_marker_so_lazy_path_can_re_register(
+ def test_delete_forgets_the_registration_so_ensure_can_re_register(
self,
) -> None:
- """An in-band delete must clear the Redis registration marker —
- otherwise ``ensure_dream_system_scheduled`` sees the stored tz
- still matching and refuses to re-register for up to 7 days
- while no cron exists in APScheduler."""
+ """An in-band delete must forget the job in the registry (its id on
+ the scope's row, and the Redis marker) — otherwise
+ ``ensure_scope_scheduled`` reads the recorded job id and never
+ re-registers a cron that no longer exists in APScheduler."""
s = _stub_scheduler()
s.scheduler.get_job.return_value = MagicMock()
with (
patch("backend.executor.scheduler.run_async") as run_async_mock,
- patch("backend.executor.scheduler.clear_registration_marker") as clear_mock,
+ patch("backend.executor.scheduler.forget_registration") as forget_mock,
):
assert s.delete_community_rebuild_schedule("abc") is True
- clear_mock.assert_called_once_with("abc", "community_rebuild_registered")
+ scope, job = forget_mock.call_args.args
+ assert scope == MemoryScope.for_user("abc")
+ assert job.job_id_prefix == "community_rebuild"
run_async_mock.assert_called_once()
@@ -535,18 +563,21 @@ class TestAddNightlyBatchSchedule:
assert kwargs["max_instances"] == 1
assert kwargs["replace_existing"] is True
assert kwargs["jobstore"] == Jobstores.EXECUTION.value
+ # The account's cron keeps its per-user kwargs, so existing jobs,
+ # re-registered ones and a rolled-back scheduler agree.
+ assert kwargs["kwargs"] == {"user_id": "abc"}
trigger_repr = repr(kwargs["trigger"])
# Daily 03:00 cron — same as the former dream pass cron, but
# carries the consolidated submitter set.
assert "hour='3'" in trigger_repr
assert result["id"] == "dream_nightly_batch_abc"
+ assert result["scope_key"] == "abc"
assert result.get("skipped") is not True
def test_flag_off_returns_skipped_dict_without_calling_add_job(self) -> None:
"""Layer 2 of the 3-layer flag gating — direct callers
- (admin endpoint, ad-hoc scripts) that bypass
- ``ensure_dream_system_scheduled`` must STILL be refused when
- the flag is off."""
+ (admin endpoint, ad-hoc scripts) that bypass the registry must
+ STILL be refused when the flag is off."""
s = _stub_scheduler()
with patch("backend.executor.scheduler.run_async", return_value=False):
result = s.add_nightly_batch_schedule(user_id="abc")
@@ -554,6 +585,7 @@ class TestAddNightlyBatchSchedule:
assert result == {
"id": None,
"user_id": "abc",
+ "scope_key": "abc",
"user_timezone": "UTC",
"next_run_time": None,
"skipped": True,
@@ -561,6 +593,68 @@ class TestAddNightlyBatchSchedule:
}
+class TestAddScopeNightlyBatchAndCommunityRebuildSchedules:
+ """The scope-keyed registrations the registry calls."""
+
+ EXPERT = MemoryScope.for_expert("abc", "expert-1")
+
+ def _register(self, method: str, scope: MemoryScope) -> dict:
+ s = _stub_scheduler()
+ s.scheduler.add_job.side_effect = lambda *a, **kw: MagicMock(
+ id=kw["id"], next_run_time=None
+ )
+ with patch("backend.executor.scheduler.run_async", return_value=True):
+ getattr(s, method)(scope=scope, user_timezone="Asia/Tokyo")
+ return s.scheduler.add_job.call_args.kwargs
+
+ @pytest.mark.parametrize(
+ "method, prefix",
+ [
+ ("add_scope_nightly_batch_schedule", "dream_nightly_batch"),
+ ("add_scope_community_rebuild_schedule", "community_rebuild"),
+ ],
+ )
+ def test_expert_scope_is_keyed_by_its_scope_key(
+ self, method: str, prefix: str
+ ) -> None:
+ kwargs = self._register(method, self.EXPERT)
+
+ assert kwargs["id"] == f"{prefix}_{self.EXPERT.scope_key}"
+ assert kwargs["kwargs"] == {
+ "user_id": "abc",
+ "expert_id": "expert-1",
+ "func": scheduler_module._MEMORY_CRONS[prefix][0],
+ }
+ assert "Asia/Tokyo" in repr(kwargs["trigger"])
+
+ def test_scope_methods_are_reachable_over_rpc(self) -> None:
+ """The RPC request schema is built from the signature: a
+ ``MemoryScope`` argument has to be constructible there."""
+ s = _stub_scheduler()
+ s._create_fastapi_endpoint(s.add_scope_nightly_batch_schedule)
+ s._create_fastapi_endpoint(s.add_scope_community_rebuild_schedule)
+ s._create_fastapi_endpoint(s.remove_scope_memory_jobs)
+
+
+class TestRemoveScopeMemoryJobsForNightlyBatchAndCommunityRebuild:
+ def test_removes_only_that_scopes_crons(self) -> None:
+ scope = MemoryScope.for_expert("abc", "expert-1")
+ s = _stub_scheduler()
+ nightly = MagicMock(id=f"dream_nightly_batch_{scope.scope_key}")
+ s.scheduler.get_job.side_effect = lambda job_id, jobstore: (
+ nightly if job_id == nightly.id else None
+ )
+
+ assert s.remove_scope_memory_jobs(scope=scope) == [nightly.id]
+
+ nightly.remove.assert_called_once()
+ looked_up = [call.args[0] for call in s.scheduler.get_job.call_args_list]
+ assert looked_up == [
+ f"community_rebuild_{scope.scope_key}",
+ f"dream_nightly_batch_{scope.scope_key}",
+ ]
+
+
class TestDeleteNightlyBatchSchedule:
def test_returns_true_when_job_exists(self) -> None:
s = _stub_scheduler()
@@ -568,7 +662,7 @@ class TestDeleteNightlyBatchSchedule:
s.scheduler.get_job.return_value = fake_job
with (
patch("backend.executor.scheduler.run_async"),
- patch("backend.executor.scheduler.clear_registration_marker"),
+ patch("backend.executor.scheduler.forget_registration"),
):
assert s.delete_nightly_batch_schedule("abc") is True
s.scheduler.get_job.assert_called_once_with(
@@ -581,34 +675,215 @@ class TestDeleteNightlyBatchSchedule:
s.scheduler.get_job.return_value = None
assert s.delete_nightly_batch_schedule("abc") is False
- def test_delete_clears_registration_marker_so_lazy_path_can_re_register(
+ def test_the_registry_forgets_the_cron_before_it_is_removed(self) -> None:
+ """A cron a row records is restored by the reconcile, so the record
+ goes first; a forget that fails keeps the cron and fails the delete."""
+ s = _stub_scheduler()
+ job = MagicMock()
+ s.scheduler.get_job.return_value = job
+ order: list[str] = []
+ job.remove.side_effect = lambda: order.append("remove")
+
+ def forgot(*args, **kwargs):
+ order.append("forget")
+ return True
+
+ with (
+ patch("backend.executor.scheduler.run_async", side_effect=forgot),
+ patch("backend.executor.scheduler.forget_registration"),
+ ):
+ assert s.delete_nightly_batch_schedule("abc") is True
+ assert order == ["forget", "remove"]
+
+ job.remove.reset_mock()
+ with (
+ patch("backend.executor.scheduler.run_async", return_value=False),
+ patch("backend.executor.scheduler.forget_registration"),
+ ):
+ assert s.delete_nightly_batch_schedule("abc") is False
+ job.remove.assert_not_called()
+
+ def test_delete_forgets_the_registration_so_ensure_can_re_register(
self,
) -> None:
"""Same contract as the community-rebuild delete: removing the
- cron in-band must also drop the dedup marker so the next memory
- write can lazily re-register without waiting out the TTL."""
+ cron in-band must also forget it in the registry so the next
+ memory write registers it again."""
s = _stub_scheduler()
s.scheduler.get_job.return_value = MagicMock()
with (
patch("backend.executor.scheduler.run_async") as run_async_mock,
- patch("backend.executor.scheduler.clear_registration_marker") as clear_mock,
+ patch("backend.executor.scheduler.forget_registration") as forget_mock,
):
assert s.delete_nightly_batch_schedule("abc") is True
- clear_mock.assert_called_once_with("abc", "dream_nightly_batch_registered")
+ scope, job = forget_mock.call_args.args
+ assert scope == MemoryScope.for_user("abc")
+ assert job.job_id_prefix == "dream_nightly_batch"
run_async_mock.assert_called_once()
+class TestAddSkillLearningSchedule:
+ """The account's skill-learning cron (#14551), on the registry's
+ scope-keyed pattern: recorded like the scope's other crons, the
+ account's alone."""
+
+ def _stub(self, existing=None):
+ s = _stub_scheduler()
+ s.scheduler.get_job.return_value = existing
+ s.scheduler.add_job.side_effect = lambda *a, **kw: MagicMock(
+ id=kw["id"], next_run_time=None
+ )
+ return s
+
+ def test_registers_the_accounts_cron_caught_up_only_the_same_morning(
+ self,
+ ) -> None:
+ s = self._stub()
+ with patch("backend.executor.scheduler.run_async", return_value=True):
+ result = s.add_skill_learning_schedule(
+ user_id="abc", user_timezone="Asia/Tokyo"
+ )
+
+ kwargs = s.scheduler.add_job.call_args.kwargs
+ assert kwargs["id"] == "skill_learning_nightly_abc"
+ assert kwargs["kwargs"] == {"user_id": "abc"}
+ assert kwargs["max_instances"] == 1
+ assert kwargs["misfire_grace_time"] == 6 * 3600
+ assert "hour='3'" in repr(kwargs["trigger"])
+ assert "Asia/Tokyo" in repr(kwargs["trigger"])
+ assert result["id"] == "skill_learning_nightly_abc"
+ assert result.get("skipped") is not True
+
+ def test_the_other_memory_crons_keep_the_schedulers_catch_up(self) -> None:
+ s = self._stub()
+ with patch("backend.executor.scheduler.run_async", return_value=True):
+ s.add_nightly_batch_schedule(user_id="abc", user_timezone="UTC")
+
+ assert s.scheduler.add_job.call_args.kwargs["misfire_grace_time"] is None
+
+ def test_an_experts_scope_is_refused(self) -> None:
+ """One pass reviews the account's skills and its experts': an expert
+ has no skill-learning cron, whoever asks."""
+ s = self._stub()
+ with patch(
+ "backend.executor.scheduler.run_async", return_value=True
+ ) as run_async_mock:
+ result = s.add_scope_skill_learning_schedule(
+ scope=MemoryScope.for_expert("abc", "expert-1"), user_timezone="UTC"
+ )
+
+ s.scheduler.add_job.assert_not_called()
+ run_async_mock.assert_not_called()
+ assert (result["skipped"], result["reason"]) == (True, "account_only")
+
+ def test_flag_off_returns_skipped_without_calling_add_job(self) -> None:
+ s = self._stub()
+ with patch("backend.executor.scheduler.run_async", return_value=False):
+ result = s.add_skill_learning_schedule(user_id="abc")
+
+ s.scheduler.add_job.assert_not_called()
+ assert (result["skipped"], result["reason"]) == (
+ True,
+ "skill_learning_disabled",
+ )
+
+ def test_a_cron_in_that_timezone_is_kept_and_reported_registered(self) -> None:
+ """The registry records the id it gets back, so a kept cron must not
+ come back as a skip; its overdue next run is not pushed back."""
+ overdue = datetime(2026, 8, 7, 8, tzinfo=timezone.utc)
+ existing = MagicMock(
+ id="skill_learning_nightly_abc",
+ next_run_time=overdue,
+ trigger=CronTrigger.from_crontab("0 3 * * *", timezone="Asia/Tokyo"),
+ )
+ s = self._stub(existing)
+ with patch("backend.executor.scheduler.run_async", return_value=True):
+ result = s.add_skill_learning_schedule(
+ user_id="abc", user_timezone="Asia/Tokyo"
+ )
+
+ s.scheduler.add_job.assert_not_called()
+ assert result["id"] == "skill_learning_nightly_abc"
+ assert result["next_run_time"] == overdue.isoformat()
+ assert result.get("skipped") is not True
+
+ def test_reachable_over_rpc(self) -> None:
+ s = _stub_scheduler()
+ s._create_fastapi_endpoint(s.add_scope_skill_learning_schedule)
+
+
+class TestRemoveScopeMemoryJobsWithSkillLearning:
+ def test_the_accounts_learning_cron_goes_with_its_scope(self) -> None:
+ scope = MemoryScope.for_user("abc")
+ s = _stub_scheduler()
+ learning = MagicMock(id="skill_learning_nightly_abc")
+ s.scheduler.get_job.side_effect = lambda job_id, jobstore: (
+ learning if job_id == learning.id else None
+ )
+
+ assert s.remove_scope_memory_jobs(scope=scope) == [learning.id]
+
+ learning.remove.assert_called_once()
+ looked_up = [call.args[0] for call in s.scheduler.get_job.call_args_list]
+ assert looked_up == [
+ "community_rebuild_abc",
+ "dream_nightly_batch_abc",
+ "skill_learning_nightly_abc",
+ ]
+
+
+class TestDeleteSkillLearningSchedule:
+ def test_the_registry_forgets_the_cron_before_it_is_removed(self) -> None:
+ s = _stub_scheduler()
+ job = MagicMock()
+ s.scheduler.get_job.return_value = job
+ order: list[str] = []
+ job.remove.side_effect = lambda: order.append("remove")
+
+ def forgot(*args, **kwargs):
+ order.append("forget")
+ return True
+
+ with (
+ patch("backend.executor.scheduler.run_async", side_effect=forgot),
+ patch("backend.executor.scheduler.forget_registration") as forget_mock,
+ ):
+ assert s.delete_skill_learning_schedule("abc") is True
+ assert order == ["forget", "remove"]
+ s.scheduler.get_job.assert_called_once_with(
+ "skill_learning_nightly_abc", jobstore=Jobstores.EXECUTION.value
+ )
+ scope, forgotten = forget_mock.call_args.args
+ assert scope == MemoryScope.for_user("abc")
+ assert forgotten.job_id_prefix == "skill_learning_nightly"
+
+ def test_returns_false_when_no_job(self) -> None:
+ s = _stub_scheduler()
+ s.scheduler.get_job.return_value = None
+ assert s.delete_skill_learning_schedule("abc") is False
+
+
# ---------------------------------------------------------------------------
-# Execution-time flag gate for the nightly batch cron (layer 3)
+# Execution-time gates for the nightly batch and community crons
# ---------------------------------------------------------------------------
+def _run_coroutine(coro, timeout=None, *, cancel_on_timeout=False):
+ """``run_async`` stand-in: run the (mocked) coroutine to completion on a
+ private loop, leaving the test session's loop alone."""
+ loop = asyncio.new_event_loop()
+ try:
+ return loop.run_until_complete(coro)
+ finally:
+ loop.close()
+
+
class TestExecuteNightlyBatchSyncRuntimeGate:
def test_flag_off_short_circuits_before_calling_submitter_fanout(self) -> None:
"""``DREAM_PASS_ENABLED`` flipped off after registration → the
cron still fires but never calls ``run_nightly_batch_submit``,
so no submitter runs."""
- from backend.executor.scheduler import execute_nightly_batch_sync
+ from backend.executor.scheduler import _submit_nightly_batch
with (
patch(
@@ -618,7 +893,7 @@ class TestExecuteNightlyBatchSyncRuntimeGate:
"backend.copilot.dream.nightly_batch.run_nightly_batch_submit"
) as fanout_mock,
):
- execute_nightly_batch_sync("abc")
+ assert _submit_nightly_batch("abc") is None
# Exactly one run_async call (the flag check). The fan-out
# function never gets invoked.
@@ -626,6 +901,270 @@ class TestExecuteNightlyBatchSyncRuntimeGate:
fanout_mock.assert_not_called()
+class TestNightlyBatchScopeGate:
+ """The cron body runs only while the registry lets the scope fire, and
+ stamps the scope's last clean run."""
+
+ def _fire(self, *, gate: bool, result, expert_id: str | None = None):
+ from backend.executor.scheduler import execute_nightly_batch_sync
+
+ with (
+ patch("backend.executor.scheduler.run_async", new=_run_coroutine),
+ patch(
+ "backend.executor.scheduler._memory_scope_may_fire",
+ return_value=gate,
+ ),
+ patch(
+ "backend.executor.scheduler._submit_nightly_batch",
+ return_value=result,
+ ) as submit,
+ patch(
+ "backend.executor.scheduler.record_scope_run", new=AsyncMock()
+ ) as stamp,
+ ):
+ returned = execute_nightly_batch_sync("abc", expert_id)
+ return returned, submit, stamp
+
+ def test_a_scope_the_registry_holds_back_does_not_run(self) -> None:
+ returned, submit, stamp = self._fire(gate=False, result=_nightly_result())
+
+ assert returned is None
+ submit.assert_not_called()
+ stamp.assert_not_awaited()
+
+ def test_a_clean_run_stamps_the_scope(self) -> None:
+ result = _nightly_result(dream=_dream_result())
+ returned, submit, stamp = self._fire(gate=True, result=result)
+
+ assert returned is result
+ submit.assert_called_once_with("abc", None)
+ stamp.assert_awaited_once_with(MemoryScope.for_user("abc"), "nightly")
+
+ def test_an_errored_run_is_not_stamped(self) -> None:
+ result = _nightly_result(dream=_dream_result(error="phase 1 LLM down"))
+ _, _, stamp = self._fire(gate=True, result=result)
+
+ stamp.assert_not_awaited()
+
+ def test_an_expert_cron_runs_the_expert_scope(self) -> None:
+ _, submit, stamp = self._fire(
+ gate=True, result=_nightly_result(), expert_id="expert-1"
+ )
+
+ submit.assert_called_once_with("abc", "expert-1")
+ stamp.assert_awaited_once_with(
+ MemoryScope.for_expert("abc", "expert-1"), "nightly"
+ )
+
+
+class TestSkillLearningScopeGate:
+ """The learning cron body runs only while the registry lets the
+ account's scope fire: a cron straggling past a pause or a wipe does
+ nothing."""
+
+ def _fire(self, *, gate: bool, flag: bool = True):
+ learned = SkillLearningResult(
+ user_id="abc",
+ run_id="learn-1",
+ trigger="nightly",
+ started_at=datetime.now(timezone.utc),
+ )
+ with (
+ patch("backend.executor.scheduler.run_async", new=_run_coroutine),
+ patch(
+ "backend.executor.scheduler._memory_scope_may_fire",
+ return_value=gate,
+ ) as gate_mock,
+ patch(
+ "backend.util.feature_flag.is_feature_enabled",
+ new=AsyncMock(return_value=flag),
+ ),
+ patch(
+ "backend.copilot.learning.nightly.run_skill_learning_pass",
+ new=AsyncMock(return_value=learned),
+ ) as learn,
+ ):
+ returned = execute_skill_learning_sync("abc")
+ return returned, learned, gate_mock, learn
+
+ def test_an_account_the_registry_holds_back_does_not_learn(self) -> None:
+ returned, _, gate_mock, learn = self._fire(gate=False)
+
+ assert returned is None
+ gate_mock.assert_called_once_with(MemoryScope.for_user("abc"))
+ learn.assert_not_awaited()
+
+ def test_an_account_the_registry_lets_fire_learns(self) -> None:
+ returned, learned, _, learn = self._fire(gate=True)
+
+ assert returned is learned
+ learn.assert_awaited_once_with("abc", trigger="nightly")
+
+ def test_the_flag_off_still_short_circuits(self) -> None:
+ returned, _, _, learn = self._fire(gate=True, flag=False)
+
+ assert returned is None
+ learn.assert_not_awaited()
+
+
+class TestCommunityRebuildScopeGate:
+ def _fire(self, *, gate: bool, result: dict | None, expert_id=None):
+ from backend.executor.scheduler import execute_community_rebuild
+
+ with (
+ patch("backend.executor.scheduler.run_async", new=_run_coroutine),
+ patch(
+ "backend.executor.scheduler._memory_scope_may_fire",
+ return_value=gate,
+ ),
+ patch(
+ "backend.executor.scheduler._rebuild_scope_communities",
+ return_value=result,
+ ) as rebuild,
+ patch(
+ "backend.executor.scheduler.record_scope_run", new=AsyncMock()
+ ) as stamp,
+ ):
+ execute_community_rebuild("abc", expert_id)
+ return rebuild, stamp
+
+ def test_a_scope_the_registry_holds_back_does_not_run(self) -> None:
+ rebuild, stamp = self._fire(gate=False, result={"error": None})
+
+ rebuild.assert_not_called()
+ stamp.assert_not_awaited()
+
+ def test_a_clean_rebuild_of_an_expert_stamps_its_scope(self) -> None:
+ rebuild, stamp = self._fire(
+ gate=True, result={"error": None}, expert_id="expert-1"
+ )
+
+ scope = MemoryScope.for_expert("abc", "expert-1")
+ rebuild.assert_called_once_with(scope)
+ stamp.assert_awaited_once_with(scope, "community")
+
+ def test_an_errored_or_flag_skipped_rebuild_is_not_stamped(self) -> None:
+ for result in ({"error": "OpenRouterError: 502"}, None):
+ _, stamp = self._fire(gate=True, result=result)
+ stamp.assert_not_awaited()
+
+
+class _SchedulerLoop:
+ """A real event loop on its own thread, installed as the scheduler's
+ shared loop so ``run_async`` bridges to it as in production."""
+
+ def __enter__(self) -> asyncio.AbstractEventLoop:
+ self.loop = asyncio.new_event_loop()
+ self.thread = threading.Thread(target=self.loop.run_forever, daemon=True)
+ self.thread.start()
+ self.patch = patch.object(scheduler_module, "_event_loop", self.loop)
+ self.patch.start()
+ return self.loop
+
+ def __exit__(self, *exc) -> None:
+ self.patch.stop()
+ self.loop.call_soon_threadsafe(self.loop.stop)
+ self.thread.join(timeout=2)
+ self.loop.close()
+
+
+def _short_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(deadline, "REGISTRY_CALL_TIMEOUT_SECONDS", 0.05)
+ monkeypatch.setattr(deadline, "BRIDGE_MARGIN_SECONDS", 0.05)
+
+
+class _HangRecorder:
+ def __init__(self) -> None:
+ self.started = threading.Event()
+ self.cancelled = threading.Event()
+
+ async def forever(self, *args, **kwargs):
+ self.started.set()
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ self.cancelled.set()
+ raise
+
+
+class TestMemoryScopeGateBridge:
+ """The scheduler side of the registry gate: the gate's answer passes
+ through, and a gate the bridge gives up on skips the run and is
+ cancelled instead of raising after 300 s."""
+
+ def test_the_gate_gets_the_expert_lifecycle_lookup(self) -> None:
+ gate = AsyncMock(return_value=True)
+ scope = MemoryScope.for_expert("abc", "expert-1")
+ with (
+ patch("backend.executor.scheduler.run_async", new=_run_coroutine),
+ patch("backend.executor.scheduler.memory_scope_may_fire", new=gate),
+ ):
+ assert _memory_scope_may_fire(scope) is True
+ gate.assert_awaited_once_with(scope, scheduler_module._expert_scope_status)
+
+ def test_a_gate_the_bridge_gives_up_on_skips_and_is_cancelled(
+ self, monkeypatch
+ ) -> None:
+ _short_deadline(monkeypatch)
+ hang = _HangRecorder()
+ monkeypatch.setattr(scheduler_module, "memory_scope_may_fire", hang.forever)
+ with _SchedulerLoop():
+ started = time.monotonic()
+ fires = _memory_scope_may_fire(MemoryScope.for_user("abc"))
+ elapsed = time.monotonic() - started
+ assert hang.cancelled.wait(timeout=2)
+
+ assert fires is False
+ assert elapsed < 1.0
+
+ @pytest.mark.parametrize("bridge", ["stamp", "forget"])
+ def test_the_other_memory_bridges_cancel_what_they_give_up_on(
+ self, monkeypatch, bridge
+ ) -> None:
+ """Like the gate, the run stamp and the in-band delete's bookkeeping
+ are safe to stop anywhere, so they ask ``run_async`` to cancel."""
+ _short_deadline(monkeypatch)
+ hang = _HangRecorder()
+ monkeypatch.setattr(scheduler_module, "record_scope_run", hang.forever)
+ monkeypatch.setattr(scheduler_module, "forget_registration", hang.forever)
+ scope = MemoryScope.for_user("abc")
+ with _SchedulerLoop():
+ if bridge == "stamp":
+ scheduler_module._stamp_scope_run(scope, "nightly")
+ else:
+ scheduler_module._forget_dream_registration(scope, DREAM_SYSTEM_JOBS[0])
+ assert hang.cancelled.wait(timeout=2)
+
+ @pytest.mark.parametrize(
+ "body",
+ [
+ scheduler_module.execute_nightly_batch_sync,
+ scheduler_module.execute_community_rebuild,
+ ],
+ )
+ def test_a_hanging_registry_skips_the_cron_promptly(self, monkeypatch, body):
+ """Codex's reproduction through the real bridge and the real gate: a
+ registry read that never answers used to raise TimeoutError out of
+ the cron body after 300 s and leave the read running."""
+ _short_deadline(monkeypatch)
+ hang = _HangRecorder()
+ registry = SimpleNamespace(get_scope_schedule=hang.forever)
+ monkeypatch.setattr(scope_crons, "memory_schedule_db", lambda: registry)
+ with (
+ _SchedulerLoop(),
+ patch("backend.executor.scheduler._submit_nightly_batch") as submit,
+ patch("backend.executor.scheduler._rebuild_scope_communities") as rebuild,
+ ):
+ started = time.monotonic()
+ body("abc")
+ elapsed = time.monotonic() - started
+ assert hang.cancelled.wait(timeout=2)
+
+ assert elapsed < 1.0
+ submit.assert_not_called()
+ rebuild.assert_not_called()
+
+
# ---------------------------------------------------------------------------
# JobStatus transitions for the admin-triggered nightly fan-out wrapper
# ---------------------------------------------------------------------------
@@ -667,14 +1206,15 @@ def _run_nightly_wrapper(result):
``mark_*`` / ``update_status_phase`` are imported inside the wrapper,
so patching them at their definition module intercepts the call-time
import. ``run_async`` is stubbed so the (mocked, non-coroutine)
- status writes don't hit an event loop.
+ status writes don't hit an event loop. The admin trigger runs the
+ fan-out directly, past the cron body's registry gate.
"""
from backend.executor.scheduler import execute_nightly_batch_with_status
with (
patch("backend.executor.scheduler.run_async"),
patch(
- "backend.executor.scheduler.execute_nightly_batch_sync",
+ "backend.executor.scheduler._submit_nightly_batch",
return_value=result,
),
patch("backend.copilot.dream.job_status.mark_complete") as complete_mock,
@@ -903,7 +1443,7 @@ class TestExecuteCommunityRebuildWithStatus:
class TestExecuteCommunityRebuildRuntimeGate:
def test_flag_off_short_circuits_before_rebuild_runs(self) -> None:
- from backend.executor.scheduler import execute_community_rebuild
+ from backend.executor.scheduler import _rebuild_scope_communities
# First run_async returns False (flag check). If we let the gate
# pass, a second call would invoke rebuild_communities_for_user;
@@ -916,11 +1456,31 @@ class TestExecuteCommunityRebuildRuntimeGate:
"backend.executor.scheduler.rebuild_communities_for_user"
) as rebuild_mock,
):
- execute_community_rebuild("abc")
+ assert _rebuild_scope_communities(MemoryScope.for_user("abc")) is None
run_async_mock.assert_called_once()
rebuild_mock.assert_not_called()
+ def test_an_expert_scope_rebuilds_the_expert_graph(self) -> None:
+ from backend.executor.scheduler import _rebuild_scope_communities
+
+ result = {"error": None, "communities_built": 2}
+ with (
+ patch("backend.executor.scheduler.run_async", side_effect=[True, result]),
+ patch(
+ "backend.copilot.graphiti.config.is_communities_enabled_for_user",
+ new=MagicMock(return_value="flag"),
+ ),
+ patch(
+ "backend.executor.scheduler.rebuild_communities_for_user",
+ new=MagicMock(return_value="rebuild"),
+ ) as rebuild_mock,
+ ):
+ scope = MemoryScope.for_expert("abc", "expert-1")
+ assert _rebuild_scope_communities(scope) == result
+
+ rebuild_mock.assert_called_once_with("abc", expert_id="expert-1")
+
@pytest.mark.asyncio(loop_scope="session")
async def test_copilot_turn_schedule_one_shot(server: SpinTestServer):
diff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
index 056a337798..dc588c5c9d 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
@@ -9,6 +9,7 @@ import asyncio
import logging
import threading
import time
+from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from typing import NamedTuple
@@ -17,9 +18,11 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.date import DateTrigger
+from apscheduler.triggers.interval import IntervalTrigger
from backend.api.features.experts.models import ExpertRoutine
from backend.copilot.credential_selection import CredentialPin
+from backend.copilot.dream.scope_reconcile import RECONCILE_INTERVAL_MINUTES
from backend.copilot.executor.utils import ScheduledTurnOrigin
from backend.copilot.permissions import (
CAPABILITY_GATE_NAMES,
@@ -52,6 +55,7 @@ from backend.executor.scheduler import (
_routine_turn_permissions,
_self_delete_copilot_turn_schedule,
_self_delete_morning_briefing_schedule,
+ reconcile_memory_scope_crons,
reconcile_stripe_tiers,
refresh_near_expiry_credentials,
)
@@ -1679,17 +1683,20 @@ class _StartupRun(NamedTuple):
add_job_calls: list
embedding_backfill: MagicMock
backfill_calls_at_readiness: int
+ scheduler_calls: list
def _registered_jobs(
monkeypatch,
interval_hours: int,
startup_embedding_backfill: bool = True,
+ on_start: Callable[[], object] | None = None,
existing_idle_job=None,
) -> _StartupRun:
"""Drive ``Scheduler.run_service`` with every heavy dependency stubbed and
a mock APScheduler, recording what it registered and what it ran before
- handing over to ``AppService.run_service``."""
+ handing over to ``AppService.run_service``. *on_start* runs as the mock
+ APScheduler starts, as a job due then would."""
monkeypatch.setattr(
f"{_SCHEDULER_PATH}.config.stripe_tier_reconcile_interval_hours",
interval_hours,
@@ -1699,6 +1706,8 @@ def _registered_jobs(
startup_embedding_backfill,
)
mock_scheduler = MagicMock()
+ mock_scheduler.start.side_effect = on_start
+
mock_scheduler.get_job.side_effect = lambda job_id, **_: (
existing_idle_job
if mock_scheduler.start.called and job_id == "refresh_near_expiry_credentials"
@@ -1730,7 +1739,10 @@ def _registered_jobs(
):
Scheduler(register_system_tasks=True).run_service()
return _StartupRun(
- mock_scheduler.add_job.call_args_list, embedding_backfill, at_readiness[0]
+ mock_scheduler.add_job.call_args_list,
+ embedding_backfill,
+ at_readiness[0],
+ mock_scheduler.mock_calls,
)
@@ -1751,6 +1763,47 @@ def test_reconcile_stripe_tiers_job_registered_with_interval_and_single_instance
assert call.kwargs["seconds"] == 6 * 3600
+def test_memory_cron_reconcile_registered_after_start_on_its_interval(
+ monkeypatch,
+):
+ """The level-triggered backstop of the memory registry runs every
+ ``RECONCILE_INTERVAL_MINUTES``, one at a time per scheduler, a missed run
+ coalesced. Each process gets an immediate pass in its own memory store."""
+ calls = _registered_jobs(monkeypatch, interval_hours=6).scheduler_calls
+
+ names = [name for name, _, _ in calls]
+ [at] = [
+ i
+ for i, (name, args, _) in enumerate(calls)
+ if name == "add_job" and args and args[0] is reconcile_memory_scope_crons
+ ]
+ _, args, kwargs = calls[at]
+ assert names.index("start") < at
+ assert str(args[1]) == str(IntervalTrigger(minutes=RECONCILE_INTERVAL_MINUTES))
+ assert RECONCILE_INTERVAL_MINUTES == 30
+ assert kwargs["id"] == "reconcile_memory_scope_crons"
+ assert kwargs["max_instances"] == 1
+ assert kwargs["coalesce"] is True
+ assert kwargs["jobstore"] == scheduler_module.Jobstores.MEMORY_CRON_RECONCILE.value
+ assert kwargs["next_run_time"] <= datetime.now(timezone.utc)
+
+
+def test_a_system_job_due_at_start_reaches_the_scheduler(monkeypatch):
+ """An immediately due system job reaches the service only through
+ ``_scheduler_instance``, so that is published before ``start``; published
+ after it, the pass was skipped as "scheduler not running" and the crons
+ waited a whole interval more."""
+ monkeypatch.setattr(f"{_SCHEDULER_PATH}._scheduler_instance", None)
+ reconciled = MagicMock()
+ monkeypatch.setattr(Scheduler, "_reconcile_memory_crons", reconciled)
+
+ _registered_jobs(
+ monkeypatch, interval_hours=6, on_start=reconcile_memory_scope_crons
+ )
+
+ reconciled.assert_called_once_with()
+
+
@pytest.mark.parametrize("offset_days", [-2, 1, None])
def test_idle_credentials_startup_preserves_existing_daily_job(
monkeypatch, offset_days
diff --git a/autogpt_platform/backend/migrations/20260926090000_add_memory_scope_schedule/migration.sql b/autogpt_platform/backend/migrations/20260926090000_add_memory_scope_schedule/migration.sql
new file mode 100644
index 0000000000..921d88d826
--- /dev/null
+++ b/autogpt_platform/backend/migrations/20260926090000_add_memory_scope_schedule/migration.sql
@@ -0,0 +1,44 @@
+-- The durable registry of memory scopes whose nightly dream and weekly
+-- community-rebuild crons the scheduler runs: one row per scope (the account,
+-- or one hired expert). Replaces the seven-day Redis registration markers as
+-- the source of truth for "is this scope scheduled". A new, empty table: no
+-- backfill runs here; `poetry run memory-schedule-backfill` fills it.
+
+-- CreateEnum
+CREATE TYPE "MemoryScopeScheduleState" AS ENUM ('ACTIVE', 'PAUSED', 'WIPED');
+
+-- CreateTable
+CREATE TABLE "MemoryScopeSchedule" (
+ "scopeKey" TEXT NOT NULL,
+ "userId" TEXT NOT NULL,
+ "expertId" TEXT,
+ "timezone" TEXT NOT NULL,
+ "state" "MemoryScopeScheduleState" NOT NULL DEFAULT 'ACTIVE',
+ "communityJobId" TEXT,
+ "nightlyJobId" TEXT,
+ "lastNightlyRunAt" TIMESTAMP(3),
+ "lastCommunityRunAt" TIMESTAMP(3),
+ "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ CONSTRAINT "MemoryScopeSchedule_pkey" PRIMARY KEY ("scopeKey"),
+ -- The account's scope key is its user id; job ids and the scheduler's
+ -- per-user wrappers rely on that. Prisma cannot express this, so the
+ -- database holds it.
+ CONSTRAINT "MemoryScopeSchedule_account_key_is_user_id" CHECK (("expertId" IS NOT NULL) OR ("scopeKey" = "userId"))
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "MemoryScopeSchedule_expertId_key" ON "MemoryScopeSchedule"("expertId");
+
+-- CreateIndex
+CREATE INDEX "MemoryScopeSchedule_userId_idx" ON "MemoryScopeSchedule"("userId");
+
+-- CreateIndex
+CREATE INDEX "MemoryScopeSchedule_state_idx" ON "MemoryScopeSchedule"("state");
+
+-- AddForeignKey
+ALTER TABLE "MemoryScopeSchedule" ADD CONSTRAINT "MemoryScopeSchedule_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "MemoryScopeSchedule" ADD CONSTRAINT "MemoryScopeSchedule_expertId_fkey" FOREIGN KEY ("expertId") REFERENCES "Expert"("id") ON DELETE CASCADE ON UPDATE CASCADE;
diff --git a/autogpt_platform/backend/migrations/20261015120000_memory_scope_schedule_learning_job/migration.sql b/autogpt_platform/backend/migrations/20261015120000_memory_scope_schedule_learning_job/migration.sql
new file mode 100644
index 0000000000..a8d8f922fa
--- /dev/null
+++ b/autogpt_platform/backend/migrations/20261015120000_memory_scope_schedule_learning_job/migration.sql
@@ -0,0 +1,4 @@
+-- The account's skill-learning cron (copilot/dream/scheduling.py), recorded
+-- like the scope's other crons. An expert's scope has none, so its row leaves
+-- this null.
+ALTER TABLE "MemoryScopeSchedule" ADD COLUMN "learningJobId" TEXT;
diff --git a/autogpt_platform/backend/poetry.lock b/autogpt_platform/backend/poetry.lock
index b47e1bfa9a..d1c8cb17bb 100644
--- a/autogpt_platform/backend/poetry.lock
+++ b/autogpt_platform/backend/poetry.lock
@@ -9002,4 +9002,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.14"
-content-hash = "9593d89a2421c29246b9df75453e8cb2a51b046e50f0554cf73ce77f2d5ffe9f"
+content-hash = "a9ef6a6105a9381f83bab7e914c5f2ec8e3252166a34df04f8036898075633ca"
diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.toml
index 987d59d9c4..208fe17d44 100644
--- a/autogpt_platform/backend/pyproject.toml
+++ b/autogpt_platform/backend/pyproject.toml
@@ -9,6 +9,7 @@ packages = [{ include = "backend", format = "sdist" }]
[tool.poetry.dependencies]
python = ">=3.10,<3.14"
+async-timeout = {version = "^5.0.1", python = "<3.11"}
aio-pika = "^9.5.5"
aiohttp = "^3.10.0"
aiodns = ">=3.5,<5.0"
@@ -153,6 +154,7 @@ export-api-schema = "backend.cli.generate_openapi_json:main"
gen-prisma-stub = "scripts.gen_prisma_types_stub:main"
brain-dump-eval = "backend.api.features.onboarding_dump.brain_dump_eval:main"
expert-style-eval = "backend.copilot.eval.style.runner:main"
+memory-schedule-backfill = "backend.copilot.dream.registry_backfill:main"
oauth-tool = "backend.cli.oauth_tool:cli"
publish-skills-catalog = "backend.cli.publish_skills_catalog:main"
diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prisma
index 3c081678ad..9dedb59438 100644
--- a/autogpt_platform/backend/schema.prisma
+++ b/autogpt_platform/backend/schema.prisma
@@ -180,6 +180,7 @@ model User {
ExpertRoutines ExpertRoutine[]
+ MemoryScopeSchedules MemoryScopeSchedule[]
Profile Profile[]
UserOnboarding UserOnboarding?
@@ -1280,6 +1281,7 @@ model Expert {
PauseEvents ExpertPauseEvent[]
BundledSkills ExpertSkillListing[]
Routines ExpertRoutine[]
+ MemorySchedule MemoryScopeSchedule?
// Owner-scoped grouping into named "pods" (distinct from org-tenancy Team).
podId String?
@@ -1771,6 +1773,56 @@ model ExpertPod {
@@unique([userId, name])
}
+// One row per memory scope (the account, or one hired expert) that the
+// nightly dream and weekly community-rebuild crons serve. It is the source of
+// truth for "is this scope scheduled" (copilot/dream/registry.py): whether its
+// crons may run, the timezone they were registered in, and their APScheduler
+// job ids. Replaces the Redis registration markers.
+model MemoryScopeSchedule {
+ // MemoryScope.scope_key: the raw user id for the account, expert_<sha256>
+ // for an expert. Also the suffix of both job ids.
+ scopeKey String @id
+
+ userId String
+ User User @relation(fields: [userId], references: [id], onDelete: Cascade)
+
+ // Null for the account scope, whose scopeKey equals userId (a CHECK in the
+ // migration holds that).
+ expertId String? @unique
+ Expert Expert? @relation(fields: [expertId], references: [id], onDelete: Cascade)
+
+ // The IANA timezone the recorded jobs were registered in.
+ timezone String
+ state MemoryScopeScheduleState @default(ACTIVE)
+
+ // Null while that cron is not registered: its flag is off, registration
+ // failed, or the scope is paused or wiped. Skill learning runs per account,
+ // so an expert's scope never records a learning job.
+ communityJobId String?
+ nightlyJobId String?
+ learningJobId String?
+
+ // When each job body last ran its pass without an error.
+ lastNightlyRunAt DateTime?
+ lastCommunityRunAt DateTime?
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @default(now()) @updatedAt
+
+ @@index([userId])
+ @@index([state])
+}
+
+// ACTIVE: the crons are registered and run. PAUSED: the expert is archived or
+// its schedules are paused; the crons are removed until it resumes. WIPED: the
+// scope's memory was erased; the crons are removed and no registry operation
+// reactivates the scope (the wipe work decides what brings one back).
+enum MemoryScopeScheduleState {
+ ACTIVE
+ PAUSED
+ WIPED
+}
+
////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////
//////// AGENT DEFINITION AND EXECUTION TABLES ////////
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 7de0e52239..e0589ce715 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -28454,6 +28454,10 @@
"NightlyBatchResult": {
"properties": {
"user_id": { "type": "string", "title": "User Id" },
+ "expert_id": {
+ "anyOf": [{ "type": "string" }, { "type": "null" }],
+ "title": "Expert Id"
+ },
"nightly_id": {
"type": "string",
"title": "Nightly Id",