Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): dream runtime correctness — phase timeouts, ingestion drain, honest registration (#13338)

Nicholas Tindle2c97b6718f79
diff --git a/autogpt_platform/backend/backend/copilot/dream/apply.py b/autogpt_platform/backend/backend/copilot/dream/apply.py
index 9ade412571..8a3cc158aa 100644
--- a/autogpt_platform/backend/backend/copilot/dream/apply.py
+++ b/autogpt_platform/backend/backend/copilot/dream/apply.py
@@ -19,12 +19,17 @@ from __future__ import annotations
import logging
import uuid as uuidlib
+from collections.abc import Mapping
from datetime import datetime, timezone
from backend.copilot.graphiti.client import derive_group_id
from backend.copilot.graphiti.config import graphiti_config
from backend.copilot.graphiti.falkordb_driver import AutoGPTFalkorDriver
-from backend.copilot.graphiti.ingest import enqueue_episode
+from backend.copilot.graphiti.ingest import (
+ IngestionCompletion,
+ enqueue_episode,
+ wait_for_ingestion,
+)
from backend.copilot.graphiti.memory_model import (
MemoryEnvelope,
MemoryKind,
@@ -38,6 +43,7 @@ from backend.copilot.tools.graphiti_forget import (
from backend.util.feature_flag import Flag, is_feature_enabled
from .batch_submit import read_input_bundle
+from .locks import DreamLockHandle, DreamLockLostError
from .schemas import (
ConsolidatedFact,
DemotionSummary,
@@ -46,12 +52,70 @@ from .schemas import (
DreamOperationsSnapshot,
EntityInvalidation,
EntityInvalidationSummary,
+ IngestionDrainStatus,
ProposedFinding,
WriteSummary,
)
logger = logging.getLogger(__name__)
+# Upper bound on waiting for the per-user ingestion queue to drain before
+# apply_operations returns. Dev numbers put a single add_episode at 7-131s,
+# so a full pass (up to 30 writes + 20 proposals) can take far longer than
+# any defensible in-lock wait. The caller holds the dream lock until apply
+# returns (locks.DEFAULT_LOCK_TTL_SECONDS=1800, shared with the three LLM
+# phases budgeted at ~1320s), so 300s keeps the whole pass inside the lock
+# TTL envelope. Past the cap the enqueued episodes keep processing
+# fire-and-forget in this process: apply warns and reports ``timed_out``
+# rather than failing the pass.
+INGESTION_DRAIN_TIMEOUT_SECONDS = 300
+
+# Fresh dream-lock TTL granted right before the ingestion drain on the sync
+# path (see ``apply_operations``' ``lock_handle``). The drain is the longest
+# non-LLM tail of the pass, and the plain lock TTL — shared with the three
+# LLM phases — can be nearly exhausted by the time apply runs. Renewing the
+# lock to this dedicated budget guarantees it cannot expire mid-write and
+# admit a concurrent pass onto the same user's graph. Sized to cover the
+# drain cap plus the demotions / entity invalidations / summary write that
+# follow it inside apply.
+LOCK_DRAIN_RENEWAL_SECONDS = INGESTION_DRAIN_TIMEOUT_SECONDS + 180
+
+# Drain bound for the Anthropic batch path. apply runs there inside
+# ``handle_dream_batch_result``, which ``BatchExecutor.walk_once`` awaits
+# SERIALLY in its single poll loop (one pending entry at a time). A 300s
+# in-line drain would stall the poll/dispatch of every OTHER user's pending
+# batch for the whole window — and per the drain math above a full pass
+# almost never drains in 300s anyway, so the cost is paid for near-zero
+# benefit while risking MAX_BATCH_LIFETIME_SECONDS expiry on the batches
+# stuck behind it. The batch path therefore SKIPS the drain (0 = no wait):
+# the enqueued episodes still process fire-and-forget in the executor
+# process, and the pass honestly reports ``IngestionDrainStatus.skipped`` so
+# the non-drained state is visible downstream instead of masked as success.
+BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS = 0
+
+
+def drain_status_from_stats(
+ stats: Mapping[str, object],
+) -> IngestionDrainStatus:
+ """Coerce the apply-stats drain flag into the typed tri-state enum.
+
+ Shared by both the sync orchestrator and the batch callback so the
+ read lives in exactly one place. Fail-closed: a missing or malformed
+ key reads as ``timed_out`` (writes potentially at risk), never as a
+ confirmed drain — missing observability must not present as success.
+ The batch path always writes ``skipped`` explicitly, so the default
+ only bites when apply produced no stats at all (e.g. an upstream bug).
+ """
+ raw = stats.get("ingestion_drain_status")
+ if isinstance(raw, IngestionDrainStatus):
+ return raw
+ if isinstance(raw, str):
+ try:
+ return IngestionDrainStatus(raw)
+ except ValueError:
+ return IngestionDrainStatus.timed_out
+ return IngestionDrainStatus.timed_out
+
def _provenance(pass_id: str, phase: str) -> str:
"""Provenance string written into the MemoryEnvelope.
@@ -92,6 +156,7 @@ async def _write_consolidated_fact(
counter: int,
fact: ConsolidatedFact,
session_id: str,
+ completion: IngestionCompletion,
) -> bool:
envelope = MemoryEnvelope(
content=fact.content,
@@ -113,6 +178,7 @@ async def _write_consolidated_fact(
),
is_json=True,
edge_metadata=_edge_metadata(envelope),
+ completion=completion,
)
@@ -122,6 +188,7 @@ async def _write_proposed_finding(
counter: int,
finding: ProposedFinding,
session_id: str,
+ completion: IngestionCompletion,
) -> bool:
envelope = MemoryEnvelope(
content=finding.content,
@@ -145,6 +212,7 @@ async def _write_proposed_finding(
source_description="; ".join(description_parts),
is_json=True,
edge_metadata=_edge_metadata(envelope),
+ completion=completion,
)
@@ -385,19 +453,80 @@ async def _write_dream_summary_message(
)
+async def _drain_ingestion(
+ pass_id: str, completion: IngestionCompletion, timeout_seconds: float
+) -> IngestionDrainStatus:
+ """Wait for the dream's OWN enqueued episodes to land in the graph.
+
+ ``enqueue_episode`` returning True only proves the episode reached the
+ in-process asyncio queue; the real write (LLM extraction + embedding in
+ ``_ingestion_worker``) happens later. The caller of ``apply_operations``
+ holds the dream lock until apply returns, so draining here keeps the
+ writes inside the lock envelope — without it, a scheduler pod restart
+ silently discards queued writes while the pass stays recorded
+ successful.
+
+ Scoped to ``completion`` — only the episodes THIS pass enqueued. The
+ per-user queue is shared with live-chat ingestion, so a whole-queue
+ barrier would let a user's concurrent chat activity extend the in-lock
+ hold up to the full timeout (and items enqueued after the drain starts
+ would never let it resolve). Tracking the pass's own episodes makes the
+ drain resolve the instant they land, regardless of other queue traffic.
+
+ Returns:
+ * ``drained`` — nothing was enqueued (vacuous) or all of the pass's
+ episodes landed within the timeout.
+ * ``skipped`` — ``timeout_seconds <= 0``; the batch path uses this to
+ avoid stalling the shared, serial ``BatchExecutor.walk_once`` loop
+ (see ``BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS``). The episodes still
+ process fire-and-forget in the executor process.
+ * ``timed_out`` — the episodes did not all land within the timeout.
+ They keep processing fire-and-forget; apply warns rather than
+ failing the pass — partial visibility beats a failed pass.
+ """
+ if not completion.registered:
+ return IngestionDrainStatus.drained
+ if timeout_seconds <= 0:
+ logger.info(
+ "Dream pass %s: ingestion drain skipped (no wait) — %d episode(s) "
+ "processing fire-and-forget; reporting drain status=skipped",
+ pass_id,
+ completion.registered,
+ )
+ return IngestionDrainStatus.skipped
+ drained = await wait_for_ingestion(completion, timeout_seconds)
+ if drained:
+ return IngestionDrainStatus.drained
+ logger.warning(
+ "Dream pass %s: own ingestion episodes did not drain within %.0fs — "
+ "reported write/proposal counts include episodes still queued "
+ "in-process (lost if this pod restarts)",
+ pass_id,
+ timeout_seconds,
+ )
+ return IngestionDrainStatus.timed_out
+
+
async def apply_operations(
user_id: str,
pass_id: str,
ops: DreamOperations,
*,
known_fact_uuids: set[str] | None = None,
-) -> dict[str, int | str | DreamOperationsSnapshot]:
+ ingestion_drain_timeout: float = INGESTION_DRAIN_TIMEOUT_SECONDS,
+ lock_handle: DreamLockHandle | None = None,
+) -> dict[str, int | str | IngestionDrainStatus | DreamOperationsSnapshot]:
"""Apply a sanitized DreamOperations to Graphiti + Postgres.
Returns a small stats dict the orchestrator can fold into
``DreamPassResult``. Includes a ``snapshot`` key carrying the
detailed ``DreamOperationsSnapshot`` payload for consumers that
- need per-operation rollups (eval, admin UI, future P9 SSE event).
+ need per-operation rollups (eval, admin UI, future P9 SSE event),
+ and an ``ingestion_drain_status`` (``IngestionDrainStatus``) —
+ ``timed_out`` means the write/proposal counts were reported while
+ episodes were still queued in-process, ``skipped`` is the by-design
+ batch skip, ``drained`` is a fully-landed pass (see
+ ``_drain_ingestion``). Read it back via ``drain_status_from_stats``.
``known_fact_uuids`` is the set of edge uuids the dream pass
actually fetched (``DreamInput.known_fact_uuids``); demotions
@@ -406,6 +535,18 @@ async def apply_operations(
up the persisted input bundle by pass_id" — the batch path's
callbacks rely on that fallback.
+ ``ingestion_drain_timeout`` bounds the in-line wait for the enqueued
+ episodes to land (see ``_drain_ingestion``). The sync path keeps the
+ full ``INGESTION_DRAIN_TIMEOUT_SECONDS``; the batch path passes
+ ``BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS`` (0) so it never stalls the
+ shared, serial ``BatchExecutor.walk_once`` loop.
+
+ ``lock_handle`` is the sync path's held dream lock. It is renewed to a
+ fresh ``LOCK_DRAIN_RENEWAL_SECONDS`` budget right before the drain so
+ the lock cannot expire while the writes are still landing (which would
+ admit a concurrent pass onto the same graph). ``None`` on the batch
+ path — it already disowned the lock to its callback with a 24h TTL.
+
Postgres writes route through ``chat_db()`` / equivalent
accessors. The dream pass runs in the Scheduler subprocess where
Prisma is intentionally NOT locally connected — those accessors
@@ -425,12 +566,19 @@ async def apply_operations(
# empty dream rather than a 'completed' narrative with no memory.
session_id = await _create_dream_session(user_id=user_id, pass_id=pass_id)
+ # Tracks completion of only the episodes THIS pass enqueues, so the
+ # drain below waits on the dream's own writes and not on unrelated
+ # live-chat ingestion sharing the same per-user queue. Registered once
+ # per successful enqueue; the worker signals each as it lands.
+ completion = IngestionCompletion()
+
written = 0
write_summaries: list[WriteSummary] = []
for i, fact in enumerate(ops.writes):
if await _write_consolidated_fact(
- user_id, pass_id, i, fact, session_id=session_id
+ user_id, pass_id, i, fact, session_id=session_id, completion=completion
):
+ completion.register()
written += 1
write_summaries.append(
WriteSummary(
@@ -446,8 +594,9 @@ async def apply_operations(
proposal_summaries: list[WriteSummary] = []
for i, prop in enumerate(ops.proposals):
if await _write_proposed_finding(
- user_id, pass_id, i, prop, session_id=session_id
+ user_id, pass_id, i, prop, session_id=session_id, completion=completion
):
+ completion.register()
proposed += 1
proposal_summaries.append(
WriteSummary(
@@ -460,6 +609,47 @@ async def apply_operations(
)
)
+ # One episode was registered per successful enqueue, so the tracker's
+ # count is exactly the writes + proposals we report — the drain waits on
+ # precisely those and nothing else. A mismatch does not endanger the
+ # writes (the drain would just resolve early or wait out its cap), so
+ # log loudly instead of failing a pass that has already written.
+ if completion.registered != written + proposed:
+ logger.error(
+ "Dream pass %s: ingestion tracker registered %d episode(s) but "
+ "reported %d write(s) + %d proposal(s) — drain barrier is not "
+ "scoped to exactly the reported writes",
+ pass_id,
+ completion.registered,
+ written,
+ proposed,
+ )
+
+ # Renew the dream lock right before the longest non-LLM tail (the
+ # ingestion drain plus the demotions / summary write that follow) so the
+ # lock cannot expire mid-write and let a second pass touch the same
+ # graph. Gated on there being any mutating work left — enqueued episodes
+ # to drain OR demotions / entity invalidations to apply, the most
+ # destructive ops in the pass, which a writes-free pass would otherwise
+ # run under a near-exhausted TTL. The batch path passes no handle (it
+ # disowned the lock to its callback). A failed renewal means the lock
+ # already expired — a newer pass may own the graph — so abort before the
+ # drain and the destructive writes below. The episodes already enqueued
+ # above keep processing fire-and-forget (they cannot be recalled), but
+ # the pass is reported errored instead of pretending exclusive ownership.
+ if lock_handle is not None and (
+ completion.registered or ops.demotions or ops.entity_invalidations
+ ):
+ if not await lock_handle.extend(LOCK_DRAIN_RENEWAL_SECONDS):
+ raise DreamLockLostError(user_id)
+
+ # Drain the in-process ingestion queue before anything downstream
+ # treats the writes as landed (and before we return and the caller
+ # releases the dream lock). See ``_drain_ingestion``.
+ ingestion_drain_status = await _drain_ingestion(
+ pass_id, completion, ingestion_drain_timeout
+ )
+
demotions = await _filter_demotions_to_known_facts(
pass_id, ops.demotions, known_fact_uuids
)
@@ -486,7 +676,8 @@ async def apply_operations(
logger.info(
"Dream pass %s applied for user %s: "
- "writes=%d proposals=%d demoted=%d (failed=%d) entity_edges=%d",
+ "writes=%d proposals=%d demoted=%d (failed=%d) entity_edges=%d "
+ "ingestion_drain_status=%s",
pass_id,
user_id[:12],
written,
@@ -494,6 +685,7 @@ async def apply_operations(
demoted_ok,
demoted_fail,
entity_edges_demoted,
+ ingestion_drain_status.value,
)
snapshot = DreamOperationsSnapshot(
@@ -510,5 +702,6 @@ async def apply_operations(
"demotion_count": demoted_ok,
"demotion_failed_count": demoted_fail,
"entity_invalidation_count": entity_edges_demoted,
+ "ingestion_drain_status": ingestion_drain_status,
"snapshot": snapshot,
}
diff --git a/autogpt_platform/backend/backend/copilot/dream/apply_test.py b/autogpt_platform/backend/backend/copilot/dream/apply_test.py
index c22fb9bc40..33b24874f1 100644
--- a/autogpt_platform/backend/backend/copilot/dream/apply_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/apply_test.py
@@ -16,13 +16,17 @@ from unittest.mock import AsyncMock
import pytest
+from backend.copilot.graphiti.ingest import IngestionCompletion
+
from . import apply as apply_mod
from .fetch import DreamInput
+from .locks import DreamLockLostError
from .schemas import (
ConsolidatedFact,
DreamDemotion,
DreamOperations,
EntityInvalidation,
+ IngestionDrainStatus,
ProposedFinding,
)
@@ -41,6 +45,12 @@ def _bundle_with_known_facts(*uuids: str) -> DreamInput:
def _stub_boundaries(mocker):
"""Wire up the apply.py side-effects with AsyncMocks once per test."""
mocker.patch.object(apply_mod, "enqueue_episode", AsyncMock(return_value=True))
+ # enqueue_episode is stubbed True above, so every write/proposal registers
+ # on the pass's IngestionCompletion — but no worker runs in tests, so an
+ # unstubbed drain would block the full INGESTION_DRAIN_TIMEOUT_SECONDS
+ # (300s) per test and time out the CI job. Default the drain to an
+ # instant success; drain-behavior tests re-patch this explicitly.
+ mocker.patch.object(apply_mod, "wait_for_ingestion", AsyncMock(return_value=True))
# The driver constructor + close — apply.py opens a FalkorDB driver for
# demotions and entity invalidations. Patch where it's used.
driver = mocker.MagicMock()
@@ -67,6 +77,17 @@ def _stub_boundaries(mocker):
AsyncMock(return_value=mocker.MagicMock(session_id="s1")),
)
mocker.patch("backend.copilot.db.add_chat_message", AsyncMock(return_value=None))
+ # _create_dream_session's tenant lookup. Unmocked it runs REAL Prisma
+ # queries on this test's function-scoped event loop whenever an earlier
+ # test already connected Prisma (its except swallows the failure when
+ # not connected, so the leak is invisible locally). Those connections
+ # stay in the shared Prisma httpx pool bound to a dead loop and the
+ # next session-loop test touching Prisma dies with "Event loop is
+ # closed" (test_chatsession_redis_storage in CI).
+ mocker.patch(
+ "backend.api.features.orgs.db.get_user_default_team",
+ AsyncMock(return_value=(None, None)),
+ )
# Entity invalidation is gated on DREAM_PASS_INVALIDATE_ENTITY. Default
# the flag ON so the existing entity tests exercise the apply path; the
# flag-off behavior has its own dedicated test below.
@@ -388,6 +409,233 @@ async def test_summary_written_after_memory_ops(mocker):
assert calls == ["write", "summary"], calls
+# ---------------------------------------------------------------------------
+# Ingestion drain — enqueue_episode returning True only proves the episode
+# reached the in-process asyncio queue; the real graph write (LLM extraction
+# + embedding in _ingestion_worker) happens later. apply_operations must
+# await the queue drain BEFORE returning, because the caller holds the dream
+# lock only until apply returns — otherwise a scheduler pod restart silently
+# discards queued writes while the pass stays recorded successful.
+# ---------------------------------------------------------------------------
+
+
+async def test_apply_waits_for_ingestion_drain_before_reporting_counts(mocker):
+ """With writes enqueued, apply must await wait_for_ingestion (scoped to
+ the pass's own episodes, bounded by the drain timeout) and report
+ ingestion_drain_status=drained on success."""
+ drain = mocker.patch.object(
+ apply_mod, "wait_for_ingestion", AsyncMock(return_value=True)
+ )
+ ops = DreamOperations(
+ writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],
+ summary_for_user="ok",
+ )
+ stats = await apply_mod.apply_operations(
+ user_id="u-drain", pass_id="p-drain", ops=ops
+ )
+
+ drain.assert_awaited_once()
+ # Waited on the pass's own completion tracker, not the whole queue, with
+ # the sync drain timeout.
+ completion, timeout = drain.await_args.args
+ assert isinstance(completion, IngestionCompletion)
+ assert completion.registered == 1
+ assert timeout == apply_mod.INGESTION_DRAIN_TIMEOUT_SECONDS
+ assert stats["ingestion_drain_status"] is IngestionDrainStatus.drained
+
+
+async def test_drain_timeout_reports_partial_visibility_not_failure(mocker, caplog):
+ """When the pass's episodes don't drain inside the cap, the pass must
+ still succeed (partial visibility beats a failed pass): counts are
+ returned, ingestion_drain_status=timed_out flags the overflow, and a
+ WARNING records the revert to fire-and-forget behavior."""
+ mocker.patch.object(apply_mod, "wait_for_ingestion", AsyncMock(return_value=False))
+ ops = DreamOperations(
+ writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],
+ summary_for_user="ok",
+ )
+ with caplog.at_level(logging.WARNING, logger=apply_mod.logger.name):
+ stats = await apply_mod.apply_operations(
+ user_id="u-slow", pass_id="p-slow", ops=ops
+ )
+
+ assert stats["ingestion_drain_status"] is IngestionDrainStatus.timed_out
+ assert stats["consolidated_count"] == 1
+ assert any(
+ "did not drain" in record.getMessage()
+ for record in caplog.records
+ if record.levelno == logging.WARNING
+ )
+
+
+async def test_zero_drain_timeout_skips_wait_and_reports_skipped(mocker, caplog):
+ """The batch path passes ``ingestion_drain_timeout=0`` so apply never
+ stalls the shared, serial BatchExecutor.walk_once loop: even with writes
+ enqueued, wait_for_ingestion is NOT awaited, and the pass reports
+ ingestion_drain_status=skipped (a by-design skip, not a failure)."""
+ drain = mocker.patch.object(
+ apply_mod, "wait_for_ingestion", AsyncMock(return_value=True)
+ )
+ ops = DreamOperations(
+ writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],
+ summary_for_user="ok",
+ )
+ with caplog.at_level(logging.INFO, logger=apply_mod.logger.name):
+ stats = await apply_mod.apply_operations(
+ user_id="u-batch",
+ pass_id="p-batch",
+ ops=ops,
+ ingestion_drain_timeout=apply_mod.BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS,
+ )
+
+ drain.assert_not_awaited()
+ assert stats["ingestion_drain_status"] is IngestionDrainStatus.skipped
+ assert stats["consolidated_count"] == 1
+ assert any("drain skipped" in record.getMessage() for record in caplog.records)
+
+
+async def test_no_enqueued_writes_skips_ingestion_drain(mocker):
+ """A dream with no writes/proposals must not block on the shared
+ per-user queue (live chat episodes could be in flight) — the drain is
+ skipped and reported vacuously drained."""
+ drain = mocker.patch.object(
+ apply_mod, "wait_for_ingestion", AsyncMock(return_value=True)
+ )
+ ops = DreamOperations(summary_for_user="Nothing new today.")
+ stats = await apply_mod.apply_operations(
+ user_id="u-empty", pass_id="p-empty", ops=ops
+ )
+
+ drain.assert_not_awaited()
+ assert stats["ingestion_drain_status"] is IngestionDrainStatus.drained
+
+
+async def test_sync_path_renews_lock_before_drain(mocker):
+ """The dream lock is renewed to a fresh budget right before the drain so
+ it cannot expire while the pass's writes are still landing (which would
+ admit a concurrent pass onto the same graph)."""
+ mocker.patch.object(apply_mod, "wait_for_ingestion", AsyncMock(return_value=True))
+ lock_handle = mocker.MagicMock()
+ lock_handle.extend = AsyncMock(return_value=True)
+ ops = DreamOperations(
+ writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],
+ summary_for_user="ok",
+ )
+ await apply_mod.apply_operations(
+ user_id="u-lock", pass_id="p-lock", ops=ops, lock_handle=lock_handle
+ )
+
+ lock_handle.extend.assert_awaited_once_with(apply_mod.LOCK_DRAIN_RENEWAL_SECONDS)
+
+
+async def test_failed_lock_renewal_aborts_before_drain_and_demotions(mocker):
+ """A failed renewal means the lock expired (a newer pass may own the
+ graph), so apply must abort before the drain and the destructive
+ demotions/summary writes instead of continuing as if it held exclusive
+ ownership. The orchestrator's catch-all turns the raise into an errored
+ ``DreamPassResult``."""
+ drain = mocker.patch.object(apply_mod, "_drain_ingestion", AsyncMock())
+ demote = mocker.patch.object(
+ apply_mod, "_filter_demotions_to_known_facts", AsyncMock()
+ )
+ lock_handle = mocker.MagicMock()
+ lock_handle.extend = AsyncMock(return_value=False)
+ ops = DreamOperations(
+ writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],
+ summary_for_user="ok",
+ )
+
+ with pytest.raises(DreamLockLostError):
+ await apply_mod.apply_operations(
+ user_id="u-lost", pass_id="p-lost", ops=ops, lock_handle=lock_handle
+ )
+
+ drain.assert_not_awaited()
+ demote.assert_not_awaited()
+
+
+async def test_demotions_only_pass_renews_lock_before_destructive_tail(mocker):
+ """Demotions and entity invalidations are the most destructive ops in a
+ pass, and a sanitizer output can contain them with zero writes. Such a
+ pass registers no episodes, so the renewal must not be gated on the
+ ingestion tracker alone or it would run under a near-exhausted TTL."""
+ lock_handle = mocker.MagicMock()
+ lock_handle.extend = AsyncMock(return_value=True)
+ ops = DreamOperations(
+ demotions=[DreamDemotion(edge_uuid="a", reason="stale")],
+ entity_invalidations=[EntityInvalidation(entity_uuid="ent-x", reason="gone")],
+ summary_for_user="tidied up",
+ )
+ await apply_mod.apply_operations(
+ user_id="u-demote", pass_id="p-demote", ops=ops, lock_handle=lock_handle
+ )
+
+ lock_handle.extend.assert_awaited_once_with(apply_mod.LOCK_DRAIN_RENEWAL_SECONDS)
+
+
+async def test_empty_pass_does_not_renew_lock(mocker):
+ """A pass with nothing to drain and nothing to demote needn't touch the
+ lock TTL."""
+ mocker.patch.object(apply_mod, "wait_for_ingestion", AsyncMock(return_value=True))
+ lock_handle = mocker.MagicMock()
+ lock_handle.extend = AsyncMock(return_value=None)
+ ops = DreamOperations(summary_for_user="Nothing new today.")
+ await apply_mod.apply_operations(
+ user_id="u-empty", pass_id="p-empty", ops=ops, lock_handle=lock_handle
+ )
+
+ lock_handle.extend.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# drain_status_from_stats — the single read point for the drain outcome,
+# shared by the sync orchestrator and the batch callback. Deserialized or
+# partial stats must never read as a confirmed drain.
+# ---------------------------------------------------------------------------
+
+
+def test_drain_status_from_stats_passes_through_enum():
+ assert (
+ apply_mod.drain_status_from_stats(
+ {"ingestion_drain_status": IngestionDrainStatus.skipped}
+ )
+ is IngestionDrainStatus.skipped
+ )
+
+
+def test_drain_status_from_stats_coerces_valid_string():
+ """Stats that round-tripped through JSON carry the enum's string value."""
+ assert (
+ apply_mod.drain_status_from_stats({"ingestion_drain_status": "drained"})
+ is IngestionDrainStatus.drained
+ )
+
+
+ "raw",
+ ["", "DRAINED", "nonsense", 1, True, None, ["drained"]],
+)
+def test_drain_status_from_stats_fails_closed(raw):
+ """Malformed, wrong-typed and missing values all read as ``timed_out`` —
+ lost observability must not present as a confirmed drain."""
+ assert (
+ apply_mod.drain_status_from_stats({"ingestion_drain_status": raw})
+ is IngestionDrainStatus.timed_out
+ )
+
+
+def test_drain_status_from_stats_missing_key_fails_closed():
+ assert apply_mod.drain_status_from_stats({}) is IngestionDrainStatus.timed_out
+
+
# ---------------------------------------------------------------------------
# Prisma auto-connect regression (scheduler service starts without an open
# Prisma connection; apply_operations must open one before any DB writes).
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
index 075f10d60f..b8d7bfa05c 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
@@ -51,7 +51,12 @@ from .batch_submit import (
from .billing import record_phase_cost
from .locks import release_dream_lock
from .model_pricing import compute_cost_usd
-from .schemas import DreamOperations, DreamOperationsSnapshot, PhaseUsage
+from .schemas import (
+ DreamOperations,
+ DreamOperationsSnapshot,
+ IngestionDrainStatus,
+ PhaseUsage,
+)
if TYPE_CHECKING:
from backend.executor.batch_executor import PendingEntry
@@ -208,6 +213,10 @@ async def _finalize_stuck_duplicate(
proposal_count=len(ops.proposals),
demotion_count=len(ops.demotions),
entity_invalidation_count=len(ops.entity_invalidations),
+ # Batch apply never drains in-line by design; the first
+ # delivery's writes (if any) landed fire-and-forget. ``skipped``
+ # marks this as a healthy by-design skip, NOT a drain failure.
+ ingestion_drain_status=IngestionDrainStatus.skipped,
summary_for_user=note + (ops.summary_for_user or ""),
),
)
@@ -575,7 +584,11 @@ async def _finalize_complete(
) -> None:
"""Sanitize phase has landed. Run apply + cost log + complete."""
try:
- from .apply import apply_operations
+ from .apply import (
+ BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS,
+ apply_operations,
+ drain_status_from_stats,
+ )
from .orchestrator import _clamp_operations
except Exception:
logger.exception("Failed to import dream apply for pass=%s", pass_id)
@@ -669,13 +682,21 @@ async def _finalize_complete(
await _best_effort_cleanup(pass_id)
return
- apply_stats: dict[str, int | str | DreamOperationsSnapshot] = {}
+ apply_stats: dict[
+ str, int | str | IngestionDrainStatus | DreamOperationsSnapshot
+ ] = {}
try:
# Thread the demotion allowlist from the bundle already in memory —
# letting apply re-read it would do a second Redis GET + full JSON
# deserialize and could fail open if the bundle's TTL lapses between
# the two reads. None (bundle expired) keeps apply's documented
# fail-open fallback.
+ #
+ # Skip the ingestion drain on this path: apply runs inside this
+ # handler, which BatchExecutor.walk_once awaits serially in its single
+ # poll loop — a 300s in-line drain would stall the poll/dispatch of
+ # every other user's pending batch. See
+ # ``BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS``.
apply_stats = await apply_operations(
user_id,
pass_id,
@@ -683,6 +704,7 @@ async def _finalize_complete(
known_fact_uuids=(
input_bundle.known_fact_uuids if input_bundle is not None else None
),
+ ingestion_drain_timeout=BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS,
)
except Exception as exc:
logger.exception(
@@ -734,6 +756,12 @@ async def _finalize_complete(
raw_session_id = apply_stats.get("session_id")
session_id = raw_session_id if isinstance(raw_session_id, str) else None
+ # The batch path skips the drain by design, so apply reports
+ # ``skipped`` whenever the pass enqueued writes (``drained`` only
+ # for an empty pass). Read it via the shared, fail-closed helper
+ # rather than re-deriving the coercion here.
+ ingestion_drain_status = drain_status_from_stats(apply_stats)
+
pass_result = DreamPassResult(
user_id=user_id,
pass_id=pass_id,
@@ -743,6 +771,7 @@ async def _finalize_complete(
demotion_count=_count("demotion_count"),
entity_invalidation_count=_count("entity_invalidation_count"),
dream_session_id=session_id,
+ ingestion_drain_status=ingestion_drain_status,
operations=snapshot,
# Carry the user-facing narrative like the sync path does —
# without it the Memory Visualizer renders a blank summary for
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
index 8629b453a5..1638cf2299 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
@@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from backend.copilot.dream.batch_callbacks import handle_dream_batch_result
+from backend.copilot.dream.schemas import IngestionDrainStatus
from backend.executor.batch_executor import PendingEntry
from backend.util.llm.providers import BatchResultRow
@@ -251,7 +252,14 @@ class TestPhaseChaining:
row=_row(custom_id="p1:recombine", content=_RECOMBINE_CONTENT),
)
- apply = AsyncMock(return_value={"writes": 0, "snapshot": "..."})
+ # Real batch-path apply reports ``skipped`` (drain not run by design).
+ apply = AsyncMock(
+ return_value={
+ "writes": 0,
+ "snapshot": "...",
+ "ingestion_drain_status": IngestionDrainStatus.skipped,
+ }
+ )
mark_complete = AsyncMock()
record_cost = AsyncMock()
release_lock = AsyncMock()
@@ -271,10 +279,21 @@ class TestPhaseChaining:
# The demotion allowlist is threaded from the bundle already loaded
# for the clamp — apply must not re-read the bundle from Redis.
assert apply.call_args.kwargs["known_fact_uuids"] == {"fact-1"}
+ # The batch path must NOT run the 300s in-line ingestion drain: apply
+ # executes inside this handler, which BatchExecutor.walk_once awaits
+ # serially — a long drain would stall every other user's batch poll.
+ assert apply.call_args.kwargs["ingestion_drain_timeout"] == 0
mark_complete.assert_awaited_once()
# The sanitizer's user-facing narrative must ride on the result so the
# Memory Visualizer isn't blank for batch-completed dreams.
assert mark_complete.call_args.kwargs["result"].summary_for_user == "ok"
+ # Drain was skipped by design on the batch path — the tri-state says
+ # so (``skipped``), distinguishing it from a real sync-path failure
+ # (``timed_out``) rather than masking either as success.
+ assert (
+ mark_complete.call_args.kwargs["result"].ingestion_drain_status
+ is IngestionDrainStatus.skipped
+ )
# The batch path disowned the dream lock to this callback; the
# terminal handler must release it with the ownership token the
# input bundle carried — compare-and-delete, never a blind DEL.
@@ -861,6 +880,9 @@ class TestLockTokenWiring:
# summary must say so.
assert final.result["summary_for_user"].endswith("ok")
assert "duplicate delivery" in final.result["summary_for_user"]
+ # The batch path never drains in-line, so the finalized result marks
+ # the writes as a by-design skip (``skipped``), not a drain failure.
+ assert final.result["ingestion_drain_status"] == IngestionDrainStatus.skipped
@pytest.mark.asyncio
async def test_duplicate_dispatch_leaves_terminal_job_untouched(self, fake_redis):
diff --git a/autogpt_platform/backend/backend/copilot/dream/llm.py b/autogpt_platform/backend/backend/copilot/dream/llm.py
index 3b35398649..398468e250 100644
--- a/autogpt_platform/backend/backend/copilot/dream/llm.py
+++ b/autogpt_platform/backend/backend/copilot/dream/llm.py
@@ -87,6 +87,7 @@ async def structured_completion(
response_model: type[T],
temperature: float = 0.2,
max_output_tokens: int = 4096,
+ timeout_seconds: float = DEFAULT_REQUEST_TIMEOUT_SECONDS,
) -> StructuredCompletion[T]:
"""Call the LLM in JSON mode and parse into ``response_model``.
@@ -94,6 +95,13 @@ async def structured_completion(
and a ``CompletionUsage`` block so the dream orchestrator can roll
token counts + cost up into a ``DreamPassUsage``.
+ ``timeout_seconds`` is the wall-clock budget for the provider call.
+ The shared 120s default is sized for short chat completions; dream
+ phases with large ``max_output_tokens`` budgets (recombine/sanitize
+ at 16384) must pass their own ceiling — see the per-phase
+ ``*_TIMEOUT_SECONDS`` constants in ``orchestrator.py`` — or real
+ long responses die on TimeoutError before they finish decoding.
+
Raises ``DreamLLMError`` if the response is empty, unparseable, or
fails Pydantic validation. Callers should treat that as "this
phase failed" — the orchestrator either skips downstream phases
@@ -112,7 +120,7 @@ async def structured_completion(
max_tokens=max_output_tokens,
temperature=temperature,
force_json_output=True,
- timeout_seconds=DEFAULT_REQUEST_TIMEOUT_SECONDS,
+ timeout_seconds=timeout_seconds,
# ``call_provider`` only honors ``ollama_host`` when
# ``provider="ollama"``; passing it on cloud transports is
# harmless. ``routing.base_url`` is the ``CHAT_BASE_URL``
diff --git a/autogpt_platform/backend/backend/copilot/dream/llm_test.py b/autogpt_platform/backend/backend/copilot/dream/llm_test.py
index 9e8f38b427..5c3de24ecc 100644
--- a/autogpt_platform/backend/backend/copilot/dream/llm_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/llm_test.py
@@ -15,7 +15,7 @@ import pytest
from pydantic import BaseModel
from backend.copilot.transport_routing import ProviderRoutingKwargs
-from backend.util.llm.providers import ProviderResponse
+from backend.util.llm.providers import DEFAULT_REQUEST_TIMEOUT_SECONDS, ProviderResponse
from .llm import (
DreamLLMError,
@@ -231,6 +231,62 @@ class TestStructuredCompletionDelegation:
# work.
assert kwargs["force_json_output"] is True
+ @pytest.mark.asyncio
+ async def test_caller_supplied_phase_timeout_reaches_call_provider(self):
+ """Recombine/sanitize carry 16384-token output budgets because
+ real responses exceed 8192 tokens — at real decode speeds those
+ responses take far longer than the shared 120s default, so the
+ orchestrator passes a per-phase wall-clock budget. The wrapper
+ must thread it through to ``call_provider`` verbatim, otherwise
+ the token-cap raise is dead letter: any long response dies on
+ TimeoutError instead of finishing."""
+ fake = ProviderResponse(
+ content='{"facts": [{"content": "x", "confidence": 0.9}]}',
+ prompt_tokens=1,
+ completion_tokens=1,
+ )
+ call_provider_mock = AsyncMock(return_value=fake)
+
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_openrouter_routing(),
+ ), patch("backend.copilot.dream.llm.call_provider", call_provider_mock):
+ await structured_completion(
+ model="anthropic/claude-opus-4-7",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ timeout_seconds=600,
+ )
+
+ assert call_provider_mock.call_args.kwargs["timeout_seconds"] == 600
+
+ @pytest.mark.asyncio
+ async def test_omitted_timeout_falls_back_to_shared_default(self):
+ """Callers that don't pass a phase budget (none in-tree, but the
+ signature allows it) keep the conservative shared default rather
+ than an unbounded request."""
+ fake = ProviderResponse(
+ content='{"facts": [{"content": "x", "confidence": 0.9}]}',
+ prompt_tokens=1,
+ completion_tokens=1,
+ )
+ call_provider_mock = AsyncMock(return_value=fake)
+
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_openrouter_routing(),
+ ), patch("backend.copilot.dream.llm.call_provider", call_provider_mock):
+ await structured_completion(
+ model="anthropic/claude-sonnet-4-6",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ )
+
+ assert (
+ call_provider_mock.call_args.kwargs["timeout_seconds"]
+ == DEFAULT_REQUEST_TIMEOUT_SECONDS
+ )
+
@pytest.mark.asyncio
async def test_routes_to_ollama_under_local_transport(self):
"""Local transport: dispatch becomes ``provider="ollama"`` with
diff --git a/autogpt_platform/backend/backend/copilot/dream/locks.py b/autogpt_platform/backend/backend/copilot/dream/locks.py
index 1297492f67..74c46ace3b 100644
--- a/autogpt_platform/backend/backend/copilot/dream/locks.py
+++ b/autogpt_platform/backend/backend/copilot/dream/locks.py
@@ -76,6 +76,19 @@ class DreamLockHeld(Exception):
self.user_id = user_id
+class DreamLockLostError(Exception):
+ """Raised when a held dream lock could not be renewed — the key expired
+ (and may already belong to a newer pass), so continuing to write to the
+ user's graph risks two passes mutating it concurrently."""
+
+ def __init__(self, user_id: str) -> None:
+ super().__init__(
+ f"Dream lock lost before renewal for user {user_id[:12]} — "
+ "aborting before further graph writes"
+ )
+ self.user_id = user_id
+
+
class DreamLockHandle:
"""Yielded by ``dream_lock``. Lets the async batch path hand lock
ownership to the callback chain: ``extend`` stretches the TTL to the
@@ -98,7 +111,7 @@ class DreamLockHandle:
def disown(self) -> None:
self.release_on_exit = False
- async def extend(self, ttl_seconds: int) -> None:
+ async def extend(self, ttl_seconds: int) -> bool:
"""Stretch the lock TTL only while the key still holds our token.
Single-key Lua compare-and-extend (mirrors ``_UNLOCK_SCRIPT``): a
@@ -106,8 +119,12 @@ class DreamLockHandle:
lock expired and a newer pass re-acquired the key, it would
hijack that pass's token and stretch its TTL. The compare also
refuses to recreate an expired key — an expired lock is never
- resurrected — and returns 0 so we can log that ownership was
- lost.
+ resurrected.
+
+ Returns ``True`` when the TTL was extended, ``False`` when
+ ownership was lost (key expired or re-acquired by a newer pass).
+ Callers about to perform writes that assume exclusive graph
+ ownership must abort on ``False`` (see ``DreamLockLostError``).
"""
# ``cast`` because redis-py's stubs type ``eval`` as a bare
# ``str`` — same workaround as the release paths below.
@@ -123,6 +140,7 @@ class DreamLockHandle:
"not resurrecting it; another pass may already own the key",
self.user_id[:12],
)
+ return bool(extended)
@asynccontextmanager
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
index a8aa72a525..385e79f2ea 100644
--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
@@ -27,7 +27,7 @@ from typing import TypeVar
from backend.copilot.config import ChatConfig
from backend.util.feature_flag import Flag, is_feature_enabled
-from .apply import apply_operations
+from .apply import apply_operations, drain_status_from_stats
from .billing import check_dream_budget, record_phase_cost
from .fetch import DreamInput, gather_dream_input
from .llm import (
@@ -85,6 +85,49 @@ CONSOLIDATE_MAX_TOKENS = 4096
RECOMBINE_MAX_TOKENS = 16384
SANITIZE_MAX_TOKENS = 16384
+# Per-phase LLM wall-clock ceilings, passed through structured_completion
+# → call_provider. The shared DEFAULT_REQUEST_TIMEOUT_SECONDS (120s) is
+# sized for short chat completions; recombine/sanitize carry 16384-token
+# output budgets precisely because real responses exceed 8192 tokens, and
+# at real decode speeds (Opus-class ~40-60 tok/s via OpenRouter) an 8-16K
+# response takes 150-400s — the 120s wall killed exactly the responses
+# the token-cap raise above was meant to save.
+#
+# A conservative ~20 tok/s decode floor would ask for max_output_tokens/20
+# ≈ 205s / 819s / 819s per phase, but that sums to 1843s of LLM budget
+# alone — past both SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS (1800s,
+# scheduler.py) and DEFAULT_LOCK_TTL_SECONDS (1800s, locks.py) before
+# fetch/apply even run. So the phases get values that fit the envelope
+# instead. Budget math (three explicit line items, with real slack):
+#
+# consolidate 240s + recombine 600s + sanitize 480s = 1320s LLM ceiling
+# + apply.INGESTION_DRAIN_TIMEOUT_SECONDS 300s drain cap
+# + DREAM_NON_LLM_HEADROOM_SECONDS 120s (budget check + fetch + enqueue
+# + demotions + summary + cost logging, EXCLUDING the drain)
+# = 1740s <= 1800s SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS (60s slack)
+#
+# The drain no longer counts against the lock TTL: apply renews the dream
+# lock to a fresh budget right before the drain (see
+# apply.LOCK_DRAIN_RENEWAL_SECONDS), so the lock only has to cover
+# LLM + non-drain headroom = 1440s <= 0.9 * DEFAULT_LOCK_TTL_SECONDS (1620s)
+# with genuine margin, and the drain itself runs under the renewed lock.
+# Both invariants are pinned by
+# test_phase_timeouts_plus_headroom_fit_scheduler_and_lock_envelope —
+# bumping any value here fails that test until the budget is re-balanced.
+#
+# Recombine (fast_advanced_model, Opus-class — the slowest decoder) gets
+# the largest share: 600s covers 16384 tokens at ~27 tok/s. Sanitize runs
+# on the faster fast_standard_model: 480s covers 16384 at ~34 tok/s.
+# Consolidate's 4096-token budget fits 240s at ~17 tok/s.
+CONSOLIDATE_TIMEOUT_SECONDS = 240
+RECOMBINE_TIMEOUT_SECONDS = 600
+SANITIZE_TIMEOUT_SECONDS = 480
+# Reserved for the non-LLM segments of the pass OTHER than the ingestion
+# drain (budget check, fetch, enqueue, demotions, summary write, cost
+# logging). The drain is budgeted separately as
+# apply.INGESTION_DRAIN_TIMEOUT_SECONDS and covered by a lock renewal.
+DREAM_NON_LLM_HEADROOM_SECONDS = 120
+
# Entity invalidation is the most destructive op the sanitizer can emit:
# each one single-hop demotes EVERY :RELATES_TO edge on the entity, so a
# hub entity multiplies the blast radius far past the demotion caps. We
@@ -480,6 +523,7 @@ async def _run_consolidate(
response_model=ConsolidationOutput,
temperature=CONSOLIDATE_TEMP,
max_output_tokens=CONSOLIDATE_MAX_TOKENS,
+ timeout_seconds=CONSOLIDATE_TIMEOUT_SECONDS,
)
@@ -496,6 +540,7 @@ async def _run_recombine(
response_model=RecombinationOutput,
temperature=RECOMBINE_TEMP,
max_output_tokens=RECOMBINE_MAX_TOKENS,
+ timeout_seconds=RECOMBINE_TIMEOUT_SECONDS,
)
@@ -517,6 +562,7 @@ async def _run_sanitize(
response_model=DreamOperations,
temperature=SANITIZE_TEMP,
max_output_tokens=SANITIZE_MAX_TOKENS,
+ timeout_seconds=SANITIZE_TIMEOUT_SECONDS,
)
@@ -772,6 +818,7 @@ async def _execute_dream_pass_async(
pass_id,
ops,
known_fact_uuids=input_bundle.known_fact_uuids,
+ lock_handle=dream_lock_handle,
)
completed_at = datetime.now(timezone.utc)
@@ -781,6 +828,10 @@ async def _execute_dream_pass_async(
v = apply_stats.get(key, 0)
return int(v) if isinstance(v, (int, str)) and v else 0
+ # Fail-closed: a missing/malformed drain flag reads as
+ # ``timed_out`` (writes at risk), never a confirmed drain.
+ ingestion_drain_status = drain_status_from_stats(apply_stats)
+
return DreamPassResult(
user_id=user_id,
pass_id=pass_id,
@@ -793,6 +844,7 @@ async def _execute_dream_pass_async(
demotion_count=_as_int("demotion_count"),
entity_invalidation_count=_as_int("entity_invalidation_count"),
summary_for_user=ops.summary_for_user,
+ ingestion_drain_status=ingestion_drain_status,
dream_session_id=str(apply_stats.get("session_id") or ""),
operations=(
snapshot if isinstance(snapshot, DreamOperationsSnapshot) else None
@@ -955,7 +1007,27 @@ async def _submit_dream_pass_batch(
# Phase 1 is enqueued — hand the dream lock to the batch callback so it
# spans the full async lifetime (apply runs hours later). Extend the TTL
# to the batch window first; the callback releases it on terminal/failure.
- await dream_lock_handle.extend(BATCH_LOCK_TTL_SECONDS)
+ # A failed extend means the lock expired before the handoff — a newer
+ # pass may already own the graph, so the just-submitted batch must never
+ # be applied: revoke the pending entry (the poller then never dispatches
+ # the callback chain) and drop the input bundle. The provider batch is
+ # orphaned; its results are discarded. The lock is NOT disowned, so the
+ # context manager's compare-and-delete release stays a safe no-op.
+ if not await dream_lock_handle.extend(BATCH_LOCK_TTL_SECONDS):
+ from backend.executor.batch_executor import remove_pending
+
+ from .batch_submit import delete_input_bundle
+
+ await remove_pending(submission.provider_batch_id)
+ await delete_input_bundle(pass_id)
+ return _failure_result(
+ user_id,
+ pass_id,
+ started_at,
+ monotonic_start,
+ execution_path,
+ "anthropic_batch: dream lock lost before handoff — batch revoked",
+ )
dream_lock_handle.disown()
return DreamPassResult(
user_id=user_id,
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
index 2ea955a7dd..778952fed3 100644
--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
@@ -14,15 +14,21 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
+from backend.executor.scheduler import SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS
+from backend.util.llm.providers import DEFAULT_REQUEST_TIMEOUT_SECONDS
+
from . import orchestrator as orchestrator_mod
+from .apply import INGESTION_DRAIN_TIMEOUT_SECONDS, LOCK_DRAIN_RENEWAL_SECONDS
from .fetch import DreamInput, EpisodeRow, FactRow
from .llm import CompletionUsage, DreamLLMError, StructuredCompletion
+from .locks import DEFAULT_LOCK_TTL_SECONDS
from .schemas import (
ConsolidatedFact,
ConsolidationOutput,
DreamDemotion,
DreamOperations,
EntityInvalidation,
+ IngestionDrainStatus,
ProposedFinding,
RecombinationOutput,
)
@@ -226,6 +232,9 @@ async def test_happy_path_runs_three_steps_and_applies(mocker):
"demotion_count": 0,
"demotion_failed_count": 0,
"entity_invalidation_count": 0,
+ # Non-default value so the assertion below proves the status
+ # is threaded from apply_stats, not left at the schema default.
+ "ingestion_drain_status": IngestionDrainStatus.timed_out,
}
),
)
@@ -238,9 +247,190 @@ async def test_happy_path_runs_three_steps_and_applies(mocker):
assert result.proposal_count == 1
assert result.summary_for_user == "Dream consolidated 1 fact."
assert result.dream_session_id == "s1"
+ assert result.ingestion_drain_status is IngestionDrainStatus.timed_out
apply_mock.assert_awaited_once()
+async def test_held_dream_lock_handle_is_threaded_into_apply(mocker):
+ """apply renews the dream lock before the drain + demotions, which it can
+ only do with the handle the orchestrator holds. Dropping that kwarg would
+ silently reinstate the lock-expiry-during-drain window, so pin it."""
+ sentinel_handle = object()
+
+ @asynccontextmanager
+ async def _handle_lock(*args, **kwargs):
+ yield sentinel_handle
+
+ mocker.patch.object(orchestrator_mod, "dream_lock", _handle_lock)
+ mocker.patch.object(
+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_build_input())
+ )
+ consolidated = ConsolidationOutput(
+ facts=[ConsolidatedFact(content="A likes B", confidence=0.8)]
+ )
+ mocker.patch.object(
+ orchestrator_mod,
+ "structured_completion",
+ AsyncMock(
+ side_effect=[
+ _wrap(consolidated),
+ _wrap(RecombinationOutput(proposals=[])),
+ _wrap(DreamOperations(writes=consolidated.facts)),
+ ]
+ ),
+ )
+ apply_mock = mocker.patch.object(
+ orchestrator_mod,
+ "apply_operations",
+ AsyncMock(
+ return_value={
+ "session_id": "s1",
+ "consolidated_count": 1,
+ "ingestion_drain_status": IngestionDrainStatus.drained,
+ }
+ ),
+ )
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error is None
+ apply_mock.assert_awaited_once()
+ assert apply_mock.await_args.kwargs["lock_handle"] is sentinel_handle
+
+
+async def test_missing_drain_key_folds_to_fail_closed_timed_out(mocker):
+ """When apply_stats omits the drain key entirely (e.g. an upstream bug),
+ the result must fail closed to ``timed_out`` — writes potentially at
+ risk — never silently read as a confirmed ``drained`` success."""
+ mocker.patch.object(
+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_build_input())
+ )
+ consolidated = ConsolidationOutput(
+ facts=[ConsolidatedFact(content="A likes B", confidence=0.8)]
+ )
+ recombined = RecombinationOutput(proposals=[])
+ sanitized = DreamOperations(summary_for_user="quiet night")
+ mocker.patch.object(
+ orchestrator_mod,
+ "structured_completion",
+ AsyncMock(
+ side_effect=[_wrap(consolidated), _wrap(recombined), _wrap(sanitized)]
+ ),
+ )
+ mocker.patch.object(
+ orchestrator_mod,
+ "apply_operations",
+ # No ``ingestion_drain_status`` key at all.
+ AsyncMock(return_value={"session_id": "s1", "consolidated_count": 1}),
+ )
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error is None
+ assert result.ingestion_drain_status is IngestionDrainStatus.timed_out
+
+
+async def test_each_phase_threads_its_own_llm_timeout_into_structured_completion(
+ mocker,
+):
+ """Recombine/sanitize got 16384-token output budgets because real
+ responses exceed 8192 tokens; at real decode speeds those responses
+ outlive the shared 120s ``call_provider`` default, so each phase
+ must hand ``structured_completion`` its own wall-clock budget —
+ otherwise the timeout kills exactly the responses the token-cap
+ raise was meant to save."""
+ mocker.patch.object(
+ orchestrator_mod,
+ "gather_dream_input",
+ AsyncMock(return_value=_build_input()),
+ )
+ consolidated = ConsolidationOutput(
+ facts=[ConsolidatedFact(content="A likes B", confidence=0.8)]
+ )
+ recombined = RecombinationOutput(proposals=[])
+ sanitized = DreamOperations(
+ writes=[],
+ proposals=[],
+ demotions=[],
+ entity_invalidations=[],
+ summary_for_user="quiet night",
+ )
+ llm_mock = mocker.patch.object(
+ orchestrator_mod,
+ "structured_completion",
+ AsyncMock(
+ side_effect=[_wrap(consolidated), _wrap(recombined), _wrap(sanitized)]
+ ),
+ )
+ mocker.patch.object(
+ orchestrator_mod,
+ "apply_operations",
+ AsyncMock(return_value={"session_id": "s1"}),
+ )
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error is None
+ timeouts = [call.kwargs["timeout_seconds"] for call in llm_mock.call_args_list]
+ assert timeouts == [
+ orchestrator_mod.CONSOLIDATE_TIMEOUT_SECONDS,
+ orchestrator_mod.RECOMBINE_TIMEOUT_SECONDS,
+ orchestrator_mod.SANITIZE_TIMEOUT_SECONDS,
+ ]
+
+
+def test_long_output_phase_timeouts_exceed_the_shared_request_default():
+ """Regression pin: every phase used to run on the shared 120s
+ ``DEFAULT_REQUEST_TIMEOUT_SECONDS``, which cannot decode the 16384
+ output tokens recombine/sanitize are budgeted for. If these ever
+ drop back to (or below) the default, the token-cap raise becomes
+ dead letter again."""
+ assert orchestrator_mod.RECOMBINE_TIMEOUT_SECONDS > DEFAULT_REQUEST_TIMEOUT_SECONDS
+ assert orchestrator_mod.SANITIZE_TIMEOUT_SECONDS > DEFAULT_REQUEST_TIMEOUT_SECONDS
+ assert (
+ orchestrator_mod.CONSOLIDATE_TIMEOUT_SECONDS >= DEFAULT_REQUEST_TIMEOUT_SECONDS
+ )
+
+
+def test_phase_timeouts_plus_headroom_fit_scheduler_and_lock_envelope():
+ """Budget-math invariant with the drain budgeted as its OWN line item.
+
+ The scheduler abandons the whole pass at
+ ``SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS`` (1800s), so the full pass
+ — LLM phases + the ingestion drain + the remaining non-LLM headroom —
+ must fit inside it with slack. The dream lock (``DEFAULT_LOCK_TTL_SECONDS``,
+ 1800s) is a separate, data-integrity boundary: apply RENEWS the lock to
+ a fresh budget right before the drain (``LOCK_DRAIN_RENEWAL_SECONDS``),
+ so the drain no longer counts against the original lock TTL — only the
+ LLM phases + non-drain headroom do, and those must clear a 90%-of-TTL
+ bar so a near-worst-case pass never reaches the lock expiry while writes
+ are landing. A future bump to any phase timeout (or a cut to either
+ envelope) fails here loudly."""
+ llm = (
+ orchestrator_mod.CONSOLIDATE_TIMEOUT_SECONDS
+ + orchestrator_mod.RECOMBINE_TIMEOUT_SECONDS
+ + orchestrator_mod.SANITIZE_TIMEOUT_SECONDS
+ )
+ non_drain_headroom = orchestrator_mod.DREAM_NON_LLM_HEADROOM_SECONDS
+ drain = INGESTION_DRAIN_TIMEOUT_SECONDS
+
+ # Whole pass (including the drain) must fit the scheduler abandonment
+ # boundary with real margin.
+ worst_case = llm + drain + non_drain_headroom
+ assert worst_case < SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS
+
+ # The lock only has to cover everything UP TO the drain, because the
+ # drain runs under a freshly renewed lock. Require 10% slack so the lock
+ # never expires mid-write.
+ assert llm + non_drain_headroom <= 0.9 * DEFAULT_LOCK_TTL_SECONDS
+
+ # The renewal must comfortably outlast the drain cap it protects.
+ assert LOCK_DRAIN_RENEWAL_SECONDS > drain
+
+
@pytest.mark.asyncio
async def test_consolidate_llm_failure_surfaces_error_and_skips_apply(mocker):
"""A failure in the consolidation step must surface as
@@ -304,7 +494,9 @@ async def test_clamps_oversized_sanitizer_output(mocker):
captured: dict[str, DreamOperations] = {}
- async def fake_apply(user_id, pass_id, ops, *, known_fact_uuids=None):
+ async def fake_apply(
+ user_id, pass_id, ops, *, known_fact_uuids=None, lock_handle=None
+ ):
captured["ops"] = ops
return {
"session_id": "s",
@@ -365,7 +557,9 @@ async def test_demotions_capped_at_five_percent_of_active_facts(mocker):
captured: dict[str, DreamOperations] = {}
- async def fake_apply(user_id, pass_id, ops, *, known_fact_uuids=None):
+ async def fake_apply(
+ user_id, pass_id, ops, *, known_fact_uuids=None, lock_handle=None
+ ):
captured["ops"] = ops
return {
"session_id": "s",
@@ -475,7 +669,9 @@ async def test_sync_path_filters_hallucinated_demotion_before_cap(mocker):
captured: dict[str, DreamOperations] = {}
- async def fake_apply(user_id, pass_id, ops, *, known_fact_uuids=None):
+ async def fake_apply(
+ user_id, pass_id, ops, *, known_fact_uuids=None, lock_handle=None
+ ):
captured["ops"] = ops
return {
"session_id": "s",
@@ -990,3 +1186,50 @@ class TestNearDuplicateWriteDedup:
assert len(contents) == 2
assert any("Revenue grew" in c for c in contents)
assert sum("churn rate rose" in c.lower() for c in contents) == 1
+
+
+async def test_batch_handoff_revokes_batch_when_lock_extend_fails(mocker):
+ """If the dream lock cannot be extended to the batch window, the
+ just-submitted batch must never be applied: the pending entry is
+ removed (poller never dispatches callbacks), the input bundle is
+ dropped, the lock is NOT disowned, and the pass reports failure."""
+ submission = mocker.MagicMock(provider_batch_id="batch-xyz")
+ mocker.patch("backend.copilot.dream.batch_submit.persist_input_bundle", AsyncMock())
+ mocker.patch(
+ "backend.copilot.dream.batch_submit.phase_models_for_config",
+ return_value=mocker.MagicMock(),
+ )
+ mocker.patch(
+ "backend.copilot.dream.batch_submit.submit_phase",
+ AsyncMock(return_value=submission),
+ )
+ remove_pending = mocker.patch(
+ "backend.executor.batch_executor.remove_pending", AsyncMock()
+ )
+ delete_bundle = mocker.patch(
+ "backend.copilot.dream.batch_submit.delete_input_bundle", AsyncMock()
+ )
+ handle = mocker.MagicMock()
+ handle.extend = AsyncMock(return_value=False)
+ handle.token = "tok"
+
+ config = mocker.MagicMock()
+ config.direct_anthropic_api_key = "key"
+
+ result = await orchestrator_mod._submit_dream_pass_batch(
+ user_id="u-lock-lost",
+ pass_id="p-lock-lost",
+ started_at=datetime.now(timezone.utc),
+ monotonic_start=0.0,
+ execution_path="anthropic_batch",
+ config=config,
+ input_bundle=_build_input(),
+ status_id=None,
+ dream_lock_handle=handle,
+ )
+
+ remove_pending.assert_awaited_once_with("batch-xyz")
+ delete_bundle.assert_awaited_once_with("p-lock-lost")
+ handle.disown.assert_not_called()
+ assert result.error and "lock lost" in result.error
diff --git a/autogpt_platform/backend/backend/copilot/dream/scheduling.py b/autogpt_platform/backend/backend/copilot/dream/scheduling.py
index 58d6d750d6..339d9c21ca 100644
--- a/autogpt_platform/backend/backend/copilot/dream/scheduling.py
+++ b/autogpt_platform/backend/backend/copilot/dream/scheduling.py
@@ -67,6 +67,13 @@ logger = logging.getLogger(__name__)
# shorter wastes scheduler RPC calls when nothing changed.
REGISTRATION_TTL_SECONDS = 7 * 24 * 3600
+# Redis dedup key prefixes, exported so the scheduler's delete_*
+# @expose methods can clear the matching marker when a cron is removed
+# in-band (see :func:`clear_registration_marker`). Must stay in sync
+# with the registry rows below — pinned by a test.
+COMMUNITY_REBUILD_REGISTRATION_PREFIX = "community_rebuild_registered"
+NIGHTLY_BATCH_REGISTRATION_PREFIX = "dream_nightly_batch_registered"
+
# A SchedulerClient is the caller's handle to the scheduler service.
# We don't import the concrete type here to avoid a circular import
@@ -135,7 +142,7 @@ DREAM_SYSTEM_JOBS: list[DreamSystemJob] = [
DreamSystemJob(
name="Community rebuild",
job_id_prefix="community_rebuild",
- registration_key_prefix="community_rebuild_registered",
+ registration_key_prefix=COMMUNITY_REBUILD_REGISTRATION_PREFIX,
flag=Flag.GRAPHITI_COMMUNITIES_ENABLED,
skip_reason="graphiti_communities_disabled",
register=_register_community_rebuild,
@@ -147,7 +154,7 @@ DREAM_SYSTEM_JOBS: list[DreamSystemJob] = [
# crons — those keys (``dream_pass_registered``,
# ``ratification_pass_registered``) are orphaned by the
# consolidation and naturally expire via their 7-day TTL.
- registration_key_prefix="dream_nightly_batch_registered",
+ registration_key_prefix=NIGHTLY_BATCH_REGISTRATION_PREFIX,
# The nightly batch cron carries dream pass + ratification
# supersession + future P2/P3/P4/P11 work. All ride the same
# master gate; finer-grained flags inside individual submitters
@@ -258,6 +265,31 @@ async def _write_registration_tz(
)
+async def clear_registration_marker(user_id: str, key_prefix: str) -> None:
+ """Delete the Redis registration marker for one dream-system cron.
+
+ Called from the scheduler's ``delete_*_schedule`` @expose methods so
+ an in-band cron removal immediately re-opens lazy registration
+ instead of leaving the marker to block it for the remainder of its
+ 7-day TTL. Single-key DEL so it routes on Redis Cluster.
+
+ Best-effort — on Redis failure the marker simply expires via TTL,
+ which is the pre-existing out-of-band-deletion recovery window.
+ """
+ try:
+ from backend.data.redis_client import get_redis_async
+
+ redis = await get_redis_async()
+ await redis.delete(f"{key_prefix}:{user_id}")
+ except Exception:
+ logger.warning(
+ "Redis delete failed for %s:%s; marker will expire via TTL",
+ key_prefix,
+ user_id[:12],
+ exc_info=True,
+ )
+
+
async def ensure_dream_system_scheduled(
user_id: str, *, force_refresh: bool = False
) -> dict[str, Any]:
@@ -361,6 +393,25 @@ async def ensure_dream_system_scheduled(
client_cached = get_scheduler_client()
result = await job.register(client_cached, user_id, tz_cached)
+ if result.get("skipped"):
+ # The scheduler's own layer-2 gate refused without
+ # creating the job — layer 1 already said ON, so this
+ # is always a disagreement (LD cold start in the
+ # scheduler pod, transient LD error, targeting-context
+ # divergence). Writing the marker here would leave
+ # Redis claiming "registered" with no APScheduler job
+ # behind it for the full 7-day TTL. Surface it and let
+ # the next cycle retry.
+ logger.warning(
+ "Dream-system: scheduler skipped %s for user %s "
+ "(reason=%s) despite the local flag check passing — "
+ "not marking registered; next cycle retries",
+ job.name,
+ user_id[:12],
+ result.get("reason"),
+ )
+ results[job.job_id_prefix] = result
+ continue
await _write_registration_tz(
user_id, job.registration_key_prefix, tz_cached
)
diff --git a/autogpt_platform/backend/backend/copilot/dream/scheduling_test.py b/autogpt_platform/backend/backend/copilot/dream/scheduling_test.py
index d2a2fba603..7286880885 100644
--- a/autogpt_platform/backend/backend/copilot/dream/scheduling_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/scheduling_test.py
@@ -23,6 +23,7 @@ Contracts pinned here:
from __future__ import annotations
+import logging
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -475,6 +476,119 @@ async def test_one_cron_rpc_failure_does_not_block_the_other():
assert result["dream_nightly_batch"]["id"] == "dream_nightly_batch_abc"
+# ---------------------------------------------------------------------------
+# Layer-1/layer-2 disagreement — a skipped RPC result must NOT stamp the
+# 7-day Redis marker (no APScheduler job exists; the marker would silently
+# block re-registration until the TTL expires)
+# ---------------------------------------------------------------------------
+
+
+async def test_layer2_skipped_rpc_result_writes_no_marker_and_warns(
+ caplog: pytest.LogCaptureFixture,
+):
+ """Layer-1 said ON but the scheduler's layer-2 gate returned a
+ ``{"skipped": True, ...}`` dict without creating the job (e.g. LD
+ cold start in the scheduler pod evaluating the flag to its False
+ default). Writing the marker would leave Redis saying 'registered'
+ with no cron behind it for 7 days. The skip must be recorded in the
+ results, surfaced at WARNING (a layer-2 skip after layer-1 said ON
+ is always a disagreement), and the marker left unwritten so the
+ next cycle retries."""
+ client = MagicMock()
+ client.add_community_rebuild_schedule = AsyncMock(
+ return_value={
+ "id": "community_rebuild_abc",
+ "user_id": "abc",
+ "next_run_time": None,
+ }
+ )
+ skipped_rpc = {
+ "id": None,
+ "user_id": "abc",
+ "user_timezone": "UTC",
+ "next_run_time": None,
+ "skipped": True,
+ "reason": "dream_pass_disabled",
+ }
+ client.add_nightly_batch_schedule = AsyncMock(return_value=skipped_rpc)
+ write_spy = AsyncMock()
+
+ with patch(_PATH_FLAG, new=_flag_mock(_all_flags_on())), patch(
+ _PATH_TZ, new=AsyncMock(return_value="UTC")
+ ), patch(_PATH_READ_TZ, new=AsyncMock(return_value=None)), patch(
+ _PATH_WRITE_TZ, new=write_spy
+ ), patch(
+ _PATH_CLIENT, return_value=client
+ ), caplog.at_level(
+ logging.WARNING, logger="backend.copilot.dream.scheduling"
+ ):
+ result = await ensure_dream_system_scheduled("abc")
+
+ # The scheduler's skip dict is recorded verbatim for auditability.
+ assert result["dream_nightly_batch"] == skipped_rpc
+ # Marker written ONLY for the cron that actually registered.
+ written_prefixes = [call.args[1] for call in write_spy.await_args_list]
+ assert written_prefixes == ["community_rebuild_registered"]
+ # The disagreement is surfaced at WARNING, not logged as 'registered'.
+ assert any(
+ record.levelno == logging.WARNING and "skipped" in record.getMessage()
+ for record in caplog.records
+ )
+
+
+# ---------------------------------------------------------------------------
+# Registration-marker clearing — used by the scheduler's delete_* RPCs so an
+# in-band delete immediately re-opens lazy registration
+# ---------------------------------------------------------------------------
+
+
+def test_exported_key_prefixes_match_registry_rows():
+ """The scheduler's delete_* methods import these constants to clear
+ markers — drift between them and the registry would clear the wrong
+ Redis key and leave the real marker blocking re-registration."""
+ by_prefix = {j.job_id_prefix: j.registration_key_prefix for j in DREAM_SYSTEM_JOBS}
+ assert (
+ by_prefix["community_rebuild"]
+ == scheduling.COMMUNITY_REBUILD_REGISTRATION_PREFIX
+ )
+ assert (
+ by_prefix["dream_nightly_batch"] == scheduling.NIGHTLY_BATCH_REGISTRATION_PREFIX
+ )
+
+
+async def test_clear_registration_marker_deletes_the_single_per_cron_key():
+ """Single-key DEL (Redis-cluster-safe) on the per-cron marker."""
+ redis = AsyncMock()
+ with patch(
+ "backend.data.redis_client.get_redis_async",
+ new=AsyncMock(return_value=redis),
+ ):
+ await scheduling.clear_registration_marker(
+ "abc", "dream_nightly_batch_registered"
+ )
+ redis.delete.assert_awaited_once_with("dream_nightly_batch_registered:abc")
+
+
+async def test_clear_registration_marker_swallows_redis_failure(caplog):
+ """Best-effort: a Redis outage during clear must never break the
+ delete RPC — the marker self-heals via its 7-day TTL. The except block
+ must still emit the WARNING (with exc_info) that documents the fallback,
+ so a silent-swallow regression is caught."""
+ with patch(
+ "backend.data.redis_client.get_redis_async",
+ new=AsyncMock(side_effect=ConnectionError("redis down")),
+ ), caplog.at_level(logging.WARNING, logger=scheduling.logger.name):
+ await scheduling.clear_registration_marker("abc", "x_registered")
+
+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+ assert any("marker will expire via TTL" in r.getMessage() for r in warnings)
+ # The fallback warning must carry the traceback for operators.
+ assert any(r.exc_info is not None for r in warnings)
+
+
# ---------------------------------------------------------------------------
# Shared per-call caching
# ---------------------------------------------------------------------------
diff --git a/autogpt_platform/backend/backend/copilot/dream/schemas.py b/autogpt_platform/backend/backend/copilot/dream/schemas.py
index 7a4d0f63db..d10fe8b74c 100644
--- a/autogpt_platform/backend/backend/copilot/dream/schemas.py
+++ b/autogpt_platform/backend/backend/copilot/dream/schemas.py
@@ -12,6 +12,7 @@ from __future__ import annotations
import logging
from datetime import datetime
+from enum import Enum
from typing import Any, Literal
from pydantic import BaseModel, Field, model_validator
@@ -20,6 +21,31 @@ from backend.copilot.graphiti.memory_model import MemoryKind
logger = logging.getLogger(__name__)
+
+class IngestionDrainStatus(str, Enum):
+ """Fate of a dream pass's enqueued graph writes at the moment the pass
+ was reported complete.
+
+ Three distinct states, so consumers (and a future Memory Visualizer)
+ can tell a healthy by-design skip from a real failure:
+
+ * ``drained`` — the pass's own episodes were confirmed landed in the
+ graph before the pass returned (sync path), or the pass enqueued
+ nothing to drain. Healthy.
+ * ``skipped`` — the drain was intentionally not run to avoid stalling
+ a shared serial loop (batch path). The episodes process
+ fire-and-forget; healthy by design, NOT a failure signal.
+ * ``timed_out`` — the sync-path drain was attempted but did not finish
+ within its budget. The reported write/proposal counts include
+ episodes still queued in-process, which are lost if the worker's pod
+ restarts. This is the only value that warrants an operator warning.
+ """
+
+ drained = "drained"
+ skipped = "skipped"
+ timed_out = "timed_out"
+
+
# ---------------------------------------------------------------------------
# Phase 1 — Consolidation
# ---------------------------------------------------------------------------
@@ -309,6 +335,20 @@ class DreamPassResult(BaseModel):
summary_for_user: str = ""
dream_session_id: str | None = None
+ # Fate of the dream's enqueued graph writes when the pass was reported
+ # complete — see ``IngestionDrainStatus``. ``drained`` is a fully-landed
+ # (or nothing-to-drain) pass, ``skipped`` is the by-design batch skip
+ # (healthy), ``timed_out`` is a sync-path drain that overran its budget
+ # with writes still queued in-process (at risk on pod restart). Defaults
+ # to ``drained`` so a payload that omits the field still validates as a
+ # fully-landed pass. Note this default is deliberately the opposite of
+ # ``apply.drain_status_from_stats``, which fails closed to ``timed_out``
+ # for a missing LIVE stats key: an absent key on a fresh pass means lost
+ # observability, while an absent field on a stored record predates the
+ # field. Consumers reading legacy records should treat ``drained`` as
+ # "unknown", not a confirmed durability guarantee.
+ ingestion_drain_status: IngestionDrainStatus = IngestionDrainStatus.drained
+
# Detailed per-operation rollup. ``None`` when the pass was skipped
# or errored before phase 3 produced operations; ``DreamOperationsSnapshot()``
# with empty lists when the pass ran but produced no operations.
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/context.py b/autogpt_platform/backend/backend/copilot/graphiti/context.py
index 7f47a9fd81..293e3dd853 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/context.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/context.py
@@ -95,6 +95,22 @@ async def _fetch(user_id: str, message: str) -> str | None:
return _format_context(edges, episodes)
+# Strong refs to in-flight hit tasks — the event loop holds only weak
+# references, so an unretained fire-and-forget task can be GC'd
+# mid-execution and silently drop the hit recording. Same pattern as
+# ``backend/data/user.py``'s ``_background_tasks``.
+_pending_hit_tasks: set[asyncio.Task] = set()
+
+
+def _on_hit_task_done(task: asyncio.Task) -> None:
+ _pending_hit_tasks.discard(task)
+ if task.cancelled():
+ return
+ exc = task.exception()
+ if exc is not None:
+ logger.warning("Ratification hit task %s failed", task.get_name(), exc_info=exc)
+
+
def _spawn_ratification_hits(user_id: str, edges) -> None:
"""Fire-and-forget the ratification hit-hook for retrieved edges.
@@ -109,10 +125,12 @@ def _spawn_ratification_hits(user_id: str, edges) -> None:
from backend.copilot.dream.ratification import try_ratify_on_hit
- asyncio.create_task(
+ task = asyncio.create_task(
try_ratify_on_hit(user_id, edge_uuids),
name=f"ratify-hits-{user_id[:12]}",
)
+ _pending_hit_tasks.add(task)
+ task.add_done_callback(_on_hit_task_done)
def _format_context(edges, episodes) -> str | None:
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/context_test.py b/autogpt_platform/backend/backend/copilot/graphiti/context_test.py
index aab44bcfb3..62d6363795 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/context_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/context_test.py
@@ -1,6 +1,7 @@
"""Tests for Graphiti warm context retrieval."""
import asyncio
+import logging
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
@@ -373,3 +374,70 @@ class TestRatificationHitHookFiresFireAndForget:
user_id, uuids = captured_calls[0]
assert user_id == "user-xyz"
assert uuids == ["edge-a", "edge-b"]
+
+
+class TestRatificationHitTaskRetention:
+ """The event loop holds only weak references to tasks — the spawn
+ helper must keep a strong reference until the task completes, or GC
+ pressure can collect the hit-recording task mid-flight and silently
+ drop hits the nightly sweep can never see."""
+
+ def test_spawned_hit_task_retained_until_done_then_discarded(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from backend.copilot.dream import ratification as ratification_mod
+
+ async def driver():
+ context._pending_hit_tasks.clear()
+ release = asyncio.Event()
+
+ async def fake_try_ratify(user_id: str, edge_uuids: list[str]):
+ await release.wait()
+
+ monkeypatch.setattr(ratification_mod, "try_ratify_on_hit", fake_try_ratify)
+ context._spawn_ratification_hits(
+ "user-xyz", edges=[SimpleNamespace(uuid="edge-a")]
+ )
+ # Strong ref held while the task is in flight.
+ assert len(context._pending_hit_tasks) == 1
+ task = next(iter(context._pending_hit_tasks))
+
+ release.set()
+ await task
+ # One more tick so the done-callback (call_soon) runs.
+ await asyncio.sleep(0)
+ assert context._pending_hit_tasks == set()
+
+ asyncio.run(driver())
+
+ def test_failed_hit_task_logs_exception_and_is_discarded(
+ self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """The done-callback must observe the exception (no 'Task
+ exception was never retrieved' noise) and log it at WARNING."""
+ from backend.copilot.dream import ratification as ratification_mod
+
+ async def driver():
+ context._pending_hit_tasks.clear()
+
+ async def fake_try_ratify(user_id: str, edge_uuids: list[str]):
+ raise RuntimeError("falkordb down")
+
+ monkeypatch.setattr(ratification_mod, "try_ratify_on_hit", fake_try_ratify)
+ context._spawn_ratification_hits(
+ "user-xyz", edges=[SimpleNamespace(uuid="edge-a")]
+ )
+ task = next(iter(context._pending_hit_tasks))
+ await asyncio.gather(task, return_exceptions=True)
+ await asyncio.sleep(0)
+
+ with caplog.at_level(
+ logging.WARNING, logger="backend.copilot.graphiti.context"
+ ):
+ asyncio.run(driver())
+
+ assert context._pending_hit_tasks == set()
+ assert any(
+ record.levelno == logging.WARNING and "failed" in record.getMessage()
+ for record in caplog.records
+ )
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/ingest.py b/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
index 12605807ad..ec6c993080 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
@@ -51,6 +51,64 @@ def _get_loop_state() -> _LoopIngestState:
# Idle workers are cleaned up after this many seconds of inactivity.
_WORKER_IDLE_TIMEOUT = 60
+
+class IngestionCompletion:
+ """Tracks completion of a specific set of enqueued episodes.
+
+ The per-user ingestion queue is SHARED between live-chat ingestion and
+ dream-pass writes. A caller that must wait for only its own episodes to
+ land (dream-pass apply) creates one of these, passes it to each
+ ``enqueue_episode`` it makes, and awaits it. Unrelated activity on the
+ same queue — chat episodes enqueued before, during, or after — never
+ registers on this tracker, so it cannot extend the wait.
+
+ Single-loop discipline: ``register`` (caller side) and ``complete_one``
+ (worker side) both run on the one event loop the queue is bound to, so
+ the counters need no locking. Counts are monotonic (registered /
+ completed) rather than a single decrementing balance, so a transient
+ interleave can never drive the outstanding count negative.
+ """
+
+ __slots__ = ("_registered", "_completed", "_event")
+
+ def __init__(self) -> None:
+ self._registered = 0
+ self._completed = 0
+ self._event = asyncio.Event()
+ self._event.set() # nothing outstanding yet
+
+ @property
+ def registered(self) -> int:
+ return self._registered
+
+ def register(self) -> None:
+ """Record one more episode this tracker is waiting on."""
+ self._registered += 1
+ self._event.clear()
+
+ def complete_one(self) -> None:
+ """Called by the worker once an episode has been fully processed."""
+ self._completed += 1
+ if self._completed >= self._registered:
+ self._event.set()
+
+ async def wait(self, timeout_seconds: float) -> bool:
+ """Block until every registered episode is processed, or timeout.
+
+ Returns ``True`` when all registered episodes completed (or none
+ were registered), ``False`` on timeout. On timeout nothing is
+ cancelled — the outstanding episodes keep processing
+ fire-and-forget.
+ """
+ if self._completed >= self._registered:
+ return True
+ try:
+ await asyncio.wait_for(self._event.wait(), timeout_seconds)
+ return True
+ except asyncio.TimeoutError:
+ return False
+
+
# Hard cap on a single episode body accepted by ``enqueue_episode``.
# Dream-pass writers queue ``MemoryEnvelope.model_dump_json()`` built from
# unvalidated LLM output, so the only other bound is the indirect
@@ -193,6 +251,11 @@ async def _ingestion_worker(user_id: str, queue: asyncio.Queue) -> None:
except asyncio.TimeoutError:
break # idle — clean up below
+ # Sidecar completion tracker — present only when the enqueuer
+ # (dream-pass apply) needs to await this specific episode. Popped
+ # up front so it is signalled in the finally below even if the
+ # graph write raises. See ``IngestionCompletion``.
+ completion: IngestionCompletion | None = payload.pop("_completion", None)
try:
group_id = derive_group_id(user_id)
client = await get_graphiti_client(group_id)
@@ -228,6 +291,11 @@ async def _ingestion_worker(user_id: str, queue: asyncio.Queue) -> None:
)
finally:
queue.task_done()
+ # Signal completion for the enqueuer's drain barrier even on
+ # failure — a failed write is still "no longer pending", and
+ # leaving it outstanding would hang the caller's wait.
+ if completion is not None:
+ completion.complete_one()
except asyncio.CancelledError:
logger.debug("Ingestion worker cancelled for user %s", user_id[:12])
raise
@@ -332,6 +400,7 @@ async def enqueue_episode(
source_description: str = "Conversation memory",
is_json: bool = False,
edge_metadata: dict | None = None,
+ completion: IngestionCompletion | None = None,
) -> bool:
"""Enqueue an arbitrary episode for background ingestion.
@@ -349,8 +418,16 @@ async def enqueue_episode(
deterministically (graphiti's extractor can't recover it from
the episode text). ``None`` (conversation turns / memory-store)
leaves edges at MemoryFact defaults — no behavior change.
+ completion: Optional ``IngestionCompletion`` the worker signals once
+ this episode is processed. Dream-pass apply passes one so it can
+ await ONLY its own episodes (scoped drain), not everything on the
+ shared per-user queue. ``None`` for chat / memory-store writes
+ that are pure fire-and-forget.
Returns ``True`` if the episode was queued, ``False`` if it was dropped.
+ The caller registers the episode on ``completion`` iff this returns
+ ``True`` — a dropped episode is never enqueued and the worker never
+ completes it.
"""
if not user_id:
return False
@@ -387,6 +464,9 @@ async def enqueue_episode(
# Sidecar — popped by the worker before the add_episode
# spread; carries dream metadata for post-write stamping.
"_edge_metadata": edge_metadata,
+ # Sidecar — the worker calls ``complete_one`` on it after
+ # processing so a scoped-drain caller can await this episode.
+ "_completion": completion,
}
)
return True
@@ -398,6 +478,31 @@ async def enqueue_episode(
return False
+async def wait_for_ingestion(
+ completion: IngestionCompletion, timeout_seconds: float
+) -> bool:
+ """Block until a specific set of enqueued episodes have all landed.
+
+ ``enqueue_episode`` returning ``True`` only proves the episode reached
+ the in-process queue; the real graph write (LLM extraction + embedding
+ inside ``_ingestion_worker``) happens later. Callers that must not
+ report success while their own writes are still pending (dream-pass
+ apply) register each episode on ``completion`` and await this.
+
+ Scoped to exactly the episodes registered on ``completion`` — the
+ per-user queue is shared with live-chat ingestion, so waiting on the
+ whole queue would let unrelated chat activity extend (or never resolve)
+ the wait. ``IngestionCompletion`` decouples the barrier from queue
+ ordering: it resolves as soon as the caller's own episodes are done,
+ even while other items remain queued.
+
+ Returns ``True`` when the caller's episodes drained (or none were
+ registered — vacuously drained), ``False`` on timeout. On timeout
+ nothing is cancelled: pending items keep processing fire-and-forget.
+ """
+ return await completion.wait(timeout_seconds)
+
+
async def _ensure_worker(user_id: str) -> asyncio.Queue:
"""Create a queue and worker for *user_id* if one doesn't exist.
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/ingest_test.py b/autogpt_platform/backend/backend/copilot/graphiti/ingest_test.py
index d0813d2ea0..6cedcd73e2 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/ingest_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/ingest_test.py
@@ -9,6 +9,20 @@ import pytest
from . import ingest
+
[email protected](autouse=True)
+def _stub_dream_registration(mocker):
+ """_ensure_worker fires ensure_dream_system_scheduled fire-and-forget for
+ every first-seen user. Unmocked it runs a REAL Prisma timezone lookup on
+ this test's function-scoped event loop whenever an earlier test already
+ connected Prisma, leaving a pool connection bound to a dead loop that
+ later kills a session-loop test with "Event loop is closed"."""
+ mocker.patch(
+ "backend.copilot.dream.scheduling.ensure_dream_system_scheduled",
+ AsyncMock(return_value=None),
+ )
+
+
# Per-loop state in ingest.py auto-isolates between tests: pytest-asyncio
# creates a fresh event loop per test function, and the WeakKeyDictionary
# forgets the previous loop's state when it is GC'd. No manual reset needed.
@@ -54,6 +68,188 @@ class TestIngestionWorkerExceptionHandling:
# Worker processed the item (task_done called) and exited.
assert queue.empty()
+ @pytest.mark.asyncio
+ async def test_worker_marks_task_done_even_when_ingestion_fails(self) -> None:
+ """``queue.join()`` (the basis of ``wait_for_ingestion``) only
+ completes if the worker calls ``task_done()`` for every item —
+ including items whose graph write raised."""
+ queue: asyncio.Queue = asyncio.Queue(maxsize=10)
+ queue.put_nowait(
+ {
+ "name": "ep1",
+ "episode_body": "hello",
+ "source": "message",
+ "source_description": "test",
+ "reference_time": None,
+ "group_id": "user_test",
+ }
+ )
+
+ with (
+ patch.object(ingest, "derive_group_id", return_value="user_test"),
+ patch.object(
+ ingest,
+ "get_graphiti_client",
+ new_callable=AsyncMock,
+ side_effect=RuntimeError("connection failed"),
+ ),
+ ):
+ original_timeout = ingest._WORKER_IDLE_TIMEOUT
+ ingest._WORKER_IDLE_TIMEOUT = 0.05
+ try:
+ await ingest._ingestion_worker("test-user", queue)
+ finally:
+ ingest._WORKER_IDLE_TIMEOUT = original_timeout
+
+ # join() resolves immediately only if task_done() was called for
+ # the failed item; a hang here means the worker leaked the count.
+ await asyncio.wait_for(queue.join(), timeout=0.1)
+
+
+class TestIngestionCompletion:
+ """``IngestionCompletion`` — the per-pass barrier that lets a caller wait
+ on ONLY its own enqueued episodes, decoupled from the shared queue.
+
+ These are pure in-memory and touch no process-global ``user_queues``,
+ so they leak no shared state across tests."""
+
+ @pytest.mark.asyncio
+ async def test_no_registered_items_is_vacuously_drained(self) -> None:
+ assert await ingest.IngestionCompletion().wait(0.01) is True
+
+ @pytest.mark.asyncio
+ async def test_wait_true_once_all_registered_complete(self) -> None:
+ c = ingest.IngestionCompletion()
+ c.register()
+ c.register()
+ assert c.registered == 2
+ c.complete_one()
+ c.complete_one()
+ assert await c.wait(0.01) is True
+
+ @pytest.mark.asyncio
+ async def test_wait_false_while_items_still_outstanding(self) -> None:
+ c = ingest.IngestionCompletion()
+ c.register()
+ c.register()
+ c.complete_one() # only 1 of 2 landed
+ assert await c.wait(0.02) is False
+
+ @pytest.mark.asyncio
+ async def test_over_completion_never_hangs_the_wait(self) -> None:
+ """Counters are monotonic rather than a decrementing balance, so more
+ completions than registrations cannot drive the outstanding count
+ negative and strand the waiter."""
+ c = ingest.IngestionCompletion()
+ c.register()
+ c.complete_one()
+ c.complete_one() # spurious extra completion
+ assert await c.wait(0.01) is True
+
+ @pytest.mark.asyncio
+ async def test_registration_after_a_completion_still_resolves(self) -> None:
+ """The worker can complete an early episode while the caller is still
+ enqueueing later ones; the barrier must resolve once the counts meet
+ again, not stay latched on the interleave."""
+ c = ingest.IngestionCompletion()
+ c.register()
+ c.complete_one()
+ c.register() # enqueued after the first landed
+ assert await c.wait(0.02) is False
+ c.complete_one()
+ assert await c.wait(0.01) is True
+
+
+class TestWaitForIngestion:
+ """``wait_for_ingestion`` — the drain barrier dream-pass apply uses so
+ 'enqueued' can be upgraded to 'written' before counts are reported.
+ Scoped to the caller's own episodes via ``IngestionCompletion``."""
+
+ @pytest.mark.asyncio
+ async def test_no_registered_episodes_returns_true(self) -> None:
+ """Nothing enqueued — vacuously drained."""
+ assert (
+ await ingest.wait_for_ingestion(ingest.IngestionCompletion(), 0.1) is True
+ )
+
+ @pytest.mark.asyncio
+ async def test_drains_own_episodes_while_unrelated_chat_still_in_flight(
+ self,
+ ) -> None:
+ """The drain resolves as soon as the pass's OWN episodes land, even
+ while an unrelated live-chat episode is still being processed on the
+ same shared per-user queue — a whole-queue ``join()`` would still be
+ blocked at that instant."""
+ user_id = "scoped-user"
+ queue: asyncio.Queue = asyncio.Queue(maxsize=10)
+ completion = ingest.IngestionCompletion()
+ chat_blocked = asyncio.Event()
+
+ async def fake_add_episode(**kwargs):
+ # Block only on the unrelated chat episode; the dream episodes
+ # return immediately so the completion tracker can resolve.
+ if kwargs.get("name") == "chat_1":
+ await chat_blocked.wait()
+ return None
+
+ mock_client = MagicMock()
+ mock_client.add_episode = AsyncMock(side_effect=fake_add_episode)
+
+ for name in ("dream_1", "dream_2"):
+ queue.put_nowait(
+ {
+ "name": name,
+ "episode_body": "x",
+ "source": "message",
+ "source_description": "d",
+ "reference_time": None,
+ "group_id": "user_scoped",
+ "_completion": completion,
+ }
+ )
+ completion.register()
+ # Unrelated chat episode enqueued behind the dream writes, NOT tracked.
+ queue.put_nowait(
+ {
+ "name": "chat_1",
+ "episode_body": "y",
+ "source": "message",
+ "source_description": "c",
+ "reference_time": None,
+ "group_id": "user_scoped",
+ }
+ )
+
+ with (
+ patch.object(ingest, "derive_group_id", return_value="user_scoped"),
+ patch.object(
+ ingest,
+ "get_graphiti_client",
+ new_callable=AsyncMock,
+ return_value=mock_client,
+ ),
+ ):
+ worker = asyncio.create_task(ingest._ingestion_worker(user_id, queue))
+ try:
+ result = await ingest.wait_for_ingestion(completion, 5)
+ assert result is True
+ # The scoped drain returned while chat_1 is still in flight:
+ # a whole-queue barrier would time out here.
+ with pytest.raises(asyncio.TimeoutError):
+ await asyncio.wait_for(queue.join(), 0.05)
+ finally:
+ chat_blocked.set()
+ worker.cancel()
+ await asyncio.gather(worker, return_exceptions=True)
+
+ @pytest.mark.asyncio
+ async def test_returns_false_when_own_episodes_never_land(self) -> None:
+ """A stalled/dead worker (crashed mid add_episode, never signalling
+ completion) must not hang the caller — give up after the timeout."""
+ completion = ingest.IngestionCompletion()
+ completion.register() # one episode enqueued, never completed
+ assert await ingest.wait_for_ingestion(completion, 0.05) is False
+
class TestEnqueueConversationTurn:
@pytest.mark.asyncio
@@ -559,3 +755,27 @@ class TestEnqueueEpisodeEdgeMetadata:
)
payload = q.get_nowait()
assert payload["_edge_metadata"] is None
+ # Non-scoped-drain writes carry no completion tracker.
+ assert payload["_completion"] is None
+
+ @pytest.mark.asyncio
+ async def test_completion_rides_payload_sidecar(self) -> None:
+ """A scoped-drain caller's completion tracker is threaded onto the
+ payload so the worker can signal it after processing."""
+ with (
+ patch.object(ingest, "derive_group_id", return_value="user_abc"),
+ patch.object(ingest, "_ensure_worker", new_callable=AsyncMock) as w,
+ ):
+ q: asyncio.Queue = asyncio.Queue(maxsize=100)
+ w.return_value = q
+ completion = ingest.IngestionCompletion()
+ await ingest.enqueue_episode(
+ user_id="abc",
+ session_id="s",
+ name="dream_ep",
+ episode_body="{}",
+ is_json=True,
+ completion=completion,
+ )
+ payload = q.get_nowait()
+ assert payload["_completion"] is completion
diff --git a/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py b/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py
index 87f5e3ee30..4ab9195114 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py
@@ -981,6 +981,18 @@ def _make_sdk_patches(
f"{_SVC}.upsert_chat_session",
dict(new_callable=AsyncMock, return_value=session),
),
+ # Unmocked, build_skills_context runs get_or_create_workspace — a
+ # REAL Prisma write — on this test's function-scoped event loop
+ # whenever an earlier test already connected Prisma. That leaves
+ # connections in the shared Prisma httpx pool bound to a dead loop,
+ # and the next session-loop test to touch Prisma dies with
+ # "RuntimeError: Event loop is closed" (seen as
+ # test_chatsession_redis_storage / test_sdk_resume_multi_turn
+ # failing depending on suite timing).
+ (
+ f"{_SVC}.build_skills_context",
+ dict(new_callable=AsyncMock, return_value=""),
+ ),
(f"{_SVC}.get_redis_async", dict(new_callable=AsyncMock)),
(
f"{_SVC}.AsyncClusterLock",
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index 85fb566cb2..29e73f5a4f 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -28,6 +28,11 @@ from pydantic import BaseModel, Field, ValidationError
from sqlalchemy import MetaData, create_engine
from backend.copilot.active_turns import ConcurrentTurnLimitError
+from backend.copilot.dream.scheduling import (
+ COMMUNITY_REBUILD_REGISTRATION_PREFIX,
+ NIGHTLY_BATCH_REGISTRATION_PREFIX,
+ clear_registration_marker,
+)
from backend.copilot.executor.utils import schedule_turn
from backend.copilot.graphiti.communities import rebuild_communities_for_user
from backend.copilot.model import create_chat_session, get_chat_session
@@ -56,6 +61,7 @@ from backend.util.exceptions import (
NotAuthorizedError,
NotFoundError,
)
+from backend.util.feature_flag import initialize_launchdarkly, shutdown_launchdarkly
from backend.util.logging import PrefixFilter
from backend.util.retry import func_retry
from backend.util.service import (
@@ -65,7 +71,7 @@ from backend.util.service import (
endpoint_to_async,
expose,
)
-from backend.util.settings import Config
+from backend.util.settings import AppEnvironment, Config
def _extract_schema_from_url(database_url) -> tuple[str, str]:
@@ -101,6 +107,29 @@ SCHEDULER_OPERATION_TIMEOUT_SECONDS = 300 # 5 minutes for scheduler operations
# legitimately take >5 min. Match the dream lock's 30 min TTL so the
# future resolves before the lock expires under the dream pass.
SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS = 1800
+
+
+def _init_launchdarkly_for_scheduler() -> None:
+ """Eagerly initialize LaunchDarkly unless running LOCAL.
+
+ Mirrors ``rest_api.py``'s ``launch_darkly_context``: the @expose flag
+ gates (e.g. ``add_nightly_batch_schedule``'s ``DREAM_PASS_ENABLED``
+ check) fail closed, so evaluating them against a lazily-initialized LD
+ client right after a pod restart would silently skip registrations
+ until the first lazy init completed. Skipped LOCAL, where LD is
+ unconfigured and flags resolve to their mock defaults.
+ """
+ if config.app_env != AppEnvironment.LOCAL:
+ initialize_launchdarkly()
+
+
+def _shutdown_launchdarkly_for_scheduler() -> None:
+ """Reverse of ``_init_launchdarkly_for_scheduler`` — only tears down the
+ LD client when it was actually initialized (non-LOCAL)."""
+ if config.app_env != AppEnvironment.LOCAL:
+ shutdown_launchdarkly()
+
+
# The Stripe tier sweep pages through every active subscription, so it needs a
# generous ceiling relative to the per-op default.
STRIPE_RECONCILE_TIMEOUT_SECONDS = 1800 # 30 minutes
@@ -909,6 +938,27 @@ def execute_community_rebuild_with_status(user_id: str, job_id: str):
)
return
+ error = result.get("error")
+ if error:
+ # ``rebuild_communities_for_user`` never raises — failures land
+ # in ``result['error']``. Same contract as the dream/nightly
+ # wrappers above: an errored rebuild must surface as errored,
+ # otherwise the Memory Visualizer toasts success on a rebuild
+ # that deleted every :Community node and then crashed.
+ try:
+ run_async(
+ mark_errored(kind="rebuild", job_id=job_id, error=error),
+ timeout=10,
+ )
+ except Exception:
+ logger.warning(
+ "Failed to mark community rebuild %s errored for user %s",
+ job_id[:12],
+ user_id[:12],
+ exc_info=True,
+ )
+ return
+
try:
run_async(
mark_complete(kind="rebuild", job_id=job_id, result=result),
@@ -923,6 +973,24 @@ def execute_community_rebuild_with_status(user_id: str, job_id: str):
)
+def _clear_dream_registration_marker(user_id: str, key_prefix: str) -> None:
+ """Bridge ``clear_registration_marker`` onto the scheduler's event loop.
+
+ Best-effort: a failed clear only delays lazy re-registration until
+ the marker's 7-day TTL expires, so it must never break the delete
+ RPC that called it.
+ """
+ try:
+ run_async(clear_registration_marker(user_id, key_prefix), timeout=10)
+ except Exception:
+ logger.warning(
+ "Failed to clear registration marker %s for user %s",
+ key_prefix,
+ user_id[:12],
+ exc_info=True,
+ )
+
+
def cleanup_oauth_tokens():
"""Clean up expired OAuth tokens from the database."""
@@ -1345,6 +1413,11 @@ class Scheduler(AppService):
def run_service(self):
load_dotenv()
+ # Eagerly initialize LaunchDarkly before any @expose flag gate can
+ # run (see the helper for why lazy init would skip registrations
+ # after a pod restart).
+ _init_launchdarkly_for_scheduler()
+
# Initialize the event loop for async jobs
global _event_loop
_event_loop = asyncio.new_event_loop()
@@ -1567,6 +1640,10 @@ class Scheduler(AppService):
logger.info("⏳ Waiting for event loop thread to finish...")
_event_loop_thread.join(timeout=SCHEDULER_OPERATION_TIMEOUT_SECONDS)
+ # Reverse order of run_service: LD was initialized before the
+ # scheduler started, so close it after all jobs have drained.
+ _shutdown_launchdarkly_for_scheduler()
+
super().cleanup()
def _persist_schedule(
@@ -1985,6 +2062,7 @@ class Scheduler(AppService):
if not job:
return False
job.remove()
+ _clear_dream_registration_marker(user_id, COMMUNITY_REBUILD_REGISTRATION_PREFIX)
logger.info("Removed community rebuild job for user %s", user_id[:12])
return True
@@ -2091,6 +2169,7 @@ class Scheduler(AppService):
if not job:
return False
job.remove()
+ _clear_dream_registration_marker(user_id, NIGHTLY_BATCH_REGISTRATION_PREFIX)
logger.info("Removed nightly batch job for user %s", user_id[:12])
return True
diff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.py
index 724750dacf..893a7da896 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py
@@ -130,7 +130,11 @@ class TestDeleteCommunityRebuildSchedule:
s = _stub_scheduler()
fake_job = MagicMock()
s.scheduler.get_job.return_value = fake_job
- assert s.delete_community_rebuild_schedule("abc") is True
+ with (
+ patch("backend.executor.scheduler.run_async"),
+ patch("backend.executor.scheduler.clear_registration_marker"),
+ ):
+ assert s.delete_community_rebuild_schedule("abc") is True
# Look up by the canonical job id
s.scheduler.get_job.assert_called_once_with(
"community_rebuild_abc", jobstore=Jobstores.EXECUTION.value
@@ -142,6 +146,23 @@ class TestDeleteCommunityRebuildSchedule:
s.scheduler.get_job.return_value = None
assert s.delete_community_rebuild_schedule("abc") is False
+ def test_delete_clears_registration_marker_so_lazy_path_can_re_register(
+ self,
+ ) -> None:
+ """An in-band delete must clear the Redis registration marker —
+ otherwise ``ensure_dream_system_scheduled`` sees the stored tz
+ still matching and refuses to re-register for up to 7 days
+ while no cron exists in APScheduler."""
+ s = _stub_scheduler()
+ s.scheduler.get_job.return_value = MagicMock()
+ with (
+ patch("backend.executor.scheduler.run_async") as run_async_mock,
+ patch("backend.executor.scheduler.clear_registration_marker") as clear_mock,
+ ):
+ assert s.delete_community_rebuild_schedule("abc") is True
+ clear_mock.assert_called_once_with("abc", "community_rebuild_registered")
+ run_async_mock.assert_called_once()
+
class TestExecuteCommunityRebuildPass:
def test_default_force_false(self) -> None:
@@ -231,7 +252,11 @@ class TestDeleteNightlyBatchSchedule:
s = _stub_scheduler()
fake_job = MagicMock()
s.scheduler.get_job.return_value = fake_job
- assert s.delete_nightly_batch_schedule("abc") is True
+ with (
+ patch("backend.executor.scheduler.run_async"),
+ patch("backend.executor.scheduler.clear_registration_marker"),
+ ):
+ assert s.delete_nightly_batch_schedule("abc") is True
s.scheduler.get_job.assert_called_once_with(
"dream_nightly_batch_abc", jobstore=Jobstores.EXECUTION.value
)
@@ -242,6 +267,22 @@ class TestDeleteNightlyBatchSchedule:
s.scheduler.get_job.return_value = None
assert s.delete_nightly_batch_schedule("abc") is False
+ def test_delete_clears_registration_marker_so_lazy_path_can_re_register(
+ self,
+ ) -> None:
+ """Same contract as the community-rebuild delete: removing the
+ cron in-band must also drop the dedup marker so the next memory
+ write can lazily re-register without waiting out the TTL."""
+ s = _stub_scheduler()
+ s.scheduler.get_job.return_value = MagicMock()
+ with (
+ patch("backend.executor.scheduler.run_async") as run_async_mock,
+ patch("backend.executor.scheduler.clear_registration_marker") as clear_mock,
+ ):
+ assert s.delete_nightly_batch_schedule("abc") is True
+ clear_mock.assert_called_once_with("abc", "dream_nightly_batch_registered")
+ run_async_mock.assert_called_once()
+
# ---------------------------------------------------------------------------
# Execution-time flag gate for the nightly batch cron (layer 3)
@@ -460,6 +501,92 @@ class TestExecuteNightlyBatchWithStatus:
errored_mock.assert_not_called()
+# ---------------------------------------------------------------------------
+# JobStatus transitions for the admin-triggered community rebuild wrapper
+# ---------------------------------------------------------------------------
+
+
+def _run_rebuild_wrapper(result):
+ """Invoke the rebuild wrapper with the work body + status writers mocked.
+
+ ``rebuild_communities_for_user`` is stubbed to return a sentinel so the
+ ``run_async`` fake can hand back the rebuild ``result`` dict for that
+ call only and ``None`` for the (mocked) status writes.
+ """
+ from backend.executor.scheduler import execute_community_rebuild_with_status
+
+ rebuild_sentinel = object()
+
+ def fake_run_async(coro, timeout=None):
+ return result if coro is rebuild_sentinel else None
+
+ with (
+ patch("backend.executor.scheduler.run_async", side_effect=fake_run_async),
+ # ``new=MagicMock(...)`` — a bare patch() would auto-detect the
+ # async target and install an AsyncMock, whose call returns a
+ # coroutine instead of the sentinel.
+ patch(
+ "backend.executor.scheduler.rebuild_communities_for_user",
+ new=MagicMock(return_value=rebuild_sentinel),
+ ),
+ patch("backend.copilot.dream.job_status.mark_complete") as complete_mock,
+ patch("backend.copilot.dream.job_status.mark_errored") as errored_mock,
+ patch("backend.copilot.dream.job_status.update_status_phase"),
+ ):
+ execute_community_rebuild_with_status("abc", "job-1")
+ return complete_mock, errored_mock
+
+
+class TestExecuteCommunityRebuildWithStatus:
+ def test_errored_rebuild_result_marks_errored_not_complete(self) -> None:
+ """``rebuild_communities_for_user`` never raises — failures land
+ in ``result['error']``. The admin row must read 'errored';
+ 'complete' would toast success on a rebuild that DETACH-DELETEd
+ every :Community node and then crashed mid-summarization."""
+ result = {
+ "user_id": "abc",
+ "error": "OpenRouterError: 502",
+ "skipped": False,
+ }
+ complete_mock, errored_mock = _run_rebuild_wrapper(result)
+
+ errored_mock.assert_called_once_with(
+ kind="rebuild", job_id="job-1", error="OpenRouterError: 502"
+ )
+ complete_mock.assert_not_called()
+
+ def test_clean_rebuild_result_marks_complete(self) -> None:
+ result = {
+ "user_id": "abc",
+ "error": None,
+ "communities_built": 4,
+ "skipped": False,
+ }
+ complete_mock, errored_mock = _run_rebuild_wrapper(result)
+
+ complete_mock.assert_called_once_with(
+ kind="rebuild", job_id="job-1", result=result
+ )
+ errored_mock.assert_not_called()
+
+ def test_skipped_rebuild_result_still_marks_complete(self) -> None:
+ """An activity-gated skip is a successful no-op — the result dict
+ carries ``skip_reason`` for the visualizer, the row stays
+ 'complete'."""
+ result = {
+ "user_id": "abc",
+ "error": None,
+ "skipped": True,
+ "skip_reason": "no_activity",
+ }
+ complete_mock, errored_mock = _run_rebuild_wrapper(result)
+
+ complete_mock.assert_called_once_with(
+ kind="rebuild", job_id="job-1", result=result
+ )
+ errored_mock.assert_not_called()
+
+
class TestExecuteCommunityRebuildRuntimeGate:
def test_flag_off_short_circuits_before_rebuild_runs(self) -> None:
from backend.executor.scheduler import execute_community_rebuild
@@ -613,3 +740,57 @@ def test_build_trigger_unix_dow_various_cases(
nxt = trigger.get_next_fire_time(None, start)
assert nxt is not None
assert nxt.strftime("%A") == expected_dow
+
+
+# ---------------------------------------------------------------------------
+# LaunchDarkly lifecycle — the scheduler eagerly inits LD in run_service (so
+# @expose flag gates don't fail-closed right after a pod restart) and tears
+# it down in cleanup, both gated on the non-LOCAL app env. Test the gate at
+# the boundary where the LD symbols are used (the scheduler module).
+# ---------------------------------------------------------------------------
+
+
+class TestLaunchDarklyLifecycle:
+ def test_init_runs_when_app_env_not_local(self) -> None:
+ from backend.executor import scheduler as sched
+ from backend.util.settings import AppEnvironment
+
+ with (
+ patch.object(sched.config, "app_env", AppEnvironment.PRODUCTION),
+ patch.object(sched, "initialize_launchdarkly") as init,
+ ):
+ sched._init_launchdarkly_for_scheduler()
+ init.assert_called_once()
+
+ def test_init_skipped_when_app_env_local(self) -> None:
+ from backend.executor import scheduler as sched
+ from backend.util.settings import AppEnvironment
+
+ with (
+ patch.object(sched.config, "app_env", AppEnvironment.LOCAL),
+ patch.object(sched, "initialize_launchdarkly") as init,
+ ):
+ sched._init_launchdarkly_for_scheduler()
+ init.assert_not_called()
+
+ def test_shutdown_runs_when_app_env_not_local(self) -> None:
+ from backend.executor import scheduler as sched
+ from backend.util.settings import AppEnvironment
+
+ with (
+ patch.object(sched.config, "app_env", AppEnvironment.PRODUCTION),
+ patch.object(sched, "shutdown_launchdarkly") as shutdown,
+ ):
+ sched._shutdown_launchdarkly_for_scheduler()
+ shutdown.assert_called_once()
+
+ def test_shutdown_skipped_when_app_env_local(self) -> None:
+ from backend.executor import scheduler as sched
+ from backend.util.settings import AppEnvironment
+
+ with (
+ patch.object(sched.config, "app_env", AppEnvironment.LOCAL),
+ patch.object(sched, "shutdown_launchdarkly") as shutdown,
+ ):
+ sched._shutdown_launchdarkly_for_scheduler()
+ shutdown.assert_not_called()
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 1cc79270ac..c4e303c7dc 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -16178,6 +16178,10 @@
"anyOf": [{ "type": "string" }, { "type": "null" }],
"title": "Dream Session Id"
},
+ "ingestion_drain_status": {
+ "$ref": "#/components/schemas/IngestionDrainStatus",
+ "default": "drained"
+ },
"operations": {
"anyOf": [
{ "$ref": "#/components/schemas/DreamOperationsSnapshot" },
@@ -17947,6 +17951,12 @@
"required": ["image_url"],
"title": "ImageURLResponse"
},
+ "IngestionDrainStatus": {
+ "type": "string",
+ "enum": ["drained", "skipped", "timed_out"],
+ "title": "IngestionDrainStatus",
+ "description": "Fate of a dream pass's enqueued graph writes at the moment the pass\nwas reported complete.\n\nThree distinct states, so consumers (and a future Memory Visualizer)\ncan tell a healthy by-design skip from a real failure:\n\n * ``drained`` — the pass's own episodes were confirmed landed in the\n graph before the pass returned (sync path), or the pass enqueued\n nothing to drain. Healthy.\n * ``skipped`` — the drain was intentionally not run to avoid stalling\n a shared serial loop (batch path). The episodes process\n fire-and-forget; healthy by design, NOT a failure signal.\n * ``timed_out`` — the sync-path drain was attempted but did not finish\n within its budget. The reported write/proposal counts include\n episodes still queued in-process, which are lost if the worker's pod\n restarts. This is the only value that warrants an operator warning."
+ },
"InputValidationErrorResponse": {
"properties": {
"type": {