AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/copilot): dream pass leases with renewal, a reaper for expired passes, retention, and in-flight batch cancellation (#14977)
Nicholas Tindle2af6febb7886
diff --git a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.pyindex b8b2367ebf..523f781278 100644--- a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py+++ b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py@@ -36,7 +36,7 @@ from backend.copilot.dream.nightly_batch import NightlyBatchResultfrom backend.copilot.dream.pass_record import dream_pass_result_from_rowfrom backend.copilot.dream.ratification import RatificationResultfrom backend.copilot.dream.schemas import DreamPassResult-from backend.copilot.dream.store import read_dream_pass+from backend.copilot.dream.store import read_dream_pass, read_user_passesfrom backend.copilot.graphiti.falkordb_driver import open_driverfrom backend.copilot.graphiti.recall import live_fact_predicatefrom backend.copilot.graphiti.scope import MemoryScope@@ -1020,6 +1020,12 @@ class DreamPassRecordResponse(BaseModel):result: DreamPassResult+class DreamPassListResponse(BaseModel):+ """A user's dream passes, every scope, newest first."""++ passes: list[DreamPassRecord]++# What a cancelled pass's record says when the admin gave no reason.ADMIN_CANCEL_REASON = "cancelled by an admin"@@ -1103,6 +1109,41 @@ async def trigger_dream_pass(return JSONResponse(status_code=202, content=payload.model_dump(mode="json"))+# Declared before ``GET /{user_id}/dream/{job_id}``, which would otherwise+# read "passes" as a job id.+ "/{user_id}/dream/passes",+ response_model=DreamPassListResponse,+)+async def list_dream_pass_records(+ request: Request,+ user_id: Annotated[str, Path(description="User id or 'me'")],+ caller_id: Annotated[str, Depends(get_user_id)],+ jwt_payload: Annotated[dict[str, Any], Security(get_jwt_payload)],+ status: Annotated[+ Literal["open", "all"],+ Query(description="Only the passes still running, or all of them"),+ ] = "all",+ limit: Annotated[+ int, Query(ge=1, le=100, description="How many passes, newest first")+ ] = 20,+) -> DreamPassListResponse:+ """List a user's dream passes, across the account and every expert,+ newest first: the durable records, each as ``GET .../record`` returns+ it. A finished pass keeps its phase outputs, operations and usage; its+ input bundle went when it closed, and its lease with it, or, for a pass+ whose close left a cleanup pending, once that cleanup finished."""+ target = _resolve_user_id(user_id, caller_id)+ _audit_cross_user_access(+ request=request,+ caller_id=caller_id,+ target_id=target,+ jwt_payload=jwt_payload,+ )+ passes = await read_user_passes(target, open_only=status == "open", limit=limit)+ return DreamPassListResponse(passes=passes)++@router.get("/{user_id}/dream/{job_id}",response_model=DreamJobStatus,@@ -1180,10 +1221,12 @@ async def cancel_dream_pass_run(The row closes at once; the pass itself stops at its next check (a phaseboundary, its batch submit, or just before apply, on either route)- without applying. A batch pass waiting on its provider stops when that- batch lands. A pass already past its last check still applies, once, and- whatever a pass applied stays. 404 for a pass that does not exist or- belongs to another user, alike; 409 once the pass has ended.+ without applying. A batch pass waiting on its provider has that batch+ cancelled too, and the batch executor drops it at its next poll, which+ releases the pass's lock. A pass already past its last check still+ applies, once, and whatever a pass applied stays. 404 for a pass that+ does not exist or belongs to another user, alike; 409 once the pass has+ ended."""target = _resolve_user_id(user_id, caller_id)_audit_cross_user_access(diff --git a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes_test.py b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes_test.pyindex 98047893a6..35c7335ed3 100644--- a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes_test.py+++ b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes_test.py@@ -1123,6 +1123,71 @@ class TestCancelDreamPass:assert resp.status_code == 500+class TestDreamPassList:+ """``GET /{user_id}/dream/passes`` lists a user's durable pass records,+ every scope, newest first, the open ones only when asked."""++ def test_lists_the_users_passes_newest_first(self) -> None:+ newer = _dream_pass_row("abc").model_copy(update={"id": "p2"})+ read = AsyncMock(return_value=[newer, _dream_pass_row("abc")])+ with patch(f"{_MOCK_MODULE}.read_user_passes", new=read):+ resp = client.get("/admin/memory/abc/dream/passes")++ assert resp.status_code == 200+ read.assert_awaited_once_with("abc", open_only=False, limit=20)+ passes = resp.json()["passes"]+ assert [p["id"] for p in passes] == ["p2", "p1"]+ assert passes[0]["status"] == "COMPLETE"++ def test_the_open_ones_only_and_a_limit_reach_the_read(self) -> None:+ read = AsyncMock(return_value=[])+ with patch(f"{_MOCK_MODULE}.read_user_passes", new=read):+ resp = client.get("/admin/memory/abc/dream/passes?status=open&limit=5")++ assert resp.status_code == 200+ assert resp.json() == {"passes": []}+ read.assert_awaited_once_with("abc", open_only=True, limit=5)++ def test_me_lists_the_callers_own_passes(self, mock_jwt_admin) -> None:+ read = AsyncMock(return_value=[])+ with patch(f"{_MOCK_MODULE}.read_user_passes", new=read):+ resp = client.get("/admin/memory/me/dream/passes")++ assert resp.status_code == 200+ assert read.call_args.args == (mock_jwt_admin["user_id"],)++ def test_passes_is_never_read_as_a_job_id(self) -> None:+ job_status = AsyncMock()+ with (+ patch(f"{_MOCK_MODULE}.read_user_passes", new=AsyncMock(return_value=[])),+ patch(f"{_MOCK_MODULE}.read_status", new=job_status),+ ):+ resp = client.get("/admin/memory/abc/dream/passes")++ assert resp.status_code == 200+ job_status.assert_not_awaited()++ @pytest.mark.parametrize(+ "query", ["status=done", "limit=0", "limit=101", "limit=many"]+ )+ def test_a_bad_status_or_limit_is_422_before_the_read(self, query) -> None:+ read = AsyncMock()+ with patch(f"{_MOCK_MODULE}.read_user_passes", new=read):+ resp = client.get(f"/admin/memory/abc/dream/passes?{query}")++ assert resp.status_code == 422+ read.assert_not_awaited()++ def test_non_admin_gets_403_before_the_read(self, mock_jwt_user) -> None:+ app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]+ read = AsyncMock()+ with patch(f"{_MOCK_MODULE}.read_user_passes", new=read):+ resp = client.get("/admin/memory/abc/dream/passes")++ assert resp.status_code == 403+ read.assert_not_awaited()++class TestAdminGating:"""Non-admin callers must get 403 on every route."""diff --git a/autogpt_platform/backend/backend/copilot/dream/apply.py b/autogpt_platform/backend/backend/copilot/dream/apply.pyindex b480a57424..151e7d9d3c 100644--- a/autogpt_platform/backend/backend/copilot/dream/apply.py+++ b/autogpt_platform/backend/backend/copilot/dream/apply.py@@ -24,6 +24,7 @@ import loggingimport uuid as uuidlibfrom collections.abc import Mappingfrom datetime import datetime, timezone+from typing import TYPE_CHECKINGfrom backend.copilot.graphiti.falkordb_driver import open_driverfrom backend.copilot.graphiti.ingest import (@@ -48,7 +49,7 @@ from backend.util.feature_flag import Flag, is_feature_enabledfrom .batch_submit import read_input_bundlefrom .fetch import DREAM_EPISODE_NAME_PREFIX-from .locks import DreamLockHandle, DreamLockLostError+from .locks import LOCK_CHECK_TIMEOUT_SECONDS, DreamLockHandle, DreamLockLostErrorfrom .schemas import (ConsolidatedFact,DemotionSummary,@@ -62,27 +63,30 @@ from .schemas import (WriteSummary,)+if TYPE_CHECKING:+ from .lease import ApplyLease+logger = logging.getLogger(__name__)# Upper bound on waiting for the per-user ingestion queue to drain before# apply_operations returns. Dev numbers put a single add_episode at 7-131s,# so a full pass (up to 30 writes + 20 proposals) can take far longer than# any defensible in-lock wait. The caller holds the dream lock until apply-# returns (locks.DEFAULT_LOCK_TTL_SECONDS=1800, shared with the three LLM-# phases budgeted at ~1320s), so 300s keeps the whole pass inside the lock-# TTL envelope. Past the cap the enqueued episodes keep processing-# fire-and-forget in this process: apply warns and reports ``timed_out``-# rather than failing the pass.+# returns, and renews it right before the drain to a budget that covers this+# cap (``LOCK_DRAIN_RENEWAL_SECONDS``). Past the cap the enqueued episodes+# keep processing fire-and-forget in this process: apply warns and reports+# ``timed_out`` rather than failing the pass.INGESTION_DRAIN_TIMEOUT_SECONDS = 300-# Fresh dream-lock TTL granted right before the ingestion drain on the sync-# path (see ``apply_operations``' ``lock_handle``). The drain is the longest-# non-LLM tail of the pass, and the plain lock TTL — shared with the three-# LLM phases — can be nearly exhausted by the time apply runs. Renewing the-# lock to this dedicated budget guarantees it cannot expire mid-write and-# admit a concurrent pass onto the same user's graph. Sized to cover the-# drain cap plus the demotions / entity invalidations / summary write that-# follow it inside apply.+# The dream-lock TTL set right before the ingestion drain on the sync path+# (see ``apply_operations``' ``lock_handle``), sized to cover the drain cap+# plus the demotions / entity invalidations / summary write that follow it+# inside apply. The pass has renewed its lock before each phase and again+# before its first write (``lease.py``); this renewal proves it still holds+# the lock before the longest and most destructive tail, and can shorten+# what the earlier ones granted. Like every renewal it is a timer, not a+# storage fence: a tail that outruns this TTL writes on after the lock+# lapsed, and a newer pass may have taken the scope by then.LOCK_DRAIN_RENEWAL_SECONDS = INGESTION_DRAIN_TIMEOUT_SECONDS + 180# Drain bound for the Anthropic batch path. apply runs there inside@@ -517,10 +521,11 @@ async def _drain_ingestion(``enqueue_episode`` returning True only proves the episode reached thein-process asyncio queue; the real write (LLM extraction + embedding in``_ingestion_worker``) happens later. The caller of ``apply_operations``- holds the dream lock until apply returns, so draining here keeps the- writes inside the lock envelope — without it, a scheduler pod restart- silently discards queued writes while the pass stays recorded- successful.+ holds the dream lock until apply returns, so draining here lands the+ writes before the pass gives its lock back (while the lock the drain+ renewal proved lasts: ``LOCK_DRAIN_RENEWAL_SECONDS``) — without it, a+ scheduler pod restart silently discards queued writes while the pass+ stays recorded successful.Scoped to *trackers* — only the episodes THIS pass enqueued. Theper-user queue is shared with live-chat ingestion, so a whole-queue@@ -577,6 +582,7 @@ async def apply_operations(known_episode_uuids: set[str] | None = None,ingestion_drain_timeout: float = INGESTION_DRAIN_TIMEOUT_SECONDS,lock_handle: DreamLockHandle | None = None,+ lease: ApplyLease | None = None,) -> dict[str, int | str | IngestionDrainStatus | DreamOperationsSnapshot]:"""Apply a sanitized DreamOperations to Graphiti + Postgres.@@ -627,11 +633,20 @@ async def apply_operations(``BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS`` (0) so it never stalls theshared, serial ``BatchExecutor.walk_once`` loop.- ``lock_handle`` is the sync path's held dream lock. It is renewed to a- fresh ``LOCK_DRAIN_RENEWAL_SECONDS`` budget right before the drain so- the lock cannot expire while the writes are still being processed (which would- admit a concurrent pass onto the same graph). ``None`` on the batch- path — it already disowned the lock to its callback with a 24h TTL.+ ``lock_handle`` is the sync path's held dream lock. It is renewed to+ ``LOCK_DRAIN_RENEWAL_SECONDS`` right before the drain, failing closed,+ so a pass that lost its lock stops before the drain and the destructive+ writes after it. The renewal does not keep the lock through those+ writes: a drain and tail that outrun the TTL go on writing after it+ lapsed (``lease.py``). ``None`` on the batch path — it already disowned+ the lock to its callback with a 24h TTL.++ ``lease`` is the pass's lease, on either path: renewed once more right+ after the session shell is created and before the first graph write,+ failing closed (``DreamLockLostError``), so a lock that lapsed while the+ session was created, and maybe went to a newer pass, stops the writes.+ It narrows the window, it does not fence each write: a lease can still+ lapse between this renewal and the writes after it (``lease.py``).Postgres writes route through ``chat_db()`` / equivalentaccessors. The dream pass runs in the Scheduler subprocess where@@ -676,6 +691,11 @@ async def apply_operations(# is written AFTER the ops (see below), so a partway failure leaves an# empty dream rather than a 'completed' narrative with no memory.session_id = await _create_dream_session(scope, pass_id)+ # Creating the session can take long enough for the lock to lapse: a pass+ # that cannot prove it still holds it writes nothing (the shell stays, as+ # after any partway failure).+ if lease is not None and not await lease.renew():+ raise DreamLockLostError(user_id)# Track only the episodes THIS pass enqueues, its writes on one tracker# and its proposals on another: the drain below waits on the dream's own@@ -735,22 +755,23 @@ async def apply_operations()# Renew the dream lock right before the longest non-LLM tail (the- # ingestion drain plus the demotions / summary write that follow) so the- # lock cannot expire mid-write and let a second pass touch the same- # graph. Gated on there being any mutating work left — enqueued episodes- # to drain OR demotions / entity invalidations to apply, the most- # destructive ops in the pass, which a writes-free pass would otherwise- # run under a near-exhausted TTL. The batch path passes no handle (it- # disowned the lock to its callback). A failed renewal means the lock- # already expired — a newer pass may own the graph — so abort before the- # drain and the destructive writes below. The episodes already enqueued- # above keep processing fire-and-forget (they cannot be recalled), but- # the pass is reported errored instead of pretending exclusive ownership.+ # ingestion drain plus the demotions / summary write that follow), to a+ # budget sized for that tail. The renewal proves ownership at the instant+ # it runs; it is a timer, not a fence, so a tail that outruns the TTL+ # writes on after the lock lapsed (``lease.py``). Gated on there being+ # any mutating work left — enqueued episodes to drain OR demotions /+ # entity invalidations to apply, the most destructive ops in the pass.+ # The batch path passes no handle (it disowned the lock to its callback).+ # A failed renewal means the lock already expired — a newer pass may own+ # the graph — so abort before the drain and the destructive writes below.+ # The episodes already enqueued above keep processing fire-and-forget+ # (they cannot be recalled), but the pass is reported errored instead of+ # pretending exclusive ownership.queued = writes_done.registered + proposals_done.registeredif lock_handle is not None and (queued or ops.demotions or ops.entity_invalidations):- if not await lock_handle.extend(LOCK_DRAIN_RENEWAL_SECONDS):+ if not await _renewed_for_the_tail(lock_handle, pass_id):raise DreamLockLostError(user_id)# Drain the in-process ingestion queue before anything downstream@@ -830,3 +851,21 @@ async def apply_operations("ingestion_drain_status": ingestion_drain_status,"snapshot": snapshot,}+++async def _renewed_for_the_tail(lock_handle: DreamLockHandle, pass_id: str) -> bool:+ """Renew the lock for the drain and the writes after it. A renewal Redis+ does not answer within ``LOCK_CHECK_TIMEOUT_SECONDS`` fails closed, as+ the lease renewals that admit apply do: ownership unknown is not+ ownership."""+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ return await lock_handle.extend(LOCK_DRAIN_RENEWAL_SECONDS)+ except TimeoutError:+ logger.warning(+ "Dream pass %s: the lock renewal before the drain did not answer "+ "within %ss; stopping before the destructive tail",+ pass_id,+ LOCK_CHECK_TIMEOUT_SECONDS,+ )+ return Falsediff --git a/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py b/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.pyindex 1e14cd2c35..60639d2143 100644--- a/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py@@ -1,8 +1,8 @@-"""The lock check right before a dream pass applies, on both routes, with the+"""The lock checks right before a dream pass applies, on both routes, with thereal dream lock, callbacks and orchestrator over the in-memory Redis andstore: a pass whose lock lapsed, and was maybe taken by a newer pass, never-applies, so at most one pass of a scope applies. Only apply's graph writes and-the models are stubbed."""+applies, and neither does one that cannot prove it still holds its lock, so at+most one pass of a scope applies. Only apply and the models are stubbed."""import asynciofrom datetime import datetime, timedelta, timezone@@ -26,8 +26,8 @@ from backend.util.llm.providers import BatchResultRowfrom . import batch_callbacks as batch_callbacks_modfrom . import batch_outcome as batch_outcome_mod-from . import cancel as cancel_modfrom . import job_status+from . import lease as lease_modfrom . import locks as locks_modfrom . import orchestrator as orchestrator_modfrom .batch_callbacks import handle_dream_batch_result@@ -150,6 +150,9 @@ class TestTheBatchRoute:async def test_a_callback_whose_lock_a_newer_pass_holds_leaves_it_alone(self, fake_dream_db, fake_dream_redis, apply, charges, mocker):+ """The lock was already a newer pass's when the last phase landed:+ the callback's lease renewal finds it and ends the pass before it+ even claims apply."""await _seed_batch_pass(fake_dream_db, fake_dream_redis)fake_dream_redis.store[_LOCK_KEY] = "newer-token"release = mocker.spy(batch_outcome_mod, "release_dream_lock")@@ -159,7 +162,7 @@ class TestTheBatchRoute:apply.assert_not_awaited()release.assert_not_called()await _assert_ended_without_applying(- fake_dream_db, fake_dream_redis, LOCK_LOST_ERROR+ fake_dream_db, fake_dream_redis, LOCK_LOST_ERROR, gate_spent=False)assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"row = fake_dream_db.rows["p1"]@@ -169,21 +172,83 @@ class TestTheBatchRoute:)assert _charged(charges) == ["consolidate", "recombine", "sanitize"]- async def test_a_lock_that_cannot_be_read_after_the_claim_fails_closed(- self, fake_dream_db, fake_dream_redis, apply, charges, mocker+ @pytest.mark.parametrize("failure", ["timeout", "error"])+ async def test_a_fence_renewal_without_an_answer_never_applies_after_its_replacement(+ self,+ fake_dream_db,+ fake_dream_redis,+ apply,+ charges,+ sync_pass,+ mocker,+ failure,):- """The pass does not apply on a lock it cannot confirm, and releases- it by compare-and-delete, which only ever deletes its own."""+ """Codex's failed-final-renewal case, the timeout at the real two+ seconds: the last callback claims its gate, and its compare-and-extend+ gets no answer while its lock lapses and a forced admin pass takes the+ scope and applies. Ownership unknown is not ownership: the callback+ ends without applying."""await _seed_batch_pass(fake_dream_db, fake_dream_redis)- mocker.patch.object(- cancel_mod, "dream_lock_held_by", AsyncMock(side_effect=TimeoutError())+ extend = lease_mod.extend_dream_lock+ at_fence, proceed = asyncio.Event(), asyncio.Event()+ renewals = 0++ async def no_answer_at_the_fence(scope, token: str, ttl_seconds: int) -> bool:+ nonlocal renewals+ renewals += 1+ if renewals == 2: # the landing renewal first, then the fence+ at_fence.set()+ await proceed.wait()+ if failure == "timeout":+ await asyncio.Event().wait()+ raise ConnectionError("redis did not answer")+ return await extend(scope, token, ttl_seconds)++ mocker.patch.object(lease_mod, "extend_dream_lock", no_answer_at_the_fence)+ loop = asyncio.get_running_loop()+ started = loop.time()+ old = asyncio.create_task(_deliver_sanitize())+ await asyncio.wait_for(at_fence.wait(), 5)+ await fake_dream_redis.delete(_LOCK_KEY)++ newer = await orchestrator_mod.execute_dream_pass(+ "u1", trigger="admin", force=True)+ proceed.set()+ await asyncio.wait_for(old, 5)++ if failure == "timeout":+ assert 1.9 < loop.time() - started < 4+ assert newer.error is None and not newer.skipped+ assert [call.args[1] for call in apply.await_args_list] == [newer.pass_id]+ error = f"expired: forced by an admin-triggered dream pass {newer.pass_id}"+ await _assert_ended_without_applying(fake_dream_db, fake_dream_redis, error)+ assert _charged(charges) == ["consolidate", "recombine", "sanitize"]++ async def test_a_fence_renewal_without_an_answer_fails_closed_and_releases(+ self, fake_dream_db, fake_dream_redis, apply, charges, mocker+ ):+ """With no newer pass about: the pass does not apply on a lock it+ cannot prove, ends with the lease it could not renew, and releases+ the lock by compare-and-delete, which only ever deletes its own."""+ await _seed_batch_pass(fake_dream_db, fake_dream_redis)+ extend = lease_mod.extend_dream_lock+ renewals = 0++ async def no_answer_at_the_fence(scope, token: str, ttl_seconds: int) -> bool:+ nonlocal renewals+ renewals += 1+ if renewals == 2:+ raise TimeoutError()+ return await extend(scope, token, ttl_seconds)++ mocker.patch.object(lease_mod, "extend_dream_lock", no_answer_at_the_fence)await _deliver_sanitize()apply.assert_not_awaited()await _assert_ended_without_applying(- fake_dream_db, fake_dream_redis, LOCK_LOST_ERROR+ fake_dream_db, fake_dream_redis, "apply: dream lease could not be renewed")assert _LOCK_KEY not in fake_dream_redis.store@@ -382,15 +447,16 @@ def _landed(phase: str, output: BaseModel) -> BatchResultRow:async def _assert_ended_without_applying(- fake_dream_db, fake_dream_redis, error: str+ fake_dream_db, fake_dream_redis, error: str, *, gate_spent: bool = True) -> None:"""Ended as ``fail_pass`` ends a pass: its job errored with *error*, its- batch state and bundle cleaned, its apply gate spent."""+ batch state and bundle cleaned, its apply gate spent when it got that+ far (else never claimed)."""status = await job_status.read_status(kind="dream_pass", job_id="j1")assert status is not None and (status.state, status.error) == ("errored", error)assert state_key("p1") not in fake_dream_redis.hashesassert input_bundle_key("p1") not in fake_dream_redis.store- assert fake_dream_redis.store["dream:applied:p1"] == "1"+ assert ("dream:applied:p1" in fake_dream_redis.store) is gate_spentassert DreamPassStatus.COMPLETE not in fake_dream_db.statuses("p1")diff --git a/autogpt_platform/backend/backend/copilot/dream/apply_test.py b/autogpt_platform/backend/backend/copilot/dream/apply_test.pyindex cf57046cea..d92f1dd250 100644--- a/autogpt_platform/backend/backend/copilot/dream/apply_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/apply_test.py@@ -10,6 +10,7 @@ to ``enqueue_episode`` / ``mark_edges_superseded`` /from __future__ import annotations+import asyncioimport loggingfrom datetime import datetime, timezonefrom unittest.mock import AsyncMock, MagicMock@@ -774,6 +775,43 @@ async def test_failed_lock_renewal_aborts_before_drain_and_demotions(mocker):demote.assert_not_awaited()+async def test_a_lock_renewal_redis_does_not_answer_aborts_before_drain(mocker):+ """The renewal before the drain is bounded like every other lock round+ trip, and fails closed: a renewal that does not answer in time is not+ taken for one that landed."""+ mocker.patch.object(apply_mod, "LOCK_CHECK_TIMEOUT_SECONDS", 0.05)+ drain = mocker.patch.object(apply_mod, "_drain_ingestion", AsyncMock())+ demote = mocker.patch.object(+ apply_mod, "_filter_demotions_to_known_facts", AsyncMock()+ )++ async def never_answers(_ttl_seconds: int) -> bool:+ await asyncio.Event().wait()+ return True++ lock_handle = mocker.MagicMock()+ lock_handle.extend = never_answers+ ops = DreamOperations(+ writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ summary_for_user="ok",+ )++ with pytest.raises(DreamLockLostError):+ await asyncio.wait_for(+ apply_mod.apply_operations(+ scope=MemoryScope.for_user("u-slow"),+ pass_id="p-slow",+ ops=ops,+ lock_handle=lock_handle,+ ),+ 5,+ )++ drain.assert_not_awaited()+ demote.assert_not_awaited()++@pytest.mark.asyncioasync def test_demotions_only_pass_renews_lock_before_destructive_tail(mocker):"""Demotions and entity invalidations are the most destructive ops in adiff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.pyindex f4b85c8bbd..7dc9158cd4 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py@@ -18,18 +18,20 @@ runs the apply step + cost log + JobStatus complete when phase 3 lands.The pass's Redis state and its at-most-once gates are ``batch_state.py``;how the pass ends (JobStatus, the durable ``DreamPass`` record, the lock)-is ``batch_outcome.py``; what its phases cost and used is ``batch_costs.py``. Each-callback also advances the record: the phase that landed and its output,-the next batch, the apply, and the end. It ends a pass cancelled or expired-meanwhile before it chains or claims the apply gate, and one that lost its-scope's lock after the claim, before it applies (``cancel.py``).+is ``batch_outcome.py``; what its phases cost and used is ``batch_costs.py``;+the deliveries that never run the chain (a closed pass's batch, a dead-end+payload, a duplicate) are ``batch_deliveries.py``. Each callback advances the+record and, once its phase has landed, renews the pass's lease (``lease.py``),+ending a pass whose lock is no longer its own. It ends a pass cancelled or+expired meanwhile before it chains or claims the apply gate (``cancel.py``),+and after the claim applies only on a renewal proving the lock is the pass's+(``lease.admit_batch_apply``)."""from __future__ import annotationsimport jsonimport logging-from contextlib import ExitStackfrom typing import TYPE_CHECKING, Anyfrom pydantic import BaseModel, ValidationError@@ -39,37 +41,33 @@ from backend.copilot.inference.context import InferenceErrorfrom backend.copilot.trial_cost_context import TrialCostContext, restore_cost_contextfrom backend.copilot.usage_activation import UsageActivationUnavailable-from .batch_costs import landed_usage, log_all_phase_costs, recorded_usage+from .batch_costs import landed_usage+from .batch_deliveries import (+ drop_closed,+ end_dead_end,+ finish_duplicate,+ should_dispatch,+)from .batch_outcome import (ApplyStats,BatchPass,+ clean_up_after,fail_pass,- finalize_stuck_duplicate,- mark_job_errored,+ lock_token_of,record_completion,release_lock,)-from .batch_state import (- best_effort_cleanup,- claim_apply_gate,- content_for,- read_state,- write_phase_to_state,-)-from .batch_submit import (- PHASE_RESPONSE_MODELS,- read_input_bundle,- read_lock_token,- submit_phase,-)-from .cancel import end_batch_pass_if_lock_lost, end_batch_pass_if_stopped, pass_closed+from .batch_state import claim_apply_gate, content_for, read_state, write_phase_to_state+from .batch_submit import PHASE_RESPONSE_MODELS, read_input_bundle, submit_phase+from .cancel import end_batch_pass_if_stopped, pass_closedfrom .clamp import clamp_operations+from .lease import ApplyLease, admit_batch_apply, renew_batch_leasefrom .llm import parse_json_with_prose_fallbackfrom .provider_batch import anthropic_api_keyfrom .schemas import DreamOperations, DreamPhase, IngestionDrainStatusfrom .store import (+ read_cost_context,record_applying,- record_batch_failed,record_next_batch,record_phase_output,)@@ -84,7 +82,11 @@ logger = logging.getLogger(__name__)NAMESPACE = "dream_pass"-# A pass whose batch carries no usage attribution it can be charged under.++# The error a batch pass closes with when the usage attribution it started+# under could not be established for one of its results (#15208): a fixed,+# content-free string. Its phases are charged only under the attribution+# its row carries, if any (``batch_costs``).UNATTRIBUTED_ERROR = "batch usage attribution unavailable"NEXT_PHASE: dict[DreamPhase, DreamPhase | None] = {@@ -97,6 +99,11 @@ NEXT_PHASE: dict[DreamPhase, DreamPhase | None] = {async def handle_dream_batch_result(entry: PendingEntry, rows: list[BatchResultRow]) -> None:+ """BatchExecutor entry, once per finished phase batch: the usage+ attribution the pass started under (#15208), read off its row+ (``_pass_attribution``), restored around the phase's handling+ (``_handle_dream_batch_result``); a pass without one, or whose batch+ carries another, ends (``_fail_unattributed_pass``)."""payload = entry.payload or {}user_id = str(payload.get("user_id") or "")if (@@ -106,28 +113,66 @@ async def handle_dream_batch_result():await _handle_dream_batch_result(entry, rows)return- with ExitStack() as attribution:- try:- raw_context = payload.get("cost_context")- if raw_context is None:- raise UsageActivationUnavailable("Dream batch has no usage attribution")- context = TrialCostContext.model_validate(raw_context)- attribution.enter_context(restore_cost_context(user_id, context))- except (UsageActivationUnavailable, ValueError):- await _fail_unattributed_pass(payload)- raise+ try:+ context = await _pass_attribution(payload)+ except (UsageActivationUnavailable, ValueError):+ await _fail_unattributed_pass(entry)+ raise+ with restore_cost_context(user_id, context):await _handle_dream_batch_result(entry, rows)-async def _fail_unattributed_pass(payload: dict[str, Any]) -> None:- """Close a pass whose batch result carries no usage attribution it can- be charged under (#15208): errored like any failed pass (its admin job,+async def _pass_attribution(payload: dict[str, Any]) -> TrialCostContext:+ """The usage attribution the pass started under: its row's+ (``store.read_cost_context``), the one source every charge of the pass+ goes by. The batch's payload carries the copy #15208 put there, which+ must agree with it. While the row cannot be read (the store is down),+ the copy stands in for this callback's own work, so a store outage does+ not end the pass; its charges (``batch_costs``) still wait on the row.++ Raises ``UsageActivationUnavailable`` when the payload carries no copy,+ the row holds no attribution (none was kept, or there is no row), or+ the two differ; ``ValueError`` when the copy does not parse or is+ another user's."""+ user_id = str(payload["user_id"])+ carried = payload.get("cost_context")+ if carried is None:+ raise UsageActivationUnavailable("Dream batch has no usage attribution")+ copy = TrialCostContext.model_validate(carried)+ if copy.user_id != user_id:+ raise ValueError("Trial cost attribution belongs to a different user")+ pass_id = str(payload["pass_id"])+ try:+ recorded = await read_cost_context(pass_id)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not read its usage attribution; the "+ "callback goes on under its batch's copy",+ exc_info=True,+ )+ return copy+ if recorded is None:+ raise UsageActivationUnavailable("Dream pass has no usage attribution")+ if recorded != copy:+ raise UsageActivationUnavailable(+ "Dream batch attribution differs from its pass's"+ )+ return recorded+++async def _fail_unattributed_pass(entry: PendingEntry) -> None:+ """Close a pass whose usage attribution could not be established for its+ batch's result (#15208): errored like any failed pass (its admin job,its record with what the landed phases used, the disowned lock, its- state), but nothing is charged against another usage context. The lock- is the pass's own scope's, read off its input bundle as the phase- handlers read it; without the bundle there is no token to release it- with, and it is left to its TTL."""- bp = BatchPass.from_payload(payload)+ state, the batch stopped at the provider), and charged only under the+ attribution its row carries, if any (``batch_costs``): never under+ another. The lock is the pass's own scope's, read off its input bundle+ as the phase handlers read it; without the bundle there is no token to+ release it with, and it is left to its TTL. The batch is the one this+ result came from: a result the executor gave up waiting for (its+ lifetime timeout) may still be running, and the row stays marked until+ the provider says it has stopped (``cleanup.py``)."""+ bp = BatchPass.from_payload(entry.payload or {})try:input_bundle = await read_input_bundle(bp.pass_id)except Exception:@@ -140,24 +185,17 @@ async def _fail_unattributed_pass(payload: dict[str, Any]) -> None:"expert_id": input_bundle.expert_id,})- await fail_pass(bp, UNATTRIBUTED_ERROR, charge=False)+ await fail_pass(bp, UNATTRIBUTED_ERROR, provider_batch_id=entry.provider_batch_id)async def _handle_dream_batch_result(entry: PendingEntry, rows: list[BatchResultRow]) -> None:- """BatchExecutor entry — called once per finished phase batch.-- Reads the phase label off ``entry.payload``, validates the- response shape, persists it to the per-pass accumulator, then- decides what to do next:-- * If error → mark JobStatus errored, clean up state + input- * If non-terminal phase → submit next phase batch, leave job- in ``submitted`` state with updated ``current_phase``- * If terminal phase (sanitize) → run apply, log costs for all- three phases, mark JobStatus complete, clean up- """+ """One finished phase batch, under its pass's attribution: validate and keep+ the phase's result, then end the pass on an error, submit the next phase+ (the job left ``submitted`` at it), or after sanitize apply, charge the+ three phases and complete the job. A payload no phase can take is a dead+ end (``batch_deliveries``)."""payload = entry.payload or {}bp = BatchPass.from_payload(payload)phase = payload.get("phase")@@ -166,28 +204,15 @@ async def _handle_dream_batch_result("Dream batch handler missing user_id/pass_id/phase — payload=%s",payload,)- await _dead_end(bp, "batch payload missing user_id/pass_id/phase")+ await end_dead_end(bp, "batch payload missing user_id/pass_id/phase")returnif phase not in NEXT_PHASE:logger.warning("Dream batch handler unknown phase=%r", phase)- await _dead_end(bp, f"unknown batch phase {phase!r}")+ await end_dead_end(bp, f"unknown batch phase {phase!r}")returnawait _handle_guarded(bp, phase, rows)-async def _dead_end(bp: BatchPass, error: str) -> None:- """A payload no phase handler can take. Close the admin job row (it- would otherwise sit queued/submitted until its TTL) and, when the pass- is known, its record with what its landed phases used; then release- the disowned lock so the user isn't locked out until the 24h TTL."""- await mark_job_errored(bp.job_id, error, dead_end=True)- if bp.pass_id:- usage = await recorded_usage(bp.pass_id, bp.phase_models)- await record_batch_failed(bp.pass_id, error, usage)- if bp.user_id:- await release_lock(bp)--async def _handle_guarded(bp: BatchPass, phase: DreamPhase, rows: list[BatchResultRow]) -> None:@@ -267,6 +292,8 @@ async def _handle_phase_result(if await _landed_output(bp, phase, rows[0]) is None:returnnext_phase = NEXT_PHASE[phase]+ if not await renew_batch_lease(bp, next_phase or "apply"):+ returnif next_phase is not None:await _chain_next_phase(bp, input_bundle, next_phase)return@@ -280,7 +307,9 @@ async def _landed_output("""The phase's validated output, kept in the pass's Redis state and itsrecord; ``None`` once an errored or malformed row has failed the pass."""if row.error:- await write_phase_to_state(pass_id=bp.pass_id, phase=phase, row=row)+ await write_phase_to_state(+ pass_id=bp.pass_id, phase=phase, row=row, model=bp.phase_models.get(phase)+ )await fail_pass(bp, f"{phase}: {row.error}")return None# Validate the row's content matches the phase's Pydantic schema@@ -294,11 +323,16 @@ async def _landed_output(output = PHASE_RESPONSE_MODELS[phase].model_validate(payload)except (InferenceError, ValidationError) as exc:row.error = f"invalid output shape — {type(exc).__name__}"- await write_phase_to_state(pass_id=bp.pass_id, phase=phase, row=row)+ await write_phase_to_state(+ pass_id=bp.pass_id, phase=phase, row=row, model=bp.phase_models.get(phase)+ )await fail_pass(bp, f"{phase}: invalid output shape — {type(exc).__name__}")return Noneawait write_phase_to_state(- pass_id=bp.pass_id, phase=phase, row=row.with_content(json.dumps(payload))+ pass_id=bp.pass_id,+ phase=phase,+ row=row.with_content(json.dumps(payload)),+ model=bp.phase_models.get(phase),)await record_phase_output(bp.pass_id, phase, output)return output@@ -383,14 +417,13 @@ async def _finalize_complete(bp: BatchPass, input_bundle: DreamInput) -> None:returnstate = await read_state(bp.pass_id)ops = await _terminal_ops(bp, state, input_bundle)- if ops is None or not await _claim_apply(bp, state, ops):+ lease = await _claim_apply(bp, ops) if ops is not None else None+ if ops is None or lease is None:returntry:- # Skip the ingestion drain on this path: apply runs inside this- # handler, which BatchExecutor.walk_once awaits serially in its single- # poll loop — a 300s in-line drain would stall the poll/dispatch of- # every other user's pending batch. See- # ``BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS``.+ # No ingestion drain here: walk_once awaits this handler serially, so+ # an in-line drain would stall every other user's pending batch+ # (``BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS``).apply_stats = await apply_operations(MemoryScope.build(bp.user_id, bp.expert_id),bp.pass_id,@@ -398,6 +431,7 @@ async def _finalize_complete(bp: BatchPass, input_bundle: DreamInput) -> None:known_fact_uuids=input_bundle.known_fact_uuids,known_episode_uuids=input_bundle.known_episode_uuids,ingestion_drain_timeout=BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS,+ lease=lease,)except Exception as exc:logger.exception(@@ -411,20 +445,18 @@ async def _finalize_complete(bp: BatchPass, input_bundle: DreamInput) -> None:)-async def _claim_apply(- bp: BatchPass, state: dict[str, dict[str, Any]], ops: DreamOperations-) -> bool:- """Record APPLYING, check for a stop, claim the apply gate, check the- lock; ``False`` once a stop, a duplicate, an unreadable gate or a lost lock- has ended this delivery. A delivery that stalls or dies before the claim- has not claimed it, so a redelivery still applies. Between the claim and- apply only the lock is read (one GET), so a newer pass that took the scope- while this one waited is never applied over.- """+async def _claim_apply(bp: BatchPass, ops: DreamOperations) -> ApplyLease | None:+ """Record APPLYING, check for a stop, claim the apply gate, admit apply+ on a renewal of the lock: the lease apply renews again before it writes,+ or ``None`` once a stop, a duplicate, an unreadable gate or an unproven+ lease has ended this delivery. A delivery that dies before the claim has+ not claimed it, so a redelivery still applies. After the claim only the+ compare-and-extend runs, so a newer pass that took the scope meanwhile is+ never applied over (``lease.py`` has what stays open)."""await record_applying(bp.pass_id, ops)- lock_token = await read_lock_token(bp.pass_id)+ lock_token = await lock_token_of(bp.pass_id)if await end_batch_pass_if_stopped(bp):- return False+ return Nonegate = await claim_apply_gate(bp.pass_id)if gate == "error":# We cannot tell first-vs-duplicate apart, and "complete with no@@ -432,11 +464,11 @@ async def _claim_apply(await fail_pass(bp, "apply: gate unavailable (redis) — cannot guarantee at-most-once")- return False+ return Noneif gate == "duplicate":- await _finish_duplicate(bp, state, ops)- return False- return not await end_batch_pass_if_lock_lost(bp, lock_token)+ await finish_duplicate(bp, ops)+ return None+ return await admit_batch_apply(bp, lock_token)async def _terminal_ops(@@ -465,35 +497,6 @@ async def _terminal_ops()-async def _finish_duplicate(- bp: BatchPass, state: dict[str, dict[str, Any]], ops: DreamOperations-) -> None:- """A repeated delivery: an earlier one claimed the apply gate, so skip- apply and keep the first delivery's results.-- Normally the first delivery closed the admin JobStatus row. If it died- between apply and ``mark_complete``, the row is stuck in 'submitted' and- is finalized here without clobbering an existing terminal result. The- pass's DreamPass record is left as it is: this delivery cannot tell- whether the first one's apply finished, so closing it is a reaper's job.- """- logger.info(- "Duplicate dispatch for pass=%s — operations already applied; "- "preserving the first delivery's job result",- bp.pass_id,- )- await finalize_stuck_duplicate(bp, ops)- await log_all_phase_costs(- user_id=bp.user_id,- expert_id=bp.expert_id,- pass_id=bp.pass_id,- state=state,- phase_models=bp.phase_models,- )- await release_lock(bp)- await best_effort_cleanup(bp.pass_id)--async def _finish_applied(bp: BatchPass,state: dict[str, dict[str, Any]],@@ -501,18 +504,11 @@ async def _finish_applied(ops: DreamOperations,ingestion_drain_status: IngestionDrainStatus,) -> None:- """Charge the landed phases, close the job and the record, release."""- # Per-phase usage log on the success path. Failure paths record the- # same usage via ``fail_pass`` (we incurred those provider tokens- # regardless); the Redis dedup gate inside ``log_all_phase_costs``- # keeps it at-most-once across both paths and any repeated delivery.- await log_all_phase_costs(- user_id=bp.user_id,- expert_id=bp.expert_id,- pass_id=bp.pass_id,- state=state,- phase_models=bp.phase_models,- )+ """Close the job and the record, marked, then clean up after the pass+ (``clean_up_after``): the landed phases charged, the lock the batch path+ disowned to this callback released so the user's next dream can run.+ Failure paths do the same through ``fail_pass``; the per-phase claims+ keep the charges at-most-once across every path."""# The batch path skips the drain by design, so apply reports# ``skipped`` whenever the pass enqueued writes (``drained`` only# for an empty pass), read via the shared, fail-closed helper.@@ -523,17 +519,19 @@ async def _finish_applied(summary_for_user=ops.summary_for_user,usage=landed_usage(state, bp.phase_models, bp.pass_id),)- # The batch path disowned the dream lock to this callback; release it now- # that the pass has terminated so the next dream for this user can run.- await release_lock(bp)- await best_effort_cleanup(bp.pass_id)+ await clean_up_after(bp)def _register() -> None:try:from backend.executor.batch_executor import register_handler- register_handler(NAMESPACE, handle_dream_batch_result)+ register_handler(+ NAMESPACE,+ handle_dream_batch_result,+ should_dispatch=should_dispatch,+ on_drop=drop_closed,+ )except Exception:logger.exception("Failed to register dream batch handler — "diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.pyindex 61b9d1f8ba..8794ccf8e0 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py@@ -25,7 +25,11 @@ from prisma.enums import (from backend.copilot.dream import job_statusfrom backend.copilot.dream.batch_callbacks import handle_dream_batch_result-from backend.copilot.dream.batch_state import state_key, write_phase_to_state+from backend.copilot.dream.batch_state import (+ read_state,+ state_key,+ write_phase_to_state,+)from backend.copilot.dream.batch_submit import input_bundle_key, persist_input_bundlefrom backend.copilot.dream.cancel import cancel_dream_passfrom backend.copilot.dream.fetch import DreamInput@@ -52,6 +56,16 @@ from backend.executor.batch_executor import (from backend.util.llm.providers import BatchResultRow[email protected](autouse=True)+def submitted_rows(fake_dream_db) -> None:+ """Every pass these callbacks name has the row its orchestrator left at+ submit, carrying the usage attribution the pass started under: the one+ the callback restores and every charge goes by. A test about the row+ seeds its own over it."""+ for pass_id in ("p1", "p-expert", "p-missing"):+ _seed_submitted_pass(fake_dream_db, pass_id)++@pytest.fixturedef fake_redis():"""In-memory redis fixture matching the BatchExecutor / JobStatus pattern."""@@ -84,13 +98,15 @@ def fake_redis():async def fake_expire(key, ttl):return 1- async def fake_eval(script, numkeys, key, token):- # The only Lua the dream path runs is the lock's single-key- # compare-and-delete; mirror its semantics on the string store.- if string_store.get(key) == token:+ async def fake_eval(script, numkeys, key, token, *argv):+ # The only Lua the dream path runs is the lock's two single-key+ # scripts, compare-and-delete and compare-and-extend (the lease+ # renewal, which passes the TTL); mirror them on the string store.+ if string_store.get(key) != token:+ return 0+ if not argv:string_store.pop(key, None)- return 1- return 0+ return 1stub = AsyncMock()stub.hset.side_effect = fake_hset@@ -197,7 +213,7 @@ class TestPhaseChaining:mark_errored,),patch(- "backend.copilot.dream.batch_outcome.release_dream_lock",+ "backend.copilot.dream.cleanup.release_dream_lock",release_lock,),):@@ -216,7 +232,10 @@ class TestPhaseChaining:mark_errored.await_args.kwargs["error"]== "batch DreamInput missing; memory scope unavailable")- release_lock.assert_awaited_once_with(MemoryScope.for_user("u1"), None)+ # No token anywhere (no bundle, no row): the pass's ownership of the+ # lock is unknown, so nothing is released; its TTL, or the reaper,+ # settles it.+ release_lock.assert_not_awaited()@pytest.mark.asyncioasync def test_autopilot_payload_cannot_apply_expert_dream(@@ -248,7 +267,7 @@ class TestPhaseChaining:mark_errored,),patch(- "backend.copilot.dream.batch_outcome.release_dream_lock",+ "backend.copilot.dream.cleanup.release_dream_lock",release_lock,),):@@ -304,7 +323,7 @@ class TestPhaseChaining:mark_errored,),patch(- "backend.copilot.dream.batch_outcome.release_dream_lock",+ "backend.copilot.dream.cleanup.release_dream_lock",release_lock,),):@@ -378,6 +397,9 @@ class TestPhaseChaining:update_status.assert_awaited_once()update_kwargs = update_status.call_args.kwargsassert update_kwargs["current_phase"] == "recombine"+ # The landed phase keeps the model it ran on, to be charged on it.+ state = await read_state("p1")+ assert state["consolidate"]["model"] == "claude-sonnet-5"@pytest.mark.asyncioasync def test_recombine_result_submits_sanitize_batch(self, fake_redis):@@ -476,9 +498,7 @@ class TestPhaseChaining:patch("backend.copilot.dream.apply.apply_operations", apply),patch("backend.copilot.dream.job_status.mark_complete", mark_complete),patch("backend.copilot.inference.record.persist_and_record_usage", persist),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", release_lock- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),):await handle_dream_batch_result(_entry(phase="sanitize"),@@ -591,7 +611,7 @@ class TestPhaseChaining:new=persist,),patch(- "backend.copilot.dream.batch_outcome.release_dream_lock",+ "backend.copilot.dream.cleanup.release_dream_lock",release_lock,),):@@ -709,9 +729,7 @@ class TestPhaseChaining:patch("backend.copilot.dream.apply.apply_operations", apply),patch("backend.copilot.dream.job_status.mark_complete", mark_complete),patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", release_lock- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),patch("backend.copilot.dream.batch_state.delete_state", AsyncMock()),patch("backend.copilot.dream.batch_state.delete_input_bundle", AsyncMock()),):@@ -796,9 +814,7 @@ class TestErrorPaths:entry = _entry(phase="consolidate")entry.payload = {"user_id": "u1", "job_id": "j-dead"}- with patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ):+ with patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()):await handle_dream_batch_result(entry, [])final = await read_status(kind="dream_pass", job_id="j-dead")@@ -819,9 +835,7 @@ class TestErrorPaths:"phase": "daydream",}- with patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ):+ with patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()):await handle_dream_batch_result(entry, [])final = await read_status(kind="dream_pass", job_id="j-odd")@@ -1028,9 +1042,7 @@ class TestErrorPaths:patch("backend.copilot.dream.batch_callbacks._chain_next_phase", chain),patch("backend.copilot.dream.job_status.mark_errored", mark_errored),patch("backend.copilot.dream.batch_costs.record_phase_cost", record_cost),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", release_lock- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),):# Must not propagate — the crash guard finalizes and swallows.await handle_dream_batch_result(@@ -1046,21 +1058,20 @@ class TestErrorPaths:class TestMalformedPayload:@pytest.mark.asyncio- async def test_missing_pass_id_releases_lock_without_token(self, fake_redis):- """Malformed payload (missing pass_id) early-returns but, since the- user_id is known, still attempts the lock release — with no pass_id- there's no persisted token to read, so the release is token-less- (release_dream_lock then defers to the lock TTL rather than- blind-deleting)."""- release = AsyncMock()+ async def test_missing_pass_id_leaves_the_lock_to_its_ttl(self, fake_redis):+ """Malformed payload (missing pass_id) early-returns: with no pass_id+ there is no persisted token to release the lock with and no row to+ mark for the reaper, so the lock is left to its TTL rather than+ blind-deleted."""+ _, _, string_store = fake_redis+ string_store["dream:inflight:u1"] = "tok-u1"entry = _entry()entry.payload["pass_id"] = ""- with patch("backend.copilot.dream.batch_outcome.release_dream_lock", release):- await handle_dream_batch_result(- entry,- [_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT)],- )- release.assert_awaited_once_with(MemoryScope.for_user("u1"), None)+ await handle_dream_batch_result(+ entry,+ [_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT)],+ )+ assert string_store["dream:inflight:u1"] == "tok-u1"@pytest.mark.asyncioasync def test_unknown_phase_label_releases_lock_with_persisted_token(@@ -1085,7 +1096,7 @@ class TestMalformedPayload:release = AsyncMock()entry = _entry()entry.payload["phase"] = "some_fake_phase"- with patch("backend.copilot.dream.batch_outcome.release_dream_lock", release):+ with patch("backend.copilot.dream.cleanup.release_dream_lock", release):await handle_dream_batch_result(entry, [_row(custom_id="x", content="y")])release.assert_awaited_once_with(MemoryScope.for_user("u1"), "tok-u1")@@ -1147,9 +1158,9 @@ class TestLockTokenWiring:):"""A Redis blip on the lock-token read in the terminal tail firesAFTER mark_complete already ran. The read must stay best-effort- (token=None → lock TTL fallback) — letting it propagate would hit- the handler's crash guard, whose fail_pass rewrites the- already-completed job to errored."""+ (no token: the lock is left, never blind-deleted) — letting it+ propagate would hit the handler's crash guard, whose fail_pass+ rewrites the already-completed job to errored."""_, _, string_store = fake_redisstring_store["dream:inflight:u1"] = "tok-u1"await self._seed_terminal_pass()@@ -1157,7 +1168,13 @@ class TestLockTokenWiring:mark_complete = AsyncMock()mark_errored = AsyncMock()release_lock = AsyncMock()- read_token = AsyncMock(side_effect=ConnectionError("redis blip"))++ async def blips_once_the_job_is_complete(pass_id: str) -> str:+ if mark_complete.await_count:+ raise ConnectionError("redis blip")+ return "tok-u1"++ read_token = AsyncMock(side_effect=blips_once_the_job_is_complete)with (patch("backend.copilot.dream.apply.apply_operations",@@ -1167,8 +1184,11 @@ class TestLockTokenWiring:patch("backend.copilot.dream.job_status.mark_errored", mark_errored),patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()),patch("backend.copilot.dream.batch_outcome.read_lock_token", read_token),+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),+ # The row never got the token: the lease renewal's write failed.patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", release_lock+ "backend.copilot.dream.lease.record_lease",+ AsyncMock(return_value=None),),):await handle_dream_batch_result(@@ -1178,9 +1198,10 @@ class TestLockTokenWiring:mark_complete.assert_awaited_once()mark_errored.assert_not_awaited()- # Token read failed → token-less release; release_dream_lock then- # defers to the lock TTL rather than blind-deleting.- release_lock.assert_awaited_once_with(MemoryScope.for_user("u1"), None)+ # The token read failed and the row keeps none either: ownership is+ # unknown, so nothing is released rather than blind-deleted.+ release_lock.assert_not_awaited()+ assert string_store["dream:inflight:u1"] == "tok-u1"@pytest.mark.asyncioasync def test_cleanup_failure_after_complete_keeps_job_completed(self, fake_redis):@@ -1204,9 +1225,7 @@ class TestLockTokenWiring:patch("backend.copilot.dream.job_status.mark_errored", mark_errored),patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()),patch("backend.copilot.dream.batch_state.delete_state", delete_state),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()),):await handle_dream_batch_result(_entry(phase="sanitize"),@@ -1242,9 +1261,7 @@ class TestLockTokenWiring:with (patch("backend.copilot.dream.apply.apply_operations", apply),patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()),):await handle_dream_batch_result(_entry(phase="sanitize"),@@ -1279,9 +1296,7 @@ class TestLockTokenWiring:patch("backend.copilot.dream.job_status.mark_complete", mark_complete),patch("backend.copilot.dream.job_status.read_status", read_existing),patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()),):await handle_dream_batch_result(_entry(phase="sanitize"),@@ -1366,9 +1381,7 @@ class TestDreamPassRecord:patch("backend.copilot.inference.record.persist_and_record_usage", AsyncMock()),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()),):for phase, content in (("consolidate", _CONSOLIDATE_CONTENT),@@ -1432,9 +1445,7 @@ class TestDreamPassRecord:)with (patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()),):await handle_dream_batch_result(_entry(phase="recombine"),@@ -1462,9 +1473,7 @@ class TestDreamPassRecord:AsyncMock(side_effect=RuntimeError("FalkorDB unreachable")),),patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()),):await handle_dream_batch_result(_entry(phase="sanitize"),@@ -1493,8 +1502,10 @@ class TestDreamPassRecord:):"""The consolidate phase landed and was billed; a payload no phasehandler can take then ends the pass. The record is closed with that- phase's usage before the disowned lock is released."""+ phase's usage before the disowned lock is released, under the token+ the row keeps: the pass never persisted a bundle to carry one."""_seed_submitted_pass(fake_dream_db)+ fake_dream_db.rows["p1"]["lease_token"] = "tok-u1"await write_phase_to_state(pass_id="p1",phase="consolidate",@@ -1510,11 +1521,11 @@ class TestDreamPassRecord:return await record(pass_id, update)release_lock = AsyncMock(side_effect=lambda *_: order.append("lock released"))+ charge = AsyncMock(side_effect=lambda ctx, usage: order.append("charged"))with (patch.object(fake_dream_db, "update_dream_pass", recorded),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", release_lock- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),+ patch("backend.copilot.dream.batch_costs.record_phase_cost", charge),):await handle_dream_batch_result(entry, [])@@ -1522,7 +1533,122 @@ class TestDreamPassRecord:assert row["status"] is DreamPassStatus.ERROREDassert row["error"] == errorassert [p.phase for p in row["usage"].phases] == ["consolidate"]- assert order == ["record ERRORED", "lock released"]+ # Like any failure: the landed phase charged once, and the pass's+ # batch state gone with its lock.+ assert order == ["record ERRORED", "charged", "lock released"]+ assert release_lock.await_args.args == (MemoryScope.for_user("u1"), "tok-u1")+ assert state_key("p1") not in fake_redis[1]++ @pytest.mark.asyncio+ async def test_a_dead_end_payload_naming_another_owner_ends_the_pass_as_its_rows(+ self, fake_redis, fake_dream_db+ ):+ """The payload is the part that went wrong: the landed phase is+ charged to the owner the row names, and that owner's lock released,+ never the payload's."""+ _seed_submitted_pass(fake_dream_db)+ fake_dream_db.rows["p1"]["lease_token"] = "tok-u1"+ await write_phase_to_state(+ pass_id="p1",+ phase="consolidate",+ row=_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT),+ )+ entry = _entry(phase="recombine", expert_id="e-other")+ entry.payload.update({"phase": "daydream", "user_id": "someone-else"})+ release_lock = AsyncMock(return_value=True)+ charge = AsyncMock()+ with (+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),+ patch("backend.copilot.dream.batch_costs.record_phase_cost", charge),+ ):+ await handle_dream_batch_result(entry, [])++ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.ERRORED+ charged_scope = charge.await_args.args[0].scope+ assert (charged_scope.user_id, charged_scope.expert_id) == ("u1", None)+ assert release_lock.await_args.args == (MemoryScope.for_user("u1"), "tok-u1")++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "row_read",+ [+ AsyncMock(side_effect=TimeoutError("dream pass database stalled")),+ AsyncMock(return_value=None),+ ],+ ids=["row_unreadable", "row_missing"],+ )+ async def test_a_dead_end_payload_without_its_row_ends_as_its_bundles_owner(+ self, fake_redis, fake_dream_db, row_read+ ):+ """No row to say whose pass it is: the input bundle says, as it does+ for the phase handler. The landed phase is charged to, and the lock+ released for, the bundle's owner, never the payload's."""+ _seed_submitted_pass(fake_dream_db)+ now = datetime.now(timezone.utc)+ await persist_input_bundle(+ "p1",+ DreamInput(+ user_id="u1", group_id="user_u1", window_start=now, window_end=now+ ),+ lock_token="tok-u1",+ )+ await write_phase_to_state(+ pass_id="p1",+ phase="consolidate",+ row=_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT),+ )+ entry = _entry(phase="recombine")+ entry.payload.update({"phase": "daydream", "user_id": "someone-else"})+ release_lock = AsyncMock(return_value=True)+ charge = AsyncMock()+ with (+ patch("backend.copilot.dream.batch_deliveries.read_pass", row_read),+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),+ patch("backend.copilot.dream.batch_costs.record_phase_cost", charge),+ ):+ await handle_dream_batch_result(entry, [])++ assert charge.await_args.args[0].scope.user_id == "u1"+ assert release_lock.await_args.args == (MemoryScope.for_user("u1"), "tok-u1")++ @pytest.mark.asyncio+ async def test_a_dead_end_payload_with_no_record_of_its_owner_is_left_to_the_reaper(+ self, fake_redis, fake_dream_db+ ):+ """Neither the row nor the input bundle can be read: nothing is charged,+ released or deleted on the payload's word. The row closes marked, the+ landed phase kept in the state, for the reaper to clean up after the+ pass as the owner the row names."""+ _seed_submitted_pass(fake_dream_db)+ await write_phase_to_state(+ pass_id="p1",+ phase="consolidate",+ row=_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT),+ )+ entry = _entry(phase="recombine")+ entry.payload.update({"phase": "daydream", "user_id": "someone-else"})+ release_lock = AsyncMock(return_value=True)+ charge = AsyncMock()+ with (+ patch(+ "backend.copilot.dream.batch_deliveries.read_pass",+ AsyncMock(side_effect=TimeoutError("dream pass database stalled")),+ ),+ patch(+ "backend.copilot.dream.batch_deliveries.read_input_bundle",+ AsyncMock(side_effect=ConnectionError("redis down")),+ ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),+ patch("backend.copilot.dream.batch_costs.record_phase_cost", charge),+ ):+ await handle_dream_batch_result(entry, [])++ charge.assert_not_awaited()+ release_lock.assert_not_awaited()+ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.ERRORED+ assert row["cleanup_pending_at"] is not None+ assert state_key("p1") in fake_redis[1]@pytest.mark.asyncioasync def test_an_unreadable_state_still_closes_the_row_errored(@@ -1613,9 +1739,7 @@ class TestDreamPassRecord:patch("backend.copilot.inference.record.persist_and_record_usage", AsyncMock()),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", AsyncMock()- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", AsyncMock()),):with patch.object(fake_dream_db, "update_dream_pass", applying_held):first = asyncio.create_task(@@ -1695,10 +1819,12 @@ class TestDreamPassRecord:"p1","p2",}- # Each callback's two writes (its output, its next batch) and its- # stop check's read, each abandoned at the deadline; the check that- # got no answer let the chain go on.- assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (6, 6)+ # Each callback's read of its usage attribution, its two writes (its+ # output, its next batch), its stop check's read and the read of its+ # row for the lock token its bundle lacks, each abandoned at the+ # deadline; the reads that got no answer let the chain go on, the+ # attribution's under the batch's copy.+ assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (10, 10)@pytest.mark.asyncioasync def test_a_store_outage_never_fails_a_batch_pass(@@ -1722,9 +1848,7 @@ class TestDreamPassRecord:patch("backend.copilot.inference.record.persist_and_record_usage", AsyncMock()),- patch(- "backend.copilot.dream.batch_outcome.release_dream_lock", release_lock- ),+ patch("backend.copilot.dream.cleanup.release_dream_lock", release_lock),):await handle_dream_batch_result(_entry(phase="sanitize"),@@ -1751,7 +1875,9 @@ class TestAStoppedPass:"""A pass whose row was cancelled, or expired by a newer pass's guard,while its batch was in flight: the callback that lands next ends it beforeit chains the next phase or claims the apply gate, through ``fail_pass``,- after asking the provider to cancel the batch the row names."""+ after asking the provider to cancel the batch the row names. (A cancel+ asks for that itself as it closes the row; each test counts only the+ callback's own request.)"""@pytest.fixturedef anthropic(self):@@ -1850,6 +1976,7 @@ class TestAStoppedPass:):lock_key = await self._in_flight(fake_dream_db, fake_dream_redis, landed=())error = await stop()+ anthropic.messages.batches.cancel.reset_mock()submit_phase = AsyncMock()with (@@ -1873,17 +2000,24 @@ class TestAStoppedPass:await self._assert_ended(fake_dream_db, fake_dream_redis, lock_key, error)@pytest.mark.asyncio- async def test_an_unattributed_callback_ends_it_charging_nothing(- self, fake_dream_db, fake_dream_redis, anthropic, charges+ @pytest.mark.parametrize(+ "row_attribution", [True, False], ids=["row_kept_it", "row_kept_none"]+ )+ async def test_an_unattributed_callback_ends_it_charging_only_by_its_row(+ self, fake_dream_db, fake_dream_redis, anthropic, charges, row_attribution):"""A cancelled pass whose next callback carries no usage attribution- (#15208) ends as any failed pass ends, through ``fail_pass``, but- charges nothing, not even the phase that landed: it has no usage- context of its own to charge it under. The row stays as the cancel- closed it."""+ (#15208) ends as any failed pass ends, through ``fail_pass``. The+ phase that landed is charged once under the attribution the pass's+ row carries, the one it started under, and not at all when the row+ carries none. The batch the result came from is stopped too (the+ cancel stopped the one the row named), and the row stays as the+ cancel closed it."""lock_key = await self._in_flight(fake_dream_db, fake_dream_redis, landed=("consolidate",))+ if not row_attribution:+ fake_dream_db.rows["p1"]["cost_context"] = Noneawait _cancel_p1()entry = _entry(phase="recombine")del entry.payload["cost_context"]@@ -1893,8 +2027,14 @@ class TestAStoppedPass:entry, [_row(custom_id="p1:recombine", content=_RECOMBINE_CONTENT)])- charges.assert_not_awaited()- anthropic.messages.batches.cancel.assert_not_awaited()+ charged = [call.args[0].job.phase for call in charges.await_args_list]+ assert charged == (["consolidate"] if row_attribution else [])+ assert [+ c.args[0] for c in anthropic.messages.batches.cancel.await_args_list+ ] == [+ "msgbatch_live",+ "msgbatch_recombine",+ ]await self._assert_ended(fake_dream_db,fake_dream_redis,@@ -1912,6 +2052,7 @@ class TestAStoppedPass:is charged or cancelled twice."""await self._in_flight(fake_dream_db, fake_dream_redis, landed=())error = await _cancel_p1()+ anthropic.messages.batches.cancel.reset_mock()entry = _entry(phase="consolidate")rows = [_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT)]with (@@ -1943,6 +2084,7 @@ class TestAStoppedPass:fake_dream_db, fake_dream_redis, landed=("consolidate", "recombine"))error = await _cancel_p1()+ anthropic.messages.batches.cancel.reset_mock()apply = AsyncMock()with patch("backend.copilot.dream.apply.apply_operations", apply):@@ -2004,6 +2146,7 @@ class TestAStoppedPass:):lock_key = await self._in_flight(fake_dream_db, fake_dream_redis, landed=())error = await _cancel_p1()+ anthropic.messages.batches.cancel.reset_mock()anthropic.messages.batches.cancel.side_effect = RuntimeError("batch has already ended")@@ -2025,13 +2168,17 @@ class TestAStoppedPass:@pytest.mark.asyncio-async def test_callback_restores_original_trial_generation_for_late_result():+async def test_callback_restores_original_trial_generation_for_late_result(+ fake_dream_db,+):entry = _entry()entry.payload["cost_context"] = {"user_id": "u1","trial_id": "trial-original","generation": None,}+ # The pass's row, which the callback restores from, carries the same.+ fake_dream_db.rows["p1"]["cost_context"] = dict(entry.payload["cost_context"])seen = []async def handle(_entry, _rows):@@ -2118,3 +2265,97 @@ async def test_dispatch_invalid_attribution_closes_job_without_billing(submit.assert_not_called()apply.assert_not_called()capture.assert_not_called()+++async def _a_pass_with_consolidate_landed(fake_redis) -> None:+ """Pass ``p1`` of u1 at sanitize: its bundle, its lock, its admin job and+ consolidate landed in its state."""+ _, state_store, string_store = fake_redis+ now = datetime.now(timezone.utc)+ await persist_input_bundle(+ "p1",+ DreamInput(user_id="u1", group_id="user_u1", window_start=now, window_end=now),+ lock_token="tok-u1",+ )+ await write_initial_status(kind="dream_pass", job_id="j1", user_id="u1")+ string_store[MemoryScope.for_user("u1").redis_key("dream_lock")] = "tok-u1"+ state_store["dream:batch:state:p1"] = {+ "consolidate": json.dumps({"input_tokens": 10, "output_tokens": 20})+ }+++ "row_attribution",+ [None, {"user_id": "u1", "trial_id": "trial-other", "generation": None}],+ ids=["row_kept_none", "row_disagrees"],+)+async def test_a_result_its_row_does_not_attribute_ends_the_pass(+ fake_redis, fake_dream_db, row_attribution+):+ """The row is the source of the pass's attribution: a row that carries+ none (written before it kept one) or another than the batch's copy ends+ the pass unapplied. Its landed phase is charged only by the row: never+ when it carries none, and under the row's own when the copy differs."""+ await _a_pass_with_consolidate_landed(fake_redis)+ fake_dream_db.rows["p1"]["cost_context"] = row_attribution+ seen = []++ async def bill(ctx, usage):+ seen.append(get_trial_cost_context("u1"))++ with (+ patch("backend.copilot.dream.batch_costs.record_phase_cost", bill),+ patch(+ "backend.copilot.dream.cleanup.provider_batch_stopped",+ AsyncMock(return_value=True),+ ),+ patch("backend.copilot.dream.batch_callbacks.submit_phase") as submit,+ patch("backend.copilot.dream.apply.apply_operations") as apply,+ ):+ with pytest.raises(UsageActivationUnavailable):+ await handle_dream_batch_result(+ _entry(phase="sanitize"),+ [_row(custom_id="p1:sanitize", content=_SANITIZE_CONTENT)],+ )++ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.ERRORED+ assert row["error"] == "batch usage attribution unavailable"+ assert [c.trial_id if c else None for c in seen] == (+ [] if row_attribution is None else ["trial-other"]+ )+ submit.assert_not_called()+ apply.assert_not_called()++[email protected]("stopped", [False, True], ids=["running", "stopped"])+async def test_an_unattributed_result_stops_its_batch_before_the_mark_goes(+ fake_redis, fake_dream_db, stopped+):+ """The executor gave up waiting on a batch (its lifetime timeout) whose+ result carries no attribution: the pass ends, and its cleanup asks the+ provider to stop the batch the result came from. The row stays marked+ for the reaper until the provider says that batch has stopped."""+ await _a_pass_with_consolidate_landed(fake_redis)+ entry = _entry(phase="recombine")+ del entry.payload["cost_context"]+ stop = AsyncMock(return_value=stopped)+ charge = AsyncMock()++ with (+ patch("backend.copilot.dream.cleanup.provider_batch_stopped", stop),+ patch("backend.copilot.dream.batch_costs.record_phase_cost", charge),+ ):+ with pytest.raises(UsageActivationUnavailable):+ await handle_dream_batch_result(+ entry, [_row(custom_id="p1:recombine", content="", error="expired")]+ )++ stop.assert_awaited_once_with("msgbatch_recombine")+ charge.assert_awaited_once()+ assert charge.await_args.args[0].job.phase == "consolidate"+ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.ERRORED+ assert (row["cleanup_pending_at"] is None) is stoppeddiff --git a/autogpt_platform/backend/backend/copilot/dream/batch_costs.py b/autogpt_platform/backend/backend/copilot/dream/batch_costs.pyindex 480fef570b..eee884010c 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_costs.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_costs.py@@ -1,45 +1,105 @@"""What a batch dream pass cost: one PlatformCostLog row per phase that-landed, charged at most once whichever way the pass ends, and the usage-of those phases for the pass's DreamPass record."""+landed, each charged at most once however many times the pass's end or its+cleanup runs, and the usage of those phases for the pass's DreamPass record.++Every such charge goes by the usage attribution (#15208) the pass started+under, read off its row: batch settlement is deferred, made in another+process and possibly much later, so the row is the one place that knows it.+A sync pass charges each phase in-process as it lands+(``billing.record_phase_cost``), under the attribution it captured as it+started (``orchestrator``'s ``attributed_usage``), the same value its row+keeps; the row rule here is for batch settlement only."""from __future__ import annotations+import asyncioimport logging-from typing import Any+from typing import Any, Literal++from pydantic import BaseModelfrom backend.copilot.inference.context import InferenceContext, InferenceScopefrom backend.copilot.inference.routing import anthropic_batch_route+from backend.copilot.trial_cost_context import restore_cost_context-from .batch_state import claim_costs_logged_gate, read_state_or_none+from .batch_state import (+ claim_per_phase_charging,+ claim_phase_charge,+ read_state_or_none,+)from .billing import record_phase_costfrom .phase_jobs import PHASE_TIERS, phase_jobfrom .schemas import DreamPassUsage, DreamPhase+from .store import read_cost_contextfrom .usage import batch_pass_usage, landed_phase_usagelogger = logging.getLogger(__name__)+# How one landed phase's charge went: claimed by this call and charged, found+# with nothing to charge (an errored phase, no model, no usage), or its one+# attempt failed (logged, not retried); claimed by an earlier call; or+# unknown, its claim neither made nor refused, so the next cleanup tries.+PhaseCharge = Literal["charged", "nothing", "failed", "claimed", "unknown"]++# How long a pass's charge admits claims, from its start: a phase not+# claimed by then is left for the next cleanup, as is the pass whose gate+# check is still waiting on Redis (cutting that check claims nothing). A+# claim issued in time is never cut by the window.+CLAIM_WINDOW_SECONDS = 30.0+# How long one phase's claim, once issued, is waited for, the Redis client's+# retries included, as the cleanup waits on a state read. A claim Redis took+# whose reply has not come by then leaves its phase claimed and uncharged:+# the at-most-once of any Redis command whose reply is lost.+CLAIM_TIMEOUT_SECONDS = 5.0++# The passes' charges under way. Each runs on whatever happens to the caller+# that started it (``charge_landed_phases``), and is kept here until it ends.+_CHARGES_IN_FLIGHT: set[asyncio.Task[PhaseCharges]] = set()+++class PhaseCharges(BaseModel):+ """What charging a pass's landed phases did: the phases this call+ charged, and whether every landed phase is settled (charged, or its one+ attempt made, by this call or an earlier one)."""-async def log_all_phase_costs(+ charged: list[DreamPhase] = []+ settled: bool+++async def charge_landed_phases(*,user_id: str,expert_id: str | None,pass_id: str,state: dict[str, dict[str, Any]],phase_models: dict[str, str],-) -> None:- """One PlatformCostLog row per phase, on the ``anthropic_batch`` route.-- Idempotent via a Redis SETNX gate keyed on ``pass_id``: a pass logs its- costs from whichever terminal path it takes, success or failure, and a- repeated delivery of a finished batch must not charge it twice. The gate- is claimed once, before the loop, and never released, so each phase- gets at most one accounting attempt while the gate is retained.- Accounting is not transactional: the trial ledger and the weekly- counter can succeed before the cost-log write raises, so a failed- attempt may leave partial side effects, and it is not retried because- no later delivery gets past the gate; the other landed phases are- still charged. A partial failure under-accounts the pass rather than- risk a second attempt for a phase.+) -> PhaseCharges:+ """One PlatformCostLog row per phase landed in *state*, on the+ ``anthropic_batch`` route, each phase charged at most once whichever+ terminal path the pass takes and however often its cleanup runs.++ The charge (the pass's gate first, ``claim_per_phase_charging``, so a+ pass an earlier build charged whole is not charged again; then each+ landed phase claimed, ``claim_phase_charge``, and charged in turn) runs+ in a task of its own, which the caller's cancellation does not reach (a+ reaper run out of budget, the executor's bound on a drop hook): a cleanup+ that has read the state makes the charge even when it is cut while Redis+ answers slowly, and no cut lands between a claim and its charge. The task+ issues no claim ``CLAIM_WINDOW_SECONDS`` after it starts, and never cuts+ one it issued for the window (each waits up to ``CLAIM_TIMEOUT_SECONDS``+ of its own); a phase it claims is charged to its end. Each phase is+ claimed on its own,+ so a charge stopped part-way resumes with the rest: the claimed ones are+ skipped, the others charged. Not *settled* when the gate or a claim could+ not be made or checked: the next cleanup tries again.++ At most once, not exactly once. A process that dies between a phase's+ claim and its cost-log write leaves that phase uncharged, as does a claim+ Redis took whose reply was lost or outlasted ``CLAIM_TIMEOUT_SECONDS``;+ and a charge that fails part-way is not retried: the trial ledger and the weekly+ counter can move before the cost-log write raises, so a second attempt+ could charge them twice. Both under-account the pass rather than risk a+ second charge for a phase; the failure is logged, the crash is not.Each phase is recorded through ``billing.record_phase_cost`` like async phase, attributed to the pass's expert, and priced from its@@ -47,27 +107,74 @@ async def log_all_phase_costs(Anthropic's additive cache buckets, less the batch path's half-pricediscount.- No-ops on per-phase failure — apply already wrote the user-facing- memory operations; a cost-log blip shouldn't take that down.+ Every charge is made under the usage attribution the pass started under+ (#15208), read off its row (``store.read_cost_context``) whoever charges+ and however late: the callback, a failed pass's end, a closed pass's+ dropped batch, the reaper. Never under the attribution the account has+ now: a conversion after the pass started does not move what it already+ ran. A pass whose row carries no attribution is never charged, settled+ all the same; one whose row cannot be read is left for the next cleanup.+ This is the batch route's settlement; a sync pass charges in-process,+ under the attribution it captured at its start (see the module)."""- if not await claim_costs_logged_gate(pass_id):- logger.info(- "Skipping batch cost log for pass=%s — already charged",- pass_id,+ try:+ context = await read_cost_context(pass_id)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not read the usage attribution to "+ "charge it under; left for the next cleanup",+ exc_info=True,)- return+ return PhaseCharges(settled=False)+ if context is None or context.user_id != user_id:+ logger.warning(+ f"Dream pass {pass_id}: its record carries no usage attribution "+ f"for user {user_id[:12]}; its landed phases are not charged"+ )+ return PhaseCharges(settled=True)+ scope = InferenceScope(user_id=user_id, expert_id=expert_id)+ with restore_cost_context(user_id, context):+ # The task takes the attribution with it: it copies the context it+ # is created in, and goes on under it after this returns.+ task = asyncio.ensure_future(_charge_pass(scope, pass_id, state, phase_models))+ _CHARGES_IN_FLIGHT.add(task)+ task.add_done_callback(_CHARGES_IN_FLIGHT.discard)+ return await asyncio.shield(task)++async def _charge_pass(+ scope: InferenceScope,+ pass_id: str,+ state: dict[str, dict[str, Any]],+ phase_models: dict[str, str],+) -> PhaseCharges:+ """The gate, then each landed phase claimed and charged; admitting+ claims until ``CLAIM_WINDOW_SECONDS`` after it starts."""+ deadline = asyncio.get_running_loop().time() + CLAIM_WINDOW_SECONDS+ try:+ async with asyncio.timeout_at(deadline):+ per_phase = await claim_per_phase_charging(pass_id)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not claim its charging; left for the "+ "next cleanup",+ exc_info=True,+ )+ return PhaseCharges(settled=False)+ if not per_phase:+ logger.info(f"Dream pass {pass_id}: charged whole by an earlier build")+ return PhaseCharges(settled=True)+ charges: dict[DreamPhase, PhaseCharge] = {}for phase in PHASE_TIERS:row = state.get(phase)- if row is None:- continue- try:- scope = InferenceScope(user_id=user_id, expert_id=expert_id)- await _log_phase_cost(scope, pass_id, phase, row, phase_models)- except Exception:- logger.exception(- "Failed to log batch cost for pass=%s phase=%s", pass_id, phase+ if row is not None:+ charges[phase] = await _claim_and_charge(+ scope, pass_id, phase, row, phase_models, deadline)+ return PhaseCharges(+ charged=[phase for phase, how in charges.items() if how == "charged"],+ settled="unknown" not in charges.values(),+ )async def recorded_usage(@@ -98,15 +205,62 @@ def landed_usage(return None+async def _claim_and_charge(+ scope: InferenceScope,+ pass_id: str,+ phase: DreamPhase,+ row: dict[str, Any],+ phase_models: dict[str, str],+ deadline: float,+) -> PhaseCharge:+ """Claim one phase, then charge it. The claim is issued only before+ *deadline* (loop time), the charge's claim window, else the phase is left+ for the next cleanup; once issued it is waited for up to+ ``CLAIM_TIMEOUT_SECONDS`` whatever the window, which must not cut a claim+ Redis may have taken."""+ if asyncio.get_running_loop().time() >= deadline:+ logger.warning(+ f"Dream pass {pass_id}: its charge stopped claiming before its "+ f"{phase} phase; left for the next cleanup"+ )+ return "unknown"+ try:+ async with asyncio.timeout(CLAIM_TIMEOUT_SECONDS):+ claimed = await claim_phase_charge(pass_id, phase)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not claim the charge of its {phase} "+ "phase; left for the next cleanup",+ exc_info=True,+ )+ return "unknown"+ if not claimed:+ return "claimed"+ try:+ if not await _log_phase_cost(scope, pass_id, phase, row, phase_models):+ return "nothing"+ except Exception:+ # Apply already wrote the user-facing memory operations; a cost-log+ # blip shouldn't take that down, and a retry could charge twice.+ logger.exception(+ "Failed to log batch cost for pass=%s phase=%s; not retried",+ pass_id,+ phase,+ )+ return "failed"+ return "charged"++async def _log_phase_cost(scope: InferenceScope,pass_id: str,phase: DreamPhase,row: dict[str, Any],phase_models: dict[str, str],-) -> None:+) -> bool:+ """Charge one landed phase; ``False`` when there is nothing to charge."""if row.get("error"):- return+ return Falsephase_model = phase_models.get(phase)if not phase_model:logger.warning(@@ -114,13 +268,14 @@ async def _log_phase_cost(pass_id,phase,)- return+ return Falseusage = landed_phase_usage(row, phase_model)if usage is None:- return+ return Falsectx = InferenceContext(scope=scope,job=phase_job(phase, pass_id, timeout_seconds=None, pinned_model=phase_model),route=anthropic_batch_route(phase_model),)await record_phase_cost(ctx, usage)+ return Truediff --git a/autogpt_platform/backend/backend/copilot/dream/batch_costs_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_costs_test.pynew file mode 100644index 0000000000..3258e78afa--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/batch_costs_test.py@@ -0,0 +1,443 @@+"""Charging a batch pass's landed phases, over the in-memory Redis and store:+each phase is claimed on its own right before it is charged, so a cleanup cut+short resumes with the phases it did not reach and never charges one twice+(Codex's cuts: after the pass is claimed, after its first phase); a charge+the cut lands in finishes on its own; a charge that fails is not tried again;+a pass an earlier build charged whole is not charged again, and one charged+phase by phase keeps an earlier build out. The one gap left, at most once+over exactly once: a process that dies between a phase's claim and its+charge leaves that phase uncharged. The provider and the cost log are+stubbed at their edges."""++import asyncio+import logging+from unittest.mock import AsyncMock++import pytest+from prisma.enums import DreamPassRoute, DreamPassStatus, SubscriptionTier++from backend.copilot.trial_cost_context import TrialCostContext, get_trial_cost_context+from backend.data.pro_activation import UsageActivationState+from backend.util.llm.providers import BatchResultRow++from . import batch_costs as costs_mod+from . import cleanup as cleanup_mod+from . import reaper as reaper_mod+from .batch_costs import charge_landed_phases+from .batch_state import read_state, state_key, write_phase_to_state+from .reaper import reap_expired_passes+from .reaper_cleanup_test import _scale_the_budget+from .reaper_test import _charged, _dead_batch_pass, _seed++_MODELS = {p: "claude-sonnet-5" for p in ("consolidate", "recombine", "sanitize")}+_PASS_KEY = "dream:batch:costs_logged:p1"++[email protected](autouse=True)+def provider(mocker) -> AsyncMock:+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key", return_value="k"+ )+ mocker.patch.object(reaper_mod, "phase_models_for_config", return_value=_MODELS)+ return mocker.patch(+ "backend.copilot.dream.provider_batch.cancel_batch",+ AsyncMock(return_value=True),+ )++[email protected](autouse=True)+def charges(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )++[email protected](autouse=True)+def pass_row(fake_dream_db) -> None:+ """The charged pass's row, live, carrying the usage attribution every+ charge of it goes by (``store.read_cost_context``); a test about a dead+ pass seeds its own over it."""+ _seed(+ fake_dream_db,+ "p1",+ route=DreamPassRoute.ANTHROPIC_BATCH,+ status=DreamPassStatus.SUBMITTED,+ lapsed=-3600,+ )+++class TestTheAttributionAChargeGoesBy:+ """Every charge of a pass goes by the usage attribution its row carries,+ whoever makes it and whatever the account's own reads now; a row without+ one is never charged."""++ async def test_is_the_one_the_pass_started_under(+ self, mocker, fake_dream_db, fake_dream_redis, usage_snapshot+ ):+ trial = TrialCostContext(user_id="u1", trial_id="trial-1", generation=None)+ fake_dream_db.rows["p1"]["cost_context"] = trial.model_dump(mode="json")+ usage_snapshot.side_effect = lambda user_id: UsageActivationState(+ user_id=user_id, generation="paid-1", tier=SubscriptionTier.PRO, ready=True+ )+ seen: list[TrialCostContext | None] = []++ async def observe(ctx, usage):+ seen.append(get_trial_cost_context("u1"))++ mocker.patch.object(costs_mod, "record_phase_cost", observe)++ charged = await _charge(await _landed_state())++ assert charged.settled and charged.charged == ["consolidate", "recombine"]+ assert seen == [trial, trial]+ assert get_trial_cost_context("u1") is None+ usage_snapshot.assert_not_awaited()++ @pytest.mark.parametrize(+ "kept",+ [None, {"user_id": "someone-else", "trial_id": None}, {"trial_id": 7}],+ ids=["none", "another_users", "unparseable"],+ )+ async def test_a_row_without_a_usable_one_is_never_charged(+ self, fake_dream_db, fake_dream_redis, charges, kept+ ):+ fake_dream_db.rows["p1"]["cost_context"] = kept++ charged = await _charge(await _landed_state())++ assert charged.settled and charged.charged == []+ charges.assert_not_awaited()+ assert _PASS_KEY not in fake_dream_redis.store++ async def test_a_row_that_cannot_be_read_leaves_the_charge_for_later(+ self, fake_dream_db, fake_dream_redis, charges+ ):+ fake_dream_db.fail = True++ charged = await _charge(await _landed_state())++ assert not charged.settled+ charges.assert_not_awaited()+ assert _PASS_KEY not in fake_dream_redis.store+++class TestACleanupCutShortInItsCharge:+ async def test_after_the_pass_is_claimed_the_next_run_charges_every_phase(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """The run is cut while the pass's gate, set, never answers; the+ charge goes on until its claim window closes, and gives up there,+ having claimed no phase."""+ _scale_the_budget(mocker, budget=0.3, release=0.05)+ mocker.patch.object(costs_mod, "CLAIM_WINDOW_SECONDS", 0.6)+ await _two_landed(fake_dream_db)+ claim = costs_mod.claim_per_phase_charging++ async def claimed_then_cut(pass_id: str) -> bool:+ await claim(pass_id)+ await asyncio.Event().wait()+ return True++ mocker.patch.object(costs_mod, "claim_per_phase_charging", claimed_then_cut)++ first = await asyncio.wait_for(reap_expired_passes(), 5)++ assert first.outcomes == {"out_of_budget": 1}+ assert costs_mod._CHARGES_IN_FLIGHT, "the charge outlives the run"+ await asyncio.wait_for(asyncio.gather(*costs_mod._CHARGES_IN_FLIGHT), 5)+ assert fake_dream_redis.store[_PASS_KEY] == "per_phase"+ charges.assert_not_awaited()+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is not None++ mocker.patch.object(costs_mod, "claim_per_phase_charging", claim)+ second = await reap_expired_passes()++ assert second.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate", "recombine"]+ assert state_key("p1") not in fake_dream_redis.hashes++ async def test_after_its_first_phase_the_next_run_charges_the_second_only(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """The claim window closes while the first phase's cost write runs:+ that phase is charged, the second's claim is given up, and the next+ run charges the second only."""+ _scale_the_budget(mocker, budget=0.3, release=0.05)+ mocker.patch.object(costs_mod, "CLAIM_WINDOW_SECONDS", 0.05)+ await _two_landed(fake_dream_db)+ log = costs_mod._log_phase_cost++ async def slow_consolidate(scope, pass_id, phase, row, models) -> bool:+ if phase == "consolidate":+ await asyncio.sleep(0.1)+ return await log(scope, pass_id, phase, row, models)++ mocker.patch.object(costs_mod, "_log_phase_cost", slow_consolidate)++ first = await asyncio.wait_for(reap_expired_passes(), 5)++ assert first.outcomes == {"retry": 1}+ assert _charged(charges) == ["consolidate"]+ assert "dream:batch:charged:p1:recombine" not in fake_dream_redis.store++ mocker.patch.object(costs_mod, "_log_phase_cost", log)+ second = await reap_expired_passes()++ assert second.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate", "recombine"]+ assert state_key("p1") not in fake_dream_redis.hashes++ async def test_a_charge_the_cut_lands_in_finishes_on_its_own(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Codex's probe cut the run inside a phase's cost write: the write+ runs in a task of its own, so it finishes after the run has ended,+ and the next run finds the phase charged."""+ _scale_the_budget(mocker, budget=0.3, release=0.05)+ await _two_landed(fake_dream_db)+ log = costs_mod._log_phase_cost++ async def slow_recombine(scope, pass_id, phase, row, models) -> bool:+ if phase == "recombine":+ await asyncio.sleep(0.5)+ return await log(scope, pass_id, phase, row, models)++ mocker.patch.object(costs_mod, "_log_phase_cost", slow_recombine)++ first = await asyncio.wait_for(reap_expired_passes(), 5)++ assert first.outcomes == {"out_of_budget": 1}+ assert _charged(charges) == ["consolidate"]+ await asyncio.wait_for(asyncio.gather(*costs_mod._CHARGES_IN_FLIGHT), 5)+ assert _charged(charges) == ["consolidate", "recombine"]++ second = await reap_expired_passes()++ assert second.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate", "recombine"]++ async def test_by_its_own_bound_the_claimed_phase_is_charged_on_its_own(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """A cost write that stalls keeps the cleanup no longer than its+ charge step's bound: the step is left unfinished (the state kept for+ it), the write lands on its own once it can, and the next run finds+ the phase charged and finishes."""+ mocker.patch.object(cleanup_mod, "CHARGE_STEP_TIMEOUT_SECONDS", 0.05)+ await _dead_batch_pass(fake_dream_db)+ stalled = asyncio.Event()++ async def stalls(*_args, **_kwargs) -> None:+ await stalled.wait()++ charges.side_effect = stalls++ first = await asyncio.wait_for(reap_expired_passes(), 5)++ assert first.outcomes == {"retry": 1}+ assert state_key("p1") in fake_dream_redis.hashes+ stalled.set()+ await asyncio.wait_for(asyncio.gather(*costs_mod._CHARGES_IN_FLIGHT), 5)++ second = await reap_expired_passes()++ assert second.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes++ async def test_a_crash_between_a_claim_and_its_charge_loses_that_phase(+ self, fake_dream_db, fake_dream_redis, charges+ ):+ """The gap left, written down: a process that died right after+ claiming recombine never charged it, and the claim keeps every later+ cleanup from charging it too."""+ await _two_landed(fake_dream_db)+ fake_dream_redis.store[_PASS_KEY] = "per_phase"+ fake_dream_redis.store["dream:batch:charged:p1:recombine"] = "1"++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ assert _charged(charges) == ["consolidate"]+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None++ async def test_a_claim_issued_in_time_is_charged_when_its_reply_comes_late(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Codex's probe: the gate answers late in the claim window, so the+ phase's claim goes out in time; Redis takes it, and its reply comes+ after the window has closed. The window only admits claims, so the+ claim is waited for and the phase charged; the next run finds it+ charged and finishes."""+ mocker.patch.object(costs_mod, "CLAIM_WINDOW_SECONDS", 0.3)+ mocker.patch.object(costs_mod, "CLAIM_TIMEOUT_SECONDS", 0.5)+ mocker.patch.object(cleanup_mod, "CHARGE_STEP_TIMEOUT_SECONDS", 0.1)+ await _dead_batch_pass(fake_dream_db)+ gate = costs_mod.claim_per_phase_charging++ async def slow_gate(pass_id: str) -> bool:+ await asyncio.sleep(0.2)+ return await gate(pass_id)++ claim = "dream:batch:charged:p1:consolidate"+ set_ = fake_dream_redis.set+ late: list[str] = []++ async def late_reply(key: str, *args, **kwargs):+ taken = await set_(key, *args, **kwargs)+ if key == claim and taken:+ await asyncio.sleep(0.15)+ late.append(key)+ return taken++ mocker.patch.object(costs_mod, "claim_per_phase_charging", slow_gate)+ mocker.patch.object(fake_dream_redis, "set", late_reply)++ first = await asyncio.wait_for(reap_expired_passes(), 5)+ await asyncio.wait_for(asyncio.gather(*costs_mod._CHARGES_IN_FLIGHT), 5)++ assert first.outcomes == {"retry": 1}+ assert late == [claim]+ assert _charged(charges) == ["consolidate"]++ mocker.patch.object(costs_mod, "claim_per_phase_charging", gate)+ second = await reap_expired_passes()++ assert second.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate"]+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None+++class TestAPhaseCharge:+ @pytest.mark.parametrize(+ "error", ["max_tokens", "invalid output shape", "provider error"]+ )+ async def test_failed_model_phases_are_absorbed_when_the_reaper_settles(+ self, fake_dream_db, fake_dream_redis, charges, error+ ):+ await _dead_batch_pass(fake_dream_db)+ await write_phase_to_state(+ pass_id="p1",+ phase="recombine",+ row=BatchResultRow(+ custom_id="p1_recombine",+ content="{}",+ input_tokens=30,+ output_tokens=40,+ error=error,+ ),+ model=_MODELS["recombine"],+ )++ await reap_expired_passes()+ await reap_expired_passes()++ assert _charged(charges) == ["consolidate"]+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None+ assert state_key("p1") not in fake_dream_redis.hashes++ async def test_is_made_once_whoever_asks_and_however_often(+ self, fake_dream_redis, charges+ ):+ state = await _landed_state()++ first = await _charge(state)+ again = await _charge(state)++ assert (first.charged, first.settled) == (["consolidate", "recombine"], True)+ assert (again.charged, again.settled) == ([], True)+ assert _charged(charges) == ["consolidate", "recombine"]++ async def test_that_fails_is_not_tried_again(+ self, fake_dream_redis, charges, caplog+ ):+ """The trial ledger or the weekly counter may have moved before the+ cost-log write raised; a second attempt could charge them twice."""+ state = await _landed_state()+ charges.side_effect = [RuntimeError("cost log down"), None]++ with caplog.at_level(logging.ERROR):+ first = await _charge(state)+ again = await _charge(state)++ assert (first.charged, first.settled) == (["recombine"], True)+ assert (again.charged, again.settled) == ([], True)+ assert charges.await_count == 2+ assert "Failed to log batch cost for pass=p1 phase=consolidate" in caplog.text++ async def test_whose_claim_cannot_be_made_is_left_for_the_next_cleanup(+ self, monkeypatch, fake_dream_redis, charges+ ):+ state = await _landed_state()+ claim = costs_mod.claim_phase_charge++ async def down_for_recombine(pass_id: str, phase: str) -> bool:+ if phase == "recombine":+ raise ConnectionError("redis down")+ return await claim(pass_id, phase)++ monkeypatch.setattr(costs_mod, "claim_phase_charge", down_for_recombine)+ first = await _charge(state)+ monkeypatch.setattr(costs_mod, "claim_phase_charge", claim)+ again = await _charge(state)++ assert (first.charged, first.settled) == (["consolidate"], False)+ assert (again.charged, again.settled) == (["recombine"], True)+ assert _charged(charges) == ["consolidate", "recombine"]+++class TestTwoBuildsSideBySide:+ async def test_a_pass_an_earlier_build_charged_whole_is_not_charged_again(+ self, fake_dream_redis, charges+ ):+ fake_dream_redis.store[_PASS_KEY] = "1"++ charged = await _charge(await _landed_state())++ assert (charged.charged, charged.settled) == ([], True)+ charges.assert_not_awaited()++ async def test_an_earlier_build_finds_a_pass_charged_phase_by_phase_taken(+ self, fake_dream_redis, charges+ ):+ """An earlier build claims the pass-level key with SETNX before it+ charges every phase: this build's claim of it, under its own value,+ makes that SETNX fail, so the earlier build charges nothing."""+ await _charge(await _landed_state())++ earlier_build = await fake_dream_redis.set(_PASS_KEY, "1", nx=True)++ assert earlier_build is None+ assert fake_dream_redis.store[_PASS_KEY] == "per_phase"+ assert charges.await_count == 2+++async def _two_landed(fake_dream_db) -> None:+ """A dead batch pass with consolidate and recombine in its state."""+ await _dead_batch_pass(fake_dream_db)+ await write_phase_to_state(+ pass_id="p1",+ phase="recombine",+ row=BatchResultRow(+ custom_id="p1_recombine", content="{}", input_tokens=30, output_tokens=40+ ),+ )+++async def _landed_state() -> dict:+ """The state of a pass with consolidate and recombine landed."""+ for phase in ("consolidate", "recombine"):+ await write_phase_to_state(+ pass_id="p1",+ phase=phase,+ row=BatchResultRow(+ custom_id=f"p1_{phase}", content="{}", input_tokens=10, output_tokens=20+ ),+ )+ return await read_state("p1")+++async def _charge(state: dict) -> costs_mod.PhaseCharges:+ return await charge_landed_phases(+ user_id="u1", expert_id=None, pass_id="p1", state=state, phase_models=_MODELS+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_deliveries.py b/autogpt_platform/backend/backend/copilot/dream/batch_deliveries.pynew file mode 100644index 0000000000..eac53c923c--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/batch_deliveries.py@@ -0,0 +1,264 @@+"""The dream batch deliveries that never run the phase chain.++The BatchExecutor hands each finished dream batch to ``batch_callbacks``'s+phase handler, which lands the phase and chains the next one or applies.+Three kinds of delivery end the pass instead, each the way ``fail_pass`` ends+one: the admin job closed, the row closed and marked for the cleanup after+the pass, then that cleanup (``cleanup.py``): the landed phases charged once+each, the lock released by compare-and-delete, the batch state and bundle+deleted, the mark cleared once every step has finished.++ * A batch whose pass has already ended: cancelled or expired while the+ batch was in flight, or closed any other way, and marked by whatever+ closed it. Before the executor polls it, it asks ``should_dispatch``,+ which only reads the pass's row, bounded: a closed row means nobody+ waits for the batch, and the executor drops it, claimed off its queue+ before anything else happens. The walker whose claim took it then runs+ ``drop_closed``: the batch stopped at the provider and the pass ended+ out, nothing chained, nothing applied, nothing charged beyond the phases+ that landed; a walker that lost the claim does nothing. A drop hook that+ fails, times out or never runs leaves the row marked, and the reaper+ finishes the cleanup. An open row, a missing one or one the store cannot+ read in time dispatches as before, and the phase handler keeps its own+ stop checks.+ * A payload no phase handler can take (``end_dead_end``).+ * A repeat of a delivery whose apply already claimed the gate+ (``finish_duplicate``): apply is never run twice, and the row, which a+ first delivery that died after its claim left APPLYING, is closed+ EXPIRED rather than left for the reaper. A claimed gate proves neither+ that apply ran nor that its writes landed: the first delivery may have+ died before, during or after them.+"""++from __future__ import annotations++import logging++from backend.data.dream_pass_models import OPEN_STATUSES, DreamPassRecord+from backend.executor.batch_executor import PendingEntry++from . import job_status+from .batch_costs import recorded_usage+from .batch_outcome import BatchPass, clean_up_after, fail_pass, mark_job_errored+from .batch_submit import read_input_bundle+from .fetch import DreamInput+from .pass_record import stop_error+from .provider_batch import provider_batch_stopped+from .schemas import DreamOperations, DreamPassResult, IngestionDrainStatus+from .store import read_pass, record_batch_failed, record_expired++logger = logging.getLogger(__name__)++# The check's row read, shorter than a record write's deadline: the executor+# walks its queue serially and asks at every poll.+DISPATCH_CHECK_READ_TIMEOUT_SECONDS = 3.0++# How a pass whose last batch came back twice closes: the first delivery+# claimed apply and never finished.+DUPLICATE_ERROR = (+ "apply: an earlier delivery claimed apply and never finished; closed "+ "without applying again, and what it wrote may have landed"+)+++async def should_dispatch(entry: PendingEntry) -> bool:+ """The executor's check before it polls one of a dream pass's batches:+ ``False`` once the pass's row has closed, ``True`` otherwise. It only+ reads; ending the pass is ``drop_closed``'s, once the drop is claimed."""+ bp = BatchPass.from_payload(entry.payload or {})+ if not bp.pass_id:+ return True+ row = await _row(bp.pass_id)+ return row is None or row.status in OPEN_STATUSES+++async def drop_closed(entry: PendingEntry) -> None:+ """The executor's drop hook, run only by the walker whose claim took a+ closed pass's batch off the queue: the pass ended out as the owner a+ record of it names (``_owner_of``), its batch stopped at the provider as+ part of the cleanup. The row refuses the failure ``fail_pass`` writes:+ it keeps how it closed. With no owner to go by, the batch is still+ stopped by its id (a pass that took back a batch whose stop the provider+ did not confirm leaves it here, its bundle gone); the rest is the+ reaper's, from the closed row."""+ bp = BatchPass.from_payload(entry.payload or {})+ row, owner = await _owner_of(bp)+ reason = "pass already closed"+ if row is not None:+ reason = stop_error(row) or f"pass already {row.status.value.lower()}"+ logger.info(+ f"Dream batch {entry.provider_batch_id} of pass {bp.pass_id} dropped "+ f"unpolled: {reason}"+ )+ if owner is not None:+ await fail_pass(owner, reason, provider_batch_id=entry.provider_batch_id)+ return+ if not await provider_batch_stopped(entry.provider_batch_id):+ logger.warning(+ f"Dream batch {entry.provider_batch_id} of pass {bp.pass_id}: its "+ "stop is not confirmed and nothing names its owner"+ )+++async def end_dead_end(bp: BatchPass, error: str) -> None:+ """A payload no phase handler can take ends its pass as the owner a+ record of it names (``_owner_of``), like any failure: the payload is the+ part that went wrong, so the owner it names is never the one charged, nor+ whose lock is released. With no owner to go by, the admin job closes and+ so does the record, marked, nothing charged, unlocked or deleted: the+ reaper cleans up after the pass from its row, which names the owner to+ charge and keeps the token to release the lock with. Without a pass+ there is nothing more to end."""+ owner = (await _owner_of(bp))[1] if bp.pass_id else None+ if owner is not None:+ await fail_pass(owner, error)+ return+ await mark_job_errored(bp.job_id, error, dead_end=True)+ if bp.pass_id:+ usage = await recorded_usage(bp.pass_id, bp.phase_models)+ await record_batch_failed(bp.pass_id, error, usage)+++async def finish_duplicate(bp: BatchPass, ops: DreamOperations) -> None:+ """A repeat of a delivery that already claimed the pass's apply: apply is+ skipped and the first delivery's results kept. That delivery normally+ closed the job and the row; one that died after its claim left both+ open, so the job is finalized with the attempted counts and the row+ closed EXPIRED (``DUPLICATE_ERROR``), marked. Then the cleanup after it,+ as after any end (``clean_up_after``): the landed phases charged once,+ the lock released, the state and bundle deleted."""+ logger.info(+ "Duplicate dispatch for pass=%s — apply already claimed, not run "+ "again; preserving the first delivery's job result",+ bp.pass_id,+ )+ await _finalize_stuck_duplicate(bp, ops)+ await record_expired(bp.pass_id, DUPLICATE_ERROR)+ await clean_up_after(bp)+++async def _owner_of(+ bp: BatchPass,+) -> tuple[DreamPassRecord | None, BatchPass | None]:+ """The ended pass's row, when it can be read, and *bp* as the owner a+ record of the pass names: its row, else its input bundle, the scope the+ phase handler checks a payload against. ``None`` for the owner when+ neither can be read: a batch's payload is only a copy, never enough to+ charge an account or release its lock."""+ row = await _row_again(bp.pass_id)+ if row is not None:+ return row, _as_owner(bp, row.user_id, row.expert_id, "its row")+ bundle = await _input_bundle(bp.pass_id)+ if bundle is not None:+ return None, _as_owner(bp, bundle.user_id, bundle.expert_id, "its input bundle")+ logger.warning(+ f"Dream pass {bp.pass_id}: neither its row nor its input bundle could be "+ "read; nothing charged or released on its batch's payload alone"+ )+ return None, None+++async def _row_again(pass_id: str) -> DreamPassRecord | None:+ """The ended pass's row again; ``None`` when there is none or the store+ cannot say in time."""+ try:+ return await read_pass(pass_id, timeout=DISPATCH_CHECK_READ_TIMEOUT_SECONDS)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not read its row again to end it",+ exc_info=True,+ )+ return None+++async def _input_bundle(pass_id: str) -> DreamInput | None:+ """The ended pass's input bundle; ``None`` when there is none or Redis+ cannot give it."""+ try:+ return await read_input_bundle(pass_id)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not read its input bundle to end it",+ exc_info=True,+ )+ return None+++def _as_owner(+ bp: BatchPass, user_id: str, expert_id: str | None, source: str+) -> BatchPass:+ """*bp* with the owner *source* names: a record of the pass is the+ authority on whose pass it is, a batch's payload only a copy."""+ if bp.user_id and (bp.user_id, bp.expert_id) != (user_id, expert_id):+ logger.warning(+ f"Dream pass {bp.pass_id}: its batch's payload names another owner "+ f"than {source}; ending it as {source} says"+ )+ return bp.model_copy(update={"user_id": user_id, "expert_id": expert_id})+++async def _row(pass_id: str) -> DreamPassRecord | None:+ """The pass's row, or ``None`` when there is none or the store cannot say+ in time, and the batch is dispatched as before."""+ try:+ return await read_pass(pass_id, timeout=DISPATCH_CHECK_READ_TIMEOUT_SECONDS)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not read its row before polling its "+ "batch; dispatching as before",+ exc_info=True,+ )+ return None+++async def _finalize_stuck_duplicate(bp: BatchPass, ops: DreamOperations) -> None:+ """On a duplicate delivery, finalize the admin job row iff the first+ delivery crashed between apply and ``mark_complete`` and left it+ non-terminal. An already-terminal row is left untouched so the first+ delivery's real apply stats are never overwritten.++ Best-effort: a status read/write failure here must not crash the+ duplicate tail (lock release + cleanup still need to run)."""+ if not bp.job_id:+ return+ try:+ existing = await job_status.read_status(kind="dream_pass", job_id=bp.job_id)+ if existing is None or existing.state in ("complete", "errored"):+ return+ logger.warning(+ "Duplicate dispatch found job %s stuck in state=%s — "+ "finalizing with the clamped op counts",+ bp.job_id[:12],+ existing.state,+ )+ await job_status.mark_complete(+ kind="dream_pass", job_id=bp.job_id, result=_attempted_result(bp, ops)+ )+ except Exception:+ logger.exception(+ "Failed to finalize stuck duplicate for job %s", bp.job_id[:12]+ )+++def _attempted_result(bp: BatchPass, ops: DreamOperations) -> DreamPassResult:+ """The first delivery's per-edge outcomes (and dream session id) died+ with it, so the counts here are the clamped *attempted* ops — annotated+ so the admin UI doesn't present them as confirmed apply results."""+ note = (+ "[finalized after duplicate delivery — counts reflect attempted "+ "operations; the original delivery may or may not have written them] "+ )+ return DreamPassResult(+ user_id=bp.user_id,+ pass_id=bp.pass_id,+ execution_path="anthropic_batch",+ consolidated_count=len(ops.writes),+ proposal_count=len(ops.proposals),+ demotion_count=len(ops.demotions),+ entity_invalidation_count=len(ops.entity_invalidations),+ # Batch apply never drains in-line by design; the first+ # delivery's writes (if any) landed fire-and-forget. ``skipped``+ # marks this as a healthy by-design skip, NOT a drain failure.+ ingestion_drain_status=IngestionDrainStatus.skipped,+ summary_for_user=note + (ops.summary_for_user or ""),+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_deliveries_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_deliveries_test.pynew file mode 100644index 0000000000..db936444f9--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/batch_deliveries_test.py@@ -0,0 +1,485 @@+"""The dream batch deliveries that never run the phase chain, over the+in-memory store and Redis: the executor's check that drops a closed pass's+batch before it is polled and the drop that ends the pass (two walkers at+once included), a dead-end payload, and a duplicate of the last phase. The+provider and the charges are stubbed at their edges."""++import asyncio+import logging+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)+from pydantic import BaseModel++from backend.copilot.graphiti.scope import MemoryScope+from backend.data.dream_pass_models import DreamPassDraft+from backend.data.redis_helpers import _CLAIM_BATCH_DISPATCH_LUA+from backend.executor import batch_executor as executor+from backend.executor.batch_executor import PendingEntry+from backend.util.llm.providers import BatchResultRow++from . import batch_deliveries as deliveries_mod+from . import job_status+from . import reaper as reaper_mod+from .batch_callbacks import handle_dream_batch_result+from .batch_deliveries import DUPLICATE_ERROR, drop_closed, should_dispatch+from .batch_state import state_key, write_phase_to_state+from .batch_submit import input_bundle_key, persist_input_bundle+from .cancel import cancel_dream_pass+from .fetch import DreamInput+from .pass_record import expired+from .reaper import REAP_GRACE_SECONDS, reap_expired_passes+from .schemas import (+ ConsolidationOutput,+ DreamOperations,+ DreamPhase,+ RecombinationOutput,+)+from .store import write_stop++_SCOPE = MemoryScope.for_user("u1")+_LOCK_KEY = _SCOPE.redis_key("dream_lock")+_MODELS = {p: "claude-sonnet-5" for p in ("consolidate", "recombine", "sanitize")}++[email protected](autouse=True)+def provider_cancel(mocker) -> AsyncMock:+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key", return_value="k"+ )+ return mocker.patch(+ "backend.copilot.dream.provider_batch.cancel_batch",+ AsyncMock(return_value=True),+ )++[email protected](autouse=True)+def charges(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )+++class TestShouldDispatch:+ async def test_a_cancelled_pass_is_dropped_and_its_drop_ends_it(+ self, fake_dream_db, fake_dream_redis, provider_cancel, charges+ ):+ """Cancelled while recombine's batch was in flight: the executor's+ check only says not to poll it; the drop then cancels that batch,+ charges the phase that landed, releases the lock and cleans up."""+ await _in_flight(fake_dream_db, fake_dream_redis)+ assert (await cancel_dream_pass("p1", user_id="u1", reason="testing")).cancelled+ provider_cancel.reset_mock()++ assert await should_dispatch(_entry("recombine")) is False+ provider_cancel.assert_not_awaited()+ charges.assert_not_awaited()+ assert state_key("p1") in fake_dream_redis.hashes++ await drop_closed(_entry("recombine"))++ provider_cancel.assert_awaited_once()+ assert provider_cancel.await_args.kwargs["provider_batch_id"] == "b-rec"+ assert _charged(charges) == ["consolidate"]+ assert _LOCK_KEY not in fake_dream_redis.store+ assert state_key("p1") not in fake_dream_redis.hashes+ assert input_bundle_key("p1") not in fake_dream_redis.store+ status = await job_status.read_status(kind="dream_pass", job_id="j1")+ assert status is not None+ assert (status.state, status.error) == ("errored", "cancelled: testing")+ # The job keeps what the landed phase used, which the closed row+ # refused.+ assert status.result is not None+ phases = status.result["usage"]["phases"]+ assert [phase["phase"] for phase in phases] == ["consolidate"]+ row = fake_dream_db.rows["p1"]+ assert (row["status"], row["error"]) == (DreamPassStatus.CANCELLED, "testing")+ assert row.get("usage") is None+ # Every step finished, so the drop cleared the mark the cancel set.+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)++ async def test_a_closed_pass_with_no_owner_on_record_still_has_its_batch_stopped(+ self, mocker, fake_dream_db, fake_dream_redis, provider_cancel, charges+ ):+ """The batch a take-back left on the queue (its stop unconfirmed,+ its row refusing it): neither the row nor the bundle can be read+ when the drop runs, so nobody is charged or unlocked on the payload's+ word, but the batch is still asked to stop, by its id."""+ await _in_flight(fake_dream_db, fake_dream_redis)+ assert (await cancel_dream_pass("p1", user_id="u1", reason="testing")).cancelled+ provider_cancel.reset_mock()+ mocker.patch.object(+ deliveries_mod,+ "read_pass",+ AsyncMock(side_effect=TimeoutError("dream pass database stalled")),+ )+ mocker.patch.object(+ deliveries_mod,+ "read_input_bundle",+ AsyncMock(side_effect=ConnectionError("redis down")),+ )++ await drop_closed(_entry("recombine"))++ provider_cancel.assert_awaited_once()+ assert provider_cancel.await_args.kwargs["provider_batch_id"] == "b-rec"+ charges.assert_not_awaited()++ async def test_an_expired_pass_whose_lock_a_newer_pass_holds_keeps_that_lock(+ self, fake_dream_db, fake_dream_redis+ ):+ await _in_flight(fake_dream_db, fake_dream_redis)+ assert await write_stop("p1", expired("stale", not_updated_since=None))+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"++ assert await should_dispatch(_entry("recombine")) is False+ await drop_closed(_entry("recombine"))++ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.EXPIRED++ async def test_an_open_pass_is_dispatched_untouched(+ self, fake_dream_db, fake_dream_redis, provider_cancel, charges+ ):+ await _in_flight(fake_dream_db, fake_dream_redis)++ assert await should_dispatch(_entry("recombine")) is True++ provider_cancel.assert_not_awaited()+ charges.assert_not_awaited()+ assert fake_dream_redis.store[_LOCK_KEY] == "tok"+ assert state_key("p1") in fake_dream_redis.hashes++ async def test_a_row_the_store_cannot_read_is_dispatched_as_before(+ self, fake_dream_db, fake_dream_redis, provider_cancel, caplog+ ):+ await _in_flight(fake_dream_db, fake_dream_redis)+ fake_dream_db.fail = True++ with caplog.at_level(logging.WARNING):+ assert await should_dispatch(_entry("recombine")) is True++ provider_cancel.assert_not_awaited()+ assert "could not read its row before polling its batch" in caplog.text++ async def test_a_stalled_row_read_gives_up_at_the_checks_own_deadline(+ self, monkeypatch, stalled_dream_db, provider_cancel+ ):+ """The executor walks its queue serially, so the check's read gets a+ deadline of its own, shorter than a record write's."""+ monkeypatch.setattr(+ "backend.copilot.dream.store.RECORD_WRITE_TIMEOUT_SECONDS", 30+ )+ monkeypatch.setattr(+ "backend.copilot.dream.batch_deliveries."+ "DISPATCH_CHECK_READ_TIMEOUT_SECONDS",+ 0.05,+ )++ dispatched = await asyncio.wait_for(should_dispatch(_entry("recombine")), 5)++ assert dispatched is True+ assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (1, 1)+ provider_cancel.assert_not_awaited()++ @pytest.mark.parametrize("pass_id", ["", "never-inserted"])+ async def test_a_batch_with_no_row_to_read_is_dispatched(+ self, fake_dream_db, provider_cancel, pass_id+ ):+ entry = _entry("recombine")+ entry.payload["pass_id"] = pass_id++ assert await should_dispatch(entry) is True++ provider_cancel.assert_not_awaited()+++class TestTwoWalkersDropOneClosedBatch:+ async def test_only_the_claimant_ends_the_pass_and_its_usage_stays(+ self, monkeypatch, fake_dream_db, fake_dream_redis, provider_cancel, charges+ ):+ """Codex's interleaving, through two real executor walks of the same+ due entry: both read the closed row, the second held until the first+ has claimed the entry and ended the pass. The second finds the entry+ claimed and does nothing: one drop, one provider cancel, one charge,+ and the job keeps the usage the first recorded."""+ _emulate_the_dispatch_claim(monkeypatch, fake_dream_redis)+ await _in_flight(fake_dream_db, fake_dream_redis)+ entry = _entry("recombine")+ await executor.enqueue_pending(entry)+ executor.register_handler(+ "dream_pass",+ handle_dream_batch_result,+ should_dispatch=should_dispatch,+ on_drop=drop_closed,+ )+ assert (await cancel_dream_pass("p1", user_id="u1", reason="stop")).cancelled+ provider_cancel.reset_mock()+ poll = AsyncMock(side_effect=AssertionError("a closed batch is not polled"))+ monkeypatch.setattr(executor, "poll_batch", poll)+ drops = AsyncMock(wraps=deliveries_mod.fail_pass)+ monkeypatch.setattr(deliveries_mod, "fail_pass", drops)+ first_read, second_read, first_done = (asyncio.Event() for _ in range(3))+ read = deliveries_mod._row+ reads = 0++ async def staggered(pass_id: str):+ nonlocal reads+ row = await read(pass_id)+ reads += 1+ if reads == 1:+ first_read.set()+ await second_read.wait()+ else:+ second_read.set()+ await first_done.wait()+ return row++ monkeypatch.setattr(deliveries_mod, "_row", staggered)+ now = datetime.now(timezone.utc)++ async def first_walk() -> None:+ await executor._walk_entry(entry, now=now, api_key_for=lambda _: "k")+ first_done.set()++ first = asyncio.create_task(first_walk())+ await asyncio.wait_for(first_read.wait(), 5)+ second = executor._walk_entry(entry, now=now, api_key_for=lambda _: "k")+ await asyncio.wait_for(asyncio.gather(first, second), 5)++ status = await job_status.read_status(kind="dream_pass", job_id="j1")+ assert status is not None and status.state == "errored"+ assert status.result is not None+ usage = status.result["usage"]+ assert (usage["total_input_tokens"], usage["total_output_tokens"]) == (10, 20)+ assert usage["total_cost_usd"] == pytest.approx(0.00011)+ drops.assert_awaited_once()+ provider_cancel.assert_awaited_once()+ charges.assert_awaited_once()+ poll.assert_not_awaited()+ assert await executor.list_pending() == []+++class TestADeadEnd:+ async def test_charges_the_landed_phases_and_cleans_the_pass_up(+ self, fake_dream_db, fake_dream_redis, charges+ ):+ await _in_flight(fake_dream_db, fake_dream_redis)+ entry = _entry("recombine")+ entry.payload["phase"] = "daydream"++ await handle_dream_batch_result(entry, [])++ row = fake_dream_db.rows["p1"]+ assert (row["status"], row["error"]) == (+ DreamPassStatus.ERRORED,+ "unknown batch phase 'daydream'",+ )+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes+ assert input_bundle_key("p1") not in fake_dream_redis.store+ assert _LOCK_KEY not in fake_dream_redis.store++ async def test_without_an_owner_is_ended_as_its_rows(+ self, fake_dream_db, fake_dream_redis, charges+ ):+ """A payload that names the pass but not its owner is ended as the+ owner the row names: the landed phase charged to that owner, its lock+ released, the pass cleaned up."""+ await _in_flight(fake_dream_db, fake_dream_redis)+ entry = _entry("recombine")+ entry.payload["user_id"] = ""++ await handle_dream_batch_result(entry, [])++ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.ERRORED+ assert _charged(charges) == ["consolidate"]+ assert charges.await_args.args[0].scope.user_id == "u1"+ assert _LOCK_KEY not in fake_dream_redis.store+ assert state_key("p1") not in fake_dream_redis.hashes+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)++ async def test_with_no_record_of_its_owner_leaves_the_cleanup_to_the_reaper(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Neither the row nor the input bundle can be read when the dead end+ lands: the job and the row close, marked, and nobody is charged on the+ payload's word; the reaper, which reads the owner and the lock's token+ off the row, charges that owner, releases the lock and cleans up."""+ await _in_flight(fake_dream_db, fake_dream_redis)+ entry = _entry("recombine")+ entry.payload.update({"phase": "daydream", "user_id": "someone-else"})+ mocker.patch.object(+ deliveries_mod,+ "read_pass",+ AsyncMock(side_effect=TimeoutError("dream pass database stalled")),+ )+ mocker.patch.object(+ deliveries_mod,+ "read_input_bundle",+ AsyncMock(side_effect=ConnectionError("redis down")),+ )++ await handle_dream_batch_result(entry, [])++ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.ERRORED+ assert (row["lease_token"], row["cleanup_pending_at"] is None) == (+ "tok",+ False,+ )+ charges.assert_not_awaited()+ assert state_key("p1") in fake_dream_redis.hashes+ assert fake_dream_redis.store[_LOCK_KEY] == "tok"++ mocker.patch.object(reaper_mod, "phase_models_for_config", return_value=_MODELS)+ run = await reap_expired_passes(now=_past_the_grace())++ assert run.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate"]+ assert charges.await_args.args[0].scope.user_id == "u1"+ assert _LOCK_KEY not in fake_dream_redis.store+ assert state_key("p1") not in fake_dream_redis.hashes+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+++class TestADuplicateOfTheLastPhase:+ async def test_closes_the_row_a_dead_first_delivery_left_applying(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """The first delivery claimed apply and died: the duplicate never+ applies, and closes the row instead of leaving it APPLYING."""+ apply = mocker.patch(+ "backend.copilot.dream.apply.apply_operations", AsyncMock()+ )+ await _in_flight(fake_dream_db, fake_dream_redis, landed=2)+ fake_dream_redis.store["dream:applied:p1"] = "1"++ await handle_dream_batch_result(+ _entry("sanitize"), [_row("sanitize", DreamOperations())]+ )++ apply.assert_not_awaited()+ row = fake_dream_db.rows["p1"]+ assert (row["status"], row["error"]) == (+ DreamPassStatus.EXPIRED,+ DUPLICATE_ERROR,+ )+ assert (row["lease_token"], row["lease_expires_at"]) == (None, None)+ assert _charged(charges) == ["consolidate", "recombine", "sanitize"]+ assert _LOCK_KEY not in fake_dream_redis.store++ async def test_leaves_a_row_the_first_delivery_completed(+ self, mocker, fake_dream_db, fake_dream_redis+ ):+ mocker.patch("backend.copilot.dream.apply.apply_operations", AsyncMock())+ await _in_flight(fake_dream_db, fake_dream_redis, landed=2)+ fake_dream_db.rows["p1"]["status"] = DreamPassStatus.COMPLETE+ fake_dream_redis.store["dream:applied:p1"] = "1"++ await handle_dream_batch_result(+ _entry("sanitize"), [_row("sanitize", DreamOperations())]+ )++ row = fake_dream_db.rows["p1"]+ assert (row["status"], row.get("error")) == (DreamPassStatus.COMPLETE, None)+++async def _in_flight(fake_dream_db, fake_dream_redis, *, landed: int = 1) -> None:+ """Batch pass p1 holding its lock under ``tok``: its first *landed*+ phases in its state, the next one's batch in flight, its job open."""+ now = datetime.now(timezone.utc)+ await persist_input_bundle(+ "p1",+ DreamInput(+ user_id="u1", group_id=_SCOPE.group_id, window_start=now, window_end=now+ ),+ lock_token="tok",+ )+ fake_dream_redis.store[_LOCK_KEY] = "tok"+ await job_status.write_initial_status(kind="dream_pass", job_id="j1", user_id="u1")+ outputs: tuple[tuple[DreamPhase, BaseModel], ...] = (+ ("consolidate", ConsolidationOutput()),+ ("recombine", RecombinationOutput()),+ )+ for phase, output in outputs[:landed]:+ await write_phase_to_state(pass_id="p1", phase=phase, row=_row(phase, output))+ fake_dream_db.seed(+ DreamPassDraft(+ id="p1",+ user_id="u1",+ scope_key=_SCOPE.scope_key,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ trigger=DreamPassTrigger.CRON,+ status=DreamPassStatus.SUBMITTED,+ phase=DreamPassPhase.RECOMBINE if landed == 1 else DreamPassPhase.SANITIZE,+ lease_token="tok",+ lease_expires_at=now,+ ),+ provider_batch_id="b-rec" if landed == 1 else "b-san",+ )+++def _past_the_grace() -> datetime:+ """An instant by which the cleanup of a row closed now is due."""+ return datetime.now(timezone.utc) + timedelta(seconds=REAP_GRACE_SECONDS + 60)+++def _entry(phase: str) -> PendingEntry:+ now = datetime.now(timezone.utc)+ return PendingEntry(+ provider="anthropic",+ provider_batch_id="b-rec" if phase == "recombine" else "b-san",+ callback_namespace="dream_pass",+ submitted_at=now,+ next_poll_at=now,+ payload={+ "user_id": "u1",+ "cost_context": {"user_id": "u1", "trial_id": None, "generation": None},+ "pass_id": "p1",+ "job_id": "j1",+ "phase": phase,+ "phase_models": _MODELS,+ },+ )+++def _row(phase: str, output) -> BatchResultRow:+ return BatchResultRow(+ custom_id=f"p1_{phase}",+ content=output.model_dump_json(),+ input_tokens=10,+ output_tokens=20,+ )+++def _charged(charges: AsyncMock) -> list[str]:+ return [call.args[0].job.phase for call in charges.await_args_list]+++def _emulate_the_dispatch_claim(monkeypatch, fake_dream_redis) -> None:+ """The executor's two-key claim script over the in-memory Redis, whose+ ``eval`` models only the lock scripts: a tombstone SET NX EX, then the+ pending row's HDEL, one indivisible step as Redis runs it."""+ evaluate = fake_dream_redis.eval++ async def with_the_claim(script: str, numkeys: int, *args):+ if script != _CLAIM_BATCH_DISPATCH_LUA:+ return await evaluate(script, numkeys, *args)+ pending, tombstone, batch_id, ttl = args+ if tombstone in fake_dream_redis.store:+ return 0+ fake_dream_redis.store[tombstone] = "1"+ fake_dream_redis.ttls[tombstone] = int(ttl)+ await fake_dream_redis.hdel(pending, batch_id)+ return 1++ monkeypatch.setattr(fake_dream_redis, "eval", with_the_claim)diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py b/autogpt_platform/backend/backend/copilot/dream/batch_handoff.pyindex 94a25eec47..2c3fc7e55b 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py@@ -8,9 +8,12 @@ The dream lock goes with the pass: extended to the batch window here andreleased by the callbacks when the pass ends.A pass whose row was cancelled (or expired) while it gathered submits-nothing; one whose row refuses its submit (closed meanwhile, or gone) cancels-that batch here and keeps its lock to release on the way out, rather than-handing both to callbacks that would only stop.+nothing; one whose row refuses its submit (closed meanwhile, or gone), or+does not hold it (the write failed, and a read does not find the batch on+the row), takes that batch back here and keeps its lock to release on the+way out, rather than handing both to callbacks that would only stop, or+to a reaper that would take the pass for dead while its batch ran: the+batch's lease is the submit's to record.Any caller can take this route (it depends on the flag and the deployment'skey, not the caller). ``status_id`` ties the callbacks' updates to the@@ -24,24 +27,23 @@ from backend.copilot.config import ChatConfigfrom backend.executor.batch_executor import remove_pendingfrom .batch_state import best_effort_cleanup-from .batch_submit import (- delete_input_bundle,- persist_input_bundle,- phase_models_for_config,- submit_phase,-)+from .batch_submit import persist_input_bundle, phase_models_for_config, submit_phasefrom .cancel import stopped_errorfrom .fetch import DreamInputfrom .locks import BATCH_LOCK_TTL_SECONDS, DreamLockHandle+from .pass_record import stop_errorfrom .pass_run import DreamPassRunfrom .provider_batch import cancel_provider_batchfrom .schemas import DreamPassResult-from .store import record_submitted+from .store import read_pass, record_submitted, record_taken_backlogger = logging.getLogger(__name__)# How a pass ends whose row refused its submit and cannot say why.REFUSED_SUBMIT_ERROR = "stopped: row refused the submitted write"+# How a pass ends whose submit write failed and whose row, read again, does+# not hold its batch, or cannot be read.+UNRECORDED_SUBMIT_ERROR = "anthropic_batch: batch not on the pass's record; taken back"async def submit_dream_pass_batch(@@ -100,14 +102,13 @@ async def _hand_off(A failed extend means the lock expired before the handoff — a newer passmay already own the graph, so the just-submitted batch must never be- applied: revoke the pending entry (the poller then never dispatches the- callback chain) and drop the input bundle. The provider batch is orphaned;- its results are discarded. The lock is NOT disowned, so the context- manager's compare-and-delete release stays a safe no-op.-- A row that refuses the submit (a stop closed it meanwhile, or it is gone)- gets the same revoke, the provider batch cancelled too, and keeps the lock- to release on the way out.+ applied: it is taken back (``_take_back``). The lock is NOT disowned, so+ the context manager's compare-and-delete release stays a safe no-op.++ A row that refuses the submit (a stop closed it meanwhile, or it is gone),+ or does not hold it (``_record_submit``), gets the same take-back and+ keeps the lock to release on the way out. Nothing has landed yet, so+ nothing is charged."""logger.info("Dream pass %s submitted via Anthropic batch=%s (phase=consolidate)",@@ -115,14 +116,12 @@ async def _hand_off(provider_batch_id,)if not await lock_handle.extend(BATCH_LOCK_TTL_SECONDS):- await remove_pending(provider_batch_id)- await delete_input_bundle(run.pass_id)- return run.failure(- "anthropic_batch: dream lock lost before handoff — batch revoked"- )+ error = "anthropic_batch: dream lock lost before handoff — batch revoked"+ await _take_back(run, lock_handle, provider_batch_id, error)+ return run.failure(error)stopped = await _record_submit(run, lock_handle, provider_batch_id, input_bundle)if stopped is not None:- await _revoke_stopped(run.pass_id, provider_batch_id)+ await _take_back(run, lock_handle, provider_batch_id, stopped)return run.failure(stopped)lock_handle.disown()return run.handed_off()@@ -137,11 +136,13 @@ async def _record_submit("""Record the submit on the pass's row; why the pass must not be handedoff, else ``None``.- A refused write is authoritative: the row has closed or is gone, so the- pass never hands off, its error the stop that closed the row or, when the- row cannot say, ``REFUSED_SUBMIT_ERROR``. A write that failed or ran out- of time says nothing about the row: the pass hands off, logged, unless its- row reads stopped."""+ The pass hands off only once its row holds the batch: the submit records+ the batch's day-long lease, without which the reaper would take the pass+ for dead within the hour while its batch ran. A refused write is+ authoritative: the row has closed or is gone, so the pass never hands+ off, its error the stop that closed the row or, when the row cannot say,+ ``REFUSED_SUBMIT_ERROR``. A write that failed or ran out of time says+ nothing about the row, which is read again (``_submit_on_record``)."""recorded = await record_submitted(run.pass_id,input_bundle=input_bundle,@@ -151,26 +152,98 @@ async def _record_submit()if recorded:return None+ if recorded is None:+ return await _submit_on_record(run.pass_id, provider_batch_id)stopped = await stopped_error(run.pass_id)if stopped is not None:return stopped- if recorded is False:+ logger.warning(+ f"Dream pass {run.pass_id}: its row refused batch {provider_batch_id} "+ "and does not say why; not handing it off"+ )+ return REFUSED_SUBMIT_ERROR+++async def _submit_on_record(pass_id: str, provider_batch_id: str) -> str | None:+ """Whether a submit whose write failed landed after all: ``None`` when+ the row, read again, holds *provider_batch_id*, else why the pass must+ not hand off (the stop that closed the row, or+ ``UNRECORDED_SUBMIT_ERROR`` when the row does not hold the batch or+ cannot be read). A write that committed late while the read missed it+ costs the pass, which the reaper would otherwise have closed mid-batch."""+ try:+ row = await read_pass(pass_id)+ except Exception:logger.warning(- f"Dream pass {run.pass_id}: its row refused batch {provider_batch_id} "- "and does not say why; not handing it off"+ f"Dream pass {pass_id}: its submit write failed and its row cannot "+ f"be read to find batch {provider_batch_id}; taking the batch back",+ exc_info=True,+ )+ return UNRECORDED_SUBMIT_ERROR+ stopped = stop_error(row) if row is not None else None+ if stopped is not None:+ return stopped+ if row is not None and row.provider_batch_id == provider_batch_id:+ logger.info(+ f"Dream pass {pass_id}: its submit write landed after all; handing "+ f"batch {provider_batch_id} off")- return REFUSED_SUBMIT_ERROR+ return Nonelogger.warning(- f"Dream pass {run.pass_id}: batch {provider_batch_id} is not on its "- "record; handing it to the callbacks anyway"+ f"Dream pass {pass_id}: batch {provider_batch_id} is not on its record; "+ "taking it back")- return None+ return UNRECORDED_SUBMIT_ERROR-async def _revoke_stopped(pass_id: str, provider_batch_id: str) -> None:- """Take back a stopped pass's just-submitted batch: off the executor's- queue, cancelled at the provider (best-effort), its bundle dropped."""- logger.info(f"Dream pass {pass_id} stopped at its handoff; revoking batch")+async def _take_back(+ run: DreamPassRun,+ lock_handle: DreamLockHandle,+ provider_batch_id: str,+ error: str,+) -> None:+ """Take back the batch a pass just submitted and must not hand off: the+ provider asked to stop it by the id the pass holds, then the batch off+ the executor's queue and the pass's bundle dropped.++ When the provider does not confirm the stop (it refused, or the call+ failed or ran out of time), the pass's open row closes ERRORED with+ *error*, the batch on it and marked (``pass_record.taken_back``), so the+ reaper's cleanup asks the provider again until the batch has stopped or+ ended. When that write does not land, the batch stays on the executor's+ queue, whose drop of the closed pass's batch+ (``batch_deliveries.drop_closed``), or the callback of a pass without a+ record, asks the provider once more to stop it by its id. The lock is+ left to the context manager's single release on the way out.++ Known limit: when the provider does not confirm the cancel and the+ take-back cannot be recorded on the open row (a store outage, a missing+ row, or a row a concurrent cancel already closed), the batch is left to+ the provider's 24-hour expiry. Its results are discarded and nothing is+ charged to the user; the provider may still bill that one phase."""+ pass_id = run.pass_id+ logger.info(+ f"Dream pass {pass_id} not handed off; taking batch {provider_batch_id} back"+ )+ if not await cancel_provider_batch(provider_batch_id):+ recorded = await record_taken_back(+ pass_id,+ error,+ provider_batch_id=provider_batch_id,+ lease_token=lock_handle.token,+ )+ if not recorded:+ logger.warning(+ f"Dream pass {pass_id}: batch {provider_batch_id} is not "+ "confirmed stopped and not on its record; left on the executor's "+ "queue, whose drop or callback stops it"+ )+ await best_effort_cleanup(pass_id)+ return+ logger.warning(+ f"Dream pass {pass_id}: batch {provider_batch_id} is not confirmed "+ "stopped; its record keeps it for the reaper's cleanup"+ )try:await remove_pending(provider_batch_id)except Exception:@@ -178,5 +251,4 @@ async def _revoke_stopped(pass_id: str, provider_batch_id: str) -> None:f"Dream pass {pass_id}: could not revoke batch {provider_batch_id}",exc_info=True,)- await cancel_provider_batch(provider_batch_id)await best_effort_cleanup(pass_id)diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.pyindex 9c6bf42bee..88e958ca61 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py@@ -4,6 +4,7 @@ in-memory store: a pass cancelled while it gathered submits nothing, and onecancelled while it submitted takes its batch back and releases its lockinstead of handing both to callbacks that would only stop."""+import asyncioimport loggingfrom datetime import datetime, timedelta, timezonefrom unittest.mock import AsyncMock, MagicMock@@ -12,13 +13,18 @@ import pytestfrom prisma.enums import DreamPassStatusfrom backend.copilot.graphiti.scope import MemoryScope+from backend.executor.batch_executor import PendingEntryfrom . import batch_handoff as batch_handoff_modfrom . import orchestrator as orchestrator_mod+from . import reaper as reaper_mod+from .batch_deliveries import drop_closedfrom .batch_submit import input_bundle_keyfrom .cancel import cancel_dream_passfrom .fetch import DreamInput, EpisodeRowfrom .locks import BATCH_LOCK_TTL_SECONDS+from .provider_batch import cancel_provider_batch+from .reaper import REAP_GRACE_SECONDS, reap_expired_passes_SCOPE = MemoryScope.for_user("u")_LOCK_KEY = _SCOPE.redis_key("dream_lock")@@ -124,27 +130,85 @@ async def test_a_pass_nobody_stopped_records_its_batch_then_hands_off_its_lock(provider.cancel.assert_not_awaited()-async def test_a_submit_write_that_fails_on_an_open_row_still_hands_off(- fake_dream_db, fake_dream_redis, gather, provider, mocker, caplog-):+def _submit_write_fails(mocker, fake_dream_db, *, lands: bool) -> None:+ """The submit's write fails as the pass sees it; with *lands* it reached+ the row all the same (a write that committed after the store gave up)."""record = fake_dream_db.update_dream_passasync def submit_write_fails(pass_id, update):- if update.status is DreamPassStatus.SUBMITTED:- raise ConnectionError("dream pass database unreachable")- return await record(pass_id, update)+ if update.status is not DreamPassStatus.SUBMITTED:+ return await record(pass_id, update)+ if lands:+ await record(pass_id, update)+ raise ConnectionError("dream pass database unreachable")mocker.patch.object(fake_dream_db, "update_dream_pass", submit_write_fails)- with caplog.at_level(logging.WARNING, logger=batch_handoff_mod.logger.name):++async def test_a_failed_submit_write_whose_batch_the_row_holds_hands_off(+ fake_dream_db, fake_dream_redis, gather, provider, mocker, caplog+):+ """The write landed after all: a read of the row finds the batch on it,+ with its day-long lease, so the pass hands off as usual."""+ _submit_write_fails(mocker, fake_dream_db, lands=True)++ with caplog.at_level(logging.INFO, logger=batch_handoff_mod.logger.name):result = await orchestrator_mod.execute_dream_pass("u")assert result.error is None- assert "batch batch-1 is not on its record" in caplog.text+ assert "its submit write landed after all" in caplog.text+ row = fake_dream_db.rows[result.pass_id]+ assert (row["status"], row["provider_batch_id"]) == (+ DreamPassStatus.SUBMITTED,+ "batch-1",+ )assert fake_dream_redis.ttls[_LOCK_KEY] == BATCH_LOCK_TTL_SECONDS+ provider.revoke.assert_not_awaited()provider.cancel.assert_not_awaited()+async def test_a_failed_submit_write_whose_batch_the_row_lacks_takes_it_back(+ fake_dream_db, fake_dream_redis, gather, provider, mocker+):+ """The row does not hold the batch, so nothing would renew the pass's+ half-hour lease while the batch ran, and the reaper would close the live+ pass: the pass takes its batch back (off the executor's queue, cancelled+ at the provider) and ends now, the lock released on the way out."""+ _submit_write_fails(mocker, fake_dream_db, lands=False)++ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error == batch_handoff_mod.UNRECORDED_SUBMIT_ERROR+ _assert_revoked(provider, fake_dream_redis, result.pass_id)+ assert fake_dream_db.rows[result.pass_id]["status"] is DreamPassStatus.ERRORED+++async def test_a_failed_submit_write_whose_row_cannot_be_read_takes_it_back(+ fake_dream_db, fake_dream_redis, gather, provider, mocker+):+ """The read that would find the batch fails too: the pass cannot know+ its lease is held, so it takes its batch back, even though the write+ landed here (the cost of not knowing)."""+ _submit_write_fails(mocker, fake_dream_db, lands=True)+ read = fake_dream_db.get_dream_pass+ reads_after_the_submit = 0++ async def read_fails_after_the_submit(pass_id):+ nonlocal reads_after_the_submit+ if fake_dream_db.rows[pass_id]["status"] is DreamPassStatus.SUBMITTED:+ reads_after_the_submit += 1+ raise ConnectionError("dream pass database unreachable")+ return await read(pass_id)++ mocker.patch.object(fake_dream_db, "get_dream_pass", read_fails_after_the_submit)++ result = await orchestrator_mod.execute_dream_pass("u")++ assert reads_after_the_submit >= 1+ assert result.error == batch_handoff_mod.UNRECORDED_SUBMIT_ERROR+ _assert_revoked(provider, fake_dream_redis, result.pass_id)++async def test_a_refused_submit_whose_row_cannot_be_read_is_not_handed_off(fake_dream_db, fake_dream_redis, gather, provider, mocker, caplog):@@ -231,3 +295,173 @@ def _input() -> DreamInput:window_end=now,episodes=[episode],)+++def _the_stop_is_not_confirmed(mocker, provider, how: str) -> None:+ """The provider does not confirm the take-back's cancel: it refuses it+ (``denied``), or the call outlasts its deadline (``timed_out``, through+ the real cancel)."""+ if how == "denied":+ provider.cancel.return_value = False+ return++ async def hangs(**_kwargs) -> bool:+ await asyncio.Event().wait()+ return True++ mocker.patch.object(+ batch_handoff_mod, "cancel_provider_batch", cancel_provider_batch+ )+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key", return_value="k"+ )+ mocker.patch(+ "backend.copilot.dream.provider_batch.PROVIDER_CANCEL_TIMEOUT_SECONDS", 0.05+ )+ provider.cancel = mocker.patch(+ "backend.copilot.dream.provider_batch.cancel_batch",+ AsyncMock(side_effect=hangs),+ )+++def _submit_lands_during_the_cancel(mocker, fake_dream_db, provider) -> None:+ """The submit's write fails as the pass sees it and lands later, after+ the read that looked for it: while the take-back cancels the batch."""+ record = fake_dream_db.update_dream_pass+ late: list = []++ async def lands_late(pass_id, update):+ if update.status is DreamPassStatus.SUBMITTED:+ late.append((pass_id, update))+ raise ConnectionError("dream pass database unreachable")+ return await record(pass_id, update)++ mocker.patch.object(fake_dream_db, "update_dream_pass", lands_late)+ cancel = provider.cancel.side_effect++ async def cancel_after_the_write_lands(*args, **kwargs):+ for pass_id, update in late:+ await record(pass_id, update)+ if cancel is not None:+ return await cancel(*args, **kwargs)+ return provider.cancel.return_value++ provider.cancel.side_effect = cancel_after_the_write_lands++[email protected]("stop", ["denied", "timed_out"])[email protected]("submit", ["absent", "landed_late"])+async def test_a_batch_taken_back_without_a_confirmed_stop_stays_on_its_record(+ fake_dream_db, fake_dream_redis, gather, provider, mocker, stop, submit+):+ """The take-back cannot confirm the batch stopped: the row closes+ ERRORED with the batch on it and marked, so the reaper's cleanup keeps+ asking the provider to stop it; the batch leaves the executor's queue,+ the bundle goes and the lock is released once, on the way out."""+ _the_stop_is_not_confirmed(mocker, provider, stop)+ if submit == "absent":+ _submit_write_fails(mocker, fake_dream_db, lands=False)+ else:+ _submit_lands_during_the_cancel(mocker, fake_dream_db, provider)++ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error == batch_handoff_mod.UNRECORDED_SUBMIT_ERROR+ provider.cancel.assert_awaited_once()+ row = fake_dream_db.rows[result.pass_id]+ assert (row["status"], row["provider_batch_id"]) == (+ DreamPassStatus.ERRORED,+ "batch-1",+ )+ assert row["cleanup_pending_at"] is not None+ provider.revoke.assert_awaited_once_with("batch-1")+ assert input_bundle_key(result.pass_id) not in fake_dream_redis.store+ assert _LOCK_KEY not in fake_dream_redis.store+++async def test_a_batch_neither_stopped_nor_recorded_stays_on_the_executors_queue(+ fake_dream_db, fake_dream_redis, gather, provider, mocker+):+ """The stop is not confirmed and the row will not take the batch either+ (the store down for writes): the batch stays on the executor's queue,+ whose drop or callback stops it by its id; the bundle goes and the lock+ is released on the way out."""+ _the_stop_is_not_confirmed(mocker, provider, "denied")+ record = fake_dream_db.update_dream_pass++ async def store_down_from_the_submit(pass_id, update):+ if update.status in (DreamPassStatus.SUBMITTED, DreamPassStatus.ERRORED):+ raise ConnectionError("dream pass database unreachable")+ return await record(pass_id, update)++ mocker.patch.object(fake_dream_db, "update_dream_pass", store_down_from_the_submit)++ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error == batch_handoff_mod.UNRECORDED_SUBMIT_ERROR+ provider.cancel.assert_awaited_once_with("batch-1")+ provider.revoke.assert_not_awaited()+ assert input_bundle_key(result.pass_id) not in fake_dream_redis.store+ assert _LOCK_KEY not in fake_dream_redis.store+++async def test_a_take_back_racing_a_cancel_leaves_the_batch_to_its_expiry(+ fake_dream_db, fake_dream_redis, gather, provider, mocker+):+ """The known limit at ``_take_back``: a cancel closes the row while the+ batch is submitted, and the provider confirms neither the take-back's+ stop nor the executor's drop's. The closed row does not take the batch,+ so nothing asks again and the batch is left to the provider's expiry:+ nothing raises, nothing is charged, and the row stays how the cancel+ closed it."""++ async def submitted_then_cancelled(**_kwargs) -> MagicMock:+ await _cancel_the_pass(fake_dream_db)+ return MagicMock(provider_batch_id="batch-1")++ provider.submit.side_effect = submitted_then_cancelled+ provider.cancel.return_value = False+ stop = mocker.patch(+ "backend.copilot.dream.cleanup.provider_batch_stopped",+ AsyncMock(return_value=False),+ )+ charge = mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )++ result = await orchestrator_mod.execute_dream_pass("u")++ _assert_cancelled(fake_dream_db, result.pass_id)+ assert fake_dream_db.rows[result.pass_id].get("provider_batch_id") is None++ await drop_closed(_batch_entry(result.pass_id))+ asked = stop.await_count+ mocker.patch.object(reaper_mod, "phase_models_for_config", return_value={})+ await reap_expired_passes(+ now=datetime.now(timezone.utc) + timedelta(seconds=REAP_GRACE_SECONDS + 60)+ )++ _assert_cancelled(fake_dream_db, result.pass_id)+ assert fake_dream_db.rows[result.pass_id].get("provider_batch_id") is None+ assert stop.await_count == asked, "nothing asks the provider again"+ charge.assert_not_awaited()+++def _batch_entry(pass_id: str) -> PendingEntry:+ """The executor's entry for the pass's consolidate batch."""+ now = datetime.now(timezone.utc)+ return PendingEntry(+ provider="anthropic",+ provider_batch_id="batch-1",+ callback_namespace="dream_pass",+ submitted_at=now,+ next_poll_at=now,+ payload={+ "user_id": "u",+ "cost_context": {"user_id": "u", "trial_id": None, "generation": None},+ "pass_id": pass_id,+ "job_id": "",+ "phase": "consolidate",+ "phase_models": {},+ },+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py b/autogpt_platform/backend/backend/copilot/dream/batch_outcome.pyindex d0d710e259..8d9c6a7968 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py@@ -1,12 +1,13 @@"""How a batch dream pass ends: what it reports to the admin JobStatus row and-to its durable DreamPass record, and the lock it gives back.+to its durable DreamPass record, and the cleanup after it.A batch pass ends in a callback, in another process, long after it started:``fail_pass`` for every failure, ``record_completion`` once apply has run.Either way the record gets the usage of every phase that landed, read off thepass's Redis state and priced in ``batch_costs.py``, unless a stop closed therecord first: a closed record refuses the write, and that usage lives on only-in the cost log.+in the cost log. Either way the record is marked for the cleanup after the+pass, which ``clean_up_after`` then does (``cleanup.py``)."""from __future__ import annotations@@ -18,18 +19,18 @@ from pydantic import BaseModel, ConfigDictfrom backend.copilot.graphiti.scope import MemoryScope-from .batch_costs import landed_usage, log_all_phase_costs-from .batch_state import best_effort_cleanup, read_state_or_none+from .batch_costs import landed_usage+from .batch_state import read_state_or_nonefrom .batch_submit import read_lock_token+from .cleanup import clean_up_pass, finish_cleanupfrom .locks import release_dream_lockfrom .schemas import (- DreamOperations,DreamOperationsSnapshot,DreamPassResult,DreamPassUsage,IngestionDrainStatus,)-from .store import record_batch_complete, record_batch_failed+from .store import read_pass, record_batch_complete, record_batch_failedlogger = logging.getLogger(__name__)@@ -64,37 +65,70 @@ class BatchPass(BaseModel):async def fail_pass(- bp: BatchPass, error: str, *, holds_lock: bool = True, charge: bool = True+ bp: BatchPass,+ error: str,+ *,+ holds_lock: bool = True,+ provider_batch_id: str | None = None,) -> None:- """Close the pass errored: its admin job and its record first, then- charge the phases that landed, release the lock (unless the pass lost it,- *holds_lock* false: it is then another pass's) and clean up per-pass- state. Each step is best-effort on its own, so one that fails never- stops the ones after it. A pass that cannot be charged under its own- usage attribution passes ``charge=False``: its record still shows what- the landed phases used, and nothing is charged under another one.-- The record carries the usage of the phases that landed, read off the- pass's state, unless a stop already closed it; a state that cannot be- read leaves that usage unknown rather than the record open.+ """Close the pass errored: its admin job and its record first, the+ record marked for the cleanup after it, then that cleanup+ (``clean_up_after``): the batch *provider_batch_id* names stopped at the+ provider, the phases that landed charged under the usage attribution+ the pass started under (none when its row carries none,+ ``batch_costs``), the lock released (unless the pass lost it,+ *holds_lock* false: it is then another pass's) and the per-pass state+ deleted. Each step is best-effort on its own, so one that fails never+ stops the ones after it, and leaves the row marked for the reaper.++ The record and the job's result carry the usage of the phases that+ landed, read off the pass's state (a stop that closed the record first+ keeps it off the record, not off the job); a state that cannot be read+ leaves that usage unknown rather than the record open.We incurred the provider tokens for completed phases regardless ofwhether the whole pass landed, so they're recorded against theuser's usage — matching the sync path and the documented contract in- ``dream/billing.py``. The idempotency gate inside- ``log_all_phase_costs`` keeps this at-most-once.+ ``dream/billing.py``. The per-phase claims inside+ ``charge_landed_phases`` keep this at-most-once."""logger.warning("Dream batch pass=%s failed: %s", bp.pass_id, error)- await mark_job_errored(bp.job_id, error)state = await read_state_or_none(bp.pass_id)- await record_batch_failed(- bp.pass_id, error, landed_usage(state, bp.phase_models, bp.pass_id)+ usage = landed_usage(state, bp.phase_models, bp.pass_id)+ await mark_job_errored(bp.job_id, error, result=_failed_result(bp, error, usage))+ await record_batch_failed(bp.pass_id, error, usage)+ await clean_up_after(bp, holds_lock=holds_lock, provider_batch_id=provider_batch_id)+++async def clean_up_after(+ bp: BatchPass,+ *,+ holds_lock: bool = True,+ provider_batch_id: str | None = None,+) -> None:+ """The cleanup after the ended pass (``cleanup.clean_up_pass``), its lock+ released under the pass's token (``lock_token_of``), then its row's mark+ cleared if every step finished; a step that did not, a lock the pass has+ no token for included, leaves the mark for the reaper. Never raises: on+ the success tail an exception would fire AFTER ``mark_complete`` and the+ crash guard would rewrite a completed job to errored."""+ try:+ scope = MemoryScope.build(bp.user_id, bp.expert_id)+ except ValueError:+ logger.warning(+ f"Dream pass {bp.pass_id}: invalid memory scope for user "+ f"{bp.user_id[:12]}; its cleanup is left for the reaper"+ )+ return+ cleanup = await clean_up_pass(+ bp.pass_id,+ scope,+ phase_models=bp.phase_models,+ provider_batch_id=provider_batch_id,+ lock_token=await lock_token_of(bp.pass_id) if holds_lock else None,+ release=holds_lock,)- if state and charge:- await _charge_landed_phases(bp, state)- if holds_lock:- await release_lock(bp)- await best_effort_cleanup(bp.pass_id)+ await finish_cleanup(bp.pass_id, cleanup)async def record_completion(@@ -127,47 +161,23 @@ async def record_completion(await record_batch_complete(pass_result, usage)-async def finalize_stuck_duplicate(bp: BatchPass, ops: DreamOperations) -> None:- """On a duplicate delivery, finalize the admin job row iff the first- delivery crashed between apply and ``mark_complete`` and left it- non-terminal. An already-terminal row is left untouched so the first- delivery's real apply stats are never overwritten.-- Best-effort: a status read/write failure here must not crash the- duplicate tail (lock release + cleanup still need to run)."""- if not bp.job_id:- return- try:- from .job_status import mark_complete, read_status-- existing = await read_status(kind="dream_pass", job_id=bp.job_id)- if existing is None or existing.state in ("complete", "errored"):- return- logger.warning(- "Duplicate dispatch found job %s stuck in state=%s — "- "finalizing with the clamped op counts",- bp.job_id[:12],- existing.state,- )- await mark_complete(- kind="dream_pass", job_id=bp.job_id, result=_attempted_result(bp, ops)- )- except Exception:- logger.exception(- "Failed to finalize stuck duplicate for job %s", bp.job_id[:12]- )---async def mark_job_errored(job_id: str, error: str, *, dead_end: bool = False) -> None:- """Close the admin job row errored. Best-effort: status write failures- are logged, never raised; a *dead_end* (a payload no phase handler can- take) logs under its own message."""+async def mark_job_errored(+ job_id: str,+ error: str,+ *,+ dead_end: bool = False,+ result: DreamPassResult | None = None,+) -> None:+ """Close the admin job row errored, with the pass's *result* when there+ is one (its usage). Best-effort: status write failures are logged, never+ raised; a *dead_end* (a payload no phase handler can take) logs under its+ own message."""if not job_id:returntry:from .job_status import mark_errored- await mark_errored(kind="dream_pass", job_id=job_id, error=error)+ await mark_errored(kind="dream_pass", job_id=job_id, error=error, result=result)except Exception:if dead_end:logger.exception("Failed to mark dead-end job %s errored", job_id[:12])@@ -175,43 +185,17 @@ async def mark_job_errored(job_id: str, error: str, *, dead_end: bool = False) -logger.exception("Failed to mark dream pass job %s errored", job_id)-async def _charge_landed_phases(- bp: BatchPass, state: dict[str, dict[str, Any]]-) -> None:- """Charge the phases that landed; a failure is logged, never raised."""- try:- await log_all_phase_costs(- user_id=bp.user_id,- expert_id=bp.expert_id,- pass_id=bp.pass_id,- state=state,- phase_models=bp.phase_models,- )- except Exception:- logger.exception("Dream batch cost log failed for pass=%s", bp.pass_id)---async def release_lock(bp: BatchPass) -> None:- """Release the disowned dream lock with the ownership token persisted- alongside the input bundle. Must run before ``delete_input_bundle`` —- the token rides on that key. A missing token (bundle TTL'd out,- malformed payload) leaves the lock for its TTL to clear rather than- blind-deleting what may be a newer pass's lock.+async def release_lock(bp: BatchPass) -> bool:+ """Release the disowned dream lock under the pass's ownership token+ (``lock_token_of``), and say whether the lock is no longer the pass's+ (``release_dream_lock``). Must run before ``delete_input_bundle``, whose+ key carries the token first. With no token anywhere the lock is left+ alone rather than blind-deleting what may be a newer pass's lock.Best-effort like ``release_dream_lock`` itself: a Redis blip on thetoken read must not propagate — on the success tail it would fireAFTER ``mark_complete`` and the crash guard would rewrite a completed- job to errored. Falls back to a token-less release (lock TTL)."""- token: str | None = None- if bp.pass_id:- try:- token = await read_lock_token(bp.pass_id)- except Exception:- logger.exception(- "Failed to read dream lock token for pass=%s — "- "leaving the lock to its TTL",- bp.pass_id,- )+ job to errored."""try:scope = MemoryScope.build(bp.user_id, bp.expert_id)except ValueError:@@ -220,8 +204,55 @@ async def release_lock(bp: BatchPass) -> None:"leaving it for the TTL to clear",bp.user_id[:12],)- return- await release_dream_lock(scope, token)+ return False+ return await release_dream_lock(scope, await lock_token_of(bp.pass_id))+++async def lock_token_of(pass_id: str) -> str | None:+ """The token the batch pass holds its scope's lock under: the one its+ input bundle carries, else the one its row keeps (one bounded read, for+ a bundle that lost its token or is gone). ``None`` when neither can say:+ the pass's ownership of the lock is then unknown, never taken for "no+ lock", so it neither applies nor has its lock taken for released."""+ if not pass_id:+ return None+ token = await _bundle_lock_token(pass_id)+ if token is not None:+ return token+ token = await _row_lock_token(pass_id)+ if token is not None:+ logger.warning(+ f"Dream pass {pass_id}: no lock token with its input bundle; "+ "going by the one its row keeps"+ )+ return token+++async def _bundle_lock_token(pass_id: str) -> str | None:+ """The lock token the pass's input bundle carries; ``None`` when there is+ no bundle, it carries none, or Redis cannot say."""+ try:+ return await read_lock_token(pass_id)+ except Exception:+ logger.exception(+ "Failed to read dream lock token for pass=%s — going by its row",+ pass_id,+ )+ return None+++async def _row_lock_token(pass_id: str) -> str | None:+ """The lease token the pass's row keeps, read once under the store's+ deadline; ``None`` when there is none, or the store cannot say in time."""+ try:+ row = await read_pass(pass_id)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not read its row for its lock token",+ exc_info=True,+ )+ return None+ return row.lease_token if row is not None else Nonedef _applied_result(@@ -253,27 +284,17 @@ def _applied_result()-def _attempted_result(bp: BatchPass, ops: DreamOperations) -> DreamPassResult:- """The first delivery's per-edge outcomes (and dream session id) died- with it, so the counts here are the clamped *attempted* ops — annotated- so the admin UI doesn't present them as confirmed apply results."""- note = (- "[finalized after duplicate delivery — writes and proposals were "- "queued by the original delivery, counted pending; demotion counts "- "reflect attempted operations] "- )+def _failed_result(+ bp: BatchPass, error: str, usage: DreamPassUsage | None+) -> DreamPassResult:+ """A failed batch pass as its job reports it: the error, and what the+ phases that landed used."""return DreamPassResult(user_id=bp.user_id,pass_id=bp.pass_id,execution_path="anthropic_batch",- pending_writes=len(ops.writes) + len(ops.proposals),- demotion_count=len(ops.demotions),- entity_invalidation_count=len(ops.entity_invalidations),- # Batch apply never drains in-line by design; the first- # delivery's writes (if any) landed fire-and-forget. ``skipped``- # marks this as a healthy by-design skip, NOT a drain failure.- ingestion_drain_status=IngestionDrainStatus.skipped,- summary_for_user=note + (ops.summary_for_user or ""),+ error=error,+ usage=usage,)diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_state.py b/autogpt_platform/backend/backend/copilot/dream/batch_state.pyindex 9910a8dd93..d1c806f4c3 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_state.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_state.py@@ -11,11 +11,15 @@ Per-pass state lives in two Redis keys:token usage, so the apply step has everything it needs and the cost logcan record all three rows at once-Both are TTL'd to 24h (Anthropic's batch SLA) so a forgotten pass naturally-falls off the radar. Two SETNX gates (7-day TTL) keep the side effects-at-most-once should a finished batch be delivered twice:-``dream:applied:{pass_id}`` for the memory writes and-``dream:batch:costs_logged:{pass_id}`` for billing.+Both are TTL'd to twelve hours past the pass's lease (``INPUT_TTL_SECONDS``), so+the reaper can still charge and delete them and a forgotten pass naturally+falls off the radar. SETNX gates (7-day TTL) keep the side effects+at-most-once should a finished batch be delivered twice or a cleanup run+again: ``dream:applied:{pass_id}`` for the memory writes, and for billing+one ``dream:batch:charged:{pass_id}:{phase}`` per landed phase, claimed right+before that phase is charged. ``dream:batch:costs_logged:{pass_id}`` is the+whole-pass billing gate of earlier builds; this build claims it too, with a+value of its own (``claim_per_phase_charging``)."""from __future__ import annotations@@ -25,7 +29,7 @@ import loggingfrom collections.abc import Awaitablefrom typing import TYPE_CHECKING, Any, Literal, cast-from .batch_submit import delete_input_bundle+from .batch_submit import INPUT_TTL_SECONDS, delete_input_bundlefrom .schemas import DreamPhaseif TYPE_CHECKING:@@ -33,10 +37,8 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-# 24h matches Anthropic's batch SLA; if no phase has landed in that-# window the BatchExecutor has already issued a timeout error via-# ``MAX_BATCH_LIFETIME_SECONDS``.-STATE_TTL_SECONDS = 24 * 60 * 60+# The same as the input bundle's, see ``batch_submit.INPUT_TTL_SECONDS``.+STATE_TTL_SECONDS = INPUT_TTL_SECONDS_APPLIED_GATE_PREFIX = "dream:applied"# 7 days — same window as the costs_logged gate; no realistic@@ -44,10 +46,15 @@ _APPLIED_GATE_PREFIX = "dream:applied"# 24h) can outlive it._APPLIED_GATE_TTL_SECONDS = 7 * 24 * 60 * 60+# The pass-level billing key. Earlier builds claim it with "1" and then+# charge every landed phase; this build claims it with ``_PER_PHASE`` and+# charges each phase under its own claim._COSTS_LOGGED_PREFIX = "dream:batch:costs_logged"+_PER_PHASE = "per_phase"+_PHASE_CHARGED_PREFIX = "dream:batch:charged"# 7 days — long enough that no realistic BatchExecutor re-dispatch-# (poll backoff caps at 5 min, max lifetime 24h) can slip through and-# bill twice. Matches the spirit of the Stripe-reconcile gate's TTL.+# (poll backoff caps at 5 min, max lifetime 24h) or reaper retry can slip+# through and bill twice. Matches the spirit of the Stripe-reconcile gate's TTL._COSTS_LOGGED_TTL_SECONDS = 7 * 24 * 60 * 60@@ -96,8 +103,11 @@ async def read_state_or_none(pass_id: str) -> dict[str, dict[str, Any]] | None:async def write_phase_to_state(- *, pass_id: str, phase: DreamPhase, row: BatchResultRow+ *, pass_id: str, phase: DreamPhase, row: BatchResultRow, model: str | None = None) -> None:+ """Keep a landed phase's result in the pass's state, with *model*, the+ model the phase was submitted on, so whoever charges the phase later+ (the pass's end, or the reaper long after) prices it on that model."""from backend.data.redis_client import get_redis_asyncredis = await get_redis_async()@@ -110,12 +120,23 @@ async def write_phase_to_state("cache_read_tokens": row.cache_read_tokens,"cache_creation_tokens": row.cache_creation_tokens,"error": row.error,+ "model": model,})await cast("Awaitable[int]", redis.hset(state_key(pass_id), phase, body))await redis.expire(state_key(pass_id), STATE_TTL_SECONDS)+def recorded_phase_models(state: dict[str, dict[str, Any]]) -> dict[str, str]:+ """The model each landed phase's state row recorded; a row written before+ rows recorded their model names none."""+ return {+ phase: model+ for phase, row in state.items()+ if isinstance(model := row.get("model"), str) and model+ }++async def delete_state(pass_id: str) -> None:from backend.data.redis_client import get_redis_async@@ -131,20 +152,24 @@ def content_for(state: dict[str, dict[str, Any]], phase: str) -> str | None:return content if isinstance(content, str) else None-async def best_effort_cleanup(pass_id: str) -> None:- """Delete the per-pass state + input bundle without letting a Redis- blip propagate. These deletes run AFTER ``mark_complete`` on the- success/duplicate tails — an exception here would route through the- crash guard to ``fail_pass`` and rewrite a completed job to errored.- Both keys carry 24h TTLs, so a failed delete self-heals."""+async def best_effort_cleanup(pass_id: str) -> bool:+ """Delete the per-pass state + input bundle, and say whether both went,+ without letting a Redis blip propagate. These deletes run AFTER+ ``mark_complete`` on the success/duplicate tails — an exception here+ would route through the crash guard to ``fail_pass`` and rewrite a+ completed job to errored. A failed delete is logged and reported, so+ the cleanup it belongs to stays marked for the reaper; both keys carry+ TTLs besides."""try:await delete_state(pass_id)await delete_input_bundle(pass_id)except Exception:logger.exception(- "Per-pass cleanup failed for pass=%s — keys will expire via TTL",+ "Per-pass cleanup failed for pass=%s — left for the reaper",pass_id,)+ return False+ return Trueasync def claim_apply_gate(pass_id: str) -> Literal["claimed", "duplicate", "error"]:@@ -187,37 +212,76 @@ async def claim_apply_gate(pass_id: str) -> Literal["claimed", "duplicate", "errreturn "error"-async def claim_costs_logged_gate(pass_id: str) -> bool:- """Atomically claim the per-pass cost-charge gate. Returns True- when this caller won the race (first time costs_logged is set);- False when an earlier caller already claimed it, which does not- prove the pass was charged: that caller's accounting may have- failed part-way, or Redis may have failed this claim.+async def claim_per_phase_charging(pass_id: str) -> bool:+ """Whether the pass is charged phase by phase (``claim_phase_charge``):+ ``True`` once the pass-level key holds this build's value, set now or by+ an earlier call; ``False`` when it holds an earlier build's, which+ charged the pass whole, so nothing more is charged. Raises when Redis+ does not answer.- Modelled on ``rate_limit._maybe_reconcile_stripe_tier`` — Redis- SETNX with a long TTL is the established convention for "do this- side-effect at most once per identifier" in this codebase. The- dedup lives at the dream-batch boundary, not inside- ``record_cost_usage`` itself (chat legitimately charges every turn).- """+ Earlier builds claim that key before charging every phase, so claiming+ it here too keeps one pass from being charged both ways while two builds+ run side by side. The key is read before it is set: a cleanup that runs+ again finds it set, and waits on one Redis command for it, not two."""+ from backend.data.redis_client import get_redis_async++ redis = await get_redis_async()+ key = f"{_COSTS_LOGGED_PREFIX}:{pass_id}"+ held = await redis.get(key)+ if held is None:+ if await redis.set(key, _PER_PHASE, nx=True, ex=_COSTS_LOGGED_TTL_SECONDS):+ return True+ held = await redis.get(key)+ return held is not None and _text(held) == _PER_PHASE+++async def uncharged_phases(pass_id: str, phases: list[str]) -> list[str]:+ """Those of *phases* nobody claimed the charge of, nor the pass's whole+ charge under an earlier build; all of them when Redis cannot say. For+ reporting usage that is gone, never for deciding to charge."""from backend.data.redis_client import get_redis_asynctry:redis = await get_redis_async()- return bool(- await redis.set(- f"{_COSTS_LOGGED_PREFIX}:{pass_id}",- "1",- nx=True,- ex=_COSTS_LOGGED_TTL_SECONDS,- )- )+ whole = await redis.get(f"{_COSTS_LOGGED_PREFIX}:{pass_id}")+ if whole is not None and _text(whole) != _PER_PHASE:+ return []+ return [+ phase+ for phase in phases+ if not await redis.exists(f"{_PHASE_CHARGED_PREFIX}:{pass_id}:{phase}")+ ]except Exception:- # Fail closed: if we can't claim the gate, do not charge.- # Better to under-bill on a Redis brown-out than risk- # double-billing under retry pressure.- logger.exception(- "Failed to claim costs_logged gate for pass=%s — skipping charge",- pass_id,+ logger.warning(+ f"Dream pass {pass_id}: could not read which phases were charged",+ exc_info=True,)- return False+ return list(phases)+++def _text(value: Any) -> str:+ return value.decode() if isinstance(value, bytes) else str(value)+++async def claim_phase_charge(pass_id: str, phase: str) -> bool:+ """Claim the charge of one landed phase of the pass: ``True`` for the+ first caller, which then charges it, ``False`` once it is claimed.+ Raises when Redis does not answer, and a claim that landed all the same+ leaves the phase uncharged: never charged twice.++ Modelled on ``rate_limit._maybe_reconcile_stripe_tier`` — Redis SETNX+ with a long TTL is the established convention for "do this side-effect+ at most once per identifier" in this codebase. The dedup lives at the+ dream-batch boundary, not inside ``record_cost_usage`` itself (chat+ legitimately charges every turn)."""+ from backend.data.redis_client import get_redis_async++ redis = await get_redis_async()+ return bool(+ await redis.set(+ f"{_PHASE_CHARGED_PREFIX}:{pass_id}:{phase}",+ "1",+ nx=True,+ ex=_COSTS_LOGGED_TTL_SECONDS,+ )+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_submit.py b/autogpt_platform/backend/backend/copilot/dream/batch_submit.pyindex 9ec029af10..762f070f63 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_submit.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_submit.py@@ -47,7 +47,7 @@ from backend.util.llm.tool_use import (from .fetch import DreamInputfrom .input_bundle import input_bundle_from_dict, input_bundle_to_dict-from .locks import read_dream_lock_token+from .locks import BATCH_LOCK_TTL_SECONDS, read_dream_lock_tokenfrom .prompts import (build_consolidate_prompt,build_recombine_prompt,@@ -190,6 +190,8 @@ async def submit_phase(# multi-phase chain can legitimately outlive a TTL stamped once at the# first submit. Re-arm the bundle TTL at every submit so it stays alive# exactly as long as some phase batch may still call back.+ # The payload carries a copy of the attribution the pass runs under (the+ # one its row keeps, which the callback checks it against).cost_context = await capture_cost_context(user_id)await refresh_input_bundle_ttl(pass_id)model = phase_models[phase]@@ -335,10 +337,14 @@ def input_bundle_key(pass_id: str) -> str:return f"dream:batch:input:{pass_id}"-# 24h TTL matches Anthropic's batch SLA — if a batch hasn't completed-# in 24h we've already timed out via BatchExecutor's-# ``MAX_BATCH_LIFETIME_SECONDS`` ceiling.-INPUT_TTL_SECONDS = 24 * 60 * 60+# The pass's lease runs ``BATCH_LOCK_TTL_SECONDS`` from its submit and each+# callback; the reaper closes a pass one lock TTL after its lease lapsed and+# runs every ten minutes. The per-pass keys (this bundle and the batch state)+# outlive the lease by twelve hours, so the reaper still finds them to charge+# and delete behind a backlog of cleanups that keep failing (a pass that ends+# normally deletes them at once). A batch that never completes is timed out+# well before, by the BatchExecutor's 24h ``MAX_BATCH_LIFETIME_SECONDS``.+INPUT_TTL_SECONDS = BATCH_LOCK_TTL_SECONDS + 12 * 60 * 60async def persist_input_bundle(@@ -413,9 +419,11 @@ async def read_lock_token(pass_id: str) -> str | None:"""Dream-lock ownership token persisted alongside the input bundle.None when the bundle is gone (TTL expired or already cleaned up), is- corrupted, or was written while no lock was held — the caller then- leaves the lock to its TTL rather than risking a blind delete of a- newer pass's lock.+ corrupted, or was written while no lock was held. The batch paths then+ go by the lease token the pass's row keeps+ (``batch_outcome.lock_token_of``); with no token anywhere the pass's+ ownership of the lock is unknown, and the lock is left held rather+ than risking a blind delete of a newer pass's lock."""from backend.data.redis_client import get_redis_asyncdiff --git a/autogpt_platform/backend/backend/copilot/dream/bounded_calls_test.py b/autogpt_platform/backend/backend/copilot/dream/bounded_calls_test.pynew file mode 100644index 0000000000..ec151fc266--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/bounded_calls_test.py@@ -0,0 +1,172 @@+"""A cut that lands as a bounded call answers is kept. On Python 3.11,+``asyncio.wait_for`` runs its call in a task of its own and, cancelled from+outside once that task is done, returns the call's result: the cut is lost,+and a reaper row cut at its deadline runs on into the time of the rows behind+it. Every bounded call on the reaper's path is a timeout around a direct+await instead, which keeps it. Each test cancels the task making the call+from inside the call, as the call answers (the moment ``wait_for`` lost the+cut), and checks that the task still ends cancelled."""++import asyncio+from collections.abc import Awaitable, Callable+from typing import Any+from unittest.mock import AsyncMock++import pytest+from prisma.enums import DreamPassRoute++from backend.copilot.graphiti.scope import MemoryScope+from backend.data.dream_pass_models import DreamPassUpdate++from . import cleanup as cleanup_mod+from . import provider_batch as provider_batch_mod+from . import reaper as reaper_mod+from . import store as store_mod+from .batch_costs import PhaseCharges+from .reaper_test import _seed++_SCOPE = MemoryScope.build("u1")+++class _Cut:+ """Stands in for a call: cancels the task making it, then answers, both+ in the same step."""++ def __init__(self) -> None:+ self.task: asyncio.Task[Any] | None = None++ def answering(self, answer: object) -> Callable[..., Awaitable[object]]:+ async def call(*_args: object, **_kwargs: object) -> object:+ assert self.task is not None+ self.task.cancel()+ return answer++ return call++ async def run(self, call: Awaitable[object]) -> asyncio.Task[Any]:+ self.task = asyncio.ensure_future(call)+ await asyncio.wait([self.task])+ return self.task+++def cut() -> _Cut:+ return _Cut()+++def provider_key(mocker) -> None:+ mocker.patch.object(provider_batch_mod, "anthropic_api_key", return_value="k")+++async def test_the_provider_cancel(mocker, cut, provider_key):+ mocker.patch.object(provider_batch_mod, "cancel_batch", cut.answering(True))++ task = await cut.run(provider_batch_mod.cancel_provider_batch("b-1"))++ assert task.cancelled()+++async def test_the_provider_status_read(mocker, cut, provider_key):+ mocker.patch.object(+ provider_batch_mod, "cancel_batch", AsyncMock(return_value=False)+ )+ mocker.patch.object(provider_batch_mod, "poll_batch", cut.answering("ended"))++ task = await cut.run(provider_batch_mod.provider_batch_stopped("b-1"))++ assert task.cancelled()+++async def test_a_record_write(mocker, cut, fake_dream_db):+ mocker.patch.object(fake_dream_db, "update_dream_pass", cut.answering(True))++ task = await cut.run(store_mod.write_stop("p1", DreamPassUpdate()))++ assert task.cancelled()+++async def test_the_reapers_take_of_a_scope(monkeypatch, cut, fake_dream_redis):+ monkeypatch.setattr(fake_dream_redis, "set", cut.answering(True))++ task = await cut.run(reaper_mod._take_scope(_SCOPE))++ assert task.cancelled()+++async def test_the_reapers_note_of_the_row_it_tries(+ monkeypatch, cut, fake_dream_db, fake_dream_redis+):+ _seed(fake_dream_db, "p1", route=DreamPassRoute.SYNC)+ monkeypatch.setattr(fake_dream_redis, "hset", cut.answering(1))++ task = await cut.run(reaper_mod._note_tried(fake_dream_db.record("p1")))++ assert task.cancelled()+++async def test_the_reapers_read_of_where_its_scan_stands(+ monkeypatch, cut, fake_dream_redis+):+ monkeypatch.setattr(fake_dream_redis, "hgetall", cut.answering({}))++ task = await cut.run(reaper_mod._read_scan("open"))++ assert task.cancelled()+++async def test_the_reapers_release_of_its_hold(mocker, cut):+ mocker.patch.object(reaper_mod, "release_dream_lock", cut.answering(True))++ task = await cut.run(reaper_mod._release_within_bound(_SCOPE, "reaper:x"))++ assert task.cancelled()+++async def test_the_cleanups_state_read(mocker, cut):+ mocker.patch.object(cleanup_mod, "read_state", cut.answering({}))++ task = await cut.run(cleanup_mod._charge("p1", _SCOPE, None, ()))++ assert task.cancelled()+++async def test_the_cleanups_charges(mocker, cut):+ mocker.patch.object(+ cleanup_mod,+ "read_state",+ AsyncMock(return_value={"consolidate": {"content": "{}"}}),+ )+ mocker.patch.object(+ cleanup_mod, "charge_landed_phases", cut.answering(PhaseCharges(settled=True))+ )++ task = await cut.run(+ cleanup_mod._charge("p1", _SCOPE, {"consolidate": "claude-sonnet-5"}, ())+ )++ assert task.cancelled()+++async def test_the_cleanups_unlock(mocker, cut):+ mocker.patch.object(cleanup_mod, "release_dream_lock", cut.answering(True))++ task = await cut.run(cleanup_mod._unlock("p1", _SCOPE, "token", attribute=False))++ assert task.cancelled()+++async def test_the_cleanups_read_of_a_tokenless_rows_lock(mocker, cut):+ mocker.patch.object(cleanup_mod, "read_dream_lock_token", cut.answering(None))++ task = await cut.run(cleanup_mod._lock_is_not_the_passes("p1", _SCOPE))++ assert task.cancelled()+++async def test_the_cleanups_delete(mocker, cut):+ mocker.patch.object(cleanup_mod, "best_effort_cleanup", cut.answering(True))++ task = await cut.run(cleanup_mod._deleted("p1"))++ assert task.cancelled()diff --git a/autogpt_platform/backend/backend/copilot/dream/cancel.py b/autogpt_platform/backend/backend/copilot/dream/cancel.pyindex 48a7948e30..1a09ed4b9b 100644--- a/autogpt_platform/backend/backend/copilot/dream/cancel.py+++ b/autogpt_platform/backend/backend/copilot/dream/cancel.py@@ -1,9 +1,10 @@"""The stop path of a dream pass: cancel it, and notice a stop while it runs.``cancel_dream_pass`` closes the pass's row CANCELLED in one conditional-transition that also bumps its cancel generation, and only while the row is-open and the caller's own. It does not reach into the running pass; the pass-reads its row at its next check and stops itself:+transition that also bumps its cancel generation and marks the row for the+cleanup after the pass (``cleanup.py``), and only while the row is open and+the caller's own. It does not reach into the running pass; the pass reads+its row at its next check and stops itself:* a sync pass at each phase boundary and once more just before apply(``stop_if_stopped``, ``stop_before_apply``) ends with its failure result:@@ -13,23 +14,35 @@ reads its row at its next check and stops itself:refuses its submit cancels that batch instead of handing it on(``batch_handoff``);* a batch pass in its callback, before it chains the next phase and before- it claims the apply gate (``end_batch_pass_if_stopped``), cancels its- provider batch best-effort and ends through ``fail_pass``: its landed- phases charged, its lock released, its batch state and bundle cleaned.+ it claims the apply gate (``end_batch_pass_if_stopped``), ends through+ ``fail_pass``: its provider batch stopped, its landed phases charged,+ its lock released, its batch state and bundle cleaned, and the mark+ cleared once all of that has finished;+ * a batch pass waiting on its provider has that batch cancelled by the+ cancel itself, and the executor drops it at its next poll without+ dispatching it; the walker whose claim takes the entry off the queue+ ends the pass the same way (``batch_deliveries``).++Whatever that leaves undone (an executor that is down, a drop hook that+fails, a pass that stopped without a cleanup of its own) the reaper finishes+from the mark, once the pass has had its grace to stop (``reaper.py``).A cancel that lands after a pass's last check, while it claims apply orapplies, is too late: that apply runs, once, and the row stays CANCELLED.A row a newer pass's guard expired (``guard.py``) has its generation bumped thesame way, so its pass stops at the same checks. And right before apply each-route checks that it still holds its scope's lock (``stop_before_apply``,-``end_batch_pass_if_lock_lost``): a newer pass takes the scope only once-this one's lock has lapsed, so a pass that lost it never applies over the-newer one. A closed row takes no other write, so the failure each route-records on the way out leaves it CANCELLED (or EXPIRED). The stop checks read-the row under the store's deadline and never stop a pass on a read that fails:-the store being down must not end work that nobody cancelled. The lock check-fails closed: a pass that cannot confirm its lock does not apply.+route proves it still holds its scope's lock: the sync pass reads it+(``stop_before_apply``), then both renew it by compare-and-extend, failing+closed (``lease.admit_sync_apply``, ``lease.admit_batch_apply``). A newer+pass takes the scope only once this one's lock has lapsed, so a pass the+admission finds without it never applies over the newer one; one whose lock+lapses after its last renewal is ``lease.py``'s residual. A closed row takes+no other write, so the failure each route records on the way out leaves it+CANCELLED (or EXPIRED). The stop checks read the row under the store's+deadline and never stop a pass on a read that fails: the store being down+must not end work that nobody cancelled. The lock checks fail closed: a pass+that cannot confirm its lock does not apply."""import logging@@ -41,7 +54,7 @@ from backend.copilot.graphiti.scope import MemoryScopefrom backend.data.dream_pass_models import OPEN_STATUSES, DreamPassRecordfrom .batch_outcome import BatchPass, fail_pass-from .locks import DreamLockHandle, dream_lock_held_by+from .locks import DreamLockHandlefrom .pass_record import cancelled, stop_errorfrom .pass_run import DreamPassRun, PassEndedfrom .provider_batch import cancel_provider_batch@@ -66,13 +79,18 @@ async def cancel_dream_pass(pass_id: str, *, user_id: str, reason: str) -> DreamPassCancel:"""Cancel *user_id*'s pass *pass_id* while it is open, then read its row- back. Unlike a pass's own writes, a store call that fails or runs out of- time raises: the caller is answering a request, or about to erase the- memory the pass would write."""+ back; a batch it closed has its provider batch in flight cancelled too,+ best-effort (the executor drops that batch at its next poll whatever the+ provider says, see ``batch_deliveries.should_dispatch``). Unlike a pass's+ own writes, a store call that fails or runs out of time raises: the caller+ is answering a request, or about to erase the memory the pass would+ write."""closed = await write_stop(pass_id, cancelled(reason, owner_user_id=user_id))record = await read_dream_pass(pass_id, user_id=user_id)if closed:logger.info(f"Dream pass {pass_id} cancelled: {reason}")+ if closed and record is not None and record.provider_batch_id:+ await cancel_provider_batch(record.provider_batch_id)return DreamPassCancel(cancelled=closed, record=record)@@ -129,37 +147,16 @@ async def end_batch_pass_if_stopped(bp: BatchPass) -> bool:say whether it did; the callback's check before it chains the next phaseand before it claims the apply gate.- The batch the row names is cancelled at the provider first. By the time a- callback runs, that is usually the batch that just ended, which refuses;- it matters when the row names a later one still in flight."""+ The batch the row names is stopped at the provider as part of the+ cleanup. By the time a callback runs, that is usually the batch that just+ ended, which refuses and reads as ended; it matters when the row names a+ later one still in flight."""row = await _read_row(bp.pass_id)error = stop_error(row) if row is not None else Noneif row is None or error is None:return Falselogger.info(f"Dream batch pass {bp.pass_id} stops: {error}")- if row.provider_batch_id:- await cancel_provider_batch(row.provider_batch_id)- await fail_pass(bp, error)- return True---async def end_batch_pass_if_lock_lost(bp: BatchPass, lock_token: str | None) -> bool:- """The batch pass's check once its apply gate is claimed, with nothing- durable awaited before it: end the pass and say so when it no longer- holds its scope's lock under *lock_token*.-- Its landed phases are charged and its state and bundle cleaned. A lock- that is no longer the pass's is left alone; one that could not be read is- released by compare-and-delete, which only ever deletes the pass's own.- The job and the record say why: the stop that closed the row (a newer- pass's expiry), else that the lock was lost."""- scope = MemoryScope.build(bp.user_id, bp.expert_id)- held = await _lock_held(dream_lock_held_by(scope, lock_token), bp.pass_id)- if held:- return False- error = await stopped_error(bp.pass_id) or LOCK_LOST_ERROR- logger.warning(f"Dream batch pass {bp.pass_id} does not apply: {error}")- await fail_pass(bp, error, holds_lock=held is None)+ await fail_pass(bp, error, provider_batch_id=row.provider_batch_id)return Truediff --git a/autogpt_platform/backend/backend/copilot/dream/cancel_test.py b/autogpt_platform/backend/backend/copilot/dream/cancel_test.pyindex 54b6dc90d0..c477a1aa4f 100644--- a/autogpt_platform/backend/backend/copilot/dream/cancel_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/cancel_test.py@@ -139,6 +139,38 @@ class TestCancelDreamPass:_, update = db.update_dream_pass.await_args.argsassert update.owner_user_id == "u2"+ async def test_a_batch_it_closes_has_its_batch_in_flight_cancelled(+ self, db, mocker+ ):+ provider_cancel = mocker.patch.object(+ cancel, "cancel_provider_batch", AsyncMock()+ )+ db.get_dream_pass_for_user.return_value = _cancelled(+ route=DreamPassRoute.ANTHROPIC_BATCH, provider_batch_id="msgbatch_live"+ )++ result = await cancel_dream_pass("p1", user_id="u1", reason="testing")++ assert result.cancelled+ provider_cancel.assert_awaited_once_with("msgbatch_live")++ @pytest.mark.parametrize("closed", [True, False], ids=["sync", "already_ended"])+ async def test_nothing_is_cancelled_at_the_provider_otherwise(+ self, db, mocker, closed+ ):+ """A sync pass has no batch, and a pass that had already ended keeps+ whatever it had."""+ provider_cancel = mocker.patch.object(+ cancel, "cancel_provider_batch", AsyncMock()+ )+ db.update_dream_pass.return_value = closed+ ended = _row(status=DreamPassStatus.ERRORED, provider_batch_id="b")+ db.get_dream_pass_for_user.return_value = _cancelled() if closed else ended++ await cancel_dream_pass("p1", user_id="u1", reason="testing")++ provider_cancel.assert_not_awaited()+async def test_a_long_reason_is_capped_like_any_error(self, db):await cancel_dream_pass("p1", user_id="u1", reason="x" * (MAX_ERROR_CHARS * 2))@@ -259,9 +291,11 @@ class TestTheBatchCheck:fail = mocker.patch.object(cancel, "fail_pass", AsyncMock())return provider, fail- async def test_a_stopped_pass_cancels_its_batch_and_ends_through_fail_pass(+ async def test_a_stopped_pass_ends_through_fail_pass_which_stops_its_batch(self, db, ends):+ """The batch the row names is stopped by the cleanup ``fail_pass``+ runs, so a provider that cannot confirm it keeps the row marked."""provider, fail = endsdb.get_dream_pass.return_value = _cancelled(route=DreamPassRoute.ANTHROPIC_BATCH, provider_batch_id="msgbatch_1"@@ -269,8 +303,10 @@ class TestTheBatchCheck:assert await end_batch_pass_if_stopped(_BATCH_PASS) is True- provider.assert_awaited_once_with("msgbatch_1")- fail.assert_awaited_once_with(_BATCH_PASS, "cancelled: testing")+ provider.assert_not_awaited()+ fail.assert_awaited_once_with(+ _BATCH_PASS, "cancelled: testing", provider_batch_id="msgbatch_1"+ )async def test_a_stopped_row_without_a_batch_skips_the_provider(self, db, ends):provider, fail = ends@@ -279,7 +315,9 @@ class TestTheBatchCheck:assert await end_batch_pass_if_stopped(_BATCH_PASS) is Trueprovider.assert_not_awaited()- fail.assert_awaited_once()+ fail.assert_awaited_once_with(+ _BATCH_PASS, "cancelled: testing", provider_batch_id=None+ )async def test_a_pass_nobody_stopped_goes_on(self, db, ends):provider, fail = endsdiff --git a/autogpt_platform/backend/backend/copilot/dream/cleanup.py b/autogpt_platform/backend/backend/copilot/dream/cleanup.pynew file mode 100644index 0000000000..baa4c1130c--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/cleanup.py@@ -0,0 +1,284 @@+"""The cleanup after a dream pass whose row has closed: what a batch pass+leaves behind (its provider batch, its landed phases' state and charges, its+lock), and the lock of a sync pass that stopped without giving it back.++Four steps, in this order, each idempotent, each bounded, and each saying+whether it finished:++ charge every phase landed in the pass's Redis state charged once,+ under the usage attribution its row carries and never another+ (``batch_costs.charge_landed_phases``); a state that cannot be+ read is not an empty one, and one that is gone while the row+ says phases landed is usage lost, logged as an error+ unlock the pass's dream lock released by compare-and-delete on its+ token, or found no longer the pass's; with no token anywhere+ the pass's ownership is unknown, and the lock is left held and+ the step unfinished, until the reaper finds the lock gone or+ another open pass's+ delete the state and the input bundle deleted, once every landed phase+ is charged: a charge left unfinished keeps what it needs+ provider the batch the pass has in flight stopped at Anthropic: the cancel+ acknowledged, or the batch found ended+ (``provider_batch.provider_batch_stopped``)++The charge goes first because the state it reads lapses. The provider goes+last because it is the slowest (two calls of up to+``PROVIDER_CANCEL_TIMEOUT_SECONDS``, spread over two cleanups when they do+not fit one, ``provider_batch.py``) and none of the other steps needs it+done first: the executor drops a closed pass's batch unpolled whether or not+Anthropic has stopped it. A provider that hangs then costs a cleanup its+last step, retried, and not the release of the user's lock.++Every transition that may leave this cleanup behind marks the row for it+(``cleanupPendingAt``) and keeps its lease token (``pass_record.py``): a stop+from outside, a batch pass's end. Whoever cleans up after the pass, the pass+itself as it ends (``batch_outcome.clean_up_after``) or the reaper, clears+the mark only once every step has finished; until then the row stays on the+reaper's list, and its next run does every step again, the finished ones+doing nothing (``reaper.py``).+"""++import asyncio+import logging+from collections.abc import Sequence+from typing import Literal++from pydantic import BaseModel++from backend.copilot.graphiti.scope import MemoryScope++from .batch_costs import PhaseCharges, charge_landed_phases+from .batch_state import (+ best_effort_cleanup,+ read_state,+ recorded_phase_models,+ uncharged_phases,+)+from .locks import LOCK_CHECK_TIMEOUT_SECONDS, read_dream_lock_token, release_dream_lock+from .phase_jobs import PHASE_TIERS+from .provider_batch import provider_batch_stopped+from .schemas import DreamPhase+from .store import read_open_passes, record_cleanup_finished++logger = logging.getLogger(__name__)++CleanupStep = Literal["charge", "unlock", "delete", "provider"]++# How long reading or deleting a pass's Redis state may take, the client's+# retries included: a stalled Redis costs a cleanup seconds, not its turn.+STATE_STEP_TIMEOUT_SECONDS = 5.0+# How long the cleanup waits on its landed phases' charge. The charge goes on+# without it, in a task of its own (``batch_costs.charge_landed_phases``),+# claiming for ``batch_costs.CLAIM_WINDOW_SECONDS``; what it does not reach+# waits for the next cleanup.+CHARGE_STEP_TIMEOUT_SECONDS = 10.0+++class PassCleanup(BaseModel):+ """What one cleanup did: the phases it charged, and the steps it could+ not finish, none once the cleanup is done."""++ charged: list[DreamPhase] = []+ unfinished: list[CleanupStep] = []++ @property+ def finished(self) -> bool:+ return not self.unfinished+++async def clean_up_pass(+ pass_id: str,+ scope: MemoryScope,+ *,+ phase_models: dict[str, str] | None,+ provider_batch_id: str | None,+ lock_token: str | None,+ release: bool,+ attribute_tokenless: bool = False,+ landed: Sequence[DreamPhase] = (),+) -> PassCleanup:+ """Every step of the cleanup after pass *pass_id* of *scope*, in order,+ each run whatever became of the others but the delete, which waits on+ the charge. Each is bounded: the state read and delete by+ ``STATE_STEP_TIMEOUT_SECONDS``, the charges by+ ``CHARGE_STEP_TIMEOUT_SECONDS``, each lock call by+ ``LOCK_CHECK_TIMEOUT_SECONDS``, each provider call by+ ``PROVIDER_CANCEL_TIMEOUT_SECONDS``. Never raises.++ *provider_batch_id* names the batch to stop, if any; *phase_models*+ prices the landed phases (``None``: they cannot be priced, and the+ charge stays unfinished while any landed); the lock is released under+ *lock_token* only when *release*. Without a token the unlock stays+ unfinished, unless *attribute_tokenless* (the reaper, for a row that kept+ no token) and the lock is found not to be the pass's. *landed* are the+ phases the pass's row says landed: should their state be gone before+ they were charged, that usage is reported lost (an error), not taken for+ nothing to charge. Each phase is charged under the usage attribution+ the pass started under, read off its row (``batch_costs``); a row+ without one is charged nothing, and the state goes all the same."""+ unfinished: list[CleanupStep] = []+ charges = await _charge(pass_id, scope, phase_models, landed)+ if not charges.settled:+ unfinished.append("charge")+ if release and not await _unlock(+ pass_id, scope, lock_token, attribute=attribute_tokenless+ ):+ unfinished.append("unlock")+ if not charges.settled or not await _deleted(pass_id):+ unfinished.append("delete")+ if provider_batch_id and not await provider_batch_stopped(provider_batch_id):+ unfinished.append("provider")+ return PassCleanup(charged=charges.charged, unfinished=unfinished)+++async def finish_cleanup(pass_id: str, cleanup: PassCleanup) -> None:+ """Clear the row's mark once *cleanup* has finished every step; else+ leave it for the reaper, whose next run does the cleanup again. Never+ raises: a mark left behind costs the reaper a row, nothing more."""+ if not cleanup.finished:+ logger.warning(+ f"Dream pass {pass_id}: cleanup unfinished at "+ f"{', '.join(cleanup.unfinished)}; left marked for the reaper"+ )+ return+ try:+ await record_cleanup_finished(pass_id)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not clear its cleanup mark; the "+ "reaper will",+ exc_info=True,+ )+++async def _charge(+ pass_id: str,+ scope: MemoryScope,+ phase_models: dict[str, str] | None,+ landed: Sequence[DreamPhase],+) -> PhaseCharges:+ """Charge the phases landed in the pass's state, each on the model its+ state row recorded, else the one *phase_models* names; not settled when+ the state cannot be read, a landed phase has no model to price it with+ or cannot be claimed, or the charges outlast+ ``CHARGE_STEP_TIMEOUT_SECONDS``."""+ try:+ async with asyncio.timeout(STATE_STEP_TIMEOUT_SECONDS):+ state = await read_state(pass_id)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not read its batch state to charge it",+ exc_info=True,+ )+ return PhaseCharges(settled=False)+ if not state:+ await _report_lost_usage(pass_id, scope, landed)+ return PhaseCharges(settled=True)+ models = {**(phase_models or {}), **recorded_phase_models(state)}+ unpriced = [+ phase+ for phase in PHASE_TIERS+ if (row := state.get(phase)) and not row.get("error") and phase not in models+ ]+ if unpriced:+ logger.warning(+ f"Dream pass {pass_id}: no model to price its {', '.join(unpriced)} "+ "phase with"+ )+ return PhaseCharges(settled=False)+ try:+ async with asyncio.timeout(CHARGE_STEP_TIMEOUT_SECONDS):+ return await charge_landed_phases(+ user_id=scope.owner_user_id,+ expert_id=scope.expert_id,+ pass_id=pass_id,+ state=state,+ phase_models=models,+ )+ except TimeoutError:+ logger.warning(+ f"Dream pass {pass_id}: its charge outlasted "+ f"{CHARGE_STEP_TIMEOUT_SECONDS:g} s; it goes on on its own, and "+ "what it does not reach is left for the next cleanup"+ )+ return PhaseCharges(settled=False)+++async def _report_lost_usage(+ pass_id: str, scope: MemoryScope, landed: Sequence[DreamPhase]+) -> None:+ """An error naming the landed phases whose batch state lapsed before+ anyone charged them: their usage cannot be recovered. Nothing more can+ be done for them, so the cleanup goes on as settled."""+ if not landed:+ return+ lost = await uncharged_phases(pass_id, list(landed))+ if lost:+ logger.error(+ f"Dream pass {pass_id} of user {scope.owner_user_id} "+ f"(expert {scope.expert_id or 'none'}): the batch state of landed "+ f"phase(s) {', '.join(lost)} lapsed before they were charged; their "+ "usage cannot be recovered"+ )+++async def _deleted(pass_id: str) -> bool:+ """The state and input bundle deleted (``best_effort_cleanup``), within+ ``STATE_STEP_TIMEOUT_SECONDS``."""+ try:+ async with asyncio.timeout(STATE_STEP_TIMEOUT_SECONDS):+ return await best_effort_cleanup(pass_id)+ except TimeoutError:+ logger.warning(f"Dream pass {pass_id}: deleting its batch state timed out")+ return False+++async def _unlock(+ pass_id: str, scope: MemoryScope, lock_token: str | None, *, attribute: bool+) -> bool:+ """Release the pass's lock by compare-and-delete under *lock_token*,+ within ``LOCK_CHECK_TIMEOUT_SECONDS``. Without a token the pass's+ ownership is unknown: the step finishes only when *attribute* and the+ lock is found not to be the pass's (``_lock_is_not_the_passes``);+ otherwise it stays unfinished and the lock is left alone."""+ if lock_token is not None:+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ return await release_dream_lock(scope, lock_token)+ except TimeoutError:+ logger.warning(f"Dream pass {pass_id}: releasing its lock timed out")+ return False+ if not attribute:+ logger.warning(+ f"Dream pass {pass_id}: no lock token to release its lock with; "+ "left for the reaper"+ )+ return False+ return await _lock_is_not_the_passes(pass_id, scope)+++async def _lock_is_not_the_passes(pass_id: str, scope: MemoryScope) -> bool:+ """Whether the scope's lock is surely not the tokenless pass's: gone, or+ held under the lease token of another open pass of the scope. Never+ deletes: a lock held under a token no open pass keeps may be the pass's+ own, so it is left to lapse on its TTL (24 h 10 min at most for a batch+ lock, 30 min for a sync one), and the step stays unfinished until then."""+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ holder = await read_dream_lock_token(scope)+ if holder is None:+ return True+ others = await read_open_passes(scope)+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not tell whose its scope's lock is",+ exc_info=True,+ )+ return False+ if any(row.id != pass_id and row.lease_token == holder for row in others):+ return True+ logger.warning(+ f"Dream pass {pass_id}: its scope's lock is held under a token no open "+ "pass keeps; left to lapse on its TTL"+ )+ return Falsediff --git a/autogpt_platform/backend/backend/copilot/dream/cleanup_backstop_test.py b/autogpt_platform/backend/backend/copilot/dream/cleanup_backstop_test.pynew file mode 100644index 0000000000..b020a50394--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/cleanup_backstop_test.py@@ -0,0 +1,262 @@+"""The reaper as the backstop for every stopped pass, over the in-memory+store, Redis and the executor's queue. A cancel or a guard's expiry marks the+row and keeps its lease token in the transition that closes it, so when+nothing else cleans up after the pass (the executor down and never polling;+a drop hook that raises, times out or dies after its claim) the reaper does,+once the pass has had its grace to stop: the batch stopped at the provider,+the landed phase charged once, the lock released under the kept token, the+state and bundle deleted, the mark cleared. Before that grace it leaves the+lock of a pass that may still be running alone; a pass that cleans up after+itself clears its own mark. The provider and the charges are stubbed at+their edges."""++import asyncio+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++import pytest+from prisma.enums import DreamPassStatus++from backend.executor import batch_executor as executor++from . import batch_deliveries+from . import reaper as reaper_mod+from .batch_callbacks import handle_dream_batch_result+from .batch_deliveries_test import (+ _LOCK_KEY,+ _MODELS,+ _emulate_the_dispatch_claim,+ _entry,+ _in_flight,+ _row,+)+from .batch_state import state_key+from .batch_submit import input_bundle_key+from .cancel import cancel_dream_pass+from .conftest import FakeAsyncRedis, FakeDreamDb+from .locks import BATCH_LOCK_TTL_SECONDS+from .pass_record import expired+from .reaper import REAP_GRACE_SECONDS, reap_expired_passes+from .reaper_test import _charged+from .schemas import DreamOperations+from .store import write_stop++[email protected](autouse=True)+def provider(mocker) -> AsyncMock:+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key", return_value="k"+ )+ mocker.patch.object(reaper_mod, "phase_models_for_config", return_value=_MODELS)+ return mocker.patch(+ "backend.copilot.dream.provider_batch.cancel_batch",+ AsyncMock(return_value=True),+ )++[email protected](autouse=True)+def charges(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )++[email protected](autouse=True)+def handlers():+ executor.clear_handlers_for_test()+ yield+ executor.clear_handlers_for_test()+++class TestACancelledPassNobodyPolls:+ async def test_is_left_alone_for_its_grace_then_cleaned_up_by_the_reaper(+ self, fake_dream_db, fake_dream_redis, provider, charges+ ):+ """Codex's case: a SUBMITTED pass cancelled while the executor is+ down. Its lease is a batch lease, fresh for a day; the reaper waits+ out the grace from the cancel, time for a pass still running to see+ it, then does what the drop would have."""+ await _submitted(fake_dream_db, fake_dream_redis)+ await executor.enqueue_pending(_entry("recombine"))++ assert (await cancel_dream_pass("p1", user_id="u1", reason="stop")).cancelled++ _assert_marked(fake_dream_db, DreamPassStatus.CANCELLED)+ provider.assert_awaited_once()+ within_the_grace = await reap_expired_passes(+ now=_after(REAP_GRACE_SECONDS - 60)+ )+ assert within_the_grace.listed == 0+ assert fake_dream_redis.store[_LOCK_KEY] == "tok"++ run = await reap_expired_passes(now=_after(REAP_GRACE_SECONDS + 60))++ assert run.outcomes == {"cleaned": 1}+ assert provider.await_count == 2+ assert provider.await_args.kwargs["provider_batch_id"] == "b-rec"+ _assert_cleaned_up(fake_dream_db, fake_dream_redis, charges)+++class TestADropHookThatDoesNotFinish:+ @pytest.mark.parametrize("failure", ["exception", "timeout", "crash_after_claim"])+ async def test_leaves_the_pass_to_the_reaper(+ self, monkeypatch, fake_dream_db, fake_dream_redis, charges, failure+ ):+ """Codex's three: the executor claims the entry off its queue, and+ its hook raises, runs out of time, or the process dies right after+ the claim. The row keeps the mark the cancel set, and the reaper+ finishes the cleanup."""+ await _submitted(fake_dream_db, fake_dream_redis)+ _emulate_the_dispatch_claim(monkeypatch, fake_dream_redis)+ entry = _entry("recombine")+ await executor.enqueue_pending(entry)+ assert (await cancel_dream_pass("p1", user_id="u1", reason="stop")).cancelled++ async def drop_that_fails(_entry) -> None:+ if failure == "exception":+ raise RuntimeError("drop hook failed")+ await asyncio.Event().wait()++ executor.register_handler(+ "dream_pass",+ handle_dream_batch_result,+ should_dispatch=batch_deliveries.should_dispatch,+ on_drop=drop_that_fails,+ )+ monkeypatch.setattr(executor, "DISPATCH_CHECK_TIMEOUT_SECONDS", 0.05)+ if failure == "crash_after_claim":+ dies = AsyncMock(side_effect=asyncio.CancelledError("process died"))+ monkeypatch.setattr(executor, "_run_drop_hook", dies)+ with pytest.raises(asyncio.CancelledError):+ await _walk(entry)+ else:+ await _walk(entry)++ assert await executor.list_pending() == []+ charges.assert_not_awaited()+ _assert_marked(fake_dream_db, DreamPassStatus.CANCELLED)++ run = await reap_expired_passes(now=_after(REAP_GRACE_SECONDS + 60))++ assert run.outcomes == {"cleaned": 1}+ _assert_cleaned_up(fake_dream_db, fake_dream_redis, charges)+++class TestAGuardExpiredBatchPass:+ async def test_is_cleaned_up_by_the_reaper_the_newer_passs_lock_kept(+ self, fake_dream_db, fake_dream_redis, provider, charges+ ):+ """A newer pass's guard expires a stale batch row while holding the+ scope's lock itself: the guard stops nothing at the provider, so the+ reaper does, charges the landed phase and cleans up; its unlock+ under the expired pass's token leaves the newer pass's lock alone."""+ await _submitted(fake_dream_db, fake_dream_redis)+ fake_dream_db.rows["p1"]["lease_expires_at"] = _after(-600)+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"++ assert await write_stop("p1", expired("stale", not_updated_since=None))++ _assert_marked(fake_dream_db, DreamPassStatus.EXPIRED)+ provider.assert_not_awaited()++ run = await reap_expired_passes(now=_after(REAP_GRACE_SECONDS + 60))++ assert run.outcomes == {"cleaned": 1}+ provider.assert_awaited_once()+ assert _charged(charges) == ["consolidate"]+ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"+ assert state_key("p1") not in fake_dream_redis.hashes+ row = fake_dream_db.rows["p1"]+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+++class TestAPassThatEndsItself:+ async def test_clears_its_own_mark_once_its_cleanup_is_done(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """The applied tail closes the row marked, cleans up, and clears the+ mark itself: nothing is left for the reaper."""+ mocker.patch(+ "backend.copilot.dream.apply.apply_operations",+ AsyncMock(return_value={"writes": 0}),+ )+ await _in_flight(fake_dream_db, fake_dream_redis, landed=2)++ await handle_dream_batch_result(+ _entry("sanitize"), [_row("sanitize", DreamOperations())]+ )++ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.COMPLETE+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+ assert _charged(charges) == ["consolidate", "recombine", "sanitize"]+ assert _LOCK_KEY not in fake_dream_redis.store+ assert (await reap_expired_passes(now=_after(86_400))).listed == 0++ async def test_whose_unlock_fails_leaves_its_mark_for_the_reaper(+ self, mocker, monkeypatch, fake_dream_db, fake_dream_redis, charges+ ):+ mocker.patch(+ "backend.copilot.dream.apply.apply_operations",+ AsyncMock(return_value={"writes": 0}),+ )+ await _in_flight(fake_dream_db, fake_dream_redis, landed=2)+ evaluate = fake_dream_redis.eval++ async def unlock_down(script: str, numkeys: int, *args):+ if '"del"' in script:+ raise ConnectionError("redis down at the unlock")+ return await evaluate(script, numkeys, *args)++ monkeypatch.setattr(fake_dream_redis, "eval", unlock_down)+ await handle_dream_batch_result(+ _entry("sanitize"), [_row("sanitize", DreamOperations())]+ )+ monkeypatch.setattr(fake_dream_redis, "eval", evaluate)++ _assert_marked(fake_dream_db, DreamPassStatus.COMPLETE)+ assert fake_dream_redis.store[_LOCK_KEY] == "tok"++ run = await reap_expired_passes(now=_after(REAP_GRACE_SECONDS + 60))++ assert run.outcomes == {"cleaned": 1}+ assert _LOCK_KEY not in fake_dream_redis.store+ assert _charged(charges) == ["consolidate", "recombine", "sanitize"]+ row = fake_dream_db.rows["p1"]+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+++async def _submitted(fake_dream_db: FakeDreamDb, fake_dream_redis: FakeAsyncRedis):+ """Pass p1 waiting on recombine's batch, consolidate landed, holding its+ lock and its lease for a batch's lifetime."""+ await _in_flight(fake_dream_db, fake_dream_redis)+ await fake_dream_redis.expire(_LOCK_KEY, BATCH_LOCK_TTL_SECONDS)+ fake_dream_db.rows["p1"]["lease_expires_at"] = _after(BATCH_LOCK_TTL_SECONDS)+++def _assert_marked(fake_dream_db: FakeDreamDb, status: DreamPassStatus) -> None:+ row = fake_dream_db.rows["p1"]+ assert row["status"] is status+ assert row["cleanup_pending_at"] is not None+ assert row["lease_token"] == "tok"+++def _assert_cleaned_up(+ fake_dream_db: FakeDreamDb, fake_dream_redis: FakeAsyncRedis, charges: AsyncMock+) -> None:+ assert _charged(charges) == ["consolidate"]+ assert _LOCK_KEY not in fake_dream_redis.store+ assert state_key("p1") not in fake_dream_redis.hashes+ assert input_bundle_key("p1") not in fake_dream_redis.store+ row = fake_dream_db.rows["p1"]+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+++async def _walk(entry) -> None:+ await executor._walk_entry(+ entry, now=datetime.now(timezone.utc), api_key_for=lambda _: "k"+ )+++def _after(seconds: float) -> datetime:+ return datetime.now(timezone.utc) + timedelta(seconds=seconds)diff --git a/autogpt_platform/backend/backend/copilot/dream/cleanup_retry_test.py b/autogpt_platform/backend/backend/copilot/dream/cleanup_retry_test.pynew file mode 100644index 0000000000..56639ab150--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/cleanup_retry_test.py@@ -0,0 +1,218 @@+"""A cleanup step that fails without raising, over the in-memory store and+Redis: a Redis error at the state read, at a charge claim, at the unlock or+at a delete; a provider that cannot confirm the batch stopped; no phase+models to price with. Each leaves the row marked and its lease token kept,+and its state too while the charge is unfinished; the run says ``retry`` at+warning, naming the steps left; and the next run finishes the cleanup once:+one charge, the lock, the state and the bundle gone, the mark and the token+cleared. The provider and the charges are stubbed at their edges; the lock,+the claims, the batch state and the row transitions are the real ones."""++import logging+from collections.abc import Callable+from datetime import datetime, timedelta, timezone+from typing import Any+from unittest.mock import AsyncMock++import pytest+from prisma.enums import DreamPassStatus++from . import reaper as reaper_mod+from .batch_state import state_key+from .batch_submit import input_bundle_key+from .conftest import FakeAsyncRedis, FakeDreamDb+from .reaper import reap_expired_passes+from .reaper_test import _LOCK_KEY, _charged, _dead_batch_pass++_MODELS = {p: "claude-sonnet-5" for p in ("consolidate", "recombine", "sanitize")}++# Each Redis call a cleanup step makes, the key it fails on, and the steps+# its failure leaves unfinished.+_REDIS_FAILURES: dict[str, tuple[str, str, str]] = {+ "state read": ("hgetall", state_key("p1"), "charge, delete"),+ "pass claim": ("set", "dream:batch:costs_logged:p1", "charge, delete"),+ "phase claim": ("set", "dream:batch:charged:p1:consolidate", "charge, delete"),+ "unlock": ("eval", _LOCK_KEY, "unlock"),+ "state delete": ("delete", state_key("p1"), "delete"),+ "bundle delete": ("delete", input_bundle_key("p1"), "delete"),+}++[email protected](autouse=True)+def provider(mocker) -> AsyncMock:+ """The provider's cancel, acknowledged unless a test says otherwise."""+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key", return_value="k"+ )+ mocker.patch.object(reaper_mod, "phase_models_for_config", return_value=_MODELS)+ return mocker.patch(+ "backend.copilot.dream.provider_batch.cancel_batch",+ AsyncMock(return_value=True),+ )++[email protected](autouse=True)+def charges(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )+++class TestARedisErrorInAStep:+ @pytest.mark.parametrize("failure", list(_REDIS_FAILURES))+ async def test_keeps_the_mark_and_the_next_run_finishes_once(+ self, monkeypatch, fake_dream_db, fake_dream_redis, charges, caplog, failure+ ):+ """Codex's three (the state read, the state delete, the unlock) and+ every other Redis call a step makes: the failure is swallowed, as+ before, but no longer taken for success."""+ await _dead_batch_pass(fake_dream_db)+ fake_dream_redis.store[_LOCK_KEY] = "dead-token"+ method, key, unfinished = _REDIS_FAILURES[failure]+ restore = _fail_on(monkeypatch, fake_dream_redis, method, key)++ with caplog.at_level(logging.WARNING):+ first = await reap_expired_passes()++ assert first.outcomes == {"retry": 1}+ _assert_marked(fake_dream_db)+ assert f"cleanup unfinished at {unfinished}; listed again" in caplog.text+ if unfinished.startswith("charge"):+ charges.assert_not_awaited()+ assert state_key("p1") in fake_dream_redis.hashes++ restore()+ await _assert_the_next_run_finishes_once(+ fake_dream_db, fake_dream_redis, charges+ )+++class TestAProviderThatCannotConfirmTheStop:+ @pytest.mark.parametrize(+ "status",+ [ConnectionError("anthropic unreachable"), "processing"],+ ids=["status unreadable", "still processing"],+ )+ async def test_keeps_the_mark_until_the_batch_has_stopped(+ self, fake_dream_db, fake_dream_redis, provider, batch_status, charges, status+ ):+ """The cancel is refused and the batch is not found ended: the other+ steps finish (one charge), the provider one is retried."""+ await _dead_batch_pass(fake_dream_db)+ provider.return_value = False+ if isinstance(status, Exception):+ batch_status.side_effect = status+ else:+ batch_status.return_value = status++ first = await reap_expired_passes()++ assert first.outcomes == {"retry": 1}+ _assert_marked(fake_dream_db)+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes++ provider.return_value = True+ await _assert_the_next_run_finishes_once(+ fake_dream_db, fake_dream_redis, charges+ )++ async def test_a_refused_cancel_of_a_batch_that_ended_is_no_failure(+ self, fake_dream_db, provider, batch_status+ ):+ await _dead_batch_pass(fake_dream_db)+ provider.return_value = False+ batch_status.return_value = "ended"++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ batch_status.assert_awaited_once()+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None++ @pytest.mark.parametrize(+ ("marked_hours_ago", "outcome"), [(1, "retry"), (26, "cleaned")]+ )+ async def test_without_a_key_a_batch_is_waited_on_only_its_window(+ self, mocker, fake_dream_db, marked_hours_ago, outcome+ ):+ """No key to cancel or read the batch with: the step stays+ unfinished until the row was marked longer ago than a batch can run,+ when the batch has ended whatever the provider could say."""+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key",+ return_value=None,+ )+ await _dead_batch_pass(fake_dream_db)+ fake_dream_db.rows["p1"].update(+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=datetime.now(timezone.utc)+ - timedelta(hours=marked_hours_ago),+ )++ run = await reap_expired_passes()++ assert run.outcomes == {outcome: 1}+ cleared = fake_dream_db.rows["p1"]["cleanup_pending_at"] is None+ assert cleared is (outcome == "cleaned")+++class TestNoPhaseModelsToPriceWith:+ async def test_keep_the_state_until_the_charge_can_be_made(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ await _dead_batch_pass(fake_dream_db)+ models = mocker.patch.object(+ reaper_mod, "phase_models_for_config", side_effect=KeyError("model")+ )++ first = await reap_expired_passes()++ assert first.outcomes == {"retry": 1}+ _assert_marked(fake_dream_db)+ charges.assert_not_awaited()+ assert state_key("p1") in fake_dream_redis.hashes++ models.side_effect = None+ models.return_value = _MODELS+ await _assert_the_next_run_finishes_once(+ fake_dream_db, fake_dream_redis, charges+ )+++def _fail_on(+ monkeypatch, redis: FakeAsyncRedis, method: str, key: str+) -> Callable[[], None]:+ """Make Redis's *method* raise for *key* until the returned undo runs."""+ real = getattr(redis, method)++ async def failing(*args: Any, **kwargs: Any) -> Any:+ if key in args:+ raise ConnectionError(f"redis down at {method} {key}")+ return await real(*args, **kwargs)++ monkeypatch.setattr(redis, method, failing)+ return lambda: monkeypatch.setattr(redis, method, real)+++def _assert_marked(fake_dream_db: FakeDreamDb) -> None:+ """The row closed EXPIRED, still marked, the dead pass's token kept."""+ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.EXPIRED+ assert row["cleanup_pending_at"] is not None+ assert row["lease_token"] == "dead-token"+++async def _assert_the_next_run_finishes_once(+ fake_dream_db: FakeDreamDb, fake_dream_redis: FakeAsyncRedis, charges: AsyncMock+) -> None:+ second = await reap_expired_passes()++ assert (second.listed, second.outcomes) == (1, {"cleaned": 1})+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes+ assert input_bundle_key("p1") not in fake_dream_redis.store+ assert _LOCK_KEY not in fake_dream_redis.store+ row = fake_dream_db.rows["p1"]+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+ assert (await reap_expired_passes()).listed == 0diff --git a/autogpt_platform/backend/backend/copilot/dream/conftest.py b/autogpt_platform/backend/backend/copilot/dream/conftest.pyindex e30e6d6b71..73088f201b 100644--- a/autogpt_platform/backend/backend/copilot/dream/conftest.py+++ b/autogpt_platform/backend/backend/copilot/dream/conftest.py@@ -14,7 +14,9 @@ in-memory stand-in by default. Tests that need their own fake keep patchingNo Postgres either, and the DatabaseManager RPC client behind ``dream_db()``would stall a write until the store's deadline, so the dream store writeseach pass's record to an in-memory ``FakeDreamDb`` that tests can read the-transitions back from. The master flag the guard reads answers on.+transitions back from. The master flag the guard reads answers on. And no+test reaches Anthropic from a pass's cleanup: there is no key to reach it+with, and a batch's status reads ended, unless a test sets its own."""from __future__ import annotations@@ -29,6 +31,7 @@ import pytestimport pytest_asynciofrom backend.copilot import usage_activation_fixtures+from backend.copilot.trial_cost_context import TrialCostContextfrom backend.data.dream_pass_models import (OPEN_STATUSES,DreamPassDraft,@@ -187,14 +190,16 @@ class FakeDreamDb:"""In-memory stand-in for ``backend.data.dream_pass`` behind ``dream_db()``.Applies an update by the rules of ``backend/data/dream_pass_update.py``:- a ``None`` field leaves its column, status and phase only move forward,+ a ``None`` field leaves its column, status, phase and the lease's+ expiry only move forward,a batch id lands only while the row has not moved past the update'sphase, ``phase_outputs`` and ``operations`` merge one field at a time, a- stop bumps the cancel generation, and a terminal row, another user's row- (for an update naming the owner) or a row written after the update's- instant is not written. ``writes`` holds every draft and update that was- written, in order, as sent. ``fail`` makes every call raise, as an- unreachable database would.+ stop bumps the cancel generation, ``clear`` empties its columns, and a+ terminal row, another user's row (for an update naming the owner) or a+ row written after the update's instant is not written; an update marked+ ``closed_row`` writes a terminal row only. ``writes`` holds+ every draft and update that was written, in order, as sent. ``fail``+ makes every call raise, as an unreachable database would."""def __init__(self) -> None:@@ -207,7 +212,7 @@ class FakeDreamDb:if draft.id in self.rows:raise ValueError(f"duplicate dream pass {draft.id}")self.writes.append((draft.id, draft))- self.seed(draft)+ self._insert(draft)async def update_dream_pass(self, pass_id: str, update: DreamPassUpdate) -> bool:self._raise_if_down()@@ -230,8 +235,11 @@ class FakeDreamDb:row[field].update({k: v for k, v in dict(value).items() if v is not None})+ elif field == "lease_expires_at" and row.get(field) is not None:+ row[field] = max(row[field], value)else:row[field] = value+ row.update({column: None for column in update.clear})row["cancel_generation"] += int(update.bump_cancel_generation)row["updated_at"] = datetime.now(timezone.utc)return True@@ -257,9 +265,88 @@ class FakeDreamDb:if row["scope_key"] == scope_key and row["status"] in OPEN_STATUSES][:limit]+ async def list_expired_dream_passes(+ self,+ expired_before: datetime,+ limit: int = 100,+ after_lapse: datetime | None = None,+ after_id: str | None = None,+ ) -> list[DreamPassRecord]:+ """Open rows whose lease lapsed before *expired_before*, in lapse+ order (the lapse, then the id), after *after_lapse* and *after_id*+ when given, at most *limit*."""+ self._raise_if_down()+ lapsed = sorted(+ (row["lease_expires_at"], pass_id)+ for pass_id, row in self.rows.items()+ if row["status"] in OPEN_STATUSES+ and row.get("lease_expires_at") is not None+ and row["lease_expires_at"] < expired_before+ )+ if after_lapse is not None and after_id is not None:+ lapsed = [key for key in lapsed if key > (after_lapse, after_id)]+ return [self.record(pass_id) for _, pass_id in lapsed[:limit]]++ async def list_dream_pass_cleanups(+ self,+ due_before: datetime,+ limit: int = 100,+ marked_through: datetime | None = None,+ after_mark: datetime | None = None,+ after_id: str | None = None,+ ) -> list[DreamPassRecord]:+ """Closed rows marked for a cleanup that is due (marked, or holding+ no lease or one that lapsed, before *due_before*), in mark order (the+ mark, then the id), marked by *marked_through* and after *after_mark*+ and *after_id* when given."""+ self._raise_if_down()+ pending = sorted(+ (row["cleanup_pending_at"], pass_id)+ for pass_id, row in self.rows.items()+ if row["status"] not in OPEN_STATUSES+ and row.get("cleanup_pending_at") is not None+ and _due(row, due_before)+ and (marked_through is None or row["cleanup_pending_at"] <= marked_through)+ )+ if after_mark is not None and after_id is not None:+ pending = [key for key in pending if key > (after_mark, after_id)]+ return [self.record(pass_id) for _, pass_id in pending[:limit]]++ async def delete_old_dream_passes(+ self, created_before: datetime, limit: int = 1000+ ) -> int:+ """Delete at most *limit* closed rows created before *created_before*,+ none whose cleanup is pending."""+ self._raise_if_down()+ old = [+ pass_id+ for pass_id, row in self.rows.items()+ if row["status"] not in OPEN_STATUSES+ and row.get("cleanup_pending_at") is None+ and row["created_at"] < created_before+ ][:limit]+ for pass_id in old:+ del self.rows[pass_id]+ return len(old)+def seed(self, draft: DreamPassDraft, **columns: Any) -> None:"""A row as an earlier step (another process) would have left it;- *columns* set the rest of it (a lease, when it was last written)."""+ *columns* set the rest of it (a lease, when it was last written).++ Unless the draft or *columns* say otherwise, the row carries the+ usage attribution its pass would have started under in these tests+ (``usage_snapshot``: the owner, ready, untrialed), as ``start_pass``+ records it; ``cost_context=None`` seeds a row that carries none."""+ if draft.cost_context is None:+ columns = {+ "cost_context": TrialCostContext(+ user_id=draft.user_id, trial_id=None+ ).model_dump(mode="json"),+ **columns,+ }+ self._insert(draft, **columns)++ def _insert(self, draft: DreamPassDraft, **columns: Any) -> None:now = datetime.now(timezone.utc)self.rows[draft.id] = {**dict(draft),@@ -296,7 +383,7 @@ class FakeDreamDb:# Update fields that are not a column written as sent: the forward-only-# columns, the generation bump and the conditions.+# columns, the generation bump, the conditions and the columns to empty._NOT_WRITTEN_AS_IS = frozenset({"status",@@ -305,15 +392,18 @@ _NOT_WRITTEN_AS_IS = frozenset("bump_cancel_generation","owner_user_id","not_updated_since",+ "clear",+ "closed_row",})def _writable(row: dict[str, Any], update: DreamPassUpdate) -> bool:- """Whether *update* may write *row*: open, the owner's when it names one,- not written after its instant when it names one."""+ """Whether *update* may write *row*: open (closed, for a ``closed_row``+ update), the owner's when it names one, not written after its instant+ when it names one."""return (- row["status"] in OPEN_STATUSES+ (row["status"] in OPEN_STATUSES) is not update.closed_rowand update.owner_user_id in (None, row["user_id"])and (update.not_updated_since is None@@ -322,6 +412,14 @@ def _writable(row: dict[str, Any], update: DreamPassUpdate) -> bool:)+def _due(row: dict[str, Any], due_before: datetime) -> bool:+ """Whether a marked row's cleanup is due, as the cleanup scan decides."""+ lease = row.get("lease_expires_at")+ return row["cleanup_pending_at"] <= due_before or (+ lease is None or lease <= due_before+ )++def _order(value: Enum | None) -> int:"""Where *value* sits in its enum's declared order, as Postgres comparesenums."""@@ -399,6 +497,19 @@ def dream_pass_flag(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:return flag[email protected](autouse=True)+def batch_status(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:+ """No Anthropic key behind a pass's cleanup, and a batch status that reads+ ended: the cleanup's provider step never reaches the network. Tests of+ that step set their own key, cancel and status."""+ monkeypatch.setattr(+ "backend.copilot.dream.provider_batch.anthropic_api_key", lambda: None+ )+ status = AsyncMock(return_value="ended")+ monkeypatch.setattr("backend.copilot.dream.provider_batch.poll_batch", status)+ return status++@pytest.fixture(autouse=True)def fake_dream_redis(monkeypatch: pytest.MonkeyPatch) -> FakeAsyncRedis:"""Give every dream test an in-memory Redis unless it patches its own."""diff --git a/autogpt_platform/backend/backend/copilot/dream/guard.py b/autogpt_platform/backend/backend/copilot/dream/guard.pyindex d452aa17f3..404d0ef968 100644--- a/autogpt_platform/backend/backend/copilot/dream/guard.py+++ b/autogpt_platform/backend/backend/copilot/dream/guard.py@@ -17,14 +17,17 @@ pass's own:A row is fresh while its lease has not lapsed or, without a lease, while it waswritten within the pass lock's TTL. Expiring is one conditional transition that-bumps the row's cancel generation, so its pass stops at its next check; it-lands only while the row is still open. A stale row's expiry is also a+bumps the row's cancel generation, so its pass stops at its next check, and+marks the row for the cleanup after that pass (its batch, its landed phases,+its state), which the reaper finishes if the pass does not (``cleanup.py``);+it lands only while the row is still open. A stale row's expiry is also acompare-and-set on the row's last write: it lands only if nothing has writtenthe row since the guard read it, and a row that moved in between is alive andblocks after all. An admin's forced expiry of a fresh row skips thatcompare-and-set: that row's pass no longer holds the lock (the forcing passtook it), and it stops at its next check or at its lock check before apply.-Rows past the limit, and rows no newer pass looks at, are left for a reaper.+Rows past the limit, and rows no newer pass looks at, are left for the reaper+(``reaper.py``).Two passes triggered together can both skip: the one that lost the lock racerecords ``lock_held`` while its row is still open, and the winner's guard sees@@ -94,10 +97,10 @@ async def _dream_pass_enabled(run: DreamPassRun) -> bool:as the cron entry that fired it (if any) already let it through."""flag = Flag.DREAM_PASS_ENABLED.valuetry:- enabled, authoritative = await asyncio.wait_for(- evaluate_feature_flag(Flag.DREAM_PASS_ENABLED, run.user_id),- timeout=FLAG_READ_TIMEOUT_SECONDS,- )+ async with asyncio.timeout(FLAG_READ_TIMEOUT_SECONDS):+ enabled, authoritative = await evaluate_feature_flag(+ Flag.DREAM_PASS_ENABLED, run.user_id+ )except Exception:logger.warning(f"Dream pass {run.pass_id}: could not read {flag}; going on",diff --git a/autogpt_platform/backend/backend/copilot/dream/job_status.py b/autogpt_platform/backend/backend/copilot/dream/job_status.pyindex 84bb48f813..5f5aef526f 100644--- a/autogpt_platform/backend/backend/copilot/dream/job_status.py+++ b/autogpt_platform/backend/backend/copilot/dream/job_status.py@@ -65,11 +65,17 @@ class JobStatus(BaseModel, Generic[ResultT]):"""Status row for one fire-and-forget admin job.Stored as a JSON-serialized Redis string at- ``dream:status:{kind}:{job_id}``. Single ``SET`` per write so there- are no read-modify-write races even if multiple workers touch the- same job_id concurrently — the last writer wins, but each writer- has read the latest state before composing its update via the- helpers in this module.+ ``dream:status:{kind}:{job_id}``. Each write is one ``SET`` of the+ whole row, so a reader never sees half of one. The helpers in this+ module read the row, compose their update and then SET it, and that+ read-modify-write is not atomic: two writers racing on one job_id can+ interleave, and the later SET wins over what the earlier wrote. What+ keeps a dream pass's job right is who writes it, not atomicity: the+ terminal guards (``mark_complete`` and ``mark_errored`` never rewrite a+ completed row), the rule that an errored row keeps the usage its+ result already carried (``_errored_result``), and the batch executor+ claiming a delivery before its handler or drop hook runs, so one walker+ ends a pass.Generic over ``ResultT`` (the work body's return shape). The Redislayer always reads and writes the row as JSON — the type parameter@@ -233,7 +239,9 @@ async def mark_errored(Refuses to overwrite a job already in ``state='complete'`` — thebatch tail runs cleanup after ``mark_complete``, and a transienterror routed through the crash guard must not rewrite a completed- job (whose writes and billing landed) as errored.+ job (whose writes and billing landed) as errored. A job errored again+ keeps the usage its result already carries when the new result has+ none: a failure replayed after cleanup knows less than the first."""existing = await read_status(kind=kind, job_id=job_id)if existing is None:@@ -259,13 +267,27 @@ async def mark_errored("updated_at": now,"completed_at": now,"error": error[:2000], # cap; error strings can be huge stack traces- "result": result.model_dump() if result is not None else existing.result,+ "result": _errored_result(existing.result, result),})await _persist(updated)return updated+def _errored_result(+ existing: dict[str, Any] | None, result: BaseModel | None+) -> dict[str, Any] | None:+ """The result an errored job keeps: *result*, but never without the+ usage the existing one carries."""+ if result is None:+ return existing+ replacement = result.model_dump()+ kept_usage = existing.get("usage") if existing is not None else None+ if replacement.get("usage") is None and kept_usage is not None:+ replacement["usage"] = kept_usage+ return replacement++async def read_status(*, kind: JobKind, job_id: str) -> JobStatus[Any] | None:"""Fetch a status row by ``(kind, job_id)``. Returns ``None`` if missing.@@ -301,7 +323,8 @@ async def read_status(*, kind: JobKind, job_id: str) -> JobStatus[Any] | None:async def _persist(status: JobStatus[Any]) -> None:- """Serialize + write with TTL. Single SET — no read-modify-write race."""+ """Serialize and write the whole row with its TTL in one SET. The write+ is atomic; the read-modify-write around it is not (``JobStatus``)."""from backend.data.redis_client import get_redis_asyncredis = await get_redis_async()diff --git a/autogpt_platform/backend/backend/copilot/dream/job_status_test.py b/autogpt_platform/backend/backend/copilot/dream/job_status_test.pyindex b4fd265838..c7bb8188c2 100644--- a/autogpt_platform/backend/backend/copilot/dream/job_status_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/job_status_test.py@@ -227,6 +227,45 @@ class TestMarkErrored:assert status.result is not Noneassert DreamPassResult.model_validate(status.result) == failure+ @pytest.mark.asyncio+ async def test_a_failure_replayed_without_usage_keeps_the_first_usage(+ self, fake_redis+ ):+ """A second failure of the same job, from a delivery that found the+ pass's state already cleaned up, knows no usage: the usage the first+ recorded stays, the rest of its result is the new one's."""+ await write_initial_status(kind="dream_pass", job_id="j1", user_id="u")+ usage = DreamPassUsage(total_input_tokens=10, total_output_tokens=20)+ first = DreamPassResult(user_id="u", pass_id="p1", error="stop", usage=usage)+ replay = DreamPassResult(user_id="u", pass_id="p1", error="stop again")++ await mark_errored(kind="dream_pass", job_id="j1", error="stop", result=first)+ await mark_errored(+ kind="dream_pass", job_id="j1", error="stop again", result=replay+ )++ status = await read_status(kind="dream_pass", job_id="j1")+ assert status is not None and status.result is not None+ kept = DreamPassResult.model_validate(status.result)+ assert (status.error, kept.error, kept.usage) == (+ "stop again",+ "stop again",+ usage,+ )++ @pytest.mark.asyncio+ async def test_a_failure_that_knows_usage_replaces_the_old(self, fake_redis):+ await write_initial_status(kind="dream_pass", job_id="j1", user_id="u")+ old = DreamPassUsage(total_input_tokens=10)+ new = DreamPassUsage(total_input_tokens=30)+ for usage in (old, new):+ result = DreamPassResult(user_id="u", pass_id="p1", usage=usage)+ await mark_errored(kind="dream_pass", job_id="j1", error="e", result=result)++ status = await read_status(kind="dream_pass", job_id="j1")+ assert status is not None and status.result is not None+ assert DreamPassResult.model_validate(status.result).usage == new+class TestReadStatus:@pytest.mark.asynciodiff --git a/autogpt_platform/backend/backend/copilot/dream/lease.py b/autogpt_platform/backend/backend/copilot/dream/lease.pynew file mode 100644index 0000000000..2facbf7fcc--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/lease.py@@ -0,0 +1,228 @@+"""A dream pass's lease: its scope's Redis lock, and the expiry its DreamPass+row records for that lock, renewed together at every step.++The lock keeps two passes of a scope apart; the row's lease is what the guard+of a newer pass and the reaper read to tell a live pass from a dead one. A+renewal compare-and-extends the lock under the pass's token, which never+resurrects a lapsed key, then writes the new expiry to the row, bounded. The+row follows the lock only as far as those writes land: one that fails is+logged and the pass goes on with its row behind its lock (the reaper may then+take it for dead, ``reaper.py``), and apply's own renewal before its ingestion+drain (``apply.py``) shortens the sync lock without writing the row, which+then overstates the lock by up to twenty-two minutes.++ sync pass written at its RUNNING insert (the token it is about to take+ the lock under, ``DEFAULT_LOCK_TTL_SECONDS`` from its start);+ renewed before each phase (``checkpoint``) and before apply+ (``admit_sync_apply``)+ batch pass renewed at submit (``batch_handoff``, to+ ``BATCH_LOCK_TTL_SECONDS``), on every callback once its phase+ has landed (``renew_batch_lease``), and once its apply gate+ is claimed (``admit_batch_apply``)+ either renewed by apply once more right before its first graph+ write (``ApplyLease``); emptied as the row closes, or, when+ the close marks the row for a cleanup (a stop, a batch+ pass's end), once that cleanup is done (``cleanup.py``)++A lock that is no longer the pass's ends the pass: another pass may hold the+scope now. A renewal Redis cannot answer in time is taken by what follows it.+Before a phase, or at a landed callback, the pass goes on, warned: nothing it+does before apply writes the graph, and apply's admission decides. The+renewals that admit apply fail closed: ownership unknown is not ownership,+and a pass that cannot prove it holds its lock does not write. A batch pass+renews under the token its input bundle carries, else the one its row keeps+(``batch_outcome.lock_token_of``); with none anywhere its ownership is+unknown, never "no lock": it does not apply, and its row stays marked until+the reaper can tell the lock is not its own.++What stays open: a renewal proves ownership at the instant Redis runs it and+keeps the lock one more TTL, not up to every write. A pass that stalls longer+than a TTL between its last renewal and its writes (a suspended process, a+reply held past the TTL) can still write after a newer pass took the scope;+apply's renewal right before its first write keeps that window to one TTL+from the writes. A cancel that lands after the last check is late+(``cancel.py``).+"""++import asyncio+import logging+from collections.abc import Awaitable++from pydantic import BaseModel, ConfigDict++from backend.copilot.graphiti.scope import MemoryScope++from .batch_outcome import BatchPass, fail_pass, lock_token_of+from .cancel import stop_before_apply, stop_if_stopped, stopped_error+from .locks import (+ BATCH_LOCK_TTL_SECONDS,+ DEFAULT_LOCK_TTL_SECONDS,+ LOCK_CHECK_TIMEOUT_SECONDS,+ DreamLockHandle,+ extend_dream_lock,+)+from .pass_run import DreamPassRun, PassEnded+from .store import record_lease++logger = logging.getLogger(__name__)+++class ApplyLease(BaseModel):+ """The lease apply renews once more right before its first graph write+ (``apply.apply_operations``): the scope's lock compare-and-extended+ under the pass's token for its route's TTL, bounded, failing closed."""++ model_config = ConfigDict(frozen=True)++ scope: MemoryScope+ token: str+ ttl_seconds: int+ pass_id: str++ async def renew(self) -> bool:+ """Whether the lock is still the pass's and now runs ``ttl_seconds``+ more; ``False`` when it is not, and when Redis could not say in time."""+ extend = extend_dream_lock(self.scope, self.token, self.ttl_seconds)+ return await _renewed(extend, self.pass_id) is True+++def lock_lost_error(step: str) -> str:+ """How a pass that lost its scope's lock before *step* ends."""+ return f"{step}: dream lock lost before {step}"+++def lease_unknown_error(step: str) -> str:+ """How a pass ends whose lease could not be renewed before *step*, where+ ownership unknown fails closed."""+ return f"{step}: dream lease could not be renewed"+++async def checkpoint(run: DreamPassRun, step: str) -> None:+ """The sync pass's check before each phase: its row shows no stop, then+ its lease is renewed (``renew_sync_lease``); raises ``PassEnded`` when+ either ends the pass."""+ await stop_if_stopped(run)+ await renew_sync_lease(run, step)+++async def renew_sync_lease(run: DreamPassRun, step: str) -> None:+ """Renew the sync pass's lease before *step*; a renewal Redis cannot+ answer goes on, warned (a phase writes nothing; apply's admission fails+ closed). Raises ``PassEnded`` once the lock is no longer the pass's."""+ if run.lock is not None:+ await _renew_sync(run, run.lock, step, fail_closed=False)+++async def admit_sync_apply(+ run: DreamPassRun, scope: MemoryScope, lock_handle: DreamLockHandle+) -> ApplyLease:+ """The sync pass's last checks before apply: its row shows no stop, its+ lock reads as its own (``stop_before_apply``) and its lease renews,+ failing closed. Returns the lease apply renews once more before it+ writes; raises ``PassEnded`` with the run's failure when it may not."""+ await stop_before_apply(run, lock_handle)+ await _renew_sync(run, lock_handle, "apply", fail_closed=True)+ return ApplyLease(+ scope=scope,+ token=lock_handle.token,+ ttl_seconds=DEFAULT_LOCK_TTL_SECONDS,+ pass_id=run.pass_id,+ )+++async def renew_batch_lease(bp: BatchPass, next_step: str) -> bool:+ """On a callback whose phase has landed: stretch the batch pass's lock+ for another ``BATCH_LOCK_TTL_SECONDS`` (a whole batch lifetime) and+ record it on the row. ``False`` once the lock turned out to be no longer+ the pass's and the pass has been ended: its landed phases charged, its+ state and bundle deleted, the lock left to whoever holds it. The token+ is the one the bundle carries, else the one the row keeps+ (``lock_token_of``). A pass with no token anywhere, or whose renewal+ Redis could not answer, goes on unrenewed: apply's admission decides+ (``admit_batch_apply``)."""+ token = await lock_token_of(bp.pass_id)+ if token is None:+ return True+ scope = MemoryScope.build(bp.user_id, bp.expert_id)+ renewed = await _renewed(+ extend_dream_lock(scope, token, BATCH_LOCK_TTL_SECONDS), bp.pass_id+ )+ if renewed is False:+ error = await stopped_error(bp.pass_id) or lock_lost_error(next_step)+ logger.warning(f"Dream batch pass {bp.pass_id} stops: {error}")+ await fail_pass(bp, error, holds_lock=False)+ return False+ if renewed:+ await record_lease(bp.pass_id, token, BATCH_LOCK_TTL_SECONDS)+ return True+++async def admit_batch_apply(bp: BatchPass, lock_token: str | None) -> ApplyLease | None:+ """The batch pass's fence once its apply gate is claimed, nothing durable+ awaited before it: compare-and-extend its lock under *lock_token* for a+ whole batch window. The extend is the proof: it finds the lock the+ pass's when Redis runs it and keeps it so for the window, where a read+ could return a stale "yours" whose reply crossed the lock's expiry.++ Fails closed: a lock that is no longer the pass's, one Redis could not+ answer for in time, and a pass with no token anywhere (its ownership+ unknown, not "no lock") all end the pass, its landed phases charged and+ its state and bundle cleaned. The lock is released by compare-and-delete,+ which only ever deletes the pass's own, unless it is known to be+ another's; with no token to release it by, the row keeps its mark and+ the reaper settles it. Returns the lease apply renews once more before it+ writes, or ``None`` once the pass has ended."""+ scope = MemoryScope.build(bp.user_id, bp.expert_id)+ renewed: bool | None = None+ if lock_token is not None:+ extend = extend_dream_lock(scope, lock_token, BATCH_LOCK_TTL_SECONDS)+ renewed = await _renewed(extend, bp.pass_id)+ if renewed and lock_token is not None:+ return ApplyLease(+ scope=scope,+ token=lock_token,+ ttl_seconds=BATCH_LOCK_TTL_SECONDS,+ pass_id=bp.pass_id,+ )+ error = await stopped_error(bp.pass_id) or _ended_error("apply", renewed)+ logger.warning(f"Dream batch pass {bp.pass_id} does not apply: {error}")+ await fail_pass(bp, error, holds_lock=renewed is not False)+ return None+++async def _renew_sync(+ run: DreamPassRun, lock: DreamLockHandle, step: str, *, fail_closed: bool+) -> None:+ """Stretch the sync pass's lock another ``DEFAULT_LOCK_TTL_SECONDS`` and+ record it on its row. Raise ``PassEnded`` with the run's failure once+ the lock is no longer the pass's (the stop that closed its row if one+ did), and, when *fail_closed*, when Redis could not say."""+ renewed = await _renewed(lock.extend(DEFAULT_LOCK_TTL_SECONDS), run.pass_id)+ if renewed:+ await record_lease(run.pass_id, lock.token, DEFAULT_LOCK_TTL_SECONDS)+ return+ if renewed is None and not fail_closed:+ return+ error = await stopped_error(run.pass_id) or _ended_error(step, renewed)+ logger.warning(f"Dream pass {run.pass_id} stops: {error}")+ raise PassEnded(run.failure(error))+++def _ended_error(step: str, renewed: bool | None) -> str:+ """Why a pass's renewal ended it: its lock lost, or ownership unknown."""+ return lock_lost_error(step) if renewed is False else lease_unknown_error(step)+++async def _renewed(extend: Awaitable[bool], pass_id: str) -> bool | None:+ """Whether *extend* found the lock still the pass's and stretched it;+ ``None`` when Redis did not answer within ``LOCK_CHECK_TIMEOUT_SECONDS``+ (the caller decides whether the pass goes on)."""+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ return await extend+ except Exception:+ logger.warning(+ f"Dream pass {pass_id}: could not renew its lease in time",+ exc_info=True,+ )+ return Nonediff --git a/autogpt_platform/backend/backend/copilot/dream/lease_admission_test.py b/autogpt_platform/backend/backend/copilot/dream/lease_admission_test.pynew file mode 100644index 0000000000..bae14ba3f8--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/lease_admission_test.py@@ -0,0 +1,363 @@+"""The renewals that admit a dream pass's writes, on both routes, through the+real orchestrator, callbacks and apply body over the in-memory Redis and+store: a final renewal that gets no answer is not ownership, a renewal whose+answer arrives after the lock lapsed is caught by apply's own renewal before+its first write, and so is a lock that lapses while apply creates its+session. In each, a newer forced pass takes the scope and writes; the old+pass writes nothing. Only the models, the budget and apply's graph and chat+writes are stubbed (the writes recorded per pass)."""++import asyncio+from datetime import datetime, timezone+from unittest.mock import AsyncMock++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)+from pydantic import BaseModel++from backend.copilot.graphiti.scope import MemoryScope+from backend.copilot.inference.complete import StructuredCompletion+from backend.copilot.inference.context import InferenceUsage, RouteDecision+from backend.data.dream_pass_models import DreamPassDraft+from backend.executor.batch_executor import PendingEntry+from backend.util.llm.providers import BatchResultRow++from . import apply as apply_mod+from . import job_status+from . import lease as lease_mod+from . import orchestrator as orchestrator_mod+from .batch_callbacks import handle_dream_batch_result+from .batch_state import write_phase_to_state+from .batch_submit import persist_input_bundle+from .fetch import DreamInput, EpisodeRow+from .locks import BATCH_LOCK_TTL_SECONDS+from .schemas import (+ ConsolidatedFact,+ ConsolidationOutput,+ DreamOperations,+ DreamPhase,+ RecombinationOutput,+)++_SCOPE = MemoryScope.for_user("u1")+_LOCK_KEY = _SCOPE.redis_key("dream_lock")+_FACT = ConsolidatedFact(content="Nick ships on Fridays", confidence=0.9)+_OPS = DreamOperations(writes=[_FACT], summary_for_user="ok")+_ANSWERS: tuple[BaseModel, ...] = (ConsolidationOutput(), RecombinationOutput(), _OPS)+_BILLED = InferenceUsage(+ model="m", input_tokens=100, output_tokens=20, payer="platform_allowance"+)+++class Graph:+ """What apply wrote, one pass id per fact; a pass named in ``hold``, or+ the next to create its session when ``hold_next``, stops in its session+ creation until ``resume`` is set."""++ def __init__(self) -> None:+ self.writes: list[str] = []+ self.hold: set[str] = set()+ self.hold_next = False+ self.in_session = asyncio.Event()+ self.resume = asyncio.Event()+++def graph(mocker) -> Graph:+ recorded = Graph()++ async def create_session(scope, pass_id: str) -> str:+ if pass_id in recorded.hold or recorded.hold_next:+ recorded.hold_next = False+ recorded.in_session.set()+ await recorded.resume.wait()+ return f"s-{pass_id}"++ async def write_fact(scope, pass_id: str, index, fact, **_kwargs) -> bool:+ recorded.writes.append(pass_id)+ return False++ mocker.patch.object(apply_mod, "_create_dream_session", create_session)+ mocker.patch.object(apply_mod, "_write_consolidated_fact", write_fact)+ mocker.patch.object(apply_mod, "_write_dream_summary_message", AsyncMock())+ return recorded++[email protected](autouse=True)+def sync_pass(mocker) -> None:+ """A sync pass for the scope, its three phases answering in turn, pass+ after pass."""+ calls = 0++ async def answer(*_args, **_kwargs) -> StructuredCompletion:+ nonlocal calls+ calls += 1+ return StructuredCompletion(value=_ANSWERS[(calls - 1) % 3], usage=_BILLED)++ mocker.patch.object(orchestrator_mod, "resolve_route", side_effect=_route)+ mocker.patch.object(+ orchestrator_mod, "resolve_dream_execution_path", return_value="sync_baseline"+ )+ mocker.patch.object(+ orchestrator_mod, "is_feature_enabled", AsyncMock(return_value=False)+ )+ mocker.patch.object(+ orchestrator_mod, "check_dream_budget", AsyncMock(return_value=(True, None))+ )+ mocker.patch.object(+ orchestrator_mod,+ "record_phase_cost",+ AsyncMock(side_effect=lambda ctx, usage: usage),+ )+ mocker.patch.object(+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_input())+ )+ mocker.patch.object(orchestrator_mod, "structured_complete", side_effect=answer)+ mocker.patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock())+++class TestTheSyncRoute:+ @pytest.mark.parametrize("failure", ["timeout", "error"])+ async def test_a_final_renewal_without_an_answer_never_applies_after_its_replacement(+ self, monkeypatch, fake_dream_db, fake_dream_redis, graph, failure+ ):+ """Codex's reproduction, the timeout at the real two seconds: the+ pass's renewal before apply gets no answer while its lock lapses and+ a forced admin pass takes the scope and writes. The old pass ends+ there, having written nothing."""+ at_renewal, proceed = asyncio.Event(), asyncio.Event()+ evaluate = fake_dream_redis.eval+ tokens: list[str] = []++ async def no_answer_before_apply(script: str, numkeys: int, *args):+ if '"expire"' in script:+ tokens.append(args[1])+ if tokens.count(tokens[0]) == 4 and args[1] == tokens[0]:+ at_renewal.set()+ await proceed.wait()+ if failure == "timeout":+ await asyncio.Event().wait()+ raise ConnectionError("redis did not answer")+ return await evaluate(script, numkeys, *args)++ monkeypatch.setattr(fake_dream_redis, "eval", no_answer_before_apply)+ loop = asyncio.get_running_loop()+ started = loop.time()+ old = asyncio.create_task(orchestrator_mod.execute_dream_pass("u1"))+ await asyncio.wait_for(at_renewal.wait(), 5)+ await fake_dream_redis.delete(_LOCK_KEY)++ newer = await orchestrator_mod.execute_dream_pass(+ "u1", trigger="admin", force=True+ )+ proceed.set()+ result = await asyncio.wait_for(old, 5)++ if failure == "timeout":+ assert 1.9 < loop.time() - started < 4+ assert newer.error is None and not newer.skipped+ assert graph.writes == [newer.pass_id]+ assert result.error == (+ f"expired: forced by an admin-triggered dream pass {newer.pass_id}"+ )+ assert fake_dream_db.rows[result.pass_id]["status"] is DreamPassStatus.EXPIRED++ async def test_a_lock_that_lapses_while_apply_starts_stops_its_writes(+ self, fake_dream_db, fake_dream_redis, graph+ ):+ """The pass is admitted and enters apply; its lock lapses while the+ session is created, and a forced admin pass takes the scope and+ writes. Apply's renewal before its first write finds the lock gone:+ the old pass writes nothing."""+ graph.hold_next = True+ old = asyncio.create_task(orchestrator_mod.execute_dream_pass("u1"))+ await asyncio.wait_for(graph.in_session.wait(), 5)+ await fake_dream_redis.delete(_LOCK_KEY)++ newer = await orchestrator_mod.execute_dream_pass(+ "u1", trigger="admin", force=True+ )+ graph.resume.set()+ result = await asyncio.wait_for(old, 5)++ assert newer.error is None+ assert graph.writes == [newer.pass_id]+ assert result.error is not None and result.error.startswith("Dream lock lost")+++class TestTheBatchRoute:+ async def test_a_lock_that_lapses_while_apply_starts_stops_its_writes(+ self, fake_dream_db, fake_dream_redis, graph+ ):+ """Codex's long-apply model: the last callback passes its fence and+ enters apply; a whole batch lease lapses while the session is+ created, and a forced admin pass takes the scope and writes. Apply's+ renewal before its first write finds the lock gone."""+ await _seed_batch_pass(fake_dream_db, fake_dream_redis)+ graph.hold.add("p1")+ old = asyncio.create_task(_deliver_sanitize())+ await asyncio.wait_for(graph.in_session.wait(), 5)+ await fake_dream_redis.delete(_LOCK_KEY)++ newer = await orchestrator_mod.execute_dream_pass(+ "u1", trigger="admin", force=True+ )+ graph.resume.set()+ await asyncio.wait_for(old, 5)++ assert newer.error is None+ assert graph.writes == [newer.pass_id]+ status = await job_status.read_status(kind="dream_pass", job_id="j1")+ assert status is not None and status.state == "errored"+ assert (status.error or "").startswith("apply: DreamLockLostError")++ async def test_a_fence_answer_held_past_the_lease_still_writes_nothing(+ self, mocker, fake_dream_db, fake_dream_redis, graph+ ):+ """PR 7's delayed-reply case on the renewal that replaced the GET:+ the fence's compare-and-extend succeeds, its answer held while the+ lock lapses (a stall past the TTL) and a forced admin pass takes the+ scope and writes. The stale "yours" admits the old callback to apply,+ and apply's renewal before its first write finds the lock gone."""+ await _seed_batch_pass(fake_dream_db, fake_dream_redis)+ extend = lease_mod.extend_dream_lock+ answered, deliver = asyncio.Event(), asyncio.Event()+ renewals = 0++ async def held_at_the_fence(scope, token: str, ttl_seconds: int) -> bool:+ nonlocal renewals+ renewals += 1+ renewed = await extend(scope, token, ttl_seconds)+ if renewals == 2:+ answered.set()+ await deliver.wait()+ return renewed++ mocker.patch.object(lease_mod, "extend_dream_lock", held_at_the_fence)+ old = asyncio.create_task(_deliver_sanitize())+ await asyncio.wait_for(answered.wait(), 5)+ assert fake_dream_redis.ttls[_LOCK_KEY] == BATCH_LOCK_TTL_SECONDS+ await fake_dream_redis.delete(_LOCK_KEY)++ newer = await orchestrator_mod.execute_dream_pass(+ "u1", trigger="admin", force=True+ )+ deliver.set()+ await asyncio.wait_for(old, 5)++ assert newer.error is None+ assert graph.writes == [newer.pass_id]++ async def test_a_pass_that_keeps_its_lock_writes_once(+ self, fake_dream_db, fake_dream_redis, graph+ ):+ await _seed_batch_pass(fake_dream_db, fake_dream_redis)++ await _deliver_sanitize()++ assert graph.writes == ["p1"]+ status = await job_status.read_status(kind="dream_pass", job_id="j1")+ assert status is not None and status.state == "complete"+ assert _LOCK_KEY not in fake_dream_redis.store+++async def _seed_batch_pass(fake_dream_db, fake_dream_redis) -> None:+ """Batch pass p1 at its last phase: its lock held under the token its+ bundle carries, consolidate and recombine landed, its job and row open."""+ now = datetime.now(timezone.utc)+ await persist_input_bundle(+ "p1",+ DreamInput(+ user_id="u1", group_id=_SCOPE.group_id, window_start=now, window_end=now+ ),+ lock_token="our-token",+ )+ fake_dream_redis.store[_LOCK_KEY] = "our-token"+ await job_status.write_initial_status(kind="dream_pass", job_id="j1", user_id="u1")+ landed: tuple[tuple[DreamPhase, BaseModel], ...] = (+ ("consolidate", ConsolidationOutput()),+ ("recombine", RecombinationOutput()),+ )+ for phase, output in landed:+ await write_phase_to_state(pass_id="p1", phase=phase, row=_row(phase, output))+ fake_dream_db.seed(+ DreamPassDraft(+ id="p1",+ user_id="u1",+ scope_key=_SCOPE.scope_key,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ trigger=DreamPassTrigger.CRON,+ status=DreamPassStatus.SUBMITTED,+ phase=DreamPassPhase.SANITIZE,+ ),+ provider_batch_id="msgbatch_sanitize",+ lease_token="our-token",+ )+++async def _deliver_sanitize() -> None:+ now = datetime.now(timezone.utc)+ entry = PendingEntry(+ provider="anthropic",+ provider_batch_id="msgbatch_sanitize",+ callback_namespace="dream_pass",+ submitted_at=now,+ next_poll_at=now,+ payload={+ "user_id": "u1",+ "cost_context": {"user_id": "u1", "trial_id": None, "generation": None},+ "pass_id": "p1",+ "job_id": "j1",+ "phase": "sanitize",+ "phase_models": {+ p: "claude-sonnet-5" for p in ("consolidate", "recombine", "sanitize")+ },+ },+ )+ await handle_dream_batch_result(entry, [_row("sanitize", _OPS)])+++def _row(phase: str, output: BaseModel) -> BatchResultRow:+ return BatchResultRow(+ custom_id=f"p1:{phase}",+ content=output.model_dump_json(),+ input_tokens=10,+ output_tokens=20,+ )+++def _route(scope, job, *, config=None) -> RouteDecision:+ return RouteDecision(+ engine="provider_sync",+ auth_provider="platform",+ provider="open_router",+ model=f"{job.tier}-model",+ payer="platform_allowance",+ execution_path="sync_baseline",+ cost_log_provider="open_router",+ reason="test",+ )+++def _input() -> DreamInput:+ now = datetime.now(timezone.utc)+ episode = EpisodeRow(+ uuid="e1",+ name=None,+ content="I ship on Fridays",+ source_description=None,+ valid_at=None,+ created_at=None,+ )+ return DreamInput(+ user_id="u1",+ group_id=_SCOPE.group_id,+ window_start=now,+ window_end=now,+ episodes=[episode],+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/lease_batch_test.py b/autogpt_platform/backend/backend/copilot/dream/lease_batch_test.pynew file mode 100644index 0000000000..3fbc660b00--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/lease_batch_test.py@@ -0,0 +1,209 @@+"""A batch pass's lease through the real callbacks, over the in-memory Redis+(its TTLs run down by hand here, the way Redis would expire the keys) and+the pass's row in the in-memory store: every callback renews the lock and the+row's lease, so a chain of three batches that runs past a day keeps its lock+and applies; a callback whose lock a newer pass took ends the pass without+chaining. Only the provider, the charges and apply are stubbed."""++import logging+from datetime import datetime, timedelta, timezone+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)+from pydantic import BaseModel++from backend.copilot.graphiti.scope import MemoryScope+from backend.data.dream_pass_models import DreamPassDraft+from backend.executor.batch_executor import PendingEntry+from backend.util.llm.providers import BatchResultRow++from . import batch_callbacks as batch_callbacks_mod+from . import lease as lease_mod+from .batch_callbacks import handle_dream_batch_result+from .batch_state import state_key+from .batch_submit import (+ input_bundle_key,+ persist_input_bundle,+ refresh_input_bundle_ttl,+)+from .fetch import DreamInput+from .locks import BATCH_LOCK_TTL_SECONDS+from .schemas import ConsolidationOutput, DreamOperations, RecombinationOutput++_SCOPE = MemoryScope.for_user("u1")+_LOCK_KEY = _SCOPE.redis_key("dream_lock")+_TOKEN = "our-token"+_HOURS_20 = 20 * 60 * 60+_OUTPUTS: dict[str, BaseModel] = {+ "consolidate": ConsolidationOutput(),+ "recombine": RecombinationOutput(),+ "sanitize": DreamOperations(summary_for_user="ok"),+}++[email protected](autouse=True)+def provider(mocker) -> AsyncMock:+ """The next phase's submit, which re-arms the bundle's TTL as the real+ one does; the provider key; the charges."""++ async def submit(**kwargs: Any) -> MagicMock:+ await refresh_input_bundle_ttl(kwargs["pass_id"])+ return MagicMock(provider_batch_id=f"msgbatch_{kwargs['phase']}")++ mocker.patch.object(batch_callbacks_mod, "anthropic_api_key", return_value="k")+ mocker.patch("backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock())+ return mocker.patch.object(+ batch_callbacks_mod, "submit_phase", AsyncMock(side_effect=submit)+ )+++def apply(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.apply.apply_operations",+ AsyncMock(return_value={"session_id": "s", "consolidated_count": 0}),+ )+++class TestTheBatchLease:+ async def test_each_callback_renews_the_lock_and_the_rows_lease(+ self, fake_dream_db, fake_dream_redis+ ):+ await _submitted(fake_dream_db, fake_dream_redis)+ fake_dream_redis.ttls[_LOCK_KEY] = 60+ before = datetime.now(timezone.utc)++ await _deliver("consolidate")++ assert fake_dream_redis.ttls[_LOCK_KEY] == BATCH_LOCK_TTL_SECONDS+ row = fake_dream_db.rows["p1"]+ assert row["lease_token"] == _TOKEN+ assert row["lease_expires_at"] >= before + timedelta(+ seconds=BATCH_LOCK_TTL_SECONDS+ )+ assert row["status"] is DreamPassStatus.SUBMITTED++ async def test_a_chain_of_three_batches_past_a_day_keeps_its_lock_and_applies(+ self, fake_dream_db, fake_dream_redis, apply, provider+ ):+ """Twenty hours per batch, sixty in all: past the lock's TTL from+ its submit twice over, yet each callback renewed it, so the last+ one still holds the scope and applies."""+ await _submitted(fake_dream_db, fake_dream_redis)+ for phase in ("consolidate", "recombine", "sanitize"):+ _elapse(fake_dream_redis, _HOURS_20)+ await _deliver(phase)++ apply.assert_awaited_once()+ assert provider.await_count == 2+ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.COMPLETE+ assert (row["lease_token"], row["lease_expires_at"]) == (None, None)+ assert _LOCK_KEY not in fake_dream_redis.store++ async def test_a_callback_whose_lock_was_taken_ends_the_pass_without_chaining(+ self, fake_dream_db, fake_dream_redis, provider, caplog+ ):+ await _submitted(fake_dream_db, fake_dream_redis)+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"++ with caplog.at_level(logging.WARNING):+ await _deliver("consolidate")++ provider.assert_not_awaited()+ row = fake_dream_db.rows["p1"]+ error = "recombine: dream lock lost before recombine"+ assert (row["status"], row["error"]) == (DreamPassStatus.ERRORED, error)+ assert [p.phase for p in row["usage"].phases] == ["consolidate"]+ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"+ assert state_key("p1") not in fake_dream_redis.hashes+ assert input_bundle_key("p1") not in fake_dream_redis.store+ assert f"Dream batch pass p1 stops: {error}" in caplog.text++ async def test_a_renewal_redis_cannot_answer_goes_on_down_the_chain(+ self, mocker, fake_dream_db, fake_dream_redis, provider, caplog+ ):+ await _submitted(fake_dream_db, fake_dream_redis)+ mocker.patch.object(+ lease_mod,+ "extend_dream_lock",+ AsyncMock(side_effect=ConnectionError("redis down")),+ )++ with caplog.at_level(logging.WARNING):+ await _deliver("consolidate")++ provider.assert_awaited_once()+ assert "Dream pass p1: could not renew its lease" in caplog.text+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.SUBMITTED+++async def _submitted(fake_dream_db, fake_dream_redis) -> None:+ """Pass p1 as its handoff leaves it: consolidate submitted, the lock+ extended to a batch lifetime under the token its bundle carries."""+ now = datetime.now(timezone.utc)+ bundle = DreamInput(+ user_id="u1", group_id=_SCOPE.group_id, window_start=now, window_end=now+ )+ await persist_input_bundle("p1", bundle, lock_token=_TOKEN)+ fake_dream_redis.store[_LOCK_KEY] = _TOKEN+ fake_dream_redis.ttls[_LOCK_KEY] = BATCH_LOCK_TTL_SECONDS+ fake_dream_db.seed(+ DreamPassDraft(+ id="p1",+ user_id="u1",+ scope_key=_SCOPE.scope_key,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ trigger=DreamPassTrigger.CRON,+ status=DreamPassStatus.SUBMITTED,+ phase=DreamPassPhase.CONSOLIDATE,+ lease_token=_TOKEN,+ lease_expires_at=now + timedelta(seconds=BATCH_LOCK_TTL_SECONDS),+ ),+ provider_batch_id="msgbatch_consolidate",+ )+++async def _deliver(phase: str) -> None:+ now = datetime.now(timezone.utc)+ entry = PendingEntry(+ provider="anthropic",+ provider_batch_id=f"msgbatch_{phase}",+ callback_namespace="dream_pass",+ submitted_at=now,+ next_poll_at=now,+ payload={+ "user_id": "u1",+ "cost_context": {"user_id": "u1", "trial_id": None, "generation": None},+ "pass_id": "p1",+ "job_id": "",+ "phase": phase,+ "phase_models": {p: "claude-sonnet-5" for p in _OUTPUTS},+ },+ )+ row = BatchResultRow(+ custom_id=f"p1_{phase}",+ content=_OUTPUTS[phase].model_dump_json(),+ input_tokens=10,+ output_tokens=20,+ )+ await handle_dream_batch_result(entry, [row])+++def _elapse(fake_dream_redis, seconds: int) -> None:+ """Let *seconds* pass on the in-memory Redis: every TTL runs down, and a+ key whose TTL runs out goes, as Redis would expire it."""+ for key, ttl in list(fake_dream_redis.ttls.items()):+ if ttl > seconds:+ fake_dream_redis.ttls[key] = ttl - seconds+ continue+ fake_dream_redis.store.pop(key, None)+ fake_dream_redis.hashes.pop(key, None)+ del fake_dream_redis.ttls[key]diff --git a/autogpt_platform/backend/backend/copilot/dream/lease_test.py b/autogpt_platform/backend/backend/copilot/dream/lease_test.pynew file mode 100644index 0000000000..56b4d70f84--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/lease_test.py@@ -0,0 +1,447 @@+"""A sync pass's lease through the real entry point, with the real dream lock+over the in-memory Redis and the pass's row in the in-memory store: written+with the row, renewed before every phase and before apply, emptied when the+row closes; a pass whose lock is no longer its own stops at its next step,+and one that cannot renew before apply does not apply. Only the models, the+budget and apply are stubbed. Every transition that closes a row drops the+lease and the input bundle."""++import logging+from collections.abc import Awaitable, Callable+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)+from pydantic import ValidationError++from backend.copilot.graphiti.scope import MemoryScope+from backend.copilot.inference.complete import StructuredCompletion+from backend.copilot.inference.context import InferenceUsage, RouteDecision+from backend.data.dream_pass_models import (+ CLOSED_ROW_CLEARS,+ CLOSED_STATUSES,+ MARKED_ROW_CLEARS,+ OPEN_STATUSES,+ DreamPassDraft,+ DreamPassUpdate,+)+from backend.data.dream_pass_update import TRANSITION_SQL, transition_args++from . import locks as locks_mod+from . import orchestrator as orchestrator_mod+from .fetch import DreamInput, EpisodeRow+from .lease import renew_sync_lease+from .locks import DEFAULT_LOCK_TTL_SECONDS+from .pass_record import cancelled, cleanup_finished, expired, failed, outcome, reaped+from .pass_run import DreamPassRun, PassEnded+from .schemas import (+ ConsolidationOutput,+ DreamOperations,+ DreamPassResult,+ RecombinationOutput,+)+from .store import write_stop++_SCOPE = MemoryScope.for_user("u")+_LOCK_KEY = _SCOPE.redis_key("dream_lock")+_LONG_AGO = datetime(2000, 1, 1, tzinfo=timezone.utc)+_BILLED = InferenceUsage(+ model="m", input_tokens=100, output_tokens=20, payer="platform_allowance"+)+_ANSWERS = (+ ConsolidationOutput(facts=[]),+ RecombinationOutput(proposals=[]),+ DreamOperations(summary_for_user="ok"),+)++[email protected](autouse=True)+def sync_pass(mocker) -> None:+ """A sync pass whose route, budget, charges and input are stubbed; the+ lock and the record run for real over the in-memory stand-ins."""+ mocker.patch.object(orchestrator_mod, "resolve_route", side_effect=_route)+ mocker.patch.object(+ orchestrator_mod, "resolve_dream_execution_path", return_value="sync_baseline"+ )+ mocker.patch.object(+ orchestrator_mod, "is_feature_enabled", AsyncMock(return_value=False)+ )+ mocker.patch.object(+ orchestrator_mod,+ "record_phase_cost",+ AsyncMock(side_effect=lambda ctx, usage: usage),+ )+ mocker.patch.object(+ orchestrator_mod, "check_dream_budget", AsyncMock(return_value=(True, None))+ )+ mocker.patch.object(+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_input())+ )+++def apply(mocker, fake_dream_redis) -> AsyncMock:+ """Apply, noting the TTL the pass's lock had when it started."""+ ttls: list[int | None] = []++ async def applied(*_args, **_kwargs):+ ttls.append(fake_dream_redis.ttls.get(_LOCK_KEY))+ return {"session_id": "s", "consolidated_count": 0}++ mock = mocker.patch.object(+ orchestrator_mod, "apply_operations", AsyncMock(side_effect=applied)+ )+ mock.lock_ttls = ttls+ return mock+++class TestTheSyncLease:+ async def test_the_row_holds_the_locks_lease_from_insert_to_close(+ self, mocker, fake_dream_db, fake_dream_redis, apply+ ):+ """Before each phase the test lets the lock run nearly out and the+ row's lease lapse; the renewal at the next step restores both."""+ seen: list[tuple[str | None, int | None, datetime | None]] = []++ async def during(call: int) -> None:+ [pass_id] = list(fake_dream_db.rows)+ row = fake_dream_db.rows[pass_id]+ lock = fake_dream_redis.store.get(_LOCK_KEY)+ seen.append(+ (lock, fake_dream_redis.ttls.get(_LOCK_KEY), row["lease_expires_at"])+ )+ fake_dream_redis.ttls[_LOCK_KEY] = 1+ row["lease_expires_at"] = _LONG_AGO++ _phases(mocker, during)+ started = datetime.now(timezone.utc)++ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error is None and result.skipped is False+ _, draft = fake_dream_db.writes[0]+ assert isinstance(draft, DreamPassDraft) and draft.lease_token+ assert result.started_at is not None+ assert draft.lease_expires_at == result.started_at + timedelta(+ seconds=DEFAULT_LOCK_TTL_SECONDS+ )+ assert [lock for lock, _, _ in seen] == [draft.lease_token] * 3+ assert [ttl for _, ttl, _ in seen] == [DEFAULT_LOCK_TTL_SECONDS] * 3+ renewed_until = started + timedelta(seconds=DEFAULT_LOCK_TTL_SECONDS)+ assert all(lease and lease >= renewed_until for _, _, lease in seen)+ assert apply.lock_ttls == [DEFAULT_LOCK_TTL_SECONDS]+ renewals = _lease_writes(fake_dream_db, result.pass_id)+ assert [update.lease_token for update in renewals] == [draft.lease_token] * 4+ row = fake_dream_db.rows[result.pass_id]+ assert row["status"] is DreamPassStatus.COMPLETE+ assert (row["lease_token"], row["lease_expires_at"]) == (None, None)++ async def test_a_pass_whose_lock_was_taken_stops_before_its_next_phase(+ self, mocker, fake_dream_db, fake_dream_redis, apply+ ):+ """A newer pass took the lock while consolidate ran: the renewal+ before recombine finds it, and the pass stops there with what it was+ billed, leaving the newer pass's lock alone."""++ async def during(call: int) -> None:+ if call == 0:+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"++ phases = _phases(mocker, during)++ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error == "recombine: dream lock lost before recombine"+ assert phases.await_count == 1+ assert result.usage is not None+ assert [p.phase for p in result.usage.phases] == ["consolidate"]+ apply.assert_not_awaited()+ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"+ row = fake_dream_db.rows[result.pass_id]+ assert (row["status"], row["error"]) == (DreamPassStatus.ERRORED, result.error)++ async def test_a_renewal_redis_cannot_answer_goes_on_until_apply(+ self, mocker, fake_dream_db, apply, caplog+ ):+ """Unsure is not lost before a phase: the pass goes on through all+ three. Before apply it is not ownership either: the pass ends there,+ with its phases' usage, and applies nothing."""+ mocker.patch.object(+ locks_mod.DreamLockHandle,+ "extend",+ AsyncMock(side_effect=ConnectionError("redis down")),+ )+ phases = _phases(mocker)++ with caplog.at_level(logging.WARNING):+ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error == "apply: dream lease could not be renewed"+ assert phases.await_count == 3+ assert result.usage is not None and len(result.usage.phases) == 3+ apply.assert_not_awaited()+ assert caplog.text.count("could not renew its lease") == 4+ assert _lease_writes(fake_dream_db, result.pass_id) == []+ row = fake_dream_db.rows[result.pass_id]+ assert (row["status"], row["error"]) == (DreamPassStatus.ERRORED, result.error)++ async def test_a_lease_the_row_will_not_take_does_not_stop_the_pass(+ self, mocker, fake_dream_db, fake_dream_redis, apply, caplog+ ):+ write = fake_dream_db.update_dream_pass++ async def refuses_leases(pass_id: str, update: DreamPassUpdate) -> bool:+ if update.lease_expires_at is not None:+ raise ConnectionError("dream pass database unreachable")+ return await write(pass_id, update)++ mocker.patch.object(fake_dream_db, "update_dream_pass", refuses_leases)+ _phases(mocker)++ with caplog.at_level(logging.WARNING):+ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error is None+ apply.assert_awaited_once()+ assert caplog.text.count("could not record the lease") == 4+ assert fake_dream_db.rows[result.pass_id]["status"] is DreamPassStatus.COMPLETE++ async def test_a_lost_lock_on_a_stopped_row_ends_with_the_stop(+ self, fake_dream_db, fake_dream_redis+ ):+ """A newer pass expired the row and took the lock between the pass's+ stop check and its renewal: the pass reports the expiry that closed+ its row, not just the lock it lost."""+ run = DreamPassRun.begin("u", "sync_baseline")+ fake_dream_db.seed(_draft(run.pass_id))+ assert await write_stop(run.pass_id, expired("stale", not_updated_since=None))+ async with locks_mod.dream_lock(_SCOPE, token=run.lease_token) as handle:+ run.hold(handle)+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"+ with pytest.raises(PassEnded) as ended:+ await renew_sync_lease(run, "apply")++ assert ended.value.result.error == "expired: stale"+ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"+++class TestAClosingRow:+ @pytest.mark.parametrize(+ "close",+ [+ lambda: outcome(_result(skipped=True), None),+ lambda: outcome(_result(), None),+ lambda: failed("boom", None, datetime.now(timezone.utc)),+ ],+ ids=["skipped", "complete", "errored"],+ )+ async def test_the_sync_routes_end_drops_its_lease_and_bundle_keeps_its_outputs(+ self, fake_dream_db, close: Callable[[], DreamPassUpdate]+ ):+ closing = close()+ fake_dream_db.seed(+ _draft("p1"),+ lease_token="tok",+ lease_expires_at=datetime.now(timezone.utc),+ input_bundle=_input(),+ phase_outputs={"consolidate": ConsolidationOutput()},+ )++ assert closing.clear == CLOSED_ROW_CLEARS+ assert closing.cleanup_pending_at is None+ assert await fake_dream_db.update_dream_pass("p1", closing)++ row = fake_dream_db.rows["p1"]+ assert [row[column] for column in sorted(CLOSED_ROW_CLEARS)] == [None] * 3+ assert row["phase_outputs"] == {"consolidate": ConsolidationOutput()}++ @pytest.mark.parametrize(+ "close",+ [+ lambda: outcome(_result(), None, marked=True),+ lambda: failed("boom", None, datetime.now(timezone.utc), marked=True),+ lambda: cancelled("testing", owner_user_id="u"),+ lambda: expired("stale", not_updated_since=None),+ ],+ ids=["batch complete", "batch errored", "cancelled", "expired"],+ )+ async def test_a_close_that_leaves_a_cleanup_marks_the_row_and_keeps_its_lease(+ self, fake_dream_db, close: Callable[[], DreamPassUpdate]+ ):+ """A stop from outside, or a batch pass's end, marks the row for the+ cleanup after the pass and keeps the lease that cleanup needs (the+ token to release the lock with, the expiry that says whether the pass+ may still run) until the write that says it finished."""+ closing = close()+ lapses = datetime.now(timezone.utc)+ fake_dream_db.seed(+ _draft("p1"),+ lease_token="tok",+ lease_expires_at=lapses,+ input_bundle=_input(),+ phase_outputs={"consolidate": ConsolidationOutput()},+ )++ assert closing.clear == MARKED_ROW_CLEARS+ assert await fake_dream_db.update_dream_pass("p1", closing)++ row = fake_dream_db.rows["p1"]+ assert (row["lease_token"], row["lease_expires_at"]) == ("tok", lapses)+ assert (row["input_bundle"], row["cleanup_pending_at"] is None) == (+ None,+ False,+ )+ assert row["phase_outputs"] == {"consolidate": ConsolidationOutput()}+ assert await fake_dream_db.update_dream_pass("p1", cleanup_finished())+ assert (+ row["cleanup_pending_at"],+ row["lease_token"],+ row["lease_expires_at"],+ ) == (None, None, None)++ def test_a_column_is_set_or_cleared_not_both(self):+ with pytest.raises(ValidationError, match="set and cleared at once"):+ DreamPassUpdate(lease_token="tok", clear=frozenset({"lease_token"}))++ def test_the_statement_empties_the_columns_the_update_clears(self):+ dropped = transition_args("p1", outcome(_result(), None))+ marked = transition_args("p1", expired("stale", not_updated_since=None))++ assert dropped[21] == ["inputBundle", "leaseExpiresAt", "leaseToken"]+ assert (marked[21], marked[22] is not None) == (["inputBundle"], True)+ assert transition_args("p1", DreamPassUpdate())[21] == []+ for column in (+ "leaseToken",+ "leaseExpiresAt",+ "inputBundle",+ "cleanupPendingAt",+ ):+ assert f"'{column}' = ANY($22::text[]) THEN NULL" in TRANSITION_SQL++ async def test_the_reapers_close_marks_it_and_only_the_mark_is_written_after(+ self, fake_dream_db+ ):+ """The reaper's expiry keeps the token its cleanup may need and marks+ the row; the write that says the cleanup finished is the one a closed+ row takes, and it only clears."""+ now = datetime.now(timezone.utc)+ fake_dream_db.seed(+ _draft("p1"), lease_token="tok", lease_expires_at=now, updated_at=now+ )++ assert await fake_dream_db.update_dream_pass(+ "p1", reaped("lapsed", not_updated_since=now)+ )+ row = fake_dream_db.rows["p1"]+ assert (row["status"], row["lease_token"]) == (DreamPassStatus.EXPIRED, "tok")+ assert row["cleanup_pending_at"] is not None+ assert not await fake_dream_db.update_dream_pass(+ "p1", failed("late", None, now)+ )++ assert await fake_dream_db.update_dream_pass("p1", cleanup_finished())+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+ assert (row["status"], row["error"]) == (DreamPassStatus.EXPIRED, "lapsed")++ def test_the_cleanup_write_goes_to_closed_rows_and_only_clears(self):+ with pytest.raises(ValidationError, match="a closed row only has columns"):+ DreamPassUpdate(closed_row=True, error="late")+ closed = transition_args("p1", cleanup_finished())+ opened = transition_args("p1", DreamPassUpdate())++ assert set(closed[17]) == {s.value for s in CLOSED_STATUSES}+ assert set(opened[17]) == {s.value for s in OPEN_STATUSES}+ assert closed[21] == ["cleanupPendingAt", "leaseExpiresAt", "leaseToken"]+ marked = transition_args(+ "p1", reaped("x", not_updated_since=datetime.now(timezone.utc))+ )+ assert marked[22] is not None and "leaseToken" not in marked[21]+++def _phases(+ mocker, during: Callable[[int], Awaitable[None]] | None = None+) -> AsyncMock:+ """Each phase answers, billed; *during* runs with the phase's index while+ it is in flight."""+ calls = 0++ async def answer(*_args, **_kwargs) -> StructuredCompletion:+ nonlocal calls+ if during is not None:+ await during(calls)+ calls += 1+ return StructuredCompletion(value=_ANSWERS[calls - 1], usage=_BILLED)++ return mocker.patch.object(+ orchestrator_mod, "structured_complete", AsyncMock(side_effect=answer)+ )+++def _lease_writes(fake_dream_db, pass_id: str) -> list[DreamPassUpdate]:+ """The renewals the pass wrote to its row, in order."""+ return [+ update+ for written, update in fake_dream_db.writes+ if written == pass_id+ and isinstance(update, DreamPassUpdate)+ and update.lease_expires_at is not None+ ]+++def _draft(pass_id: str) -> DreamPassDraft:+ return DreamPassDraft(+ id=pass_id,+ user_id="u",+ scope_key=_SCOPE.scope_key,+ route=DreamPassRoute.SYNC,+ trigger=DreamPassTrigger.CRON,+ phase=DreamPassPhase.RECOMBINE,+ )+++def _result(*, skipped: bool = False) -> DreamPassResult:+ return DreamPassResult(+ user_id="u",+ pass_id="p1",+ execution_path="sync_baseline",+ skipped=skipped,+ skip_reason="no_input" if skipped else None,+ )+++def _route(scope, job, *, config=None) -> RouteDecision:+ return RouteDecision(+ engine="provider_sync",+ auth_provider="platform",+ provider="open_router",+ model=f"{job.tier}-model",+ payer="platform_allowance",+ execution_path="sync_baseline",+ cost_log_provider="open_router",+ reason="test",+ )+++def _input() -> DreamInput:+ now = datetime.now(timezone.utc)+ episode = EpisodeRow(+ uuid="e1",+ name=None,+ content="hello",+ source_description=None,+ valid_at=None,+ created_at=None,+ )+ return DreamInput(+ user_id="u",+ group_id=_SCOPE.group_id,+ window_start=now - timedelta(days=14),+ window_end=now,+ episodes=[episode],+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/locks.py b/autogpt_platform/backend/backend/copilot/dream/locks.pyindex 8708948112..fcb053a4dc 100644--- a/autogpt_platform/backend/backend/copilot/dream/locks.py+++ b/autogpt_platform/backend/backend/copilot/dream/locks.py@@ -18,11 +18,14 @@ single-key Lua scripts, no multi-key scripts and no cross-keytransactions.The sync-path TTL is 1800 s (30 min), sized to the scheduler's job-timeout; the batch path extends it (see ``BATCH_LOCK_TTL_SECONDS``).--Right before it applies, a pass reads the key once more (``held``,-``dream_lock_held_by``): a lock that lapsed may already be a newer pass's,-and only the holder may write the scope's graph.+timeout; the batch path extends it (see ``BATCH_LOCK_TTL_SECONDS``). A pass+renews its lock at each step, and its DreamPass row records the new expiry+as the pass's lease (``lease.py``).++Right before it applies, a pass proves the key is still its own by renewing+it (``extend``), and the sync pass reads it first (``held``): a lock that+lapsed may already be a newer pass's, and only the holder may write the+scope's graph."""from __future__ import annotations@@ -40,12 +43,13 @@ logger = logging.getLogger(__name__)# Sync-path lock TTL (30 min).DEFAULT_LOCK_TTL_SECONDS = 1800-# Batch path: the dream pass is async and stays in flight up to the-# BatchExecutor's MAX_BATCH_LIFETIME_SECONDS (24h). The lock must outlive the-# whole batch so apply — which runs hours later in the callback — is still-# covered by "one dream per user". The batch callback releases it on-# terminal/failure; this TTL is only the crash backstop, kept > 24h so the-# lock can't expire before the executor times the batch out.+# Batch path: one phase's batch stays in flight up to the BatchExecutor's+# MAX_BATCH_LIFETIME_SECONDS (24h). The lock is extended to this TTL at submit+# and again at each callback whose phase landed, and once more before apply,+# so it covers the phase batch in flight (not the whole chain of three) with+# 10 minutes to spare, and apply, which runs in the last callback, is still+# covered by "one dream per user". The callbacks release it when the pass+# ends; the TTL is the backstop for a pass that died.BATCH_LOCK_TTL_SECONDS = 24 * 60 * 60 + 600# How long the ownership read right before apply may take.@@ -145,9 +149,8 @@ class DreamLockHandle:async def held(self) -> bool:"""Whether the key still holds this handle's token: one GET, raisingwhen Redis does not answer within ``LOCK_CHECK_TIMEOUT_SECONDS``."""- current = await asyncio.wait_for(- self._redis.get(self._key), timeout=LOCK_CHECK_TIMEOUT_SECONDS- )+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ current = await self._redis.get(self._key)return _as_text(current) == self.token@@ -155,6 +158,7 @@ class DreamLockHandle:async def dream_lock(scope: MemoryScope,ttl_seconds: int = DEFAULT_LOCK_TTL_SECONDS,+ token: str | None = None,):"""Acquire a per-user advisory lock for the dream pass.@@ -162,7 +166,8 @@ async def dream_lock(The lock is released on context exit via compare-and-delete on thisacquire's ownership token — a late exit can't delete a newer pass'slock. On crash the TTL provides a fallback release after- ``ttl_seconds``.+ ``ttl_seconds``. *token*, minted by the pass so its record carries the+ lease before the lock is taken, is the ownership value (uuid4 if not)."""# Lazy import so this module is cheap to import in tests that mock redis.from backend.data.redis_client import get_redis_async@@ -170,7 +175,7 @@ async def dream_lock(user_id = scope.owner_user_idredis = await get_redis_async()key = scope.redis_key("dream_lock")- token = str(uuid.uuid4())+ token = token or str(uuid.uuid4())acquired = await redis.set(key, token, nx=True, ex=ttl_seconds)if not acquired:@@ -223,16 +228,16 @@ async def read_dream_lock_token(scope: MemoryScope) -> str | None:return _as_text(await redis.get(scope.redis_key("dream_lock")))-async def dream_lock_held_by(scope: MemoryScope, token: str | None) -> bool:- """Whether the scope's lock still holds *token* (``False`` for no token):- the batch callback's check right before apply. One GET, raising when- Redis does not answer within ``LOCK_CHECK_TIMEOUT_SECONDS``."""- if token is None:- return False- current = await asyncio.wait_for(- read_dream_lock_token(scope), timeout=LOCK_CHECK_TIMEOUT_SECONDS+async def extend_dream_lock(scope: MemoryScope, token: str, ttl_seconds: int) -> bool:+ """``DreamLockHandle.extend`` for a caller holding only the token (a batch+ callback renewing a disowned lock); the caller bounds it."""+ from backend.data.redis_client import get_redis_async++ redis = await get_redis_async()+ key = scope.redis_key("dream_lock")+ return await DreamLockHandle(redis, key, scope.owner_user_id, token).extend(+ ttl_seconds)- return current == tokendef _as_text(raw: Any) -> str | None:@@ -241,17 +246,19 @@ def _as_text(raw: Any) -> str | None:return raw.decode("utf-8") if isinstance(raw, bytes) else str(raw)-async def release_dream_lock(scope: MemoryScope, token: str | None) -> None:- """Release a disowned dream lock (batch path) once the pass terminates.+async def release_dream_lock(scope: MemoryScope, token: str | None) -> bool:+ """Release a disowned dream lock (batch path) once the pass terminates,+ and say whether the lock is no longer held under *token*: released now,+ or already another's or nobody's.Compare-and-delete on ``token``: a blind delete is NOT safe here — thecallback can land close to (or after) the lock's TTL, by which point thekey may already belong to a newer pass, and deleting it would let a- third concurrent pass start. When the token is unknown (per-pass state- expired or corrupted) the key is left for its TTL to clear: the batch- lock outlives the input bundle's 24h TTL by only ~10 min, so a short- extra lockout beats releasing someone else's lock. A failed delete- likewise falls back to the TTL.+ third concurrent pass start. When the token is unknown (the input bundle+ that carries it expired, or is corrupted) the key is left alone: a+ lockout until its TTL, or until the reaper releases it with the token+ the pass's row keeps, beats releasing someone else's lock. ``False``+ then, and when the delete fails; never raises."""user_id = scope.owner_user_idif token is None:@@ -260,7 +267,7 @@ async def release_dream_lock(scope: MemoryScope, token: str | None) -> None:"leaving it for the TTL to clear",user_id[:12],)- return+ return Falsefrom backend.data.redis_client import get_redis_asynctry:@@ -288,3 +295,5 @@ async def release_dream_lock(scope: MemoryScope, token: str | None) -> None:user_id[:12],exc_info=True,)+ return False+ return Truediff --git a/autogpt_platform/backend/backend/copilot/dream/locks_test.py b/autogpt_platform/backend/backend/copilot/dream/locks_test.pyindex 6d206af952..2983594c3f 100644--- a/autogpt_platform/backend/backend/copilot/dream/locks_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/locks_test.py@@ -212,7 +212,7 @@ async def test_release_dream_lock_compare_and_deletes_with_token(mocker):redis = _redis_mock()_patch_redis(mocker, redis)- await release_dream_lock(MemoryScope.for_user("user-g"), "tok-g")+ assert await release_dream_lock(MemoryScope.for_user("user-g"), "tok-g")redis.eval.assert_awaited_once()eval_args = redis.eval.call_args.args@@ -230,7 +230,8 @@ async def test_release_skips_delete_when_token_mismatch(mocker, caplog):redis = _redis_mock(eval=AsyncMock(return_value=0))_patch_redis(mocker, redis)- await release_dream_lock(MemoryScope.for_user("user-h"), "stale-token")+ # Not ours any more: nothing left to release, so the release is done.+ assert await release_dream_lock(MemoryScope.for_user("user-h"), "stale-token")redis.eval.assert_awaited_once()redis.delete.assert_not_awaited()@@ -244,7 +245,7 @@ async def test_release_without_token_leaves_lock_for_ttl(mocker, caplog):redis = _redis_mock()_patch_redis(mocker, redis)- await release_dream_lock(MemoryScope.for_user("user-i"), None)+ assert not await release_dream_lock(MemoryScope.for_user("user-i"), None)redis.eval.assert_not_awaited()redis.delete.assert_not_awaited()@@ -256,7 +257,7 @@ async def test_release_dream_lock_swallows_redis_failure(mocker, caplog):redis = _redis_mock(eval=AsyncMock(side_effect=Exception("redis down")))_patch_redis(mocker, redis)- await release_dream_lock(MemoryScope.for_user("user-j"), "tok-j")+ assert not await release_dream_lock(MemoryScope.for_user("user-j"), "tok-j")assert "Failed to release disowned dream lock" in caplog.textdiff --git a/autogpt_platform/backend/backend/copilot/dream/missing_token_test.py b/autogpt_platform/backend/backend/copilot/dream/missing_token_test.pynew file mode 100644index 0000000000..256d524067--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/missing_token_test.py@@ -0,0 +1,182 @@+"""A batch pass whose input bundle lost its lock token (a bundle an earlier+build wrote, or a corrupted one), over the in-memory store and Redis. A+missing token is unknown ownership, never "no lock": the pass goes by the+token its row keeps, fenced and released under it as under the bundle's.+With no token to be had it does not apply, and its row stays marked with the+unlock unfinished, its lock left alone, until the reaper releases the lock by+the row's token or, for a row that kept none, finds it gone. The provider and+the charges are stubbed at their edges; the lock, the fence, the claims and+the row transitions are the real ones."""++import json+import logging+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++import pytest+from prisma.enums import DreamPassStatus++from . import reaper as reaper_mod+from .batch_callbacks import handle_dream_batch_result+from .batch_deliveries_test import _LOCK_KEY, _MODELS, _entry, _in_flight, _row+from .batch_state import state_key+from .batch_submit import input_bundle_key+from .conftest import FakeAsyncRedis, FakeDreamDb+from .locks import BATCH_LOCK_TTL_SECONDS+from .reaper import REAP_GRACE_SECONDS, reap_expired_passes+from .reaper_test import _charged+from .schemas import DreamOperations++_ALL_THREE = ["consolidate", "recombine", "sanitize"]++[email protected](autouse=True)+def provider(mocker) -> AsyncMock:+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key", return_value="k"+ )+ mocker.patch.object(reaper_mod, "phase_models_for_config", return_value=_MODELS)+ return mocker.patch(+ "backend.copilot.dream.provider_batch.cancel_batch",+ AsyncMock(return_value=True),+ )++[email protected](autouse=True)+def charges(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )+++def apply(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.apply.apply_operations",+ AsyncMock(return_value={"writes": 0}),+ )+++class TestABundleThatLostItsToken:+ async def test_goes_by_the_token_its_row_keeps(+ self, fake_dream_db, fake_dream_redis, apply, charges+ ):+ """Codex's probe: only the token is gone from an otherwise valid+ bundle. The fence renews under the row's token, apply runs once, and+ the cleanup releases the lock under that token: no lock left, no+ mark, every landed phase charged once."""+ await _last_phase_in_flight_without_a_bundle_token(+ fake_dream_db, fake_dream_redis+ )++ await _deliver_sanitize()++ apply.assert_awaited_once()+ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.COMPLETE+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+ assert _charged(charges) == _ALL_THREE+ assert _LOCK_KEY not in fake_dream_redis.store+ assert state_key("p1") not in fake_dream_redis.hashes+ assert input_bundle_key("p1") not in fake_dream_redis.store++ async def test_whose_row_cannot_say_does_not_apply_and_the_reaper_unlocks(+ self, monkeypatch, fake_dream_db, fake_dream_redis, apply, charges+ ):+ """The row's token cannot be read either (the store down for reads):+ ownership is unknown, so apply does not run; nor can the usage+ attribution the landed phases are charged under, so their charge+ waits; the row closes ERRORED, marked, its token kept, the lock left+ held. The reaper then charges the phases once, by the row, and+ releases the lock by its token."""+ await _last_phase_in_flight_without_a_bundle_token(+ fake_dream_db, fake_dream_redis+ )+ read_row = fake_dream_db.get_dream_pass+ monkeypatch.setattr(+ fake_dream_db,+ "get_dream_pass",+ AsyncMock(side_effect=ConnectionError("reads down")),+ )++ await _deliver_sanitize()++ monkeypatch.setattr(fake_dream_db, "get_dream_pass", read_row)+ apply.assert_not_awaited()+ _assert_errored_and_marked(fake_dream_db, token="tok")+ assert _charged(charges) == []+ assert fake_dream_redis.store[_LOCK_KEY] == "tok"+ assert fake_dream_redis.ttls[_LOCK_KEY] == BATCH_LOCK_TTL_SECONDS++ run = await reap_expired_passes(now=_past_the_grace())++ assert run.outcomes == {"cleaned": 1}+ assert _LOCK_KEY not in fake_dream_redis.store+ row = fake_dream_db.rows["p1"]+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+ assert _charged(charges) == _ALL_THREE+++class TestAPassWithNoTokenAnywhere:+ async def test_does_not_apply_and_its_lock_is_left_to_lapse(+ self, fake_dream_db, fake_dream_redis, apply, charges, caplog+ ):+ """Neither the bundle nor the row keeps a token (a row written before+ leases): apply does not run, the landed phases are charged once, and+ the row closes ERRORED and marked with its unlock unfinished. The+ reaper retries while the lock is held under a token no open pass+ keeps, never deleting it, and cleans the row up once it has lapsed."""+ await _last_phase_in_flight_without_a_bundle_token(+ fake_dream_db, fake_dream_redis+ )+ fake_dream_db.rows["p1"]["lease_token"] = None++ with caplog.at_level(logging.WARNING):+ await _deliver_sanitize()+ waiting = await reap_expired_passes(now=_past_the_grace())++ apply.assert_not_awaited()+ _assert_errored_and_marked(fake_dream_db, token=None)+ assert _charged(charges) == _ALL_THREE+ assert "cleanup unfinished at unlock; left marked" in caplog.text+ assert waiting.outcomes == {"retry": 1}+ assert "held under a token no open pass keeps" in caplog.text+ assert fake_dream_redis.store[_LOCK_KEY] == "tok"++ del fake_dream_redis.store[_LOCK_KEY]+ lapsed = await reap_expired_passes(now=_past_the_grace())++ assert lapsed.outcomes == {"cleaned": 1}+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None+ assert _charged(charges) == _ALL_THREE+++async def _last_phase_in_flight_without_a_bundle_token(+ fake_dream_db: FakeDreamDb, fake_dream_redis: FakeAsyncRedis+) -> None:+ """Pass p1 waiting on sanitize's batch, holding its lock under ``tok``+ for a batch lifetime, its row keeping that token, and its input bundle+ valid but for the token it lost."""+ await _in_flight(fake_dream_db, fake_dream_redis, landed=2)+ await fake_dream_redis.expire(_LOCK_KEY, BATCH_LOCK_TTL_SECONDS)+ bundle = json.loads(fake_dream_redis.store[input_bundle_key("p1")])+ del bundle["lock_token"]+ fake_dream_redis.store[input_bundle_key("p1")] = json.dumps(bundle)+++async def _deliver_sanitize() -> None:+ await handle_dream_batch_result(+ _entry("sanitize"), [_row("sanitize", DreamOperations())]+ )+++def _assert_errored_and_marked(fake_dream_db: FakeDreamDb, *, token: str | None):+ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.ERRORED+ assert row["cleanup_pending_at"] is not None+ assert row["lease_token"] == token+++def _past_the_grace() -> datetime:+ """An instant by which the cleanup of a row closed now is due."""+ return datetime.now(timezone.utc) + timedelta(seconds=REAP_GRACE_SECONDS + 60)diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator.pyindex b274d6d7ae..4ada17834d 100644--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py@@ -13,12 +13,13 @@ submits the first phase to Anthropic's Message Batches API instead andresults land.Every pass, on either route, gets a durable ``DreamPass`` row-(``store.py``): inserted at the start, advanced after each step, and-closed with how the pass ended, a write attempted before the pass releases-its lock. Like every record write it is best-effort: one that fails or-times out leaves the row open behind a free lock until a later pass's guard-or a reaper closes it. A pass holding the lock runs ``guard.py`` first, and-``cancel.py``'s checks before each phase, before its batch submit and apply.+(``store.py``): inserted with its lease at the start, advanced after each+step, and closed with how the pass ended, a write attempted before the pass+releases its lock. Like every record write it is best-effort: one that fails+leaves the row open behind a free lock until a later pass's guard or the+reaper (``reaper.py``) closes it. A pass holding the lock runs ``guard.py``+first, then before each phase and apply checks for a stop (``cancel.py``) and+renews its lease (``lease.py``); the batch route checks before its submit.The orchestrator never raises out — every failure becomes a``DreamPassResult`` with ``error`` set and, on the sync route, the usage@@ -51,7 +52,6 @@ from backend.util.feature_flag import Flag, is_feature_enabledfrom .apply import apply_operations, drain_status_from_statsfrom .batch_handoff import submit_dream_pass_batchfrom .billing import PhaseChargeError, check_dream_budget, record_phase_cost-from .cancel import stop_before_apply, stop_if_stoppedfrom .clamp import clamp_operationsfrom .fetch import (DreamInput,@@ -60,8 +60,14 @@ from .fetch import (is_dream_authored_episode,parse_episode_timestamp,)-from .guard import guard_dream_pass-from .locks import DEFAULT_LOCK_TTL_SECONDS, DreamLockHandle, DreamLockHeld, dream_lock+from .guard import GUARD_BUDGET_SECONDS, guard_dream_pass+from .lease import admit_sync_apply, checkpoint+from .locks import (+ LOCK_CHECK_TIMEOUT_SECONDS,+ DreamLockHandle,+ DreamLockHeld,+ dream_lock,+)from .pass_record import DreamTriggerfrom .pass_run import DreamPassRun, PassEndedfrom .phase_jobs import phase_job@@ -82,6 +88,7 @@ from .schemas import (RecombinationOutput,)from .store import (+ RECORD_WRITE_TIMEOUT_SECONDS,record_applying,record_gathered,record_phase_output,@@ -133,16 +140,24 @@ SANITIZE_MAX_TOKENS = 16384# consolidate 240s + recombine 600s + sanitize 480s = 1320s LLM ceiling# + apply.INGESTION_DRAIN_TIMEOUT_SECONDS 300s drain cap# + DREAM_NON_LLM_HEADROOM_SECONDS 120s (budget check + fetch + enqueue-# + demotions + summary + cost logging + the pass's record writes,-# EXCLUDING the drain)+# + demotions + summary + cost logging, EXCLUDING the drain)# = 1740s <= 1800s SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS (60s slack)#+# The pass's own bounded round trips to its record, its guard and its lock+# come on top: DREAM_STORE_IO_SECONDS, 179s should every one of them run to+# its deadline (a stalled DatabaseManager or Redis). Then a pass whose phases+# also run to their ceilings outlasts the scheduler's wait. That wait is not+# a kill: ``run_async`` stops waiting and leaves the pass running, and its+# lease renewals keep its lock and its row's lease fresh, so neither a newer+# pass's guard nor the reaper takes it for dead, and it finishes.+## The drain no longer counts against the lock TTL: apply renews the dream# lock to a fresh budget right before the drain (see# apply.LOCK_DRAIN_RENEWAL_SECONDS), so the lock only has to cover-# LLM + non-drain headroom = 1440s <= 0.9 * DEFAULT_LOCK_TTL_SECONDS (1620s)-# with genuine margin, and the drain itself runs under the renewed lock.-# Both invariants are pinned by+# LLM + non-drain headroom + store I/O = 1619s <= 0.9 * DEFAULT_LOCK_TTL_SECONDS+# (1620s), even with every store round trip at its deadline (and a lease+# renewal before each phase shortens that stretch further), and the drain+# itself runs under the renewed lock. Both invariants are pinned by# test_phase_timeouts_plus_headroom_fit_scheduler_and_lock_envelope —# bumping any value here fails that test until the budget is re-balanced.#@@ -155,11 +170,25 @@ CONSOLIDATE_TIMEOUT_SECONDS = 240RECOMBINE_TIMEOUT_SECONDS = 600SANITIZE_TIMEOUT_SECONDS = 480# Reserved for the non-LLM segments of the pass OTHER than the ingestion-# drain (budget check, fetch, enqueue, demotions, summary write, cost-# logging, and the DreamPass record writes, each capped at-# store.RECORD_WRITE_TIMEOUT_SECONDS). The drain is budgeted separately as-# apply.INGESTION_DRAIN_TIMEOUT_SECONDS and covered by a lock renewal.+# drain and the store round trips below (budget check, fetch, enqueue,+# demotions, summary write, cost logging). The drain is budgeted separately+# as apply.INGESTION_DRAIN_TIMEOUT_SECONDS and covered by a lock renewal.DREAM_NON_LLM_HEADROOM_SECONDS = 120+# The sync pass's bounded round trips to its record and its lock, each at+# its own deadline. Record reads and writes: the RUNNING insert, the+# gathered window, before each phase a stop check, a lease write and the+# phase's output, the APPLYING write, apply's admission (a stop check and a+# lease write), and the outcome. Lock round trips: the lease extend before+# each phase, apply's admission (a lock read and an extend), and apply's own+# renewals, before its first write and before its drain. Plus the guard's+# whole budget.+_RECORD_ROUND_TRIPS = 2 + 3 * 3 + 1 + 2 + 1+_LOCK_ROUND_TRIPS = 3 + 2 + 2+DREAM_STORE_IO_SECONDS = (+ GUARD_BUDGET_SECONDS+ + _RECORD_ROUND_TRIPS * RECORD_WRITE_TIMEOUT_SECONDS+ + _LOCK_ROUND_TRIPS * LOCK_CHECK_TIMEOUT_SECONDS+)# Per-user marker stamped after a successful (non-skipped) sync apply so# the next nightly pass can skip all three LLM phases when no new episode@@ -405,14 +434,9 @@ async def _execute_dream_pass_async(run = DreamPassRun.begin(user_id, await _route_for(user_id, config), force=force)try:scope = MemoryScope.build(user_id, expert_id)- await start_pass(- run.pass_id,- scope,- route=run.execution_path,- trigger=trigger,- started_at=run.started_at,- )- async with dream_lock(scope, ttl_seconds=DEFAULT_LOCK_TTL_SECONDS) as handle:+ await start_pass(run, scope, trigger=trigger)+ async with dream_lock(scope, token=run.lease_token) as handle:+ run.hold(handle)return await _run_locked(run, scope, handle, config=config, status_id=status_id)@@ -466,7 +490,8 @@ async def _run_locked(to take the lock normally finds this one's row closed. It is best-effortlike every record write: one that fails or times out is dropped and thelock is released anyway, so a free lock can still have an open row- (APPLYING, say) behind it until a reaper closes it."""+ (APPLYING, say) behind it until a later pass's guard or the reaper+ closes it."""try:result = await _dream(run, scope, lock_handle, config=config, status_id=status_id@@ -608,7 +633,7 @@ async def _phase(attempts are absorbed. A completed phase whose charge failed(``PhaseChargeError``) joins that usage too, then its error ends thepass as a crash does."""- await stop_if_stopped(run)+ await checkpoint(run, phase)try:output, usage = await call()except InferenceError as exc:@@ -635,7 +660,7 @@ async def _apply(known_fact_uuids=input_bundle.known_fact_uuids,)await record_applying(run.pass_id, ops)- await stop_before_apply(run, lock_handle)+ lease = await admit_sync_apply(run, scope, lock_handle)apply_stats = await apply_operations(scope,run.pass_id,@@ -643,12 +668,12 @@ async def _apply(known_fact_uuids=input_bundle.known_fact_uuids,known_episode_uuids=input_bundle.known_episode_uuids,lock_handle=lock_handle,+ lease=lease,)- # Apply succeeded (even as a no-op) — stamp the marker so the- # next nightly pass can skip when nothing new has landed.- # Stamped with the gather-window end so episodes that arrived- # mid-pass still count as new next time. Sync path only: batch- # apply runs hours later in batch_callbacks, which doesn't+ # Apply succeeded (even as a no-op) — stamp the marker so the next nightly+ # pass can skip when nothing new has landed. Stamped with the gather-window+ # end so episodes that arrived mid-pass still count as new next time. Sync+ # path only: batch apply runs hours later in batch_callbacks, which doesn't# stamp yet.await _stamp_last_completed_marker(scope, input_bundle.window_end)return _applied_result(run, apply_stats, ops)diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.pyindex 719f4d20af..139f978e0a 100644--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py@@ -590,16 +590,20 @@ def test_phase_timeouts_plus_headroom_fit_scheduler_and_lock_envelope():)non_drain_headroom = orchestrator_mod.DREAM_NON_LLM_HEADROOM_SECONDSdrain = INGESTION_DRAIN_TIMEOUT_SECONDS+ store_io = orchestrator_mod.DREAM_STORE_IO_SECONDS# Whole pass (including the drain) must fit the scheduler abandonment- # boundary with real margin.+ # boundary with real margin. The store round trips only reach their+ # total when every one runs to its deadline; past the boundary the+ # scheduler stops waiting, and the pass, not cancelled, finishes.worst_case = llm + drain + non_drain_headroomassert worst_case < SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS# The lock only has to cover everything UP TO the drain, because the- # drain runs under a freshly renewed lock. Require 10% slack so the lock- # never expires mid-write.- assert llm + non_drain_headroom <= 0.9 * DEFAULT_LOCK_TTL_SECONDS+ # drain runs under a freshly renewed lock: every store round trip at its+ # deadline included. Require 10% slack so the lock never expires+ # mid-write.+ assert llm + non_drain_headroom + store_io <= 0.9 * DEFAULT_LOCK_TTL_SECONDS# The renewal must comfortably outlast the drain cap it protects.assert LOCK_DRAIN_RENEWAL_SECONDS > drain@@ -676,6 +680,7 @@ async def test_clamps_oversized_sanitizer_output(mocker):known_fact_uuids=None,known_episode_uuids=None,lock_handle=None,+ lease=None,):captured["ops"] = opsreturn {@@ -745,6 +750,7 @@ async def test_demotions_capped_at_five_percent_of_active_facts(mocker):known_fact_uuids=None,known_episode_uuids=None,lock_handle=None,+ lease=None,):captured["ops"] = opsreturn {@@ -859,6 +865,7 @@ async def test_sync_path_filters_hallucinated_demotion_before_cap(mocker):known_fact_uuids=None,known_episode_uuids=None,lock_handle=None,+ lease=None,):captured["ops"] = opsreturn {@@ -2084,9 +2091,10 @@ class TestNearDuplicateWriteDedup:@pytest.mark.asyncioasync def test_batch_handoff_revokes_batch_when_lock_extend_fails(mocker):"""If the dream lock cannot be extended to the batch window, the- just-submitted batch must never be applied: the pending entry is- removed (poller never dispatches callbacks), the input bundle is- dropped, the lock is NOT disowned, and the pass reports failure."""+ just-submitted batch must never be applied: it is taken back (stopped at+ the provider, then its pending entry removed so the poller never+ dispatches callbacks, and the bundle dropped), the lock is NOT disowned,+ and the pass reports failure."""submission = mocker.MagicMock(provider_batch_id="batch-xyz")mocker.patch("backend.copilot.dream.batch_handoff.persist_input_bundle", AsyncMock()@@ -2099,11 +2107,15 @@ async def test_batch_handoff_revokes_batch_when_lock_extend_fails(mocker):"backend.copilot.dream.batch_handoff.submit_phase",AsyncMock(return_value=submission),)+ cancel = mocker.patch(+ "backend.copilot.dream.batch_handoff.cancel_provider_batch",+ AsyncMock(return_value=True),+ )remove_pending = mocker.patch("backend.copilot.dream.batch_handoff.remove_pending", AsyncMock())- delete_bundle = mocker.patch(- "backend.copilot.dream.batch_handoff.delete_input_bundle", AsyncMock()+ cleanup = mocker.patch(+ "backend.copilot.dream.batch_handoff.best_effort_cleanup", AsyncMock())handle = mocker.MagicMock()handle.extend = AsyncMock(return_value=False)@@ -2122,8 +2134,9 @@ async def test_batch_handoff_revokes_batch_when_lock_extend_fails(mocker):lock_handle=handle,)+ cancel.assert_awaited_once_with("batch-xyz")remove_pending.assert_awaited_once_with("batch-xyz")- delete_bundle.assert_awaited_once_with(run.pass_id)+ cleanup.assert_awaited_once_with(run.pass_id)handle.disown.assert_not_called()assert result.error and "lock lost" in result.error@@ -2144,7 +2157,10 @@ async def test_a_sync_pass_records_each_step_on_its_row(mocker, fake_dream_db):result = await orchestrator_mod.execute_dream_pass("u")assert result.error is None and result.skipped is False- assert fake_dream_db.writes[0] == (+ pass_id, draft = fake_dream_db.writes[0]+ assert isinstance(draft, DreamPassDraft) and draft.lease_token+ assert result.started_at is not None+ assert (pass_id, draft) == (result.pass_id,DreamPassDraft(id=result.pass_id,@@ -2153,6 +2169,12 @@ async def test_a_sync_pass_records_each_step_on_its_row(mocker, fake_dream_db):route=DreamPassRoute.SYNC,trigger=DreamPassTrigger.CRON,started_at=result.started_at,+ lease_token=draft.lease_token,+ lease_expires_at=result.started_at+ + timedelta(seconds=DEFAULT_LOCK_TTL_SECONDS),+ # The attribution the pass started under: the suite's ready,+ # untrialed usage state.+ cost_context={"user_id": "u", "trial_id": None, "generation": None},),)assert fake_dream_db.statuses(result.pass_id) == [@@ -2183,7 +2205,9 @@ async def test_a_sync_pass_records_each_step_on_its_row(mocker, fake_dream_db):assert row["usage"] == result.usageassert row["completed_at"] == result.completed_at# The sync route keeps no input bundle: nothing resumes it from the row.- assert "input_bundle" not in row+ # And a closed row holds no lease.+ assert row.get("input_bundle") is None+ assert (row["lease_token"], row["lease_expires_at"]) == (None, None)@pytest.mark.asyncio@@ -2377,8 +2401,11 @@ async def test_a_batch_handoff_that_lost_the_lock_is_recorded_errored(mocker, fake_dream_db):_batch_route(mocker, lock_extends=False)+ mocker.patch(+ "backend.copilot.dream.batch_handoff.cancel_provider_batch",+ AsyncMock(return_value=True),+ )mocker.patch("backend.copilot.dream.batch_handoff.remove_pending", AsyncMock())- mocker.patch("backend.copilot.dream.batch_handoff.delete_input_bundle", AsyncMock())mocker.patch.object(orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_build_input()))@@ -2391,8 +2418,8 @@ async def test_a_batch_handoff_that_lost_the_lock_is_recorded_errored(DreamPassStatus.ERRORED,]assert "provider_batch_id" not in fake_dream_db.rows[result.pass_id]- # The revoked batch still runs and bills at the provider, unread: what- # the pass used is unknown, not zero.+ # The batch taken back may have run, and billed, at the provider before+ # its cancel: what the pass used is unknown, not zero.assert result.usage is Noneassert "usage" not in fake_dream_db.rows[result.pass_id]@@ -2705,6 +2732,8 @@ async def test_a_late_submit_write_keeps_the_first_callbacks_progress(AsyncMock(return_value=fake_dream_redis),)_batch_route(mocker, lock_extends=True)+ # The lock the handle stands for, which the callback renews.+ fake_dream_redis.store[MemoryScope.for_user("u").redis_key("dream_lock")] = "tok"mocker.patch("backend.copilot.dream.batch_handoff.persist_input_bundle",persist_input_bundle,@@ -2760,11 +2789,11 @@ async def test_a_stalled_record_store_costs_each_write_only_its_deadline(assert result.error is None and result.dream_session_id == "s"apply_mock.assert_awaited_once()- # The insert, the guard's read of the scope's open passes, six updates- # and the four stop checks (three phases, apply), each abandoned at the- # deadline; neither the guard nor a check held the pass on a store that- # did not answer.- assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (12, 12)+ # The insert, the guard's read of the scope's open passes, six updates,+ # the four stop checks and the four lease renewals (three phases, apply),+ # each abandoned at the deadline; neither the guard nor a check nor a+ # renewal held the pass on a store that did not answer.+ assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (16, 16)@pytest.mark.asynciodiff --git a/autogpt_platform/backend/backend/copilot/dream/pass_record.py b/autogpt_platform/backend/backend/copilot/dream/pass_record.pyindex 1722ffbee7..53220afbd8 100644--- a/autogpt_platform/backend/backend/copilot/dream/pass_record.py+++ b/autogpt_platform/backend/backend/copilot/dream/pass_record.py@@ -3,9 +3,18 @@ the row reads back as.Pure: how a route, a trigger and a step are named on the row, the insert orupdate each transition makes (``store.py`` writes them, bounded), including-the two that stop a pass from outside (a cancel, an expiry), and what a row-reads back as: the ``DreamPassResult`` it describes, for the admin API and the-eval driver, and whether it says its pass was stopped.+the lease renewal and the two that stop a pass from outside (a cancel, an+expiry), and what a row reads back as: the ``DreamPassResult`` it describes,+for the admin API and the eval driver, and whether it says its pass was+stopped.++Every transition that closes a row drops its input bundle. One that may+leave a cleanup behind (Redis state, a provider batch, a lock) also marks+the row for it (``cleanup_pending_at``) and keeps its lease until the cleanup+has finished (``cleanup_finished``): every stop from outside (a cancel, an+expiry, the reaper's) and every end of a batch pass. The sync route's own+end drops its lease with its bundle: its lock goes with the pass, and it has+no batch to leave behind."""from datetime import datetime, timedelta, timezone@@ -20,8 +29,11 @@ from prisma.enums import (from pydantic import BaseModelfrom backend.copilot.graphiti.scope import MemoryScope+from backend.copilot.trial_cost_context import TrialCostContextfrom backend.data.dream_pass_models import (+ CLOSED_ROW_CLEARS,INITIAL_CANCEL_GENERATION,+ MARKED_ROW_CLEARS,DreamPassApplied,DreamPassDraft,DreamPassOperations,@@ -31,6 +43,7 @@ from backend.data.dream_pass_models import ()from .fetch import DreamInput+from .locks import DEFAULT_LOCK_TTL_SECONDSfrom .routing import ExecutionPathfrom .schemas import DreamOperations, DreamPassResult, DreamPassUsage, DreamPhase@@ -73,8 +86,14 @@ def new_pass(route: ExecutionPath,trigger: DreamTrigger,started_at: datetime,+ lease_token: str,+ cost_context: TrialCostContext | None,) -> DreamPassDraft:- """A pass's first row: running, gathering its input."""+ """A pass's first row: running, gathering its input, with the lease of+ the lock it is about to take under *lease_token* (the sync TTL from its+ start) and the usage attribution it runs under, *cost_context*: the one+ every charge of its phases goes by, whoever makes it. ``None`` leaves the+ pass with none, never to be charged."""return DreamPassDraft(id=pass_id,user_id=scope.owner_user_id,@@ -83,6 +102,19 @@ def new_pass(route=_ROUTES[route],trigger=_TRIGGERS[trigger],started_at=started_at,+ lease_token=lease_token,+ lease_expires_at=started_at + timedelta(seconds=DEFAULT_LOCK_TTL_SECONDS),+ cost_context=(+ cost_context.model_dump(mode="json") if cost_context is not None else None+ ),+ )+++def lease(lease_token: str, ttl_seconds: int) -> DreamPassUpdate:+ """The pass renewed its lock under *lease_token* for *ttl_seconds*."""+ return DreamPassUpdate(+ lease_token=lease_token,+ lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds),)@@ -150,17 +182,21 @@ def handed_to_batch(result: DreamPassResult) -> bool:)-def outcome(result: DreamPassResult, usage: DreamPassUsage | None) -> DreamPassUpdate:- """How a pass ended: skipped, failed, or applied."""+def outcome(+ result: DreamPassResult, usage: DreamPassUsage | None, *, marked: bool = False+) -> DreamPassUpdate:+ """How a pass ended: skipped, failed, or applied; a failure or an apply+ *marked* for the cleanup after it (a batch pass's end)."""finished = result.completed_at or datetime.now(timezone.utc)if result.skipped:return DreamPassUpdate(status=DreamPassStatus.SKIPPED,skip_reason=result.skip_reason,completed_at=finished,+ clear=CLOSED_ROW_CLEARS,)if result.error is not None:- return failed(result.error, usage, finished)+ return failed(result.error, usage, finished, marked=marked)return DreamPassUpdate(status=DreamPassStatus.COMPLETE,phase=DreamPassPhase.DONE,@@ -168,44 +204,113 @@ def outcome(result: DreamPassResult, usage: DreamPassUsage | None) -> DreamPassUusage=usage,applied_at=finished,completed_at=finished,+ cleanup_pending_at=finished if marked else None,+ clear=MARKED_ROW_CLEARS if marked else CLOSED_ROW_CLEARS,)def failed(- error: str, usage: DreamPassUsage | None, finished: datetime+ error: str,+ usage: DreamPassUsage | None,+ finished: datetime,+ *,+ marked: bool = False,) -> DreamPassUpdate:+ """The pass failed with *error*; *marked* for the cleanup after it (a+ batch pass's end)."""return DreamPassUpdate(status=DreamPassStatus.ERRORED,error=error[:MAX_ERROR_CHARS],usage=usage,completed_at=finished,+ cleanup_pending_at=finished if marked else None,+ clear=MARKED_ROW_CLEARS if marked else CLOSED_ROW_CLEARS,+ )+++def taken_back(+ error: str, *, provider_batch_id: str, lease_token: str+) -> DreamPassUpdate:+ """A pass took back the batch it had just submitted and the provider did+ not confirm its stop: ERRORED with *error*, the batch on the row and the+ row marked, so the reaper's cleanup keeps asking the provider to stop it+ until it has stopped or ended. The lease token stays, the key to release+ the lock with should the pass not have."""+ now = datetime.now(timezone.utc)+ return DreamPassUpdate(+ status=DreamPassStatus.ERRORED,+ phase=_PHASE_STEPS["consolidate"],+ provider_batch_id=provider_batch_id,+ lease_token=lease_token,+ error=error[:MAX_ERROR_CHARS],+ completed_at=now,+ cleanup_pending_at=now,+ clear=MARKED_ROW_CLEARS,)def cancelled(reason: str, *, owner_user_id: str) -> DreamPassUpdate:"""A cancel: the owner's open pass closes CANCELLED with *reason* as itserror, its cancel generation bumped so the running pass stops at its next- check. Written only while the row is open and the owner's."""+ check, marked for the cleanup after it. Written only while the row is+ open and the owner's."""+ now = datetime.now(timezone.utc)return DreamPassUpdate(status=DreamPassStatus.CANCELLED,error=reason[:MAX_ERROR_CHARS],- completed_at=datetime.now(timezone.utc),+ completed_at=now,bump_cancel_generation=True,owner_user_id=owner_user_id,+ cleanup_pending_at=now,+ clear=MARKED_ROW_CLEARS,)def expired(reason: str, *, not_updated_since: datetime | None) -> DreamPassUpdate:- """A newer pass's guard closing an open pass EXPIRED, its cancel- generation bumped like a cancel's. Written only while the row is open and,- given *not_updated_since* (the stale row as the guard read it), only if- nothing has written it since; an admin's forced expiry gives ``None``."""+ """An open pass closed EXPIRED from outside (a newer pass's guard, a+ duplicate delivery), its cancel generation bumped like a cancel's and+ the row marked for the cleanup after it. Written only while the row is+ open and, given *not_updated_since* (the stale row as it was read), only+ if nothing has written it since. Only an admin forcing out a fresh row,+ and a duplicate delivery, give ``None``: a forced expiry of a stale row+ keeps the compare-and-set."""+ now = datetime.now(timezone.utc)return DreamPassUpdate(status=DreamPassStatus.EXPIRED,error=reason[:MAX_ERROR_CHARS],- completed_at=datetime.now(timezone.utc),+ completed_at=now,bump_cancel_generation=True,not_updated_since=not_updated_since,+ cleanup_pending_at=now,+ clear=MARKED_ROW_CLEARS,+ )+++def reaped(reason: str, *, not_updated_since: datetime) -> DreamPassUpdate:+ """The reaper closes an open pass that outlived its lease: EXPIRED like+ ``expired``, only if nothing has written the row since *not_updated_since*,+ and marked for the cleanup the reaper is about to do, so a cleanup it+ cannot finish is resumed on its next run. The lease token stays until then,+ the key to release the dead pass's lock by compare-and-delete; the lease+ expiry goes, the pass being dead, so that cleanup is due at once."""+ now = datetime.now(timezone.utc)+ return DreamPassUpdate(+ status=DreamPassStatus.EXPIRED,+ error=reason[:MAX_ERROR_CHARS],+ completed_at=now,+ bump_cancel_generation=True,+ not_updated_since=not_updated_since,+ cleanup_pending_at=now,+ clear=frozenset({"lease_expires_at", "input_bundle"}),+ )+++def cleanup_finished() -> DreamPassUpdate:+ """The cleanup after a closed pass has finished: the mark and the lease+ the row kept for it go. The one write a closed row takes."""+ return DreamPassUpdate(+ closed_row=True,+ clear=frozenset({"cleanup_pending_at", "lease_token", "lease_expires_at"}),)diff --git a/autogpt_platform/backend/backend/copilot/dream/pass_run.py b/autogpt_platform/backend/backend/copilot/dream/pass_run.pyindex d0cdf15694..83c7a3f944 100644--- a/autogpt_platform/backend/backend/copilot/dream/pass_run.py+++ b/autogpt_platform/backend/backend/copilot/dream/pass_run.py@@ -1,11 +1,15 @@"""One dream pass while the sync entry point runs it: who and what it is,-when it started, and what its phases have used so far.+when it started, the lease it holds, and what its phases have used so far.Built once when a pass starts and handed down, so every result the pass canend with (a skip, a failure at any step, a stop from outside, an unexpectederror, the hand-off to the batch route) carries the same identity and timing,and every failure carries the usage of every phase billed before it. A stepthat ends the pass early raises ``PassEnded`` with the result it ends with.++The pass mints its lease token with its id, so its row carries the lease from+the first write; the scope's lock is then taken under that token and held by+the run (``hold``) for the renewals at each step (``lease.py``)."""import asyncio@@ -13,8 +17,9 @@ import uuidfrom datetime import datetime, timezonefrom typing import Literal-from pydantic import BaseModel, Field+from pydantic import BaseModel, Field, PrivateAttr+from .locks import DreamLockHandlefrom .routing import ExecutionPathfrom .schemas import DreamPassResult, DreamPassUsage, PhaseUsagefrom .usage import aggregate_usage@@ -40,6 +45,9 @@ class DreamPassRun(BaseModel):execution_path: ExecutionPathforce: bool = Falsephases: list[PhaseUsage] = Field(default_factory=list)+ # The ownership token of the scope's dream lock the pass takes.+ lease_token: str = Field(default_factory=lambda: str(uuid.uuid4()))+ _lock: DreamLockHandle | None = PrivateAttr(default=None)@classmethoddef begin(@@ -54,6 +62,14 @@ class DreamPassRun(BaseModel):force=force,)+ @property+ def lock(self) -> DreamLockHandle | None:+ """The scope's lock the pass holds, once it has taken it."""+ return self._lock++ def hold(self, lock: DreamLockHandle) -> None:+ self._lock = lock+def usage(self) -> DreamPassUsage:"""What the phases billed so far used, per phase and in total."""return aggregate_usage(self.phases, self.execution_path)diff --git a/autogpt_platform/backend/backend/copilot/dream/provider_batch.py b/autogpt_platform/backend/backend/copilot/dream/provider_batch.pyindex f1f5b1bb5c..2ec597ff18 100644--- a/autogpt_platform/backend/backend/copilot/dream/provider_batch.py+++ b/autogpt_platform/backend/backend/copilot/dream/provider_batch.py@@ -1,5 +1,6 @@"""Anthropic's side of a batch dream pass: the key the callbacks submit and-cancel with, and cancelling one of the pass's batches.+cancel with, cancelling one of the pass's batches, and making sure one has+stopped.The callbacks run in the BatchExecutor, a process of their own, so they lookthe key up themselves the way ``batch_executor._default_api_key_for`` does:@@ -10,13 +11,23 @@ import asyncioimport loggingfrom backend.copilot.config import ChatConfig-from backend.util.llm.providers import cancel_batch+from backend.data import redis_client+from backend.util.llm.providers import cancel_batch, poll_batchfrom backend.util.settings import Settings+from .locks import LOCK_CHECK_TIMEOUT_SECONDS+logger = logging.getLogger(__name__)-# How long a best-effort cancel may take, the client's retries included.+# How long a best-effort cancel, or a status read, may take, the client's+# retries included.PROVIDER_CANCEL_TIMEOUT_SECONDS = 10.0+# Anthropic ends every batch within 24 hours of its creation, expiring what+# it has not processed; an hour more for the end to show.+PROVIDER_BATCH_WINDOW_SECONDS = 25 * 60 * 60+# A batch whose cancel was sent without being acknowledged, kept in Redis so+# the next stop of it reads its status before sending another.+CANCEL_SENT_PREFIX = "dream:batch:cancel_sent"def anthropic_api_key() -> str | None:@@ -34,31 +45,117 @@ def anthropic_api_key() -> str | None:return None-async def cancel_provider_batch(provider_batch_id: str) -> None:+async def provider_batch_stopped(provider_batch_id: str) -> bool:+ """Whether *provider_batch_id* has stopped at Anthropic: the cancel was+ acknowledged (the batch stops within minutes), or the batch has already+ ended. ``False`` when neither can be confirmed: no key, the provider+ unreachable or out of time, or the batch still running after a refused+ cancel. Logged, never raised.++ A cancel and a status read can each take ``PROVIDER_CANCEL_TIMEOUT_SECONDS``,+ more together than a reaper attempt has, so they may be spread over two+ calls: a cancel not acknowledged is noted (``CANCEL_SENT_PREFIX``), and+ while the note stands a call reads the status without sending another.+ A status read that answers clears it, and so does one that fails: the+ note outlives only a call cut short during the read (an attempt's+ deadline), so the next one goes straight to the read it did not finish,+ and a batch still running gets the cancel again."""+ if not await _cancel_sent(provider_batch_id):+ if await cancel_provider_batch(provider_batch_id):+ return True+ await _note_cancel_sent(provider_batch_id, True)+ stopped = await _batch_ended(provider_batch_id)+ if not stopped:+ await _note_cancel_sent(provider_batch_id, False)+ return stopped+++async def _batch_ended(provider_batch_id: str) -> bool:+ """Whether Anthropic reads the batch ended; ``False`` when it reads it+ running, or cannot be asked or answer within+ ``PROVIDER_CANCEL_TIMEOUT_SECONDS``."""+ api_key = anthropic_api_key()+ if api_key is None:+ return False+ try:+ async with asyncio.timeout(PROVIDER_CANCEL_TIMEOUT_SECONDS):+ status = await poll_batch(+ provider="anthropic",+ provider_batch_id=provider_batch_id,+ api_key=api_key,+ )+ except Exception:+ logger.warning(+ f"Could not read the status of dream batch {provider_batch_id}",+ exc_info=True,+ )+ return False+ if status in ("ended", "failed"):+ logger.info(f"Dream batch {provider_batch_id} has already ended")+ return True+ logger.warning(f"Dream batch {provider_batch_id} is still {status}")+ return False+++async def _cancel_sent(provider_batch_id: str) -> bool:+ """Whether a cancel of the batch was sent and not acknowledged since its+ status last read running; ``False`` when Redis cannot say."""+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ redis = await redis_client.get_redis_async()+ return bool(await redis.exists(f"{CANCEL_SENT_PREFIX}:{provider_batch_id}"))+ except Exception:+ logger.warning(+ f"Could not read whether dream batch {provider_batch_id}'s cancel was "+ "sent",+ exc_info=True,+ )+ return False+++async def _note_cancel_sent(provider_batch_id: str, sent: bool) -> None:+ """Note, or clear, that the batch's cancel was sent and not+ acknowledged. Best-effort: without the note the next call sends the+ cancel again."""+ key = f"{CANCEL_SENT_PREFIX}:{provider_batch_id}"+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ redis = await redis_client.get_redis_async()+ if sent:+ await redis.set(key, "1", ex=PROVIDER_BATCH_WINDOW_SECONDS)+ else:+ await redis.delete(key)+ except Exception:+ logger.warning(+ f"Could not note dream batch {provider_batch_id}'s cancel",+ exc_info=True,+ )+++async def cancel_provider_batch(provider_batch_id: str) -> bool:"""Ask Anthropic to cancel *provider_batch_id* through the client's- ``messages.batches.cancel``. Best-effort: a missing key, a refusal (a- batch that has already ended cannot be cancelled) or the deadline is- logged, never raised, so it never stands in the way of the stop that- asked for it."""+ ``messages.batches.cancel``, and say whether it acknowledged. Best-effort:+ a missing key, a refusal (a batch that has already ended cannot be+ cancelled) or the deadline is logged, never raised, so it never stands in+ the way of the stop that asked for it."""api_key = anthropic_api_key()if api_key is None:logger.warning(f"No Anthropic key to cancel dream batch {provider_batch_id}")- return+ return Falsetry:- acknowledged = await asyncio.wait_for(- cancel_batch(+ async with asyncio.timeout(PROVIDER_CANCEL_TIMEOUT_SECONDS):+ acknowledged = await cancel_batch(provider="anthropic",provider_batch_id=provider_batch_id,api_key=api_key,- ),- timeout=PROVIDER_CANCEL_TIMEOUT_SECONDS,- )+ )except Exception:logger.warning(f"Cancelling dream batch {provider_batch_id} failed", exc_info=True)- return+ return Falseif acknowledged:logger.info(f"Anthropic is cancelling dream batch {provider_batch_id}")else:logger.warning(f"Anthropic did not cancel dream batch {provider_batch_id}")+ return acknowledgeddiff --git a/autogpt_platform/backend/backend/copilot/dream/provider_batch_test.py b/autogpt_platform/backend/backend/copilot/dream/provider_batch_test.pyindex e547967665..e1a7e618ab 100644--- a/autogpt_platform/backend/backend/copilot/dream/provider_batch_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/provider_batch_test.py@@ -1,6 +1,8 @@-"""Cancelling a dream pass's batch at Anthropic: the SDK call it makes, and-that nothing about it (no key, a refusal, a hang) reaches the stop that asked-for it."""+"""Cancelling a dream pass's batch at Anthropic: the SDK call it makes, that+nothing about it (no key, a refusal, a hang) reaches the stop that asked for+it, when the cleanup after a pass takes its batch to have stopped, and that+a cancel sent without an acknowledgement is not sent again before the+batch's status is read."""import asyncioimport logging@@ -9,7 +11,12 @@ from unittest.mock import AsyncMock, MagicMockimport pytestfrom . import provider_batch-from .provider_batch import anthropic_api_key, cancel_provider_batch+from .provider_batch import (+ CANCEL_SENT_PREFIX,+ anthropic_api_key,+ cancel_provider_batch,+ provider_batch_stopped,+)@pytest.fixture@@ -26,7 +33,7 @@ def client(mocker) -> MagicMock:async def test_the_batch_is_cancelled_through_the_anthropic_client(client):- await cancel_provider_batch("msgbatch_1")+ assert await cancel_provider_batch("msgbatch_1") is Trueclient.messages.batches.cancel.assert_awaited_once_with("msgbatch_1")@@ -78,3 +85,130 @@ def test_the_key_is_the_copilot_configs_before_the_shared_one(mocker):assert anthropic_api_key() == "sk-shared"settings.return_value.secrets.anthropic_api_key = ""assert anthropic_api_key() is None+++class TestWhetherABatchHasStopped:+ """What the cleanup after a pass asks: its batch has stopped once the+ cancel is acknowledged, or the batch is found ended; anything less is no+ stop, and the cleanup stays unfinished."""++ async def test_an_acknowledged_cancel_is_a_stop(self, client, batch_status):+ assert await provider_batch_stopped("msgbatch_1") is True++ batch_status.assert_not_awaited()++ @pytest.mark.parametrize(+ ("status", "stopped"), [("ended", True), ("processing", False)]+ )+ async def test_a_refused_cancel_asks_whether_the_batch_ended(+ self, client, batch_status, status, stopped+ ):+ client.messages.batches.cancel.side_effect = RuntimeError("batch has ended")+ batch_status.return_value = status++ assert await provider_batch_stopped("msgbatch_1") is stopped++ batch_status.assert_awaited_once()+ assert batch_status.await_args.kwargs["provider_batch_id"] == "msgbatch_1"++ async def test_a_status_the_provider_cannot_give_is_no_stop(+ self, client, batch_status, caplog+ ):+ client.messages.batches.cancel.side_effect = RuntimeError("unreachable")+ batch_status.side_effect = ConnectionError("unreachable")++ with caplog.at_level(logging.WARNING, logger=provider_batch.logger.name):+ assert await provider_batch_stopped("msgbatch_1") is False++ assert "Could not read the status of dream batch msgbatch_1" in caplog.text++ async def test_without_a_key_nothing_is_confirmed(self, mocker, batch_status):+ mocker.patch.object(provider_batch, "anthropic_api_key", return_value=None)++ assert await provider_batch_stopped("msgbatch_1") is False++ batch_status.assert_not_awaited()+++class TestACancelThatWasSent:+ """A cancel and a status read may not fit one reaper attempt together:+ a cancel not acknowledged is noted, so the next stop of the batch reads+ its status first, and a status that reads running sends it again."""++ async def test_is_not_sent_again_before_the_status_is_read(+ self, client, batch_status, fake_dream_redis+ ):+ """The reaper's attempt is cut during the status read: the next stop+ of the batch goes straight to the read."""+ client.messages.batches.cancel.side_effect = RuntimeError("batch has ended")+ reading = asyncio.Event()++ async def cut_during_the_read(**_kwargs) -> str:+ reading.set()+ await asyncio.Event().wait()+ return "ended"++ batch_status.side_effect = cut_during_the_read+ stopping = asyncio.ensure_future(provider_batch_stopped("msgbatch_1"))+ await asyncio.wait_for(reading.wait(), 5)+ stopping.cancel()+ await asyncio.wait([stopping])+ assert f"{CANCEL_SENT_PREFIX}:msgbatch_1" in fake_dream_redis.store++ batch_status.side_effect = None+ batch_status.return_value = "ended"+ assert await provider_batch_stopped("msgbatch_1") is True+ client.messages.batches.cancel.assert_awaited_once()++ async def test_is_sent_again_after_a_status_read_that_failed(+ self, client, batch_status, fake_dream_redis+ ):+ client.messages.batches.cancel.side_effect = RuntimeError("unreachable")+ batch_status.side_effect = ConnectionError("unreachable")++ assert await provider_batch_stopped("msgbatch_1") is False+ assert f"{CANCEL_SENT_PREFIX}:msgbatch_1" not in fake_dream_redis.store++ client.messages.batches.cancel.side_effect = None+ assert await provider_batch_stopped("msgbatch_1") is True+ assert client.messages.batches.cancel.await_count == 2++ async def test_is_sent_again_once_the_batch_reads_still_running(+ self, client, batch_status, fake_dream_redis+ ):+ client.messages.batches.cancel.side_effect = RuntimeError("unreachable")+ fake_dream_redis.store[f"{CANCEL_SENT_PREFIX}:msgbatch_1"] = "1"+ batch_status.return_value = "processing"++ assert await provider_batch_stopped("msgbatch_1") is False+ client.messages.batches.cancel.assert_not_awaited()+ assert f"{CANCEL_SENT_PREFIX}:msgbatch_1" not in fake_dream_redis.store++ client.messages.batches.cancel.side_effect = None+ assert await provider_batch_stopped("msgbatch_1") is True+ client.messages.batches.cancel.assert_awaited_once()++ async def test_acknowledged_leaves_no_note(+ self, client, batch_status, fake_dream_redis+ ):+ assert await provider_batch_stopped("msgbatch_1") is True++ assert not [+ k for k in fake_dream_redis.store if k.startswith(CANCEL_SENT_PREFIX)+ ]++ async def test_without_redis_to_say_is_sent_each_time(+ self, client, batch_status, monkeypatch, fake_dream_redis+ ):+ client.messages.batches.cancel.side_effect = RuntimeError("batch has ended")+ batch_status.return_value = "processing"++ async def down(*_args, **_kwargs):+ raise ConnectionError("redis down")++ monkeypatch.setattr(fake_dream_redis, "exists", down)+ monkeypatch.setattr(fake_dream_redis, "set", down)++ assert await provider_batch_stopped("msgbatch_1") is False+ assert await provider_batch_stopped("msgbatch_1") is False+ assert client.messages.batches.cancel.await_count == 2diff --git a/autogpt_platform/backend/backend/copilot/dream/reaper.py b/autogpt_platform/backend/backend/copilot/dream/reaper.pynew file mode 100644index 0000000000..5ff9db2adc--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/reaper.py@@ -0,0 +1,588 @@+"""The dream pass reaper: close the passes that outlived their lease, and+finish the cleanups nobody else finished.++A pass renews its lease at every step (``lease.py``), so an open row whose+lease lapsed a while ago most likely belongs to a pass that died (its process+killed, its batch callback lost, its last record write lost). A newer pass's+guard expires such a row when it looks at the scope; the reaper closes the+rest, every ``REAPER_INTERVAL_MINUTES``, across every user. The row's lease+is the authority: a pass whose Redis renewals kept landing while every row+write failed looks dead too, and is stopped at its next check.++Each run lists ``REAPER_ROW_LIMIT`` rows at most, of two kinds, each kept+to half the cap when the other has rows: the open rows whose lease lapsed+more than ``REAP_GRACE_SECONDS`` (one sync lock TTL) ago, and the closed+rows whose cleanup (``cleanup.py``) is pending and due. Each kind gets its+own share of the run's time (``_reap_both``), so neither can spend the+other's: cleanups that keep failing (a provider that hangs on every cancel)+do not keep a dead pass from being closed and charged before its batch state+lapses, and a stream of lapsing passes does not keep the pending cleanups+waiting. Within its kind, each row is tried for ``ROW_ATTEMPT_SECONDS`` at+most, so one that hangs leaves the rows behind it their turn.++Each kind is worked in scan cycles, in key order: open rows by their lapse,+pending cleanups by when they were marked, then by id. A cycle covers the+rows whose key is at most its end, the newest key when it starts, and Redis+keeps where it stands (``_SCAN_KEYS``): its end, and the key of the last row+tried, noted before that row's attempt. A run lists the rows after that one,+up to the end; once none is left, it starts the next cycle, from the first+row. So every row present when a cycle starts is tried once in it, whatever+fails, arrives or slips; a row that arrives later has a key past the end and+waits for the next cycle; and no row is written to make room. Without the+cycle's state (a Redis that cannot say) a run starts from the first row, and+fairness waits for Redis. Two reapers running at once (a rollout) can move a+cycle back, the last write winning; a row's close and its cleanup's steps+are safe to repeat, so that costs time, never a second charge.+A cleanup is due once its lease lapsed a grace ago, or it was marked a grace+ago: time for a pass stopped while it ran to reach its next check, and for+the executor's drop to clean up after it. The grace is a bound, not a fence:+an apply that runs on longer than the grace after its pass was stopped can+still be writing when the reaper releases the lock; without the grace, the+reaper would release the lock of any pass still applying.++It closes an open row EXPIRED, holding the scope's lock when it is free, its+cancel generation bumped and its error naming the phase the pass died at,+only if nothing wrote the row since it was read (else ``moved``, left+alone), and marks it in the same statement, keeping the dead pass's lease+token and dropping its lease expiry, so its cleanup is due at once. A row+left APPLYING is closed the same way and never applied again: apply is+gated, and the reaper never calls it.++Then it cleans up after the pass (``cleanup.clean_up_pass``), releasing the+lock under the token the row kept unless the reaper holds the scope itself,+and clears the mark and the lease only once every step has finished+(``expired``, ``cleaned``); else the row stays marked (``retry``), and a+later run does every step again, the finished ones doing nothing. A row+marked longer ago than a batch can run no longer waits on the provider. A+row that kept no token (one written before leases, say) cannot have its+pass's lock released: its unlock finishes once the scope's lock is gone or+held under another open pass's token, and never by deleting a lock no one+can be matched to, so such a row stays marked until its lock lapses, 24 h+10 min at most for a batch lock and 30 min for a sync one.++Bounded: a run takes at most ``REAPER_BUDGET_SECONDS``, from before its+listing to the release of its hold on a scope. Rows are worked until+``RELEASE_TIMEOUT_SECONDS`` before the end, each started only with+``ROW_RESERVE_SECONDS`` left in its kind's share, and all of its work, from+the note of its scan to the write that clears its mark, done by the end of+that share. Its cleanup is cut off after ``ROW_ATTEMPT_SECONDS``; the write+that clears its mark comes after, on a bound of its own, so a slow write+does not cost the cleanup its time, and a mark the share leaves no time to+clear is cleared by the next run, whose cleanup finds nothing left to do.+The release that may follow a cut has its own last seconds, and finishes+even if the cut lands during it. A cleanup cut short resumes where it+stopped: its steps are ordered and idempotent, each phase is charged under+its own claim, and a provider batch whose cancel was sent is next only+asked for its status (``provider_batch.py``). Every call a row makes is+bounded by a timeout around+a direct await, never ``asyncio.wait_for``, which on Python 3.11 returns the+result of a call that completes as a cut lands and so loses the cut. A row+the budget does not reach, or cuts short, is listed again next run; a phase+charge it cuts into finishes on its own (``batch_costs.py``).++Not bounded here: the scheduler's ``run_async`` stops waiting on a timeout+but does not cancel the run, so the run's own budget is what stops it; and+a batch pass's state outlives its lease by twelve hours+(``INPUT_TTL_SECONDS``), so+a pass's landed phases are still there to charge behind about seventy runs'+worth of cleanups that keep failing. One whose state lapsed all the same is+reported as lost usage, an error (``cleanup.py``).++One line per row (INFO when it expired, was cleaned or moved; WARNING when it+failed, is to be retried, or the budget left it), then one with the counts+per outcome.+"""++import asyncio+import logging+import uuid+from collections import Counter+from collections.abc import Awaitable+from datetime import datetime, timedelta, timezone+from typing import Any, Literal, cast++from prisma.enums import DreamPassRoute, DreamPassStatus+from pydantic import BaseModel++from backend.copilot.config import ChatConfig+from backend.copilot.graphiti.scope import MemoryScope+from backend.data import redis_client+from backend.data.dream_pass_models import OPEN_STATUSES, DreamPassRecord++from .batch_submit import phase_models_for_config+from .cleanup import PassCleanup, clean_up_pass+from .locks import (+ DEFAULT_LOCK_TTL_SECONDS,+ LOCK_CHECK_TIMEOUT_SECONDS,+ release_dream_lock,+)+from .pass_record import reaped+from .provider_batch import PROVIDER_BATCH_WINDOW_SECONDS+from .schemas import DreamPhase+from .store import (+ read_expired_passes,+ read_pending_cleanups,+ record_cleanup_finished,+ write_stop,+)++logger = logging.getLogger(__name__)++REAPER_INTERVAL_MINUTES = 10+REAPER_BUDGET_SECONDS = 60.0+REAPER_ROW_LIMIT = 100+# How long past its lease a pass is left before the reaper takes it for dead,+# and how long after a stop its cleanup waits for the pass to end itself.+REAP_GRACE_SECONDS = DEFAULT_LOCK_TTL_SECONDS+# The reaper's own hold on a scope while it closes one row.+REAPER_LOCK_TTL_SECONDS = 120+# The most one row's attempt may take, well under a kind's share of the+# budget, so a row that hangs leaves the others behind it their turn; and+# the budget a row needs left in its kind's share for the reaper to start it.+ROW_ATTEMPT_SECONDS = 12.0+ROW_RESERVE_SECONDS = ROW_ATTEMPT_SECONDS+# Each kind's scan cycle, a Redis hash: ``end``, the key past which rows+# wait for the next cycle, and ``last``, the key of the last row tried.+ScanKind = Literal["open", "pending"]+_SCAN_KEYS: dict[ScanKind, str] = {+ "open": "dream:reaper:open_scan",+ "pending": "dream:reaper:pending_scan",+}+SCAN_TTL_SECONDS = 24 * 60 * 60+# Giving back the reaper's hold on a scope, carved out of the budget.+RELEASE_TIMEOUT_SECONDS = 2.0++ReapOutcome = Literal["expired", "cleaned", "moved", "retry", "failed", "out_of_budget"]+# The outcomes of a row whose cleanup finished, its mark still to clear.+_FINISHED: frozenset[ReapOutcome] = frozenset({"expired", "cleaned"})+++class ReaperRun(BaseModel):+ """What one run did: how many rows it listed, and each one's outcome."""++ listed: int+ outcomes: dict[ReapOutcome, int]+++async def reap_expired_passes(*, now: datetime | None = None) -> ReaperRun:+ """One reaper run, within ``REAPER_BUDGET_SECONDS``; never raises."""+ loop = asyncio.get_running_loop()+ deadline = loop.time() + REAPER_BUDGET_SECONDS - RELEASE_TIMEOUT_SECONDS+ lapsed: list[DreamPassRecord] = []+ pending: list[DreamPassRecord] = []+ done: dict[str, ReapOutcome] = {}+ try:+ async with asyncio.timeout_at(deadline):+ lapsed, pending = await _list(now or datetime.now(timezone.utc))+ await _reap_both(lapsed, pending, done, deadline)+ except Exception:+ logger.warning("Dream pass reaper: the run stopped short", exc_info=True)+ return _summary([*lapsed, *pending], done)+++async def _list(now: datetime) -> tuple[list[DreamPassRecord], list[DreamPassRecord]]:+ """The rows to work, ``REAPER_ROW_LIMIT`` in all, each kind kept to half+ of it when the other has rows, each from its scan cycle (``_scan_rows``):+ the open rows whose lease lapsed a grace ago, and the cleanups pending+ and due."""+ due_before = now - timedelta(seconds=REAP_GRACE_SECONDS)+ lapsed = await _scan_rows("open", due_before, now)+ pending = await _scan_rows("pending", due_before, now)+ kept = min(len(lapsed), max(REAPER_ROW_LIMIT - len(pending), REAPER_ROW_LIMIT // 2))+ return lapsed[:kept], pending[: REAPER_ROW_LIMIT - kept]+++async def _scan_rows(+ kind: ScanKind, due_before: datetime, now: datetime+) -> list[DreamPassRecord]:+ """``REAPER_ROW_LIMIT`` rows of *kind* at most, in key order: those after+ the last one tried in the scan cycle under way, up to its end; when none+ is under way, or none of its rows is left, the first rows of a new cycle,+ which ends at the newest key there is: an open row's lapse is due by+ *due_before*, a cleanup is marked by *now*."""+ scan = await _read_scan(kind)+ if scan is not None:+ end, last = scan+ rows = await _rows_of(kind, due_before, end, last)+ if rows:+ return rows+ end = due_before if kind == "open" else now+ await _start_scan(kind, end)+ return await _rows_of(kind, due_before, end, None)+++async def _rows_of(+ kind: ScanKind,+ due_before: datetime,+ end: datetime,+ after: tuple[datetime, str] | None,+) -> list[DreamPassRecord]:+ if kind == "open":+ return await read_expired_passes(end, limit=REAPER_ROW_LIMIT, after=after)+ return await read_pending_cleanups(+ due_before=due_before, limit=REAPER_ROW_LIMIT, marked_through=end, after=after+ )+++async def _read_scan(+ kind: ScanKind,+) -> tuple[datetime, tuple[datetime, str] | None] | None:+ """The end of *kind*'s scan cycle under way and the key of the last row+ tried in it, if any; ``None`` when none is under way or Redis cannot+ say."""+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ redis = await redis_client.get_redis_async()+ raw = await cast(+ "Awaitable[dict[Any, Any]]", redis.hgetall(_SCAN_KEYS[kind])+ )+ scan = {_text(field): _text(value) for field, value in raw.items()}+ if "end" not in scan:+ return None+ last = scan.get("last")+ return datetime.fromisoformat(scan["end"]), _parsed_key(last) if last else None+ except Exception:+ logger.warning(+ f"Dream pass reaper: could not read where its scan of {kind} passes "+ "stands; starting a new one",+ exc_info=True,+ )+ return None+++async def _start_scan(kind: ScanKind, end: datetime) -> None:+ """Start a scan cycle of *kind* that ends at *end*, from its first row.+ Best-effort: without it, each run starts from the first row."""+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ redis = await redis_client.get_redis_async()+ key = _SCAN_KEYS[kind]+ await cast("Awaitable[int]", redis.hset(key, "end", end.isoformat()))+ await cast("Awaitable[int]", redis.hset(key, "last", ""))+ await redis.expire(key, SCAN_TTL_SECONDS)+ except Exception:+ logger.warning(+ f"Dream pass reaper: could not start a scan of {kind} passes",+ exc_info=True,+ )+++async def _note_tried(row: DreamPassRecord) -> None:+ """Note *row* as the last of its kind tried in the scan cycle under way,+ before it is tried, so the next run goes on after it however its attempt+ ends. Best-effort."""+ kind: ScanKind = "open" if row.status in OPEN_STATUSES else "pending"+ key_time = row.lease_expires_at if kind == "open" else row.cleanup_pending_at+ if key_time is None:+ return+ try:+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ redis = await redis_client.get_redis_async()+ last = f"{key_time.isoformat()}|{row.id}"+ await cast("Awaitable[int]", redis.hset(_SCAN_KEYS[kind], "last", last))+ await redis.expire(_SCAN_KEYS[kind], SCAN_TTL_SECONDS)+ except Exception:+ logger.warning(+ f"Dream pass reaper: could not note pass {row.id} as tried",+ exc_info=True,+ )+++def _parsed_key(key: str) -> tuple[datetime, str]:+ key_time, pass_id = key.split("|", 1)+ return datetime.fromisoformat(key_time), pass_id+++def _text(value: object) -> str:+ return value.decode() if isinstance(value, bytes) else str(value)+++async def _reap_both(+ lapsed: list[DreamPassRecord],+ pending: list[DreamPassRecord],+ done: dict[str, ReapOutcome],+ deadline: float,+) -> None:+ """Work both kinds, each in its own share of the time left: the lapsed+ passes up to half of it when cleanups are pending too, the pending+ cleanups up to the run's deadline, then the lapsed passes the first+ share did not reach, in whatever is left. A row slow enough to use a+ whole share, its calls each within their own deadlines, spends only its+ kind's time."""+ loop = asyncio.get_running_loop()+ halfway = loop.time() + (deadline - loop.time()) / 2+ await _reap_rows(lapsed, done, halfway if pending else deadline)+ await _reap_rows(pending, done, deadline)+ await _reap_rows(lapsed, done, deadline)+++async def _reap_rows(+ rows: list[DreamPassRecord], done: dict[str, ReapOutcome], until: float+) -> None:+ """Work the *rows* not yet done, in turn, each started only with+ ``ROW_RESERVE_SECONDS`` left before *until* and all of its work done by+ it: noted as tried in its scan cycle (``_note_tried``), cleaned up within+ ``ROW_ATTEMPT_SECONDS`` (``_attempt``), and, its cleanup finished, its+ mark cleared (``_record_finished``). A row cut off is done for this run,+ and the next one gets its turn."""+ loop = asyncio.get_running_loop()+ for row in rows:+ if row.id in done:+ continue+ if loop.time() > until - ROW_RESERVE_SECONDS:+ return+ try:+ async with asyncio.timeout_at(until):+ await _note_tried(row)+ await _attempt(row, done, min(until, loop.time() + ROW_ATTEMPT_SECONDS))+ if done.get(row.id) in _FINISHED:+ await _record_finished(row.id, done)+ except TimeoutError:+ logger.warning(+ f"Dream pass reaper: pass {row.id} cut short by its share of the "+ "budget; listed again next run"+ )+ done.setdefault(row.id, "out_of_budget")+++async def _attempt(+ row: DreamPassRecord, done: dict[str, ReapOutcome], cutoff: float+) -> None:+ """``_reap`` *row*, cut off at *cutoff*."""+ try:+ async with asyncio.timeout_at(cutoff):+ await _reap(row, done)+ except TimeoutError:+ logger.warning(+ f"Dream pass reaper: pass {row.id} cut short after "+ f"{ROW_ATTEMPT_SECONDS:g} s; listed again next run"+ )+ done.setdefault(row.id, "out_of_budget")+++def _summary(rows: list[DreamPassRecord], done: dict[str, ReapOutcome]) -> ReaperRun:+ """Log each row the budget left and the counts per outcome."""+ for row in rows:+ if row.id not in done:+ logger.warning(+ f"Dream pass reaper: pass {row.id} not finished within the "+ "budget; listed again next run"+ )+ outcomes: Counter[ReapOutcome] = Counter(done.values())+ outcomes["out_of_budget"] += len(rows) - len(done)+ run = ReaperRun(listed=len(rows), outcomes=dict(+outcomes))+ logger.info(f"Dream pass reaper: {run.listed} listed; {_counts(run)}")+ return run+++async def _reap(row: DreamPassRecord, done: dict[str, ReapOutcome]) -> None:+ """Close one dead pass and clean up after it, holding the scope's lock+ while it does when the lock is free, or finish the cleanup a closed row+ was left marked for; note the outcome in *done* before the hold is given+ back, so a budget that cuts into that release keeps it. ``failed`` when+ the store would not answer, or Redis would not let the reaper take the+ scope, and the row is listed again next run."""+ scope: MemoryScope | None = None+ hold: str | None = None+ try:+ scope = MemoryScope.build(row.user_id, row.expert_id)+ if row.status not in OPEN_STATUSES:+ done[row.id] = await _finish_cleanup(row, scope)+ return+ hold = await _take_scope(scope)+ done[row.id] = await _close(row, scope, took_scope=hold is not None)+ except Exception:+ logger.warning(+ f"Dream pass reaper: pass {row.id} failed; listed again next run",+ exc_info=True,+ )+ done[row.id] = "failed"+ finally:+ if scope is not None and hold is not None:+ await _give_back(scope, hold)+++async def _close(+ row: DreamPassRecord, scope: MemoryScope, *, took_scope: bool+) -> ReapOutcome:+ """Expire *row*, marked for cleanup, then clean up after its pass;+ ``moved`` when the row was written since it was read, and nothing is+ touched."""+ closing = reaped(_error(row), not_updated_since=row.updated_at)+ if not await write_stop(row.id, closing):+ logger.info(f"Dream pass reaper: pass {row.id} moved since it was read; left")+ return "moved"+ # A held scope may still be the dead pass's: released only if it is.+ cleanup = await _clean_up(row, scope, release=not took_scope)+ lock = "held by the reaper" if took_scope else "released if the dead pass's"+ return _logged(+ row, cleanup, "expired", f"expired ({_error(row)})", f"; scope lock {lock}"+ )+++async def _finish_cleanup(row: DreamPassRecord, scope: MemoryScope) -> ReapOutcome:+ """Clean up after a pass whose row closed marked and whose cleanup was+ not finished: every step again, the ones that already ran doing+ nothing."""+ cleanup = await _clean_up(row, scope, release=True)+ done = "cleaned" if cleanup.finished else "not cleaned up yet"+ return _logged(row, cleanup, "cleaned", f"({row.status.value.lower()}) {done}")+++async def _clean_up(+ row: DreamPassRecord, scope: MemoryScope, *, release: bool+) -> PassCleanup:+ """Clean up after *row*'s pass, releasing its lock under the token the+ row kept when *release* (for a row that kept none, finding the lock not+ the pass's). Its mark is cleared after the attempt+ (``_record_finished``). Its landed phases are charged under the usage+ attribution the pass started under, read off its row, never the one the+ account has now; a row without one is charged nothing+ (``batch_costs``)."""+ return await clean_up_pass(+ row.id,+ scope,+ phase_models=_phase_models(row),+ provider_batch_id=_batch_to_stop(row),+ lock_token=row.lease_token,+ release=release,+ attribute_tokenless=True,+ landed=_landed(row),+ )+++async def _record_finished(pass_id: str, done: dict[str, ReapOutcome]) -> None:+ """Clear the row's mark and lease once its cleanup has finished, on the+ write's own bound (``store.RECORD_WRITE_TIMEOUT_SECONDS``); ``failed``+ when it does not land, and the next run does the cleanup again, its+ finished steps doing nothing."""+ try:+ await record_cleanup_finished(pass_id)+ except Exception:+ logger.warning(+ f"Dream pass reaper: pass {pass_id}'s cleanup finished but its mark "+ "could not be cleared; listed again next run",+ exc_info=True,+ )+ done[pass_id] = "failed"+++def _logged(+ row: DreamPassRecord,+ cleanup: PassCleanup,+ outcome: ReapOutcome,+ what: str,+ lock: str = "",+) -> ReapOutcome:+ """*outcome* once *cleanup* has finished, logged at INFO; else ``retry``,+ logged at WARNING with the steps left for the next run."""+ line = (+ f"Dream pass reaper: pass {row.id} {what}; batch "+ f"{row.provider_batch_id or 'none'}; charged "+ f"{', '.join(cleanup.charged) or 'nothing'}{lock}"+ )+ if cleanup.finished:+ logger.info(line)+ return outcome+ logger.warning(+ f"{line}; cleanup unfinished at {', '.join(cleanup.unfinished)}; "+ "listed again next run"+ )+ return "retry"+++async def _take_scope(scope: MemoryScope) -> str | None:+ """Take the scope's lock under a fresh token and return it; ``None`` when+ another holds the lock. Raises when Redis does not answer within+ ``LOCK_CHECK_TIMEOUT_SECONDS``."""+ token = f"reaper:{uuid.uuid4()}"+ async with asyncio.timeout(LOCK_CHECK_TIMEOUT_SECONDS):+ redis = await redis_client.get_redis_async()+ taken = await redis.set(+ scope.redis_key("dream_lock"), token, nx=True, ex=REAPER_LOCK_TTL_SECONDS+ )+ return token if taken else None+++async def _give_back(scope: MemoryScope, hold: str) -> None:+ """Release the reaper's hold on *scope*: at most+ ``RELEASE_TIMEOUT_SECONDS``, Redis client included, and finished even+ when the run's budget cuts in meanwhile. A hold not given back lapses in+ ``REAPER_LOCK_TTL_SECONDS``."""+ release = asyncio.ensure_future(_release_within_bound(scope, hold))+ try:+ await asyncio.shield(release)+ except asyncio.CancelledError:+ await release+ raise+++async def _release_within_bound(scope: MemoryScope, hold: str) -> None:+ try:+ async with asyncio.timeout(RELEASE_TIMEOUT_SECONDS):+ await release_dream_lock(scope, hold)+ except TimeoutError:+ logger.warning(+ f"Dream pass reaper: could not give back scope {scope.scope_key} in "+ f"{RELEASE_TIMEOUT_SECONDS} s; the hold lapses in "+ f"{REAPER_LOCK_TTL_SECONDS} s"+ )+++def _phase_models(row: DreamPassRecord) -> dict[str, str] | None:+ """The deployment's batch phase models, to price those of *row*'s landed+ phases whose state rows recorded no model (rows written before they+ did); the others are priced on the model they ran on. ``None`` when they+ cannot be read, and such a phase's charge waits."""+ try:+ return phase_models_for_config(ChatConfig())+ except Exception:+ logger.warning(+ f"Dream pass reaper: no batch phase models to price pass {row.id}",+ exc_info=True,+ )+ return None+++def _landed(row: DreamPassRecord) -> list[DreamPhase]:+ """The phases of a batch pass whose output *row* recorded: landed, so+ their usage is lost should their batch state be gone before they were+ charged. None for a sync pass, which charges each phase as it lands and+ keeps no batch state."""+ if row.route != DreamPassRoute.ANTHROPIC_BATCH:+ return []+ outputs = row.phase_outputs+ recorded: dict[DreamPhase, object] = {+ "consolidate": outputs.consolidate,+ "recombine": outputs.recombine,+ "sanitize": outputs.sanitize,+ }+ return [phase for phase, output in recorded.items() if output is not None]+++def _batch_to_stop(row: DreamPassRecord) -> str | None:+ """The provider batch the cleanup stops: the one the row names, unless+ the row was marked longer ago than a batch can run, so it has ended+ whatever the provider says (or cannot say)."""+ marked = row.cleanup_pending_at+ window = timedelta(seconds=PROVIDER_BATCH_WINDOW_SECONDS)+ if marked is not None and datetime.now(timezone.utc) - marked > window:+ return None+ return row.provider_batch_id+++def _error(row: DreamPassRecord) -> str:+ """The closed row's error, naming the phase its pass died at."""+ phase = row.phase.value.lower()+ lapsed = row.lease_expires_at.isoformat() if row.lease_expires_at else "unknown"+ if row.status == DreamPassStatus.APPLYING:+ return (+ f"{phase}: the lease lapsed at {lapsed} while applying; closed by the "+ "reaper without applying again, and what it wrote may have landed"+ )+ return f"{phase}: the lease lapsed at {lapsed}; closed by the reaper"+++def _counts(run: ReaperRun) -> str:+ counts = ", ".join(f"{outcome}={n}" for outcome, n in sorted(run.outcomes.items()))+ return counts or "nothing to reap"diff --git a/autogpt_platform/backend/backend/copilot/dream/reaper_backlog_test.py b/autogpt_platform/backend/backend/copilot/dream/reaper_backlog_test.pynew file mode 100644index 0000000000..50e29d30fc--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/reaper_backlog_test.py@@ -0,0 +1,638 @@+"""The reaper behind rows it cannot finish, over the in-memory store and+Redis: a row whose attempt hangs is cut after its own ``ROW_ATTEMPT_SECONDS``,+so the rows behind it get their turn in the same run, and a healthy cleanup+behind five hanging ones is charged in the first run; slow but bounded work+reaches the charge, and a slow write of one kind does not cost the other its+share; each kind is scanned in cycles, so a row that keeps failing waits for+every other row present when the cycle started before it is tried again,+however many rows arrive meanwhile, with no row written to make room; and a+pass whose batch state lapsed before its landed phases were charged is+reported as lost usage, an error, rather than taken for one with nothing to+charge."""++import asyncio+import logging+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++import pytest+from prisma.enums import DreamPassRoute, DreamPassStatus++from backend.util.llm.providers import BatchResultRow++from . import batch_costs as batch_costs_mod+from . import cleanup as cleanup_mod+from . import locks as locks_mod+from . import provider_batch as provider_batch_mod+from . import reaper as reaper_mod+from . import store as store_mod+from .batch_state import claim_per_phase_charging, state_key, write_phase_to_state+from .reaper import (+ REAP_GRACE_SECONDS,+ REAPER_ROW_LIMIT,+ SCAN_TTL_SECONDS,+ reap_expired_passes,+)+from .reaper_cleanup_test import _scale_the_budget+from .reaper_test import _charged, _seed+from .schemas import ConsolidationOutput, RecombinationOutput++_MODELS = {p: "claude-sonnet-5" for p in ("consolidate", "recombine", "sanitize")}+_OPEN_SCAN = "dream:reaper:open_scan"++[email protected](autouse=True)+def phase_models(mocker) -> None:+ mocker.patch.object(reaper_mod, "phase_models_for_config", return_value=_MODELS)++[email protected](autouse=True)+def charges(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )+++class TestARowWhoseAttemptHangs:+ async def test_leaves_the_rows_behind_it_their_turn_in_the_same_run(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Five pending cleanups whose state read never answers, marked before+ a healthy one with a landed phase. Each hanging attempt is cut after+ its own few seconds, not the rest of the run's time, so the healthy+ cleanup is reached in the first run and charged once."""+ _scale_the_budget(mocker, budget=1.0, release=0.02, reserve=0.1)+ mocker.patch.object(reaper_mod, "ROW_ATTEMPT_SECONDS", 0.1)+ stuck = [f"stuck{index}" for index in range(5)]+ _pending(fake_dream_db, [*stuck, "healthy"])+ await _consolidate_landed("healthy")+ tried = _hang_the_state_read_of(mocker, *stuck)++ run = await asyncio.wait_for(reap_expired_passes(), 5)++ assert tried == [*stuck, "healthy"]+ assert run.outcomes == {"out_of_budget": 5, "cleaned": 1}+ assert _charged(charges) == ["consolidate"]+ assert state_key("healthy") not in fake_dream_redis.hashes+ assert fake_dream_db.rows["healthy"]["cleanup_pending_at"] is None+ assert all(fake_dream_db.rows[p]["cleanup_pending_at"] for p in stuck)+++class TestASlowCleanup:+ async def test_reaches_its_charge_and_clears_its_mark(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Every bound scaled to a tenth: the state read takes 4 s of its 5,+ and the write that clears the mark 9 s of its 10. Neither runs over,+ and the write comes after the 12 s attempt, on a bound of its own: one+ charge, the state gone, the mark cleared, in the first run."""+ _scale_every_bound(mocker, 0.1)+ _pending(fake_dream_db, ["slow"])+ await _consolidate_landed("slow")+ write = fake_dream_db.update_dream_pass+ read = cleanup_mod.read_state++ async def slow_write(pass_id, update) -> bool:+ await asyncio.sleep(0.9)+ return await write(pass_id, update)++ async def slow_read(pass_id: str) -> dict:+ await asyncio.sleep(0.4)+ return await read(pass_id)++ mocker.patch.object(fake_dream_db, "update_dream_pass", slow_write)+ mocker.patch.object(cleanup_mod, "read_state", slow_read)++ run = await asyncio.wait_for(reap_expired_passes(), 10)++ assert run.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate"]+ assert state_key("slow") not in fake_dream_redis.hashes+ assert fake_dream_db.rows["slow"]["cleanup_pending_at"] is None++ async def test_whose_provider_answers_slowly_finishes_over_two_runs(+ self, mocker, fake_dream_db, fake_dream_redis+ ):+ """Every bound scaled to a tenth: Anthropic refuses the cancel in 8 s+ of its 10 (the batch has ended) and answers the status read in 8 s+ too. Both do not fit one 12 s attempt; the cancel is noted as sent,+ so the next run only reads the status, and the cleanup finishes."""+ _scale_every_bound(mocker, 0.1)+ mocker.patch.object(provider_batch_mod, "anthropic_api_key", return_value="k")+ _pending(fake_dream_db, ["slow-provider"], provider_batch_id="b-ended")+ calls: list[str] = []++ async def refused(**_kwargs) -> bool:+ await asyncio.sleep(0.8)+ calls.append("cancel")+ return False++ async def ended(**_kwargs) -> str:+ await asyncio.sleep(0.8)+ calls.append("status")+ return "ended"++ mocker.patch.object(provider_batch_mod, "cancel_batch", refused)+ mocker.patch.object(provider_batch_mod, "poll_batch", ended)++ first = await asyncio.wait_for(reap_expired_passes(), 10)+ second = await asyncio.wait_for(reap_expired_passes(), 10)++ assert (first.outcomes, second.outcomes) == (+ {"out_of_budget": 1},+ {"cleaned": 1},+ )+ assert calls == ["cancel", "status"]+ assert fake_dream_db.rows["slow-provider"]["cleanup_pending_at"] is None++ async def test_whose_billing_gate_answers_slowly_charges_once_and_finishes(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Codex's probe, every bound scaled to a tenth: an earlier cleanup+ set the pass's billing gate and stopped before its phase's claim; the+ state read takes 4 s, and each Redis command on the gate 4.5 s. Read+ before it is set, the gate costs the retry one command, not two: one+ charge and the mark cleared in the first run, nothing left after."""+ _scale_every_bound(mocker, 0.1)+ _pending(fake_dream_db, ["gated"])+ await _consolidate_landed("gated")+ assert await claim_per_phase_charging("gated")+ _slow_the_gate(mocker, fake_dream_redis, "gated", seconds=0.45)+ _slow_the_state_read(mocker, seconds=0.4)++ runs = [await asyncio.wait_for(reap_expired_passes(), 10) for _ in range(3)]++ assert [run.outcomes for run in runs] == [{"cleaned": 1}, {}, {}]+ assert _charged(charges) == ["consolidate"]+ assert fake_dream_db.rows["gated"]["cleanup_pending_at"] is None++ async def test_whose_billing_gate_outlasts_the_attempt_is_charged_anyway(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Every bound scaled to a tenth: the state read takes 4 s and the+ gate's read 9 s, past the 12 s attempt. The charge runs on after the+ cut and charges the phase; the next run, cut the same way, charges+ nothing again, and the mark waits for a run the gate answers in."""+ _scale_every_bound(mocker, 0.1)+ _pending(fake_dream_db, ["gated"])+ await _consolidate_landed("gated")+ assert await claim_per_phase_charging("gated")+ _slow_the_gate(mocker, fake_dream_redis, "gated", seconds=0.9)+ _slow_the_state_read(mocker, seconds=0.4)++ first = await asyncio.wait_for(reap_expired_passes(), 10)+ await asyncio.wait_for(asyncio.gather(*batch_costs_mod._CHARGES_IN_FLIGHT), 5)++ assert first.outcomes == {"out_of_budget": 1}+ assert _charged(charges) == ["consolidate"]++ second = await asyncio.wait_for(reap_expired_passes(), 10)+ await asyncio.wait_for(asyncio.gather(*batch_costs_mod._CHARGES_IN_FLIGHT), 5)++ assert second.outcomes == {"out_of_budget": 1}+ assert _charged(charges) == ["consolidate"]+ assert fake_dream_db.rows["gated"]["cleanup_pending_at"] is not None++ async def test_of_one_kind_leaves_the_other_its_share(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Every bound scaled to a tenth, and every listing, and every write+ to a newly lapsed row, takes 9.5 s of its 10; the open rows' cycle+ has just ended, so they are listed twice. The open row's close and the+ write clearing its mark both end with the open rows' share, the write+ cut short there (the next run clears the mark), so the pending+ cleanup still starts within its own, and is charged in the first+ run."""+ _scale_every_bound(mocker, 0.1)+ _pending(fake_dream_db, ["healthy"])+ await _consolidate_landed("healthy")+ _seed(fake_dream_db, "new", route=DreamPassRoute.SYNC)+ ended = datetime.now(timezone.utc) - timedelta(days=1)+ fake_dream_redis.hashes[_OPEN_SCAN] = {"end": ended.isoformat(), "last": ""}+ list_open = fake_dream_db.list_expired_dream_passes+ list_pending = fake_dream_db.list_dream_pass_cleanups+ write = fake_dream_db.update_dream_pass++ async def slow_open(*args, **kwargs):+ await asyncio.sleep(0.95)+ return await list_open(*args, **kwargs)++ async def slow_pending(*args, **kwargs):+ await asyncio.sleep(0.95)+ return await list_pending(*args, **kwargs)++ async def slow_for_the_new_row(pass_id, update) -> bool:+ if pass_id == "new":+ await asyncio.sleep(0.95)+ return await write(pass_id, update)++ mocker.patch.object(fake_dream_db, "list_expired_dream_passes", slow_open)+ mocker.patch.object(fake_dream_db, "list_dream_pass_cleanups", slow_pending)+ mocker.patch.object(fake_dream_db, "update_dream_pass", slow_for_the_new_row)++ run = await asyncio.wait_for(reap_expired_passes(), 10)++ assert run.outcomes.get("cleaned") == 1+ assert _charged(charges) == ["consolidate"]+ assert fake_dream_db.rows["healthy"]["cleanup_pending_at"] is None+++class TestAScanCycle:+ async def test_tries_every_open_row_once_before_any_twice(+ self, mocker, fake_dream_db, fake_dream_redis+ ):+ """Nine open rows whose stop write never answers, the oldest lapses,+ ahead of a healthy one: more than one run can try. No row is tried+ twice before every row was tried once, and the healthy row is closed.+ Every other write to the stuck rows lands, and none is made: their+ lease and their last write are their pass's alone."""+ _scale_the_budget(mocker, budget=0.6, release=0.02, reserve=0.1)+ mocker.patch.object(reaper_mod, "ROW_ATTEMPT_SECONDS", 0.1)+ stuck = [f"stuck{index}" for index in range(9)]+ rows = [*stuck, "healthy"]+ for index, pass_id in enumerate(rows):+ _seed(+ fake_dream_db, pass_id, route=DreamPassRoute.SYNC, lapsed=4000 - index+ )+ before = {pass_id: dict(fake_dream_db.rows[pass_id]) for pass_id in stuck}+ tried = _hang_the_stop_of(mocker, fake_dream_db, *stuck)++ for _ in range(3):+ await asyncio.wait_for(reap_expired_passes(), 5)++ assert len(tried) > len(rows), "every row tried, and some again"+ assert tried[: len(rows)] == rows, "none tried twice before all once"+ assert fake_dream_db.rows["healthy"]["status"] is DreamPassStatus.EXPIRED+ assert {pass_id: fake_dream_db.rows[pass_id] for pass_id in stuck} == before++ async def test_reaches_open_rows_past_the_row_limit(+ self, mocker, fake_dream_db, fake_dream_redis+ ):+ """A hundred open rows whose close fails at once fill a run's list; the+ healthy row lapsed after them is the hundred and first. The next run+ lists from after the last row tried, and closes it."""+ stuck = [f"stuck{index:03}" for index in range(REAPER_ROW_LIMIT)]+ for index, pass_id in enumerate([*stuck, "healthy"]):+ _seed(+ fake_dream_db, pass_id, route=DreamPassRoute.SYNC, lapsed=4000 - index+ )+ write = fake_dream_db.update_dream_pass++ async def fails_for_the_stuck(pass_id, update) -> bool:+ if pass_id in stuck and update.status is DreamPassStatus.EXPIRED:+ raise TimeoutError("this row's close keeps failing")+ return await write(pass_id, update)++ mocker.patch.object(fake_dream_db, "update_dream_pass", fails_for_the_stuck)++ first = await reap_expired_passes()++ assert first.outcomes == {"failed": REAPER_ROW_LIMIT}+ assert fake_dream_db.rows["healthy"]["status"] is DreamPassStatus.RUNNING++ second = await reap_expired_passes()++ assert second.outcomes == {"expired": 1}+ assert fake_dream_db.rows["healthy"]["status"] is DreamPassStatus.EXPIRED++ async def test_reaches_a_cleanup_behind_ones_that_keep_failing(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """Every bound scaled to a tenth: six cleanups whose write clearing the+ mark fails after 9 s (a row another transaction keeps locked), marked+ before a healthy one, more than a run can try. The scan goes on past+ them whatever their writes do, and the healthy cleanup is charged in+ the second run."""+ _scale_every_bound(mocker, 0.1)+ locked = [f"locked{index}" for index in range(6)]+ _pending(fake_dream_db, [*locked, "healthy"])+ await _consolidate_landed("healthy")+ write = fake_dream_db.update_dream_pass+ tried: list[str] = []++ async def locked_rows_time_out(pass_id, update) -> bool:+ tried.append(pass_id)+ if pass_id in locked:+ await asyncio.sleep(0.9)+ raise TimeoutError("canceling statement due to lock timeout")+ return await write(pass_id, update)++ mocker.patch.object(fake_dream_db, "update_dream_pass", locked_rows_time_out)++ first = await asyncio.wait_for(reap_expired_passes(), 10)+ assert _charged(charges) == []+ second = await asyncio.wait_for(reap_expired_passes(), 10)++ assert first.outcomes["failed"] >= 4+ assert second.outcomes.get("cleaned") == 1+ assert _charged(charges) == ["consolidate"]+ assert tried[: len(locked) + 1] == [*locked, "healthy"]++ async def test_lets_no_arrival_delay_a_row_it_started_with(+ self, mocker, fake_dream_db, fake_dream_redis+ ):+ """Five open rows whose stop write never answers and a healthy one are+ there when the cycle starts; two more rows lapse before every run.+ The arrivals have keys past the cycle's end and wait for the next+ cycle, so the healthy row is closed once the five have had their+ turn, and every row the cycle started with is tried before any+ arrival."""+ _scale_the_budget(mocker, budget=0.6, release=0.02, reserve=0.1)+ mocker.patch.object(reaper_mod, "ROW_ATTEMPT_SECONDS", 0.1)+ stuck = [f"stuck{index}" for index in range(5)]+ started_with = [*stuck, "healthy"]+ for index, pass_id in enumerate(started_with):+ _seed(+ fake_dream_db, pass_id, route=DreamPassRoute.SYNC, lapsed=4000 - index+ )+ tried = _hang_the_stop_of(mocker, fake_dream_db, *stuck)+ base = datetime.now(timezone.utc)+ arrivals: list[str] = []++ for run in range(3):+ now = base + timedelta(seconds=601 * run)+ for index in range(2):+ pass_id = f"arrival{run}-{index}"+ _seed(fake_dream_db, pass_id, route=DreamPassRoute.SYNC)+ fake_dream_db.rows[pass_id]["lease_expires_at"] = now - timedelta(+ seconds=REAP_GRACE_SECONDS + 2 - index+ )+ arrivals.append(pass_id)+ await asyncio.wait_for(reap_expired_passes(now=now), 5)++ assert fake_dream_db.rows["healthy"]["status"] is DreamPassStatus.EXPIRED+ first_cycle = started_with + arrivals[:2]+ assert tried[: len(first_cycle)] == first_cycle+ assert not set(arrivals[2:]) & set(tried[: tried.index("healthy")])++ async def test_covers_the_rows_there_when_it_starts(+ self, fake_dream_db, fake_dream_redis+ ):+ """The cycle lists its rows after the last one tried, up to its end; a+ row lapsing later waits, and once the cycle's rows are done, the next+ one starts from the first row."""+ now = datetime.now(timezone.utc)+ for pass_id in ("a", "b"):+ _seed(fake_dream_db, pass_id, route=DreamPassRoute.SYNC, lapsed=7200)++ lapsed, _ = await reaper_mod._list(now)+ assert [row.id for row in lapsed] == ["a", "b"]+ assert fake_dream_redis.ttls[_OPEN_SCAN] == SCAN_TTL_SECONDS++ await reaper_mod._note_tried(fake_dream_db.record("a"))+ _seed(fake_dream_db, "later", route=DreamPassRoute.SYNC)+ fake_dream_db.rows["later"]["lease_expires_at"] = now - timedelta(+ seconds=REAP_GRACE_SECONDS - 60+ )+ lapsed, _ = await reaper_mod._list(now + timedelta(minutes=10))+ assert [row.id for row in lapsed] == ["b"]++ await reaper_mod._note_tried(fake_dream_db.record("b"))+ lapsed, _ = await reaper_mod._list(now + timedelta(minutes=20))+ assert [row.id for row in lapsed] == ["a", "b", "later"]++ async def test_without_redis_to_say_starts_from_the_first_row(+ self, monkeypatch, fake_dream_db, fake_dream_redis, caplog+ ):+ for index, pass_id in enumerate(("s1", "s2")):+ _seed(+ fake_dream_db, pass_id, route=DreamPassRoute.SYNC, lapsed=4000 - index+ )++ read, write = fake_dream_redis.hgetall, fake_dream_redis.hset++ async def hgetall(name: str):+ if name.startswith("dream:reaper:"):+ raise ConnectionError("redis down")+ return await read(name)++ async def hset(name: str, *args, **kwargs):+ if name.startswith("dream:reaper:"):+ raise ConnectionError("redis down")+ return await write(name, *args, **kwargs)++ monkeypatch.setattr(fake_dream_redis, "hgetall", hgetall)+ monkeypatch.setattr(fake_dream_redis, "hset", hset)++ with caplog.at_level(logging.WARNING):+ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 2}+ assert "could not read where its scan of open passes stands" in caplog.text+ assert "could not note pass s1 as tried" in caplog.text+++class TestABatchStateThatLapsed:+ @pytest.mark.parametrize(+ ("claims", "lost"),+ [+ ({}, "consolidate, recombine"),+ (+ {+ "dream:batch:costs_logged:p1": "per_phase",+ "dream:batch:charged:p1:consolidate": "1",+ },+ "recombine",+ ),+ ],+ ids=["neither charged", "one charged"],+ )+ async def test_before_its_landed_phases_were_charged_is_reported_lost(+ self, fake_dream_db, fake_dream_redis, charges, caplog, claims, lost+ ):+ """The row says consolidate and recombine landed, and their state is+ gone: an error names the pass, its owner and the phases nobody+ charged. Nothing more can be done for them, so the cleanup ends."""+ _closed_with_two_landed(fake_dream_db)+ fake_dream_redis.store.update(claims)++ with caplog.at_level(logging.ERROR):+ run = await reap_expired_passes()++ assert run.outcomes == {"cleaned": 1}+ charges.assert_not_awaited()+ assert _errors(caplog) == [+ f"Dream pass p1 of user u1 (expert none): the batch state of landed "+ f"phase(s) {lost} lapsed before they were charged; their usage "+ "cannot be recovered"+ ]++ async def test_of_a_sync_pass_reports_nothing(+ self, fake_dream_db, fake_dream_redis, charges, caplog+ ):+ """A sync pass charges each phase as it lands and keeps no batch+ state: its recorded outputs, with no state and no batch claims, are+ how an already charged pass looks, not lost usage."""+ _seed(+ fake_dream_db,+ "p1",+ route=DreamPassRoute.SYNC,+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(hours=1),+ phase_outputs={"consolidate": ConsolidationOutput()},+ )++ with caplog.at_level(logging.ERROR):+ run = await reap_expired_passes()++ assert run.outcomes == {"cleaned": 1}+ charges.assert_not_awaited()+ assert _errors(caplog) == []++ @pytest.mark.parametrize(+ "claims",+ [+ {+ "dream:batch:costs_logged:p1": "per_phase",+ "dream:batch:charged:p1:consolidate": "1",+ "dream:batch:charged:p1:recombine": "1",+ },+ {"dream:batch:costs_logged:p1": "1"},+ ],+ ids=["charged phase by phase", "charged whole by an earlier build"],+ )+ async def test_after_its_landed_phases_were_charged_reports_nothing(+ self, fake_dream_db, fake_dream_redis, caplog, claims+ ):+ """The pass's own end charged them, then lost the write that clears+ the mark: their state is gone because it was done with."""+ _closed_with_two_landed(fake_dream_db)+ fake_dream_redis.store.update(claims)++ with caplog.at_level(logging.ERROR):+ run = await reap_expired_passes()++ assert run.outcomes == {"cleaned": 1}+ assert _errors(caplog) == []+++def _pending(fake_dream_db, pass_ids: list[str], **columns) -> None:+ """Closed batch rows marked for their cleanup, the first marked first, an+ hour or so ago."""+ now = datetime.now(timezone.utc)+ for index, pass_id in enumerate(pass_ids):+ _seed(+ fake_dream_db,+ pass_id,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=now - timedelta(minutes=60 - index),+ **columns,+ )+++def _hang_the_stop_of(mocker, fake_dream_db, *hanging: str) -> list[str]:+ """Make the stop write of the *hanging* rows never answer; returns the+ rows whose stop was tried, in order."""+ write = fake_dream_db.update_dream_pass+ tried: list[str] = []++ async def stop(pass_id, update) -> bool:+ if update.status is DreamPassStatus.EXPIRED:+ tried.append(pass_id)+ if pass_id in hanging:+ await asyncio.Event().wait()+ return await write(pass_id, update)++ mocker.patch.object(fake_dream_db, "update_dream_pass", stop)+ return tried+++def _hang_the_state_read_of(mocker, *hanging: str) -> list[str]:+ """Make the cleanup's state read never answer for the *hanging* passes,+ and read the rest's; returns the passes it was asked for, in order."""+ tried: list[str] = []+ read = cleanup_mod.read_state++ async def read_state(pass_id: str) -> dict:+ tried.append(pass_id)+ if pass_id in hanging:+ await asyncio.Event().wait()+ return await read(pass_id)++ mocker.patch.object(cleanup_mod, "read_state", read_state)+ return tried+++def _slow_the_gate(mocker, fake_dream_redis, pass_id: str, *, seconds: float) -> None:+ """Every Redis read and write of the pass's billing gate takes *seconds*."""+ gate = f"dream:batch:costs_logged:{pass_id}"+ get, set_ = fake_dream_redis.get, fake_dream_redis.set++ async def slow_get(key: str):+ if key == gate:+ await asyncio.sleep(seconds)+ return await get(key)++ async def slow_set(key: str, *args, **kwargs):+ if key == gate:+ await asyncio.sleep(seconds)+ return await set_(key, *args, **kwargs)++ mocker.patch.object(fake_dream_redis, "get", slow_get)+ mocker.patch.object(fake_dream_redis, "set", slow_set)+++def _slow_the_state_read(mocker, *, seconds: float) -> None:+ read = cleanup_mod.read_state++ async def slow_read(pass_id: str) -> dict:+ await asyncio.sleep(seconds)+ return await read(pass_id)++ mocker.patch.object(cleanup_mod, "read_state", slow_read)+++async def _consolidate_landed(pass_id: str) -> None:+ await write_phase_to_state(+ pass_id=pass_id,+ phase="consolidate",+ row=BatchResultRow(+ custom_id=f"{pass_id}_consolidate",+ content=ConsolidationOutput().model_dump_json(),+ input_tokens=10,+ output_tokens=20,+ ),+ model="claude-sonnet-5",+ )+++def _closed_with_two_landed(fake_dream_db) -> None:+ """Batch pass p1, closed EXPIRED and marked for its cleanup an hour ago,+ its row recording consolidate's and recombine's output; no batch state."""+ _seed(+ fake_dream_db,+ "p1",+ route=DreamPassRoute.ANTHROPIC_BATCH,+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(hours=1),+ phase_outputs={+ "consolidate": ConsolidationOutput(),+ "recombine": RecombinationOutput(),+ },+ )+++def _errors(caplog) -> list[str]:+ return [r.getMessage() for r in caplog.records if r.levelno >= logging.ERROR]+++def _scale_every_bound(mocker, scale: float) -> None:+ """Every bound on the reaper's path, scaled by *scale*."""+ for module, name, seconds in (+ (reaper_mod, "REAPER_BUDGET_SECONDS", 60),+ (reaper_mod, "RELEASE_TIMEOUT_SECONDS", 2),+ (reaper_mod, "ROW_RESERVE_SECONDS", 12),+ (reaper_mod, "ROW_ATTEMPT_SECONDS", 12),+ (reaper_mod, "LOCK_CHECK_TIMEOUT_SECONDS", 2),+ (store_mod, "RECORD_WRITE_TIMEOUT_SECONDS", 10),+ (cleanup_mod, "STATE_STEP_TIMEOUT_SECONDS", 5),+ (cleanup_mod, "CHARGE_STEP_TIMEOUT_SECONDS", 10),+ (batch_costs_mod, "CLAIM_WINDOW_SECONDS", 30),+ (batch_costs_mod, "CLAIM_TIMEOUT_SECONDS", 5),+ (cleanup_mod, "LOCK_CHECK_TIMEOUT_SECONDS", 2),+ (locks_mod, "LOCK_CHECK_TIMEOUT_SECONDS", 2),+ (provider_batch_mod, "LOCK_CHECK_TIMEOUT_SECONDS", 2),+ (provider_batch_mod, "PROVIDER_CANCEL_TIMEOUT_SECONDS", 10),+ ):+ mocker.patch.object(module, name, seconds * scale)diff --git a/autogpt_platform/backend/backend/copilot/dream/reaper_cleanup_test.py b/autogpt_platform/backend/backend/copilot/dream/reaper_cleanup_test.pynew file mode 100644index 0000000000..b2d3d67169--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/reaper_cleanup_test.py@@ -0,0 +1,655 @@+"""The reaper's cleanup after the passes it closes, and its budget, over the+in-memory store and Redis: a cleanup cut short at any step (the process dying+there), or by the budget, is resumed by the next run and finishes once (one+charge, the state and bundle gone, the mark and the kept token cleared; a+step that fails without raising is ``cleanup_retry_test.py``'s); a row that+kept no token has its lock left alone unless it is gone or another open+pass's; a run never outlasts its budget, the release of its hold on a scope+included, and that release finishes when the budget cuts into it; the budget+runs from before the listing; every row gets its own line. The provider and+the charges are stubbed at their edges; the lock, the gates and the row+transitions are the real ones."""++import asyncio+import logging+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++import pytest+from prisma.enums import DreamPassRoute, DreamPassStatus++from . import cleanup as cleanup_mod+from . import provider_batch as provider_batch_mod+from . import reaper as reaper_mod+from . import store as store_mod+from .batch_state import state_key+from .batch_submit import input_bundle_key+from .reaper import reap_expired_passes+from .reaper_test import _LOCK_KEY, _charged, _dead_batch_pass, _seed++# Each cleanup step, by the module that calls it and the name it calls it by.+_STEPS = (+ (cleanup_mod, "provider_batch_stopped"),+ (cleanup_mod, "charge_landed_phases"),+ (cleanup_mod, "release_dream_lock"),+ (cleanup_mod, "best_effort_cleanup"),+ (reaper_mod, "record_cleanup_finished"),+)++[email protected](autouse=True)+def provider_cancel(mocker) -> AsyncMock:+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key", return_value="k"+ )+ mocker.patch.object(+ reaper_mod,+ "phase_models_for_config",+ return_value={p: "claude-sonnet-5" for p in ("consolidate", "recombine")},+ )+ return mocker.patch(+ "backend.copilot.dream.provider_batch.cancel_batch",+ AsyncMock(return_value=True),+ )++[email protected](autouse=True)+def charges(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )+++class TestACleanupCutShort:+ @pytest.mark.parametrize("step", _STEPS, ids=[name for _, name in _STEPS])+ async def test_at_any_step_is_finished_by_the_next_run_once(+ self, mocker, fake_dream_db, fake_dream_redis, charges, step+ ):+ """The dead pass still holds its lock, so every step has work to do;+ the run is cut at *step*, after the row closed. The next run lists+ the row by its mark and does every step again, the done ones doing+ nothing: one charge in all, the lock and the state gone, the mark and+ the kept token cleared."""+ await _dead_batch_pass(fake_dream_db)+ fake_dream_redis.store[_LOCK_KEY] = "dead-token"+ _fail_once(mocker, *step)++ first = await reap_expired_passes()++ assert first.outcomes == {"failed": 1}+ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.EXPIRED+ assert row["cleanup_pending_at"] is not None+ assert row["lease_token"] == "dead-token"++ second = await reap_expired_passes()++ assert (second.listed, second.outcomes) == (1, {"cleaned": 1})+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes+ assert input_bundle_key("p1") not in fake_dream_redis.store+ assert _LOCK_KEY not in fake_dream_redis.store+ row = fake_dream_db.rows["p1"]+ assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)+ assert row["status"] is DreamPassStatus.EXPIRED+ assert (await reap_expired_passes()).listed == 0++ @pytest.mark.parametrize(+ "step", ["_charge", "_unlock", "_deleted", "provider_batch_stopped"]+ )+ async def test_after_a_step_it_finished_resumes_from_the_next(+ self, mocker, fake_dream_db, fake_dream_redis, charges, step+ ):+ """The process stops right after *step* finished (a cut there, a+ deploy): the steps are ordered and idempotent, and each phase is+ charged under its own claim, so the next run finishes the cleanup+ from where it stopped. One charge in all; the lock, the state and the+ mark gone."""+ await _dead_batch_pass(fake_dream_db)+ fake_dream_db.rows["p1"].update(+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(hours=1),+ )+ fake_dream_redis.store[_LOCK_KEY] = "dead-token"+ finished = getattr(cleanup_mod, step)++ async def stops_after(*args, **kwargs):+ await finished(*args, **kwargs)+ raise asyncio.CancelledError(f"the process stopped after {step}")++ mocker.patch.object(cleanup_mod, step, stops_after)+ with pytest.raises(asyncio.CancelledError):+ await reap_expired_passes()+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is not None++ mocker.patch.object(cleanup_mod, step, finished)+ run = await reap_expired_passes()++ assert run.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes+ assert _LOCK_KEY not in fake_dream_redis.store+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None++ async def test_by_the_budget_after_the_close_is_resumed_next_run(+ self, mocker, fake_dream_db, fake_dream_redis, provider_cancel, charges+ ):+ """Codex's case: the budget runs out while the provider cancel of a+ row the run just closed hangs. The cancel is the cleanup's last step,+ so the landed phase was charged and the state deleted before it; the+ row keeps its mark, and the next run stops the batch, charging+ nothing more."""+ _scale_the_budget(mocker, budget=0.3, release=0.05)+ await _dead_batch_pass(fake_dream_db)++ async def never_answers(**_kwargs) -> bool:+ await asyncio.Event().wait()+ return True++ provider_cancel.side_effect = never_answers++ first = await asyncio.wait_for(reap_expired_passes(), 5)++ assert first.outcomes == {"out_of_budget": 1}+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.EXPIRED+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is not None+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes++ provider_cancel.side_effect = None+ second = await reap_expired_passes()++ assert second.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate"]+ assert provider_cancel.await_count == 2+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None+++class TestAnUnattributedPass:+ @pytest.mark.parametrize(+ "status", [DreamPassStatus.ERRORED, DreamPassStatus.CANCELLED]+ )+ async def test_is_cleaned_up_without_a_charge(+ self, fake_dream_db, fake_dream_redis, charges, status+ ):+ """A batch pass whose row carries no usage attribution (#15208: one+ written before the row kept it, or whose capture failed) closed, and+ left its cleanup to the reaper, however it closed: the reaper does+ it, the lock and the state gone, but charges none of its phases, the+ one that landed included, under no attribution of its own either."""+ await _dead_batch_pass(fake_dream_db)+ fake_dream_db.rows["p1"].update(+ status=status,+ cost_context=None,+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(hours=1),+ )+ fake_dream_redis.store[_LOCK_KEY] = "dead-token"++ run = await reap_expired_passes()++ assert run.outcomes == {"cleaned": 1}+ charges.assert_not_awaited()+ assert state_key("p1") not in fake_dream_redis.hashes+ assert _LOCK_KEY not in fake_dream_redis.store+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None+++class TestTheListing:+ async def test_lapsed_passes_and_pending_cleanups_take_turns_within_the_cap(+ self, mocker, fake_dream_db+ ):+ """The oldest lapse, then the pending cleanup: the newer lapse is past+ the cap and waits for the next run."""+ mocker.patch.object(reaper_mod, "REAPER_ROW_LIMIT", 2)+ for pass_id, lapsed_hours in (("new", 1), ("old", 5)):+ _seed(+ fake_dream_db,+ pass_id,+ route=DreamPassRoute.SYNC,+ lapsed=lapsed_hours * 3600,+ )+ _seed(+ fake_dream_db,+ "closed",+ route=DreamPassRoute.SYNC,+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(minutes=10),+ )++ run = await reap_expired_passes()++ assert (run.listed, run.outcomes) == (2, {"cleaned": 1, "expired": 1})+ statuses = {pid: row["status"] for pid, row in fake_dream_db.rows.items()}+ assert statuses["old"] is DreamPassStatus.EXPIRED+ assert statuses["new"] is DreamPassStatus.RUNNING+ assert fake_dream_db.rows["closed"]["cleanup_pending_at"] is None++ async def test_cleanups_that_keep_hanging_do_not_keep_a_lapsed_pass_waiting(+ self, mocker, fake_dream_db, provider_cancel+ ):+ """Three closed rows whose batch cancel never answers would take every+ run if the pending cleanups came first, and a dead pass would never+ be closed and charged before its batch state lapsed. It takes its+ turn ahead of them."""+ _scale_the_budget(mocker, budget=0.5, release=0.05)+ for pass_id in ("stuck1", "stuck2", "stuck3"):+ _seed(+ fake_dream_db,+ pass_id,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ status=DreamPassStatus.EXPIRED,+ provider_batch_id=f"b-{pass_id}",+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(minutes=10),+ )+ _seed(fake_dream_db, "lapsed", route=DreamPassRoute.SYNC)++ async def never_answers(**_kwargs) -> bool:+ await asyncio.Event().wait()+ return True++ provider_cancel.side_effect = never_answers++ run = await asyncio.wait_for(reap_expired_passes(), 5)++ assert run.outcomes == {"expired": 1, "out_of_budget": 3}+ assert fake_dream_db.rows["lapsed"]["status"] is DreamPassStatus.EXPIRED+ assert fake_dream_db.rows["lapsed"]["cleanup_pending_at"] is None++ async def test_a_cleanup_left_unfinished_goes_behind_the_others(+ self, mocker, fake_dream_db, provider_cancel, batch_status+ ):+ """One row a run: the cleanup whose batch will not stop is tried first,+ left marked and put behind the other pending cleanup, which the next+ run reaches instead of trying the same row again."""+ mocker.patch.object(reaper_mod, "REAPER_ROW_LIMIT", 1)+ now = datetime.now(timezone.utc)+ for pass_id, written_minutes_ago in (("stuck", 30), ("waiting", 20)):+ _seed(+ fake_dream_db,+ pass_id,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ status=DreamPassStatus.EXPIRED,+ provider_batch_id=f"b-{pass_id}",+ cleanup_pending_at=now - timedelta(minutes=40),+ )+ fake_dream_db.rows[pass_id]["updated_at"] = now - timedelta(+ minutes=written_minutes_ago+ )++ async def refuses_the_stuck_one(*, provider_batch_id: str, **_kwargs) -> bool:+ return provider_batch_id != "b-stuck"++ provider_cancel.side_effect = refuses_the_stuck_one+ batch_status.return_value = "in_progress"++ first = await reap_expired_passes()+ second = await reap_expired_passes()++ assert (first.outcomes, second.outcomes) == ({"retry": 1}, {"cleaned": 1})+ assert fake_dream_db.rows["waiting"]["cleanup_pending_at"] is None+ assert fake_dream_db.rows["stuck"]["cleanup_pending_at"] is not None+ assert fake_dream_db.rows["stuck"]["status"] is DreamPassStatus.EXPIRED++ async def test_a_cleanup_the_budget_cuts_off_goes_behind_the_others(+ self, mocker, fake_dream_db+ ):+ """The oldest pending cleanup's state read never answers, so the+ budget cuts its attempt off every time; it was moved behind the other+ before it was tried, so a later run reaches the other."""+ _scale_the_budget(mocker, budget=0.15, release=0.01)+ now = datetime.now(timezone.utc)+ for pass_id, written_minutes_ago in (("stuck", 60), ("waiting", 30)):+ _seed(+ fake_dream_db,+ pass_id,+ route=DreamPassRoute.SYNC,+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=now - timedelta(hours=1),+ )+ fake_dream_db.rows[pass_id]["updated_at"] = now - timedelta(+ minutes=written_minutes_ago+ )+ tried = _hang_the_state_read_of(mocker, "stuck")++ for _ in range(3):+ await asyncio.wait_for(reap_expired_passes(), 5)++ assert tried[:2] == ["stuck", "waiting"]+ assert fake_dream_db.rows["waiting"]["cleanup_pending_at"] is None++ async def test_lapsing_passes_leave_the_pending_cleanups_their_share(+ self, mocker, fake_dream_db+ ):+ """Every run a new pass lapses whose cleanup never answers. It gets its+ share of the run's time and no more: the pending cleanup gets the+ rest, every run, instead of waiting behind the next lapsed pass."""+ _scale_the_budget(mocker, budget=0.15, release=0.01)+ _seed(+ fake_dream_db,+ "waiting",+ route=DreamPassRoute.SYNC,+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(hours=1),+ )+ tried = _hang_the_state_read_of(mocker, "lapsed0", "lapsed1", "lapsed2")++ for index in range(3):+ _seed(fake_dream_db, f"lapsed{index}", route=DreamPassRoute.SYNC)+ await asyncio.wait_for(reap_expired_passes(), 5)++ assert tried[:2] == ["lapsed0", "waiting"]+ assert fake_dream_db.rows["waiting"]["cleanup_pending_at"] is None+ assert all(+ fake_dream_db.rows[f"lapsed{index}"]["status"] is DreamPassStatus.EXPIRED+ for index in range(3)+ )++ async def test_slow_calls_within_their_deadlines_leave_both_kinds_their_share(+ self, mocker, fake_dream_db, provider_cancel, batch_status+ ):+ """Scaled to a hundredth: every listing, close, clear of a mark,+ provider cancel and status read takes 9 s of a 10 s deadline. Each+ run's lapsed pass spends its own half of the budget; the pending+ cleanup is still reached."""+ _scale_the_budget(mocker, budget=0.6, release=0.02, reserve=0.15)+ mocker.patch.object(store_mod, "RECORD_WRITE_TIMEOUT_SECONDS", 0.1)+ mocker.patch.object(provider_batch_mod, "PROVIDER_CANCEL_TIMEOUT_SECONDS", 0.1)+ for name in (+ "read_expired_passes",+ "read_pending_cleanups",+ "write_stop",+ "record_cleanup_finished",+ ):+ mocker.patch.object(reaper_mod, name, _slowed(getattr(reaper_mod, name)))++ async def refuses_slowly(**_kwargs) -> bool:+ await asyncio.sleep(0.09)+ return False++ async def still_running(**_kwargs) -> str:+ await asyncio.sleep(0.09)+ return "in_progress"++ provider_cancel.side_effect = refuses_slowly+ batch_status.side_effect = still_running+ _seed(+ fake_dream_db,+ "waiting",+ route=DreamPassRoute.SYNC,+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(hours=1),+ )++ for index in range(3):+ _seed(+ fake_dream_db,+ f"lapsed{index}",+ route=DreamPassRoute.ANTHROPIC_BATCH,+ status=DreamPassStatus.SUBMITTED,+ provider_batch_id=f"b-{index}",+ )+ await asyncio.wait_for(reap_expired_passes(), 5)++ assert fake_dream_db.rows["waiting"]["cleanup_pending_at"] is None+ assert fake_dream_db.rows["lapsed0"]["status"] is DreamPassStatus.EXPIRED+++class TestTheBudget:+ async def test_bounds_the_release_of_the_reapers_hold_too(+ self, mocker, fake_dream_db, fake_dream_redis, provider_cancel, caplog+ ):+ """The budget cuts a row whose provider cancel hangs, and the release+ of the scope the reaper held hangs too: the run still ends at its+ budget, and the hold is left to lapse on its own TTL."""+ _scale_the_budget(mocker, budget=0.4, release=0.1)+ await _dead_batch_pass(fake_dream_db)++ async def never_answers(**_kwargs) -> bool:+ await asyncio.Event().wait()+ return True++ provider_cancel.side_effect = never_answers+ _hang_the_release_of_the_hold(mocker)+ loop = asyncio.get_running_loop()++ started = loop.time()+ with caplog.at_level(logging.WARNING):+ reaping = asyncio.create_task(reap_expired_passes())+ await asyncio.wait({reaping}, timeout=2)++ assert reaping.done(), "the run outlasted its budget"+ # Well under the 2 s wait above, which a run held by either hang+ # would exhaust; the slack absorbs a loaded test machine.+ assert loop.time() - started < 0.4 + 0.6+ assert reaping.result().outcomes == {"out_of_budget": 1}+ assert fake_dream_redis.store[_LOCK_KEY].startswith("reaper:")+ assert "could not give back scope" in caplog.text++ async def test_lets_a_release_it_cuts_into_finish(+ self, mocker, fake_dream_db, fake_dream_redis+ ):+ """The row finishes, and the budget runs out while the reaper gives+ its hold back: the release is finished all the same, inside its own+ bound, so the scope is not left held."""+ _scale_the_budget(mocker, budget=0.35, release=0.3)+ _seed(fake_dream_db, "s1", route=DreamPassRoute.SYNC)+ release = reaper_mod.release_dream_lock++ async def slow_for_the_hold(scope, token: str | None) -> None:+ if token is not None and token.startswith("reaper:"):+ await asyncio.sleep(0.15)+ await release(scope, token)++ mocker.patch.object(reaper_mod, "release_dream_lock", slow_for_the_hold)++ run = await asyncio.wait_for(reap_expired_passes(), 5)++ assert run.outcomes == {"expired": 1}+ assert _LOCK_KEY not in fake_dream_redis.store++ async def test_runs_from_before_the_listing(self, mocker, fake_dream_db):+ """A slow listing spends the budget too: with less than a row's+ reserve left once the rows are in, none is started."""+ _scale_the_budget(mocker, budget=1.0, release=0.1, reserve=0.5)+ _seed(fake_dream_db, "s1", route=DreamPassRoute.SYNC)+ list_pending = reaper_mod.read_pending_cleanups++ async def slow_listing(**kwargs):+ await asyncio.sleep(0.5)+ return await list_pending(**kwargs)++ mocker.patch.object(reaper_mod, "read_pending_cleanups", slow_listing)++ run = await reap_expired_passes()++ assert run.outcomes == {"out_of_budget": 1}+ assert fake_dream_db.rows["s1"]["status"] is DreamPassStatus.RUNNING+++class TestATokenlessRow:+ """A marked row that kept no lease token (one written before leases, say):+ its pass's lock cannot be released by compare-and-delete, and a lock the+ reaper cannot match to a pass is never deleted."""++ async def test_whose_scope_lock_is_gone_is_cleaned(+ self, fake_dream_db, fake_dream_redis, charges+ ):+ await _tokenless_marked_row(fake_dream_db)++ run = await reap_expired_passes()++ assert run.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate"]+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None++ async def test_whose_scope_lock_another_open_pass_holds_leaves_that_lock(+ self, fake_dream_db, fake_dream_redis+ ):+ """The lock is the newer pass's by the token its open row keeps: not+ the tokenless pass's, so its unlock is done and the lock untouched."""+ await _tokenless_marked_row(fake_dream_db)+ _seed(+ fake_dream_db,+ "newer",+ route=DreamPassRoute.SYNC,+ lapsed=-600,+ lease_token="newer-token",+ )+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"++ run = await reap_expired_passes()++ assert run.outcomes == {"cleaned": 1}+ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is None+ assert fake_dream_db.rows["newer"]["status"] is DreamPassStatus.RUNNING++ async def test_whose_scope_lock_no_open_pass_keeps_waits_for_it_to_lapse(+ self, fake_dream_db, fake_dream_redis, charges, caplog+ ):+ """Maybe the pass's own lock: left alone, the row retried every run+ until the lock lapses on its TTL, then cleaned; charged once."""+ await _tokenless_marked_row(fake_dream_db)+ fake_dream_redis.store[_LOCK_KEY] = "whose-token"++ with caplog.at_level(logging.WARNING):+ first = await reap_expired_passes()++ assert first.outcomes == {"retry": 1}+ assert "cleanup unfinished at unlock; listed again" in caplog.text+ assert fake_dream_redis.store[_LOCK_KEY] == "whose-token"+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is not None++ del fake_dream_redis.store[_LOCK_KEY]+ second = await reap_expired_passes()++ assert second.outcomes == {"cleaned": 1}+ assert _charged(charges) == ["consolidate"]++ async def test_whose_scope_lock_cannot_be_read_is_retried(+ self, monkeypatch, fake_dream_db, fake_dream_redis+ ):+ await _tokenless_marked_row(fake_dream_db)+ read = fake_dream_redis.get++ async def lock_read_down(key: str):+ if key == _LOCK_KEY:+ raise ConnectionError("redis down at the lock read")+ return await read(key)++ monkeypatch.setattr(fake_dream_redis, "get", lock_read_down)+ first = await reap_expired_passes()+ monkeypatch.setattr(fake_dream_redis, "get", read)+ second = await reap_expired_passes()++ assert (first.outcomes, second.outcomes) == ({"retry": 1}, {"cleaned": 1})+++async def test_every_row_gets_its_own_line(mocker, fake_dream_db, caplog):+ """Expired and moved at INFO; failed, and each row the budget did not+ reach, at WARNING; then the counts."""+ for pass_id in ("s1", "s2", "s3"):+ _seed(fake_dream_db, pass_id, route=DreamPassRoute.SYNC)+ write_stop = reaper_mod.write_stop++ async def down_for_s1_and_then_out_of_time(pass_id, update):+ if pass_id == "s1":+ raise TimeoutError("dream pass database stalled")+ mocker.patch.object(reaper_mod, "ROW_RESERVE_SECONDS", 10_000)+ return await write_stop(pass_id, update)++ mocker.patch.object(reaper_mod, "write_stop", down_for_s1_and_then_out_of_time)++ with caplog.at_level(logging.INFO):+ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1, "failed": 1, "out_of_budget": 1}+ lines = {(r.levelname, r.getMessage().split(";")[0]) for r in caplog.records}+ assert ("WARNING", "Dream pass reaper: pass s1 failed") in lines+ assert any(+ level == "INFO" and message.startswith("Dream pass reaper: pass s2 expired")+ for level, message in lines+ )+ assert (+ "WARNING",+ "Dream pass reaper: pass s3 not finished within the budget",+ ) in lines+ assert "Dream pass reaper: 3 listed; expired=1, failed=1, out_of_budget=1" in (+ caplog.text+ )+++def _fail_once(mocker, module, step: str) -> None:+ """Make *step*, as *module* calls it, raise the first time it is called,+ as a run cut short there would stop, and work as before after that."""+ real = getattr(module, step)+ calls = 0++ async def once(*args, **kwargs):+ nonlocal calls+ calls += 1+ if calls == 1:+ raise ConnectionError(f"cut short at {step}")+ return await real(*args, **kwargs)++ mocker.patch.object(module, step, once)+++def _hang_the_state_read_of(mocker, *hanging: str) -> list[str]:+ """Make the cleanup's state read never answer for the *hanging* passes,+ and answer empty for the rest; returns the passes it was asked for, in+ order."""+ tried: list[str] = []++ async def read_state(pass_id: str) -> dict:+ tried.append(pass_id)+ if pass_id in hanging:+ await asyncio.Event().wait()+ return {}++ mocker.patch.object(cleanup_mod, "read_state", read_state)+ return tried+++def _slowed(call):+ """*call*, taking 9 s scaled to a hundredth first."""++ async def slowed(*args, **kwargs):+ await asyncio.sleep(0.09)+ return await call(*args, **kwargs)++ return slowed+++def _scale_the_budget(+ mocker, *, budget: float, release: float, reserve: float = 0.01+) -> None:+ mocker.patch.object(reaper_mod, "REAPER_BUDGET_SECONDS", budget)+ mocker.patch.object(reaper_mod, "RELEASE_TIMEOUT_SECONDS", release)+ mocker.patch.object(reaper_mod, "ROW_RESERVE_SECONDS", reserve)+ mocker.patch(+ "backend.copilot.dream.provider_batch.PROVIDER_CANCEL_TIMEOUT_SECONDS", 30+ )+++def _hang_the_release_of_the_hold(mocker) -> None:+ """Only the compare-and-delete of the reaper's own hold never answers."""+ release = reaper_mod.release_dream_lock++ async def hangs_for_the_hold(scope, token: str | None) -> None:+ if token is not None and token.startswith("reaper:"):+ await asyncio.Event().wait()+ await release(scope, token)++ mocker.patch.object(reaper_mod, "release_dream_lock", hangs_for_the_hold)+++async def _tokenless_marked_row(fake_dream_db) -> None:+ """Dead batch pass p1, consolidate landed, its row closed EXPIRED and+ marked for its cleanup an hour ago, without a lease token."""+ await _dead_batch_pass(fake_dream_db)+ fake_dream_db.rows["p1"].update(+ status=DreamPassStatus.EXPIRED,+ cleanup_pending_at=datetime.now(timezone.utc) - timedelta(hours=1),+ lease_token=None,+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/reaper_test.py b/autogpt_platform/backend/backend/copilot/dream/reaper_test.pynew file mode 100644index 0000000000..d119f3ff98--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/reaper_test.py@@ -0,0 +1,449 @@+"""The dream pass reaper over the in-memory store and Redis: every branch of+closing a pass that outlived its lease. The provider and the charges are+stubbed at their edges; the lock, the gates, the batch state and the row+transitions are the real ones."""++import asyncio+import logging+from datetime import datetime, timedelta, timezone+from typing import Any+from unittest.mock import AsyncMock++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)++from backend.copilot.graphiti.scope import MemoryScope+from backend.data.dream_pass_models import DreamPassDraft+from backend.util.llm.providers import BatchResultRow++from . import reaper as reaper_mod+from .batch_state import STATE_TTL_SECONDS, state_key, write_phase_to_state+from .batch_submit import INPUT_TTL_SECONDS, input_bundle_key, persist_input_bundle+from .fetch import DreamInput+from .locks import BATCH_LOCK_TTL_SECONDS, DEFAULT_LOCK_TTL_SECONDS+from .reaper import REAP_GRACE_SECONDS, REAPER_INTERVAL_MINUTES, reap_expired_passes+from .schemas import ConsolidationOutput++_SCOPE = MemoryScope.for_user("u1")+_LOCK_KEY = _SCOPE.redis_key("dream_lock")+_MODELS = {p: "claude-sonnet-5" for p in ("consolidate", "recombine", "sanitize")}++[email protected](autouse=True)+def provider_cancel(mocker) -> AsyncMock:+ mocker.patch(+ "backend.copilot.dream.provider_batch.anthropic_api_key", return_value="k"+ )+ mocker.patch.object(reaper_mod, "phase_models_for_config", return_value=_MODELS)+ return mocker.patch(+ "backend.copilot.dream.provider_batch.cancel_batch",+ AsyncMock(return_value=True),+ )++[email protected](autouse=True)+def charges(mocker) -> AsyncMock:+ return mocker.patch(+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()+ )+++class TestAPassThatOutlivedItsLease:+ async def test_is_expired_charged_and_cleaned_up(+ self, fake_dream_db, fake_dream_redis, provider_cancel, charges, caplog+ ):+ await _dead_batch_pass(fake_dream_db)+ holders: list[str | None] = []++ async def cancel(**_kwargs) -> bool:+ holders.append(fake_dream_redis.store.get(_LOCK_KEY))+ return True++ provider_cancel.side_effect = cancel++ with caplog.at_level(logging.INFO):+ run = await reap_expired_passes()++ assert (run.listed, run.outcomes) == (1, {"expired": 1})+ # The reaper held the free scope while it cleaned up, then let go.+ assert len(holders) == 1 and (holders[0] or "").startswith("reaper:")+ row = fake_dream_db.rows["p1"]+ assert (row["status"], row["cancel_generation"]) == (+ DreamPassStatus.EXPIRED,+ 1,+ )+ assert row["error"].startswith("recombine: the lease lapsed at ")+ assert row["error"].endswith("; closed by the reaper")+ assert (row["lease_token"], row["lease_expires_at"]) == (None, None)+ assert (row["input_bundle"], row["cleanup_pending_at"]) == (None, None)+ provider_cancel.assert_awaited_once()+ assert provider_cancel.await_args.kwargs["provider_batch_id"] == "b-rec"+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes+ assert input_bundle_key("p1") not in fake_dream_redis.store+ assert _LOCK_KEY not in fake_dream_redis.store+ assert "Dream pass reaper: pass p1 expired (recombine:" in caplog.text+ assert "Dream pass reaper: 1 listed; expired=1" in caplog.text++ async def test_within_its_grace_is_left_open(self, fake_dream_db, provider_cancel):+ """One sync lock TTL past its lease a pass is taken for dead, not+ before: a pass slow to renew is left alone."""+ assert REAP_GRACE_SECONDS == DEFAULT_LOCK_TTL_SECONDS+ await _dead_batch_pass(fake_dream_db, lapsed=25 * 60)++ run = await reap_expired_passes()++ assert (run.listed, run.outcomes) == (0, {})+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.SUBMITTED+ provider_cancel.assert_not_awaited()++ async def test_whose_scope_another_pass_holds_is_closed_and_that_lock_kept(+ self, fake_dream_db, fake_dream_redis+ ):+ await _dead_batch_pass(fake_dream_db)+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.EXPIRED+ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"++ async def test_whose_own_lock_outlived_its_lease_has_it_released(+ self, fake_dream_db, fake_dream_redis+ ):+ """A renewal extended the lock but its row write was lost: the lease+ on the row is the authority, and the lock under its token goes."""+ await _dead_batch_pass(fake_dream_db)+ fake_dream_redis.store[_LOCK_KEY] = "dead-token"++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ assert _LOCK_KEY not in fake_dream_redis.store++ async def test_written_since_it_was_read_is_left_alone(+ self, mocker, fake_dream_db, fake_dream_redis, provider_cancel, charges+ ):+ """The pass renewed (or closed) between the read and the expiry: the+ compare-and-set refuses, and nothing of the pass is touched."""+ await _dead_batch_pass(fake_dream_db)+ listed = await reaper_mod.read_expired_passes(+ datetime.now(timezone.utc), limit=10+ )+ fake_dream_db.rows["p1"]["updated_at"] = datetime.now(timezone.utc)+ mocker.patch.object(+ reaper_mod, "read_expired_passes", AsyncMock(return_value=listed)+ )++ run = await reap_expired_passes()++ assert run.outcomes == {"moved": 1}+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.SUBMITTED+ provider_cancel.assert_not_awaited()+ charges.assert_not_awaited()+ assert state_key("p1") in fake_dream_redis.hashes+ assert _LOCK_KEY not in fake_dream_redis.store++ async def test_prices_a_landed_phase_on_the_model_it_ran_on(+ self, mocker, fake_dream_db, charges+ ):+ """The deployment's phase models moved since the pass was submitted,+ here to ones the batch route cannot even read: the landed phase is+ charged all the same, on the model its state row recorded."""+ mocker.patch.object(+ reaper_mod,+ "phase_models_for_config",+ side_effect=ValueError("the fast models are not Anthropic models"),+ )+ await _dead_batch_pass(fake_dream_db, model="claude-opus-5-5")++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ assert [call.args[1].model for call in charges.await_args_list] == [+ "claude-opus-5-5"+ ]++ async def test_whose_provider_cancel_fails_is_closed_all_the_same(+ self, fake_dream_db, fake_dream_redis, provider_cancel, charges+ ):+ await _dead_batch_pass(fake_dream_db)+ provider_cancel.side_effect = ConnectionError("anthropic unreachable")++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.EXPIRED+ assert _charged(charges) == ["consolidate"]+ assert state_key("p1") not in fake_dream_redis.hashes++ @pytest.mark.parametrize(+ "claimed",+ [+ # The phase's own claim, taken by an earlier cleanup.+ "dream:batch:charged:p1:consolidate",+ # The whole-pass gate an earlier build took before charging.+ "dream:batch:costs_logged:p1",+ ],+ ids=["phase claimed", "charged whole by an earlier build"],+ )+ async def test_already_charged_is_not_charged_again(+ self, fake_dream_db, fake_dream_redis, charges, caplog, claimed+ ):+ await _dead_batch_pass(fake_dream_db)+ fake_dream_redis.store[claimed] = "1"++ with caplog.at_level(logging.INFO):+ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ charges.assert_not_awaited()+ assert "; charged nothing; " in caplog.text+ assert state_key("p1") not in fake_dream_redis.hashes++ async def test_that_died_applying_is_closed_without_applying_again(+ self, mocker, fake_dream_db, fake_dream_redis, charges+ ):+ """It claimed apply, then its process died: the reaper closes the+ row, naming apply, and never calls apply itself."""+ apply = mocker.patch(+ "backend.copilot.dream.apply.apply_operations", AsyncMock()+ )+ await _dead_batch_pass(+ fake_dream_db,+ status=DreamPassStatus.APPLYING,+ phase=DreamPassPhase.APPLY,+ )+ fake_dream_redis.store["dream:applied:p1"] = "1"++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ row = fake_dream_db.rows["p1"]+ assert row["status"] is DreamPassStatus.EXPIRED+ assert row["error"].startswith("apply: the lease lapsed at ")+ assert "while applying; closed by the reaper without applying again" in (+ row["error"]+ )+ apply.assert_not_awaited()+ assert _charged(charges) == ["consolidate"]++ async def test_of_the_sync_route_is_closed_with_nothing_to_cancel(+ self, fake_dream_db, provider_cancel, charges+ ):+ _seed(fake_dream_db, "s1", route=DreamPassRoute.SYNC)++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ assert fake_dream_db.rows["s1"]["status"] is DreamPassStatus.EXPIRED+ provider_cancel.assert_not_awaited()+ charges.assert_not_awaited()+++def test_a_batch_passs_state_outlives_its_lease_until_the_reaper_comes():+ """A dead batch pass's state was written at its last callback, with its+ lease: the reaper takes it for dead a grace after the lease, and may come+ one interval later still. Its landed phases are charged and its keys+ deleted only if they are still there, so they stay for some sixty runs+ more, should a backlog of cleanups that keep failing come first."""+ interval = REAPER_INTERVAL_MINUTES * 60+ reaped_by = BATCH_LOCK_TTL_SECONDS + REAP_GRACE_SECONDS + interval+ kept = min(STATE_TTL_SECONDS, INPUT_TTL_SECONDS)+ assert (kept - reaped_by) // interval >= 60+++class TestARun:+ async def test_counts_each_outcome_and_goes_on_past_a_row_that_fails(+ self, mocker, fake_dream_db, caplog+ ):+ for pass_id in ("s1", "s2"):+ _seed(fake_dream_db, pass_id, route=DreamPassRoute.SYNC)+ write_stop = reaper_mod.write_stop++ async def down_for_s1(pass_id, update):+ if pass_id == "s1":+ raise TimeoutError("dream pass database stalled")+ return await write_stop(pass_id, update)++ mocker.patch.object(reaper_mod, "write_stop", down_for_s1)++ with caplog.at_level(logging.INFO):+ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1, "failed": 1}+ assert fake_dream_db.rows["s1"]["status"] is DreamPassStatus.RUNNING+ assert fake_dream_db.rows["s2"]["status"] is DreamPassStatus.EXPIRED+ assert "Dream pass reaper: 2 listed; expired=1, failed=1" in caplog.text++ async def test_a_cut_into_the_release_keeps_the_rows_outcome(+ self, mocker, fake_dream_db+ ):+ """The row's cutoff can land while the reaper gives the scope back.+ Its outcome was noted before that release, so it stands, and the+ row's cleanup is recorded as finished instead of being redone."""+ mocker.patch.object(reaper_mod, "ROW_ATTEMPT_SECONDS", 0.1)+ _seed(fake_dream_db, "s1", route=DreamPassRoute.SYNC)++ async def slow_release(scope, hold) -> None:+ await asyncio.sleep(0.3)++ mocker.patch.object(reaper_mod, "release_dream_lock", slow_release)++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 1}+ assert fake_dream_db.rows["s1"]["status"] is DreamPassStatus.EXPIRED+ assert fake_dream_db.rows["s1"]["cleanup_pending_at"] is None++ async def test_never_outlasts_its_budget(+ self, mocker, fake_dream_db, fake_dream_redis, provider_cancel+ ):+ """A provider cancel that never answers holds the first row: the+ budget cancels it, the row keeps its mark for the next run, the rows+ not reached are left for it too, and the scope the reaper held is let+ go."""+ mocker.patch.object(reaper_mod, "REAPER_BUDGET_SECONDS", 0.5)+ mocker.patch.object(reaper_mod, "RELEASE_TIMEOUT_SECONDS", 0.1)+ mocker.patch.object(reaper_mod, "ROW_RESERVE_SECONDS", 0.1)+ mocker.patch(+ "backend.copilot.dream.provider_batch.PROVIDER_CANCEL_TIMEOUT_SECONDS",+ 30,+ )+ for pass_id in ("p1", "p2", "p3"):+ await _dead_batch_pass(fake_dream_db, pass_id=pass_id)++ async def never_answers(**_kwargs) -> bool:+ await asyncio.Event().wait()+ return True++ provider_cancel.side_effect = never_answers+ loop = asyncio.get_running_loop()++ started = loop.time()+ run = await asyncio.wait_for(reap_expired_passes(), 10)++ assert loop.time() - started < 1+ assert (run.listed, run.outcomes) == (3, {"out_of_budget": 3})+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.EXPIRED+ assert fake_dream_db.rows["p1"]["cleanup_pending_at"] is not None+ assert fake_dream_db.rows["p2"]["status"] is DreamPassStatus.SUBMITTED+ assert _LOCK_KEY not in fake_dream_redis.store++ async def test_takes_the_oldest_lapses_first_up_to_its_limit(+ self, mocker, fake_dream_db+ ):+ mocker.patch.object(reaper_mod, "REAPER_ROW_LIMIT", 2)+ for pass_id, lapsed_hours in (("new", 1), ("old", 5), ("older", 9)):+ _seed(+ fake_dream_db,+ pass_id,+ route=DreamPassRoute.SYNC,+ lapsed=lapsed_hours * 3600,+ )++ run = await reap_expired_passes()++ assert run.outcomes == {"expired": 2}+ statuses = {pid: row["status"] for pid, row in fake_dream_db.rows.items()}+ assert statuses == {+ "new": DreamPassStatus.RUNNING,+ "old": DreamPassStatus.EXPIRED,+ "older": DreamPassStatus.EXPIRED,+ }++ async def test_a_store_that_cannot_list_ends_the_run_quietly(+ self, fake_dream_db, caplog+ ):+ fake_dream_db.fail = True++ with caplog.at_level(logging.WARNING):+ run = await reap_expired_passes()++ assert (run.listed, run.outcomes) == (0, {})+ assert "Dream pass reaper: the run stopped short" in caplog.text+++async def _dead_batch_pass(+ fake_dream_db,+ *,+ pass_id: str = "p1",+ status: DreamPassStatus = DreamPassStatus.SUBMITTED,+ phase: DreamPassPhase = DreamPassPhase.RECOMBINE,+ lapsed: int = REAP_GRACE_SECONDS + 600,+ model: str | None = None,+) -> None:+ """A batch pass that died at *phase*: consolidate landed in its state+ (recording *model*, the one it ran on, when given), recombine's batch in+ flight, its bundle kept, its lease lapsed *lapsed* seconds ago."""+ now = datetime.now(timezone.utc)+ await persist_input_bundle(+ pass_id,+ DreamInput(+ user_id="u1", group_id=_SCOPE.group_id, window_start=now, window_end=now+ ),+ lock_token="dead-token",+ )+ await write_phase_to_state(+ pass_id=pass_id,+ phase="consolidate",+ row=BatchResultRow(+ custom_id=f"{pass_id}_consolidate",+ content=ConsolidationOutput().model_dump_json(),+ input_tokens=10,+ output_tokens=20,+ ),+ model=model,+ )+ _seed(+ fake_dream_db,+ pass_id,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ status=status,+ phase=phase,+ lapsed=lapsed,+ provider_batch_id="b-rec",+ )+++def _seed(+ fake_dream_db,+ pass_id: str,+ *,+ route: DreamPassRoute,+ status: DreamPassStatus = DreamPassStatus.RUNNING,+ phase: DreamPassPhase = DreamPassPhase.RECOMBINE,+ lapsed: int = REAP_GRACE_SECONDS + 600,+ provider_batch_id: str | None = None,+ **columns: Any,+) -> None:+ """A row (open, unless *status* says otherwise) whose lease lapsed+ *lapsed* seconds ago and that nothing has written since; *columns* set+ the rest of it."""+ lease_end = datetime.now(timezone.utc) - timedelta(seconds=lapsed)+ fake_dream_db.seed(+ DreamPassDraft(+ id=pass_id,+ user_id="u1",+ scope_key=_SCOPE.scope_key,+ route=route,+ trigger=DreamPassTrigger.CRON,+ status=status,+ phase=phase,+ lease_token="dead-token",+ lease_expires_at=lease_end,+ ),+ updated_at=lease_end - timedelta(minutes=1),+ provider_batch_id=provider_batch_id,+ **columns,+ )+++def _charged(charges: AsyncMock) -> list[str]:+ return [call.args[0].job.phase for call in charges.await_args_list]diff --git a/autogpt_platform/backend/backend/copilot/dream/retention.py b/autogpt_platform/backend/backend/copilot/dream/retention.pynew file mode 100644index 0000000000..2086e5f466--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/retention.py@@ -0,0 +1,71 @@+"""How long the DreamPass table keeps a pass, and the job that enforces it.++A row sheds most of its weight as it closes: every closing transition drops+its input bundle (``pass_record.py``), keeping the phase outputs, the+operations and the usage, and its small lease until the cleanup after the+pass, when one is pending, has finished. The bundle is the bulk of an open+batch row: Codex measured one gathered from the dream fixtures (50 episodes,+500 facts, 10 session bodies) at 167 KB of JSON, which Postgres compressed to+7.2 KB, about 8.7 KB of table per row all told; real text compresses less,+and a bundle has no size cap, so an uncompressed one can run to hundreds of+KB. A closed row keeps the smaller JSON only, a few KB.++Closed rows are kept ``Config.dream_pass_retention_days`` (the+``DREAM_PASS_RETENTION_DAYS`` setting, 90 by default) from their creation;+a weekly scheduler job deletes the older ones and logs how many went. An open+row is never deleted, however old (closing it is the reaper's job), nor a+closed one whose cleanup has yet to finish.++Bounded: ``RETENTION_BATCH_SIZE`` rows per statement, each statement+``RETENTION_BATCH_TIMEOUT_SECONDS``, at most ``RETENTION_MAX_BATCHES``+statements and ``RETENTION_BUDGET_SECONDS`` a run; the next run goes on from+there. Not bounded: how much of the table a statement reads to find its+batch. There is no index on ``createdAt`` alone, so each statement scans+the table until it has its batch: cheap while the table is small, and an+index to add should it grow.+"""++import asyncio+import logging+from datetime import datetime, timedelta, timezone++from .store import delete_old_passes++logger = logging.getLogger(__name__)++RETENTION_BATCH_SIZE = 1000+RETENTION_MAX_BATCHES = 100+# One batch's delete, rows and their compressed JSON included.+RETENTION_BATCH_TIMEOUT_SECONDS = 60.0+# One run, however many batches it gets through.+RETENTION_BUDGET_SECONDS = 900.0+++async def delete_expired_records(+ retention_days: int, *, now: datetime | None = None+) -> int:+ """Delete the closed passes created more than *retention_days* ago, a+ batch at a time, and say how many went. Never raises: a batch that fails+ or a run out of budget stops it, logged, and the next run goes on."""+ cutoff = (now or datetime.now(timezone.utc)) - timedelta(days=retention_days)+ deleted = 0+ try:+ async with asyncio.timeout(RETENTION_BUDGET_SECONDS):+ for _ in range(RETENTION_MAX_BATCHES):+ batch = await delete_old_passes(+ cutoff,+ limit=RETENTION_BATCH_SIZE,+ timeout=RETENTION_BATCH_TIMEOUT_SECONDS,+ )+ deleted += batch+ if batch < RETENTION_BATCH_SIZE:+ break+ except Exception:+ logger.warning(+ f"Dream pass retention stopped after {deleted} deleted", exc_info=True+ )+ logger.info(+ f"Dream pass retention: deleted {deleted} closed pass(es) created "+ f"before {cutoff.isoformat()} ({retention_days}-day retention)"+ )+ return deleteddiff --git a/autogpt_platform/backend/backend/copilot/dream/retention_test.py b/autogpt_platform/backend/backend/copilot/dream/retention_test.pynew file mode 100644index 0000000000..4a2ad19568--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/retention_test.py@@ -0,0 +1,101 @@+"""The DreamPass retention job over the in-memory store: closed rows past+their retention go, a batch at a time, open rows never; and a run is bounded+and logged."""++import asyncio+import logging+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++from prisma.enums import DreamPassRoute, DreamPassStatus, DreamPassTrigger++from backend.data.dream_pass_models import DreamPassDraft++from . import retention as retention_mod+from .retention import delete_expired_records++_NOW = datetime.now(timezone.utc)+++class TestRetention:+ async def test_deletes_closed_passes_past_their_retention_and_keeps_the_rest(+ self, fake_dream_db, caplog+ ):+ _seed(fake_dream_db, "old-complete", DreamPassStatus.COMPLETE, days=91)+ _seed(fake_dream_db, "old-cancelled", DreamPassStatus.CANCELLED, days=120)+ _seed(fake_dream_db, "old-but-open", DreamPassStatus.SUBMITTED, days=200)+ _seed(fake_dream_db, "recent", DreamPassStatus.COMPLETE, days=10)++ with caplog.at_level(logging.INFO):+ deleted = await delete_expired_records(90, now=_NOW)++ assert deleted == 2+ assert sorted(fake_dream_db.rows) == ["old-but-open", "recent"]+ assert "Dream pass retention: deleted 2 closed pass(es) created before" in (+ caplog.text+ )++ async def test_goes_a_batch_at_a_time_until_a_batch_comes_back_short(+ self, mocker, fake_dream_db+ ):+ mocker.patch.object(retention_mod, "RETENTION_BATCH_SIZE", 2)+ delete = mocker.spy(fake_dream_db, "delete_old_dream_passes")+ for n in range(5):+ _seed(fake_dream_db, f"p{n}", DreamPassStatus.ERRORED, days=100)++ assert await delete_expired_records(90, now=_NOW) == 5++ assert [call.kwargs["limit"] for call in delete.call_args_list] == [2, 2, 2]+ assert fake_dream_db.rows == {}++ async def test_stops_after_its_batch_cap_and_leaves_the_rest_for_next_week(+ self, mocker, fake_dream_db+ ):+ mocker.patch.object(retention_mod, "RETENTION_BATCH_SIZE", 1)+ mocker.patch.object(retention_mod, "RETENTION_MAX_BATCHES", 2)+ for n in range(3):+ _seed(fake_dream_db, f"p{n}", DreamPassStatus.SKIPPED, days=100)++ assert await delete_expired_records(90, now=_NOW) == 2++ assert len(fake_dream_db.rows) == 1++ async def test_a_store_that_fails_ends_the_run_logged_not_raised(+ self, fake_dream_db, caplog+ ):+ fake_dream_db.fail = True++ with caplog.at_level(logging.WARNING):+ assert await delete_expired_records(90, now=_NOW) == 0++ assert "Dream pass retention stopped after 0 deleted" in caplog.text++ async def test_a_run_never_outlasts_its_budget(self, mocker):+ async def stalled(*_args, **_kwargs) -> int:+ await asyncio.Event().wait()+ return 0++ mocker.patch.object(retention_mod, "RETENTION_BUDGET_SECONDS", 0.2)+ mocker.patch.object(+ retention_mod, "delete_old_passes", AsyncMock(side_effect=stalled)+ )+ loop = asyncio.get_running_loop()++ started = loop.time()+ assert await asyncio.wait_for(delete_expired_records(90, now=_NOW), 10) == 0++ assert loop.time() - started < 5+++def _seed(fake_dream_db, pass_id: str, status: DreamPassStatus, *, days: int) -> None:+ fake_dream_db.seed(+ DreamPassDraft(+ id=pass_id,+ user_id="u1",+ scope_key="u1",+ route=DreamPassRoute.SYNC,+ trigger=DreamPassTrigger.CRON,+ status=status,+ ),+ created_at=_NOW - timedelta(days=days),+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/store.py b/autogpt_platform/backend/backend/copilot/dream/store.pyindex 17790ec7c5..96e2e7f910 100644--- a/autogpt_platform/backend/backend/copilot/dream/store.py+++ b/autogpt_platform/backend/backend/copilot/dream/store.py@@ -2,10 +2,13 @@Both routes write here, through ``db_accessors.dream_db()``: the Prisma modulewhere it is connected, the DatabaseManager RPC in the scheduler and the batch-executor. The sync orchestrator records the start, the gathered window, each-phase's output, the operations it is about to apply and how the pass ended.-The batch path records the submit, each phase that lands, each next batch, the-apply and the end, with what the landed phases used. What each step writes is+executor. The sync orchestrator records the start with its lease, the+gathered window, each phase's output, each renewal of the lease, the+operations it is about to apply and how the pass ended. The batch path+records the submit, each phase that lands and the lease it renewed, each next+batch, the apply and the end, with what the landed phases used; its end+marks the row for the cleanup after it, which clears the mark once done+(``cleanup.py``). What each step writes is``pass_record.py``; how a write lands on the row (forward only, JSON merged inthe database) is ``backend/data/dream_pass_update.py``.@@ -21,10 +24,12 @@ admin job status keep their keys, TTLs, scripts and formats. What changed istiming: the pass now waits on these bounded writes before it locks, inside thelock, before it chains a batch or applies, and before it cleans up.-``read_dream_pass`` is the read side, for the admin API and the eval driver.-The guard, the stop checks and the cancel read and write here too, under the+``read_dream_pass`` is the read side, for the admin API and the eval driver;+``read_cost_context`` reads the usage attribution every charge of a pass's+phases goes by (``batch_costs.py``). The guard, the stop checks and the cancel read and write here too, under thesame deadline, but a failure reaches them: each decides for itself whether to-go on (``guard.py``, ``cancel.py``).+go on (``guard.py``, ``cancel.py``). So do the reaper and the retention job,+which read and delete across users (``reaper.py``, ``retention.py``)."""import asyncio@@ -33,9 +38,10 @@ from collections.abc import Awaitable, Callablefrom datetime import datetime, timezonefrom typing import TypeVar-from pydantic import BaseModel+from pydantic import BaseModel, ValidationErrorfrom backend.copilot.graphiti.scope import MemoryScope+from backend.copilot.trial_cost_context import TrialCostContext, get_trial_cost_contextfrom backend.data.db_accessors import dream_dbfrom backend.data.dream_pass_models import DreamPassRecord, DreamPassUpdate@@ -43,16 +49,20 @@ from .fetch import DreamInputfrom .pass_record import (DreamTrigger,applying,+ cleanup_finished,+ expired,failed,gathered,handed_to_batch,+ lease,new_pass,next_batch,outcome,phase_output,submitted,+ taken_back,)-from .routing import ExecutionPath+from .pass_run import DreamPassRunfrom .schemas import DreamOperations, DreamPassResult, DreamPassUsage, DreamPhaselogger = logging.getLogger(__name__)@@ -65,26 +75,56 @@ RECORD_WRITE_TIMEOUT_SECONDS = 10.0async def start_pass(- pass_id: str,- scope: MemoryScope,- *,- route: ExecutionPath,- trigger: DreamTrigger,- started_at: datetime,+ run: DreamPassRun, scope: MemoryScope, *, trigger: DreamTrigger) -> None:- """Insert the pass's row: running, gathering its input."""+ """Insert the run's row: running, gathering its input, holding the lease+ of the lock it is about to take, and the usage attribution the pass runs+ under (#15208, captured as it started, ``orchestrator``'s+ ``attributed_usage``), which every charge of its phases then goes by."""try:draft = new_pass(- pass_id, scope, route=route, trigger=trigger, started_at=started_at+ run.pass_id,+ scope,+ route=run.execution_path,+ trigger=trigger,+ started_at=run.started_at,+ lease_token=run.lease_token,+ cost_context=get_trial_cost_context(scope.owner_user_id),)await _bounded(dream_db().create_dream_pass(draft))except Exception:logger.warning(- f"Dream pass {pass_id}: could not insert its record; the pass goes on",+ f"Dream pass {run.pass_id}: could not insert its record; the pass goes on",exc_info=True,)+async def read_cost_context(pass_id: str) -> TrialCostContext | None:+ """The usage attribution pass *pass_id* started under, off its row: the+ one every charge of its phases goes by. ``None`` when there is no row, or+ the row carries none (written before it was kept, or its capture failed),+ one that does not parse, or another user's: such a pass is never charged.+ Raises when the row cannot be read in time; the charge is then left for+ the next attempt."""+ row = await read_pass(pass_id)+ if row is None or row.cost_context is None:+ return None+ try:+ context = TrialCostContext.model_validate(row.cost_context)+ except ValidationError:+ logger.error(f"Dream pass {pass_id}: its usage attribution does not parse")+ return None+ if context.user_id != row.user_id:+ logger.error(f"Dream pass {pass_id}: its usage attribution is another user's")+ return None+ return context+++async def record_lease(pass_id: str, lease_token: str, ttl_seconds: int) -> bool | None:+ """The pass renewed its lock for *ttl_seconds* (see ``_write``)."""+ return await _write(pass_id, "the lease", lambda: lease(lease_token, ttl_seconds))++async def record_gathered(pass_id: str, input_bundle: DreamInput) -> None:await _write(pass_id, "the gathered window", lambda: gathered(input_bundle))@@ -121,6 +161,21 @@ async def record_submitted()+async def record_taken_back(+ pass_id: str, error: str, *, provider_batch_id: str, lease_token: str+) -> bool | None:+ """The pass took its just-submitted batch back without the provider+ confirming the stop: the row closes with the batch on it, marked for the+ reaper's cleanup (see ``_write``)."""+ return await _write(+ pass_id,+ "the batch taken back",+ lambda: taken_back(+ error, provider_batch_id=provider_batch_id, lease_token=lease_token+ ),+ )++async def record_next_batch(pass_id: str, phase: DreamPhase, provider_batch_id: str) -> None:@@ -141,21 +196,30 @@ async def record_batch_complete(result: DreamPassResult, usage: DreamPassUsage | None) -> None:"""A batch pass applied: *result* holds what apply reported, *usage* what- its phases used."""- await _write(result.pass_id, "the outcome", lambda: outcome(result, usage))+ its phases used; the row marked for the cleanup after it."""+ await _write(+ result.pass_id, "the outcome", lambda: outcome(result, usage, marked=True)+ )async def record_batch_failed(pass_id: str, error: str, usage: DreamPassUsage | None) -> None:- """A batch pass failed; *usage* is what its landed phases used."""+ """A batch pass failed; *usage* is what its landed phases used. The row+ is marked for the cleanup after it."""await _write(pass_id,"the failure",- lambda: failed(error, usage, datetime.now(timezone.utc)),+ lambda: failed(error, usage, datetime.now(timezone.utc), marked=True),)+async def record_expired(pass_id: str, error: str) -> None:+ """A delivery of the pass closes its row EXPIRED with *error*: one that+ found its apply claimed by an earlier delivery that never finished."""+ await _write(pass_id, "the expiry", lambda: expired(error, not_updated_since=None))++async def read_dream_pass(pass_id: str, *, user_id: str) -> DreamPassRecord | None:"""The pass's row when *user_id* owns it, else ``None``. Unlike thewrites, a read that fails or runs out of time raises: its caller is@@ -163,10 +227,13 @@ async def read_dream_pass(pass_id: str, *, user_id: str) -> DreamPassRecord | Noreturn await _bounded(dream_db().get_dream_pass_for_user(pass_id, user_id))-async def read_pass(pass_id: str) -> DreamPassRecord | None:+async def read_pass(+ pass_id: str, *, timeout: float | None = None+) -> DreamPassRecord | None:"""The pass's row, or ``None`` when it was never inserted. Raises when- the read fails or runs out of time."""- return await _bounded(dream_db().get_dream_pass(pass_id))+ the read fails or runs out of *timeout* (the write deadline by+ default)."""+ return await _bounded(dream_db().get_dream_pass(pass_id), timeout=timeout)async def read_open_passes(@@ -179,6 +246,72 @@ async def read_open_passes()+async def read_expired_passes(+ expired_before: datetime,+ *,+ limit: int,+ after: tuple[datetime, str] | None = None,+) -> list[DreamPassRecord]:+ """Open passes, of every user, whose lease lapsed before+ *expired_before*, in lapse order (the lapse, then the id): the first+ ones, or those after *after*, a lapse and an id. Raises when the read+ fails or runs out of time."""+ if after is None:+ listing = dream_db().list_expired_dream_passes(expired_before, limit=limit)+ else:+ listing = dream_db().list_expired_dream_passes(+ expired_before, limit=limit, after_lapse=after[0], after_id=after[1]+ )+ return await _bounded(listing)+++async def read_pending_cleanups(+ *,+ due_before: datetime,+ limit: int,+ marked_through: datetime | None = None,+ after: tuple[datetime, str] | None = None,+) -> list[DreamPassRecord]:+ """Closed passes, of every user, marked for a cleanup that has not+ finished and is due: marked, or their lease lapsed, before *due_before*;+ in mark order (the mark, then the id), only those marked by+ *marked_through* and after *after*, a mark and an id, when given.+ Raises when the read fails or runs out of time."""+ if marked_through is None and after is None:+ listing = dream_db().list_dream_pass_cleanups(due_before, limit=limit)+ else:+ listing = dream_db().list_dream_pass_cleanups(+ due_before,+ limit=limit,+ marked_through=marked_through,+ after_mark=after[0] if after else None,+ after_id=after[1] if after else None,+ )+ return await _bounded(listing)+++async def read_user_passes(+ user_id: str, *, open_only: bool, limit: int+) -> list[DreamPassRecord]:+ """*user_id*'s passes, every scope, newest first; only the open ones when+ *open_only*. Raises when the read fails or runs out of time."""+ return await _bounded(+ dream_db().list_dream_passes(user_id, limit=limit, open_only=open_only)+ )+++async def delete_old_passes(+ created_before: datetime, *, limit: int, timeout: float+) -> int:+ """Delete at most *limit* closed passes created before *created_before*+ and say how many went; under *timeout*, a delete being slower than a+ record write. Raises when it fails or runs out of time."""+ return await _bounded(+ dream_db().delete_old_dream_passes(created_before, limit=limit),+ timeout=timeout,+ )++async def write_stop(pass_id: str, update: DreamPassUpdate) -> bool:"""Write a stop from outside the pass (a cancel, an expiry) and saywhether it landed, which its caller acts on, unlike a pass's own writes.@@ -186,6 +319,13 @@ async def write_stop(pass_id: str, update: DreamPassUpdate) -> bool:return await _bounded(dream_db().update_dream_pass(pass_id, update))+async def record_cleanup_finished(pass_id: str) -> bool:+ """The cleanup after the closed pass has finished; whether the row took+ it. Raises when the write fails or runs out of time, and the cleanup,+ idempotent, is resumed by the reaper's next run."""+ return await _bounded(dream_db().update_dream_pass(pass_id, cleanup_finished()))++async def _write(pass_id: str, step: str, build: Callable[[], DreamPassUpdate]) -> bool | None:@@ -208,7 +348,8 @@ async def _write(return written-async def _bounded(call: Awaitable[_T]) -> _T:- """*call*, or ``TimeoutError`` once it has taken the write deadline; the- abandoned call is cancelled, not left running."""- return await asyncio.wait_for(call, timeout=RECORD_WRITE_TIMEOUT_SECONDS)+async def _bounded(call: Awaitable[_T], timeout: float | None = None) -> _T:+ """*call*, or ``TimeoutError`` once it has taken *timeout* (the write+ deadline by default); the abandoned call is cancelled, not left running."""+ async with asyncio.timeout(timeout or RECORD_WRITE_TIMEOUT_SECONDS):+ return await calldiff --git a/autogpt_platform/backend/backend/copilot/dream/store_deadline_test.py b/autogpt_platform/backend/backend/copilot/dream/store_deadline_test.pyindex e8bfb222bc..90c6cb7f9a 100644--- a/autogpt_platform/backend/backend/copilot/dream/store_deadline_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/store_deadline_test.py@@ -8,7 +8,6 @@ under it is replaced."""import asyncioimport loggingfrom collections.abc import AsyncIterator, Callable-from datetime import datetime, timezoneimport httpximport pytest@@ -18,6 +17,7 @@ from backend.data.db_manager import DatabaseManagerAsyncClientfrom backend.util.service import get_service_clientfrom . import store+from .pass_run import DreamPassRunfrom .schemas import DreamOperations_DEADLINE = 0.2@@ -110,11 +110,9 @@ async def test_a_stalled_insert_costs_the_deadline(deadline, rpc_over, caplog):started = loop.time()with caplog.at_level(logging.WARNING, logger=store.logger.name):await store.start_pass(- "p1",+ DreamPassRun.begin("u1", "sync_baseline"),MemoryScope.for_user("u1"),- route="sync_baseline",trigger="cron",- started_at=datetime.now(timezone.utc),)assert loop.time() - started < _BOUNDdiff --git a/autogpt_platform/backend/backend/copilot/dream/store_test.py b/autogpt_platform/backend/backend/copilot/dream/store_test.pyindex 8704d1de01..2df3a03abe 100644--- a/autogpt_platform/backend/backend/copilot/dream/store_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/store_test.py@@ -1,6 +1,7 @@"""The dream store: each transition the two routes write, the read side, andthat a failed write never reaches the pass."""+import asyncioimport loggingfrom datetime import datetime, timedelta, timezonefrom unittest.mock import AsyncMock, MagicMock@@ -15,6 +16,8 @@ from prisma.enums import (from backend.copilot.graphiti.scope import MemoryScopefrom backend.data.dream_pass_models import (+ CLOSED_ROW_CLEARS,+ MARKED_ROW_CLEARS,DreamPassApplied,DreamPassDraft,DreamPassOperations,@@ -25,7 +28,9 @@ from backend.data.dream_pass_models import (from . import storefrom .fetch import DreamInput+from .locks import DEFAULT_LOCK_TTL_SECONDSfrom .pass_record import dream_pass_result_from_row+from .pass_run import DreamPassRunfrom .schemas import (ConsolidationOutput,DreamOperations,@@ -53,6 +58,18 @@ def db(mocker) -> MagicMock:return database+def _run(route) -> DreamPassRun:+ """Pass p1 as the sync entry point starts it, its lease token ``tok``."""+ return DreamPassRun(+ user_id="u1",+ pass_id="p1",+ started_at=_STARTED,+ monotonic_start=0.0,+ execution_path=route,+ lease_token="tok",+ )++def _update(db: MagicMock) -> tuple[str, DreamPassUpdate]:db.update_dream_pass.assert_awaited_once()pass_id, update = db.update_dream_pass.await_args.args@@ -127,9 +144,7 @@ class TestWrites:):scope = MemoryScope.for_expert("u1", "expert-1")- await store.start_pass(- "p1", scope, route=route, trigger=trigger, started_at=_STARTED- )+ await store.start_pass(_run(route), scope, trigger=trigger)db.create_dream_pass.assert_awaited_once_with(DreamPassDraft(@@ -142,6 +157,8 @@ class TestWrites:status=DreamPassStatus.RUNNING,phase=DreamPassPhase.GATHER,started_at=_STARTED,+ lease_token="tok",+ lease_expires_at=_STARTED + timedelta(seconds=DEFAULT_LOCK_TTL_SECONDS),))@@ -261,6 +278,7 @@ class TestWrites:status=DreamPassStatus.SKIPPED,skip_reason="no_new_activity",completed_at=_FINISHED,+ clear=CLOSED_ROW_CLEARS,),)@@ -313,6 +331,9 @@ class TestWrites:_, update = _update(db)assert update.status is DreamPassStatus.COMPLETEassert update.usage == usage+ # Marked for the cleanup after the pass, its lease kept for it.+ assert update.cleanup_pending_at is not None+ assert update.clear == MARKED_ROW_CLEARSasync def test_a_failed_batch_pass_records_the_error_and_usage(self, db):usage = _usage()@@ -324,6 +345,8 @@ class TestWrites:assert update.error == "recombine: provider down"assert update.usage == usageassert update.completed_at is not None+ assert update.cleanup_pending_at == update.completed_at+ assert update.clear == MARKED_ROW_CLEARSclass TestAFailedWriteNeverFailsThePass:@@ -364,11 +387,7 @@ class TestAFailedWriteNeverFailsThePass:with caplog.at_level(logging.WARNING, logger=store.logger.name):await store.start_pass(- "p1",- MemoryScope.for_user("u1"),- route="sync_baseline",- trigger="cron",- started_at=_STARTED,+ _run("sync_baseline"), MemoryScope.for_user("u1"), trigger="cron")assert "could not insert" in caplog.text@@ -402,6 +421,96 @@ class TestAFailedWriteNeverFailsThePass:assert record.levelno == logging.DEBUG+class TestTheLeaseAndTheSweeps:+ async def test_a_renewal_writes_the_token_and_its_new_expiry(self, db):+ before = datetime.now(timezone.utc)++ assert await store.record_lease("p1", "tok", 1800) is True++ pass_id, update = _update(db)+ assert (pass_id, update.lease_token, update.status) == ("p1", "tok", None)+ assert update.lease_expires_at is not None+ assert update.lease_expires_at >= before + timedelta(seconds=1800)+ assert update.clear == frozenset()++ async def test_a_delivery_closes_its_row_expired_and_marked(self, db):+ await store.record_expired("p1", "apply: never finished")++ _, update = _update(db)+ assert (update.status, update.error) == (+ DreamPassStatus.EXPIRED,+ "apply: never finished",+ )+ assert update.bump_cancel_generation is True+ assert update.cleanup_pending_at is not None+ assert update.clear == MARKED_ROW_CLEARS++ async def test_the_sweeps_and_the_admin_list_reach_the_table(self, db):+ cutoff = _STARTED - timedelta(minutes=30)+ db.list_expired_dream_passes = AsyncMock(return_value=[_row()])+ db.list_dream_pass_cleanups = AsyncMock(return_value=[])+ db.list_dream_passes = AsyncMock(return_value=[])+ db.delete_old_dream_passes = AsyncMock(return_value=3)++ assert await store.read_expired_passes(cutoff, limit=100) == [_row()]+ assert await store.read_pending_cleanups(due_before=cutoff, limit=100) == []+ assert await store.read_user_passes("u1", open_only=True, limit=5) == []+ assert await store.delete_old_passes(cutoff, limit=1000, timeout=60) == 3++ db.list_expired_dream_passes.assert_awaited_once_with(cutoff, limit=100)+ cursor = (cutoff - timedelta(hours=1), "p0")+ assert await store.read_expired_passes(cutoff, limit=5, after=cursor) == [+ _row()+ ]+ db.list_expired_dream_passes.assert_awaited_with(+ cutoff, limit=5, after_lapse=cursor[0], after_id="p0"+ )+ db.list_dream_pass_cleanups.assert_awaited_once_with(cutoff, limit=100)+ assert (+ await store.read_pending_cleanups(+ due_before=cutoff, limit=5, marked_through=cutoff, after=cursor+ )+ == []+ )+ db.list_dream_pass_cleanups.assert_awaited_with(+ cutoff,+ limit=5,+ marked_through=cutoff,+ after_mark=cursor[0],+ after_id="p0",+ )+ db.list_dream_passes.assert_awaited_once_with("u1", limit=5, open_only=True)+ db.delete_old_dream_passes.assert_awaited_once_with(cutoff, limit=1000)++ async def test_a_finished_cleanup_clears_the_mark_on_the_closed_row(self, db):+ assert await store.record_cleanup_finished("p1") is True++ pass_id, update = _update(db)+ assert (pass_id, update.closed_row) == ("p1", True)+ assert update.clear == frozenset(+ {"cleanup_pending_at", "lease_token", "lease_expires_at"}+ )++ async def test_a_sweep_that_stalls_raises_at_its_deadline(self, db, monkeypatch):+ async def hang(*_args, **_kwargs) -> None:+ await asyncio.Event().wait()++ monkeypatch.setattr(store, "RECORD_WRITE_TIMEOUT_SECONDS", 0.05)+ db.list_expired_dream_passes = AsyncMock(side_effect=hang)+ db.list_dream_pass_cleanups = AsyncMock(side_effect=hang)+ db.delete_old_dream_passes = AsyncMock(side_effect=hang)+ db.update_dream_pass = AsyncMock(side_effect=hang)++ with pytest.raises(TimeoutError):+ await store.read_expired_passes(_STARTED, limit=1)+ with pytest.raises(TimeoutError):+ await store.read_pending_cleanups(due_before=_STARTED, limit=1)+ with pytest.raises(TimeoutError):+ await store.record_cleanup_finished("p1")+ with pytest.raises(TimeoutError):+ await store.delete_old_passes(_STARTED, limit=1, timeout=0.05)++class TestReadSide:async def test_read_is_owner_scoped(self, db):row = _row()diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.pyindex e17fe095f5..458089d39f 100644--- a/autogpt_platform/backend/backend/data/db_manager.py+++ b/autogpt_platform/backend/backend/data/db_manager.py@@ -781,15 +781,18 @@ class DatabaseManager(AppService):get_latest_user_message_in_session = _(chat_db.get_latest_user_message_in_session)# ============ Dream Passes ============ #- # The scheduler (sync passes) and the batch executor (batch callbacks)- # keep no Prisma connection; the dream store writes each pass's record- # through these via db_accessors.dream_db().+ # The scheduler (sync passes, the reaper, retention) and the batch+ # executor (batch callbacks) keep no Prisma connection; the dream store+ # reaches each pass's record through these via db_accessors.dream_db().create_dream_pass = _(dream_pass_db.create_dream_pass)update_dream_pass = _(dream_pass_db.update_dream_pass)get_dream_pass = _(dream_pass_db.get_dream_pass)get_dream_pass_for_user = _(dream_pass_db.get_dream_pass_for_user)list_open_dream_passes = _(dream_pass_db.list_open_dream_passes)list_dream_passes = _(dream_pass_db.list_dream_passes)+ list_expired_dream_passes = _(dream_pass_db.list_expired_dream_passes)+ list_dream_pass_cleanups = _(dream_pass_db.list_dream_pass_cleanups)+ delete_old_dream_passes = _(dream_pass_db.delete_old_dream_passes)# ============ Morning Briefing ============ ## Exposed so the Prisma-less scheduler process can compose, store and@@ -1305,3 +1308,6 @@ class DatabaseManagerAsyncClient(AppServiceClient):get_dream_pass_for_user = d.get_dream_pass_for_userlist_open_dream_passes = d.list_open_dream_passeslist_dream_passes = d.list_dream_passes+ list_expired_dream_passes = d.list_expired_dream_passes+ list_dream_pass_cleanups = d.list_dream_pass_cleanups+ delete_old_dream_passes = d.delete_old_dream_passesdiff --git a/autogpt_platform/backend/backend/data/db_manager_test.py b/autogpt_platform/backend/backend/data/db_manager_test.pyindex dbe58107de..ed8d2a663f 100644--- a/autogpt_platform/backend/backend/data/db_manager_test.py+++ b/autogpt_platform/backend/backend/data/db_manager_test.py@@ -10,7 +10,12 @@ from prisma.enums import (from pydantic import TypeAdapterfrom backend.copilot.dream.fetch import DreamInput-from backend.copilot.dream.pass_record import cancelled, expired+from backend.copilot.dream.pass_record import (+ cancelled,+ cleanup_finished,+ expired,+ reaped,+)from backend.copilot.dream.schemas import (ConsolidatedFact,ConsolidationOutput,@@ -60,6 +65,9 @@ _DREAM_PASS_METHODS = ("get_dream_pass_for_user","list_open_dream_passes","list_dream_passes",+ "list_expired_dream_passes",+ "list_dream_pass_cleanups",+ "delete_old_dream_passes",)@@ -111,9 +119,12 @@ def test_dream_pass_models_survive_the_rpc_round_trip() -> None:for stop in (expired("stale", not_updated_since=now.replace(microsecond=123000)),cancelled("testing", owner_user_id="u1"),+ reaped("lapsed", not_updated_since=now),+ cleanup_finished(),):sent = body.model_validate(to_dict({"pass_id": "p1", "update": stop}))assert sent.update == stop+ assert sent.update.clear == stop.clear != frozenset()record = DreamPassRecord(id="p1",@@ -129,6 +140,7 @@ def test_dream_pass_models_survive_the_rpc_round_trip() -> None:provider_batch_id="msgbatch_1",lease_token="tok",lease_expires_at=now,+ cleanup_pending_at=now,input_bundle=bundle,phase_outputs=outputs,operations=operations,@@ -147,6 +159,35 @@ def test_dream_pass_models_survive_the_rpc_round_trip() -> None:assert returned == record+def test_the_reaper_and_retention_arguments_survive_the_rpc_round_trip() -> None:+ """The scheduler lists and deletes passes over this RPC: an aware cutoff+ and a limit must arrive as they left. The endpoints are built from the+ class's functions, as the service registers them."""+ cutoff = datetime(2026, 9, 26, 3, 0, 0, 123000, tzinfo=timezone.utc)+ manager = DatabaseManager()+ # Both scans also carry where the reaper's scan cycle stands.+ open_scan = {"after_lapse": cutoff, "after_id": "p1"}+ cleanup_scan = {"marked_through": cutoff, "after_mark": cutoff, "after_id": "p1"}+ for method, args in (+ (+ DatabaseManager.list_expired_dream_passes,+ {"expired_before": cutoff, "limit": 7, **open_scan},+ ),+ (+ DatabaseManager.list_dream_pass_cleanups,+ {"due_before": cutoff, "limit": 7, **cleanup_scan},+ ),+ (+ DatabaseManager.delete_old_dream_passes,+ {"created_before": cutoff, "limit": 7},+ ),+ ):+ endpoint = manager._create_fastapi_endpoint(method)+ body = inspect.signature(endpoint).parameters["body"].annotation+ sent = body.model_validate(to_dict(args))+ assert sent.model_dump() == args++def test_the_audience_consumer_can_read_the_opt_out() -> None:assert hasattr(DatabaseManager, "is_marketing_opted_out")assert hasattr(DatabaseManagerAsyncClient, "is_marketing_opted_out")diff --git a/autogpt_platform/backend/backend/data/dream_pass.py b/autogpt_platform/backend/backend/data/dream_pass.pyindex 9a9ccbfacb..206eff3216 100644--- a/autogpt_platform/backend/backend/data/dream_pass.py+++ b/autogpt_platform/backend/backend/data/dream_pass.py@@ -9,43 +9,73 @@ forward and merges its JSON columns in the database.A pass that has reached a terminal status (complete, errored, cancelled,expired, skipped) is final: no update writes it, the way-``job_status.mark_complete`` never rewrites a finished job.+``job_status.mark_complete`` never rewrites a finished job, but for the+cleanup after it marking itself finished.++Three queries work across users: the reaper lists open passes whose lease+lapsed (``list_expired_dream_passes``, on the status and lease expiry index)+and closed passes whose cleanup is due and has not finished+(``list_dream_pass_cleanups``, on the status and cleanup index), and the+retention job deletes closed passes past their retention+(``delete_old_dream_passes``). None is exposed to a user."""+from datetime import datetime+import prisma.models+from prisma.types import DreamPassCreateInput, DreamPassWhereInputfrom backend.data.db import execute_raw_with_schemafrom backend.data.dream_pass_models import (+ CLOSED_STATUSES,OPEN_STATUSES,DreamPassDraft,DreamPassRecord,DreamPassUpdate,)from backend.data.dream_pass_update import TRANSITION_SQL, transition_args+from backend.util.json import SafeJson++# One batch of the retention delete: at most $3 closed passes created before+# $2 whose cleanup is not still pending. DELETE takes no LIMIT, so the batch's+# ids are picked in a subquery.+RETENTION_SQL = """+DELETE FROM {schema_prefix}"DreamPass" WHERE "id" IN (+ SELECT "id" FROM {schema_prefix}"DreamPass"+ WHERE NOT ("status"::text = ANY($1::text[]))+ AND "cleanupPendingAt" IS NULL+ AND "createdAt" < $2::timestamptz AT TIME ZONE 'UTC'+ LIMIT $3::int+)+"""async def create_dream_pass(draft: DreamPassDraft) -> DreamPassRecord:- row = await prisma.models.DreamPass.prisma().create(- data={- "id": draft.id,- "userId": draft.user_id,- "expertId": draft.expert_id,- "scopeKey": draft.scope_key,- "route": draft.route,- "trigger": draft.trigger,- "status": draft.status,- "phase": draft.phase,- "startedAt": draft.started_at,- }- )+ data: DreamPassCreateInput = {+ "id": draft.id,+ "userId": draft.user_id,+ "expertId": draft.expert_id,+ "scopeKey": draft.scope_key,+ "route": draft.route,+ "trigger": draft.trigger,+ "status": draft.status,+ "phase": draft.phase,+ "startedAt": draft.started_at,+ "leaseToken": draft.lease_token,+ "leaseExpiresAt": draft.lease_expires_at,+ }+ if draft.cost_context is not None:+ data["costContext"] = SafeJson(draft.cost_context)+ row = await prisma.models.DreamPass.prisma().create(data=data)return DreamPassRecord.from_db(row)async def update_dream_pass(pass_id: str, update: DreamPassUpdate) -> bool:"""Apply one transition to the pass's row, in a single statement.- ``False`` when there is no such row, it has reached a terminal status, or- it fails the update's owner or not-updated-since condition. ``True``+ ``False`` when there is no such row, it has reached a terminal status+ (it is open, for an update marked ``closed_row``), or it fails the+ update's owner or not-updated-since condition. ``True``means the row was written, not that every field moved: a status or phasebehind the row's, or a batch for a phase the row has left, is dropped inthe statement.@@ -83,9 +113,103 @@ async def list_open_dream_passes(return [DreamPassRecord.from_db(row) for row in rows]-async def list_dream_passes(user_id: str, limit: int = 20) -> list[DreamPassRecord]:- """The user's passes, across every scope, newest first."""+async def list_dream_passes(+ user_id: str, limit: int = 20, open_only: bool = False+) -> list[DreamPassRecord]:+ """The user's passes, across every scope, newest first; only those still+ open when *open_only*."""+ where: DreamPassWhereInput = {"userId": user_id}+ if open_only:+ where["status"] = {"in": list(OPEN_STATUSES)}+ rows = await prisma.models.DreamPass.prisma().find_many(+ where=where, order={"createdAt": "desc"}, take=limit+ )+ return [DreamPassRecord.from_db(row) for row in rows]+++async def list_expired_dream_passes(+ expired_before: datetime,+ limit: int = 100,+ after_lapse: datetime | None = None,+ after_id: str | None = None,+) -> list[DreamPassRecord]:+ """Open passes, of every user, whose lease lapsed before+ *expired_before*, in lapse order (the lapse, then the id), at most+ *limit*: the first ones, or, given *after_lapse* and *after_id*, those+ after that point (keyset pagination, so the reaper can go on past rows it+ tried). A pass with no lease is not listed."""+ where: DreamPassWhereInput = {+ "status": {"in": list(OPEN_STATUSES)},+ "leaseExpiresAt": {"lt": expired_before},+ }+ if after_lapse is not None and after_id is not None:+ where["OR"] = [+ {"leaseExpiresAt": {"gt": after_lapse}},+ {"leaseExpiresAt": {"equals": after_lapse}, "id": {"gt": after_id}},+ ]+ rows = await prisma.models.DreamPass.prisma().find_many(+ where=where,+ order=[{"leaseExpiresAt": "asc"}, {"id": "asc"}],+ take=limit,+ )+ return [DreamPassRecord.from_db(row) for row in rows]+++async def list_dream_pass_cleanups(+ due_before: datetime,+ limit: int = 100,+ marked_through: datetime | None = None,+ after_mark: datetime | None = None,+ after_id: str | None = None,+) -> list[DreamPassRecord]:+ """Closed passes, of every user, marked for a cleanup that has not+ finished and is due: marked before *due_before*, or holding no lease or+ one that lapsed before it. In mark order (the mark, then the id), at+ most *limit*: given *marked_through*, only those marked by then, and,+ given *after_mark* and *after_id*, only those after that point (keyset+ pagination, so the reaper can go on past cleanups it tried)."""+ where: list[DreamPassWhereInput] = [+ {+ "OR": [+ {"cleanupPendingAt": {"lte": due_before}},+ {"leaseExpiresAt": None},+ {"leaseExpiresAt": {"lte": due_before}},+ ]+ }+ ]+ if marked_through is not None:+ where.append({"cleanupPendingAt": {"lte": marked_through}})+ if after_mark is not None and after_id is not None:+ where.append(+ {+ "OR": [+ {"cleanupPendingAt": {"gt": after_mark}},+ {+ "cleanupPendingAt": {"equals": after_mark},+ "id": {"gt": after_id},+ },+ ]+ }+ )rows = await prisma.models.DreamPass.prisma().find_many(- where={"userId": user_id}, order={"createdAt": "desc"}, take=limit+ where={+ "status": {"in": list(CLOSED_STATUSES)},+ "cleanupPendingAt": {"not": None},+ "AND": where,+ },+ order=[{"cleanupPendingAt": "asc"}, {"id": "asc"}],+ take=limit,)return [DreamPassRecord.from_db(row) for row in rows]+++async def delete_old_dream_passes(created_before: datetime, limit: int = 1000) -> int:+ """Delete at most *limit* closed passes, of every user, created before+ *created_before*, and say how many went. An open pass is never deleted,+ however old, nor one whose cleanup the reaper has yet to finish."""+ return await execute_raw_with_schema(+ RETENTION_SQL,+ [status.value for status in OPEN_STATUSES],+ created_before,+ limit,+ )diff --git a/autogpt_platform/backend/backend/data/dream_pass_attribution_test.py b/autogpt_platform/backend/backend/data/dream_pass_attribution_test.pynew file mode 100644index 0000000000..1f4cc43318--- /dev/null+++ b/autogpt_platform/backend/backend/data/dream_pass_attribution_test.py@@ -0,0 +1,401 @@+"""Every charge of a batch dream pass goes by the usage attribution (#15208)+the pass started under, kept on its ``DreamPass`` row, against a real+database and Redis: the trial's spend stays the trial's after the account+converts, whichever path settles it (the reaper, the end of a pass whose+result could not be attributed, the drop of a closed pass's batch), each+phase once; and a row that keeps no attribution is never charged, under the+account's current one or any other.++A trial user is real (its trial row, then on conversion its paid usage+activation), and so are the pass's row, its Redis state and claims, the+charge's ledgers (the trial's cost, the usage counters, the cost log). The+model, the provider's batch API and apply are stubbed at their edges."""++import asyncio+import json+import uuid+from collections.abc import AsyncIterator, Iterator+from contextlib import ExitStack+from datetime import UTC, datetime, timedelta+from unittest.mock import AsyncMock, patch++import pytest+from prisma import Json+from prisma.enums import SubscriptionTier+from prisma.models import PaidUsageActivation, PlatformCostLog, SubscriptionTrial, User+from pydantic import BaseModel++from backend.copilot import token_tracking+from backend.copilot.config import ChatConfig+from backend.copilot.dream import apply as apply_mod+from backend.copilot.dream import (+ batch_callbacks,+ batch_deliveries,+ batch_submit,+ cancel,+ cleanup,+ guard,+)+from backend.copilot.dream import orchestrator as orchestrator_mod+from backend.copilot.dream import reaper+from backend.copilot.dream.fetch import DreamInput, EpisodeRow, FactRow+from backend.copilot.graphiti.scope import MemoryScope+from backend.copilot.usage_activation import UsageActivationUnavailable, usage_keys+from backend.data.db import execute_raw_with_schema+from backend.data.dream_pass import get_dream_pass+from backend.data.redis_client import get_redis_async+from backend.executor.batch_executor import PendingEntry+from backend.util.json import SafeJson+from backend.util.llm.providers import BatchResultRow, BatchSubmissionRef++pytestmark = pytest.mark.asyncio(loop_scope="session")++_CONFIG = ChatConfig(+ fast_standard_model="anthropic/claude-sonnet-5",+ fast_advanced_model="anthropic/claude-opus-5",+ direct_anthropic_api_key="test-key",+)+_OUTPUTS = {+ "consolidate": {"facts": [{"content": "A likes B", "confidence": 0.8}]},+ "recombine": {"proposals": []},+ "sanitize": {"summary_for_user": "quiet night"},+}+++class Owner(BaseModel):+ user_id: str+ trial_id: str+++class Ledger(BaseModel):+ """What the account was charged: the cost rows (each one's trial), the+ trial's own cost, and the weekly usage counters, untrialed and paid."""++ row_trials: list[str | None]+ trial_cost: int+ weekly: int+ paid_weekly: int+++async def owner() -> AsyncIterator[Owner]:+ """A user in a card-verified trial; deleting it afterwards cascades to+ its trial, its activation, its passes and its cost rows."""+ user_id = str(uuid.uuid4())+ now = datetime.now(UTC)+ await User.prisma().create(+ data={+ "id": user_id,+ "email": f"dream-pass-attribution-{user_id}@example.com",+ "topUpConfig": SafeJson({}),+ "timezone": "UTC",+ "subscriptionTier": SubscriptionTier.TRIAL,+ }+ )+ trial = await SubscriptionTrial.prisma().create(+ data={+ "userId": user_id,+ "offer": Json({}),+ "stripeCustomerId": f"cus_{user_id}",+ "checkoutSuccessUrl": "https://example.invalid/ok",+ "checkoutCancelUrl": "https://example.invalid/no",+ "status": "trialing",+ "cardVerifiedAt": now,+ "consumedAt": now,+ "startedAt": now,+ "endsAt": now + timedelta(days=7),+ }+ )+ yield Owner(user_id=user_id, trial_id=trial.id)+ await _flush_cost_rows()+ await User.prisma().delete(where={"id": user_id})+++def batch_route() -> Iterator[list[PendingEntry]]:+ """A pass that takes the batch route, everything but the model and the+ provider's batch API real: the batches it submits, queued in the order+ they were submitted."""+ queued: list[PendingEntry] = []++ async def submit(**kwargs) -> BatchSubmissionRef:+ return BatchSubmissionRef(+ provider="anthropic",+ provider_batch_id=f"batch-{uuid.uuid4()}",+ custom_id=kwargs["custom_id"],+ submitted_at=datetime.now(UTC),+ )++ async def enqueue(entry: PendingEntry) -> None:+ queued.append(entry)++ applied = AsyncMock(+ return_value={"session_id": "s", "ingestion_drain_status": "drained"}+ )+ with ExitStack() as stack:+ for target, name, value in (+ (+ orchestrator_mod,+ "check_dream_budget",+ AsyncMock(return_value=(True, None)),+ ),+ (orchestrator_mod, "_route_for", AsyncMock(return_value="anthropic_batch")),+ (orchestrator_mod, "is_feature_enabled", AsyncMock(return_value=True)),+ (orchestrator_mod, "apply_operations", applied),+ (apply_mod, "apply_operations", applied),+ (guard, "evaluate_feature_flag", AsyncMock(return_value=(True, True))),+ (batch_submit, "call_provider", submit),+ (batch_submit, "enqueue_pending", enqueue),+ (cleanup, "provider_batch_stopped", AsyncMock(return_value=True)),+ (cancel, "cancel_provider_batch", AsyncMock(return_value=True)),+ ):+ stack.enter_context(patch.object(target, name, value))+ stack.enter_context(+ patch.object(batch_callbacks, "anthropic_api_key", return_value="test-key")+ )+ yield queued+++async def test_the_reaper_charges_a_landed_phase_to_the_trial_it_ran_in(+ owner, batch_route+):+ """R1: consolidate lands, the account converts, and the pass dies; the+ reaper's cleanup charges consolidate to the trial, not the paid+ generation the account now reads."""+ pass_id = await _start(owner, batch_route)+ await _deliver(batch_route[0])+ await _convert(owner)++ await _reap(owner, pass_id)++ _assert_charged_to_the_trial(await _ledger(owner), owner, rows=1)++[email protected]("row_attribution", [True, False], ids=["kept", "none"])+async def test_a_cancelled_pass_is_charged_once_by_its_row_after_an_unattributed_result(+ owner, batch_route, row_attribution+):+ """R2: consolidate lands and the account converts; the pass is+ cancelled, its next result comes back without the batch's copy of its+ attribution, and the delete of its state fails, so the reaper cleans up+ after it. Consolidate is charged once, to the trial, by the row; never+ when the row keeps no attribution."""+ pass_id = await _start(owner, batch_route)+ await _deliver(batch_route[0])+ if not row_attribution:+ await _forget_the_attribution(pass_id)+ await _convert(owner)+ cancelled = await cancel.cancel_dream_pass(+ pass_id, user_id=owner.user_id, reason="testing"+ )+ assert cancelled.cancelled++ await _deliver_unattributed(batch_route[1])+ await _reap(owner, pass_id)++ ledger = await _ledger(owner)+ if row_attribution:+ _assert_charged_to_the_trial(ledger, owner, rows=1)+ else:+ assert ledger == _CHARGED_NOTHING++[email protected]("row_attribution", [True, False], ids=["kept", "none"])+async def test_a_dropped_batch_of_an_unattributed_pass_is_charged_once_by_its_row(+ owner, batch_route, row_attribution+):+ """R2: consolidate lands, the account converts, and the next result+ comes back without the batch's copy of the attribution: the pass ends+ ERRORED, the delete of its state failing. The executor then drops the+ closed pass's batch. Consolidate is charged once, to the trial, by the+ row; never when the row keeps no attribution."""+ pass_id = await _start(owner, batch_route)+ await _deliver(batch_route[0])+ if not row_attribution:+ await _forget_the_attribution(pass_id)+ await _convert(owner)++ await _deliver_unattributed(batch_route[1])+ await batch_deliveries.drop_closed(batch_route[1])++ row = await get_dream_pass(pass_id)+ assert row is not None and row.status.value == "ERRORED"+ ledger = await _ledger(owner)+ if row_attribution:+ _assert_charged_to_the_trial(ledger, owner, rows=1)+ else:+ assert ledger == _CHARGED_NOTHING+++async def test_a_pass_whose_row_keeps_no_attribution_is_never_charged(+ owner, batch_route+):+ """A pass in flight when rows began keeping the attribution: consolidate+ landed before, and its row keeps none. Its next result ends it unapplied,+ and its landed phase is charged nothing, then or by the reaper after the+ account converts."""+ pass_id = await _start(owner, batch_route)+ await _deliver(batch_route[0])+ await _forget_the_attribution(pass_id)++ with pytest.raises(UsageActivationUnavailable):+ await _deliver(batch_route[1])+ await _convert(owner)+ await _reap(owner, pass_id)++ row = await get_dream_pass(pass_id)+ assert row is not None and row.status.value == "ERRORED"+ assert await _ledger(owner) == _CHARGED_NOTHING+++async def _start(owner: Owner, queued: list[PendingEntry]) -> str:+ """Run a pass to its handoff: its row and its first batch, consolidate's."""+ with patch.object(+ orchestrator_mod,+ "gather_dream_input",+ AsyncMock(return_value=_input(owner.user_id)),+ ):+ result = await orchestrator_mod._execute_dream_pass_async(+ owner.user_id, expert_id=None, config=_CONFIG+ )+ assert result.error is None and len(queued) == 1, result+ return result.pass_id+++async def _deliver(entry: PendingEntry) -> None:+ """The phase's batch comes back with its output."""+ phase = entry.payload["phase"]+ await batch_callbacks.handle_dream_batch_result(+ entry,+ [+ BatchResultRow(+ custom_id=entry.payload["custom_ids"][0],+ content=json.dumps(_OUTPUTS[phase]),+ input_tokens=1000,+ output_tokens=100,+ )+ ],+ )+++async def _deliver_unattributed(entry: PendingEntry) -> None:+ """The phase's batch comes back without the batch's copy of the pass's+ attribution, and the cleanup after the pass cannot delete its state."""+ entry.payload.pop("cost_context")+ with patch.object(cleanup, "_deleted", AsyncMock(return_value=False)):+ with pytest.raises(UsageActivationUnavailable):+ await _deliver(entry)+++async def _reap(owner: Owner, pass_id: str) -> None:+ """The reaper's cleanup after the pass, as its run does it."""+ row = await get_dream_pass(pass_id)+ assert row is not None+ await reaper._clean_up(row, MemoryScope.for_user(owner.user_id), release=True)+++async def _forget_the_attribution(pass_id: str) -> None:+ """The row as one written before rows kept the attribution."""+ await execute_raw_with_schema(+ 'UPDATE {schema_prefix}"DreamPass" SET "costContext" = NULL WHERE "id" = $1',+ pass_id,+ )+++async def _convert(owner: Owner) -> None:+ """The account converts: paid, its trial converted, a ready paid usage+ generation."""+ now = datetime.now(UTC)+ await User.prisma().update(+ where={"id": owner.user_id}, data={"subscriptionTier": SubscriptionTier.PRO}+ )+ await SubscriptionTrial.prisma().update(+ where={"id": owner.trial_id}, data={"convertedAt": now, "status": "active"}+ )+ await PaidUsageActivation.prisma().create(+ data={+ "userId": owner.user_id,+ "stripeSubscriptionId": f"sub_{owner.user_id}",+ "stripeInvoiceId": f"inv_{owner.user_id}",+ "readyAt": now,+ }+ )+++async def _ledger(owner: Owner) -> Ledger:+ await _flush_cost_rows()+ rows = await PlatformCostLog.prisma().find_many(where={"userId": owner.user_id})+ trial = await SubscriptionTrial.prisma().find_unique(where={"id": owner.trial_id})+ activation = await PaidUsageActivation.prisma().find_unique(+ where={"userId": owner.user_id}+ )+ redis = await get_redis_async()+ now = datetime.now(UTC)+ weekly = await redis.get(usage_keys(owner.user_id, None, now)[1])+ paid_weekly = (+ await redis.get(usage_keys(owner.user_id, activation.id, now)[1])+ if activation is not None+ else None+ )+ return Ledger(+ row_trials=[+ dict(row.metadata or {}).get("subscription_trial_id") for row in rows+ ],+ trial_cost=int(trial.costMicrodollars) if trial is not None else 0,+ weekly=int(weekly or 0),+ paid_weekly=int(paid_weekly or 0),+ )+++def _assert_charged_to_the_trial(ledger: Ledger, owner: Owner, *, rows: int) -> None:+ """*rows* cost rows, each the trial's; the trial and its counters charged,+ the paid generation the account converted to untouched."""+ assert ledger.row_trials == [owner.trial_id] * rows, ledger+ assert ledger.trial_cost > 0 and ledger.weekly > 0, ledger+ assert ledger.paid_weekly == 0, ledger+++_CHARGED_NOTHING = Ledger(row_trials=[], trial_cost=0, weekly=0, paid_weekly=0)+++async def _flush_cost_rows() -> None:+ """The cost rows the charges scheduled, written."""+ tasks = list(token_tracking._pending_log_tasks)+ if tasks:+ await asyncio.gather(*tasks)+++def _input(user_id: str) -> DreamInput:+ now = datetime.now(UTC)+ return DreamInput(+ user_id=user_id,+ group_id=MemoryScope.for_user(user_id).group_id,+ window_start=now - timedelta(days=1),+ window_end=now,+ episodes=[+ EpisodeRow(+ uuid="e1",+ name=None,+ content="hello",+ source_description=None,+ valid_at=None,+ created_at=None,+ )+ ],+ facts=[+ FactRow(+ uuid="f1",+ source="A",+ target="B",+ name="likes",+ fact="A likes B",+ scope="real:global",+ confidence=0.7,+ status="active",+ created_at=None,+ )+ ],+ recent_sessions=[],+ known_fact_uuids={"f1"},+ known_episode_uuids={"e1"},+ )diff --git a/autogpt_platform/backend/backend/data/dream_pass_lease_test.py b/autogpt_platform/backend/backend/data/dream_pass_lease_test.pynew file mode 100644index 0000000000..07eae462a6--- /dev/null+++ b/autogpt_platform/backend/backend/data/dream_pass_lease_test.py@@ -0,0 +1,447 @@+"""The DreamPass lease, the reaper's scans and retention against a real+database: a new row carries its lease, a renewal moves it, the sync route's+own end empties it and the input bundle (the statement's ``clear``), a close+that leaves a cleanup behind (a cancel, the reaper's expiry) marks the row+and keeps its lease until the closed-row write of a finished cleanup clears+both, the reaper lists open rows whose lease lapsed (in lapse order, then by+id) and closed rows whose cleanup is due (in mark order, then by id), each+from a point it gives, so its scan goes on past rows it tried, on their+indexes, and retention deletes closed rows past their retention, a batch at+a time, none whose cleanup is pending."""++import uuid+from collections.abc import AsyncIterator+from datetime import datetime, timedelta, timezone++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)+from prisma.models import DreamPass as PrismaDreamPass+from prisma.models import User++from backend.copilot.dream.fetch import DreamInput+from backend.copilot.dream.pass_record import (+ cancelled,+ cleanup_finished,+ expired,+ lease,+ outcome,+ reaped,+)+from backend.copilot.dream.schemas import ConsolidationOutput, DreamPassResult+from backend.data.db import query_raw_with_schema+from backend.util.json import SafeJson++from .dream_pass import (+ create_dream_pass,+ delete_old_dream_passes,+ get_dream_pass,+ list_dream_pass_cleanups,+ list_dream_passes,+ list_expired_dream_passes,+ update_dream_pass,+)+from .dream_pass_models import (+ DreamPassDraft,+ DreamPassRecord,+ DreamPassUpdate,+ DreamPhaseOutputs,+)++pytestmark = pytest.mark.asyncio(loop_scope="session")++# Postgres keeps TIMESTAMP(3): whole milliseconds read back exactly.+_NOW = datetime.now(timezone.utc).replace(microsecond=0)+# Lapses far older than any other test's rows, so the global scans that+# list the oldest first reach these.+_LONG_AGO = datetime(2001, 1, 1, tzinfo=timezone.utc)+++async def owner() -> AsyncIterator[str]:+ """A throwaway user; deleting it afterwards cascades to its passes."""+ user_id = str(uuid.uuid4())+ await User.prisma().create(+ data={+ "id": user_id,+ "email": f"dream-pass-lease-{user_id}@example.com",+ "topUpConfig": SafeJson({}),+ "timezone": "UTC",+ }+ )+ yield user_id+ await User.prisma().delete(where={"id": user_id})+++async def test_a_new_row_carries_its_lease_and_a_renewal_moves_it(owner):+ pass_id = await _new_pass(owner, lease_expires_at=_NOW)++ row = await _row(pass_id)+ assert (row.lease_token, row.lease_expires_at) == ("tok", _NOW)++ assert await update_dream_pass(pass_id, lease("tok", 1800))+ renewed = await _row(pass_id)+ assert renewed.lease_token == "tok"+ assert renewed.lease_expires_at is not None+ assert renewed.lease_expires_at > _NOW + timedelta(seconds=1700)+++async def test_a_renewal_written_late_never_shortens_the_lease(owner):+ """A sync renewal the store gave up on lands after the batch submit's+ day-long lease: the row keeps the later expiry. Only ``clear`` drops+ it."""+ pass_id = await _new_pass(owner, lease_expires_at=_NOW)+ day_long = _NOW + timedelta(hours=24)+ half_hour = _NOW + timedelta(minutes=30)++ assert await update_dream_pass(+ pass_id, DreamPassUpdate(lease_token="tok", lease_expires_at=day_long)+ )+ assert await update_dream_pass(+ pass_id, DreamPassUpdate(lease_token="tok", lease_expires_at=half_hour)+ )+ assert (await _row(pass_id)).lease_expires_at == day_long++ assert await update_dream_pass(+ pass_id,+ DreamPassUpdate(error="moving on", clear=frozenset({"lease_expires_at"})),+ )+ assert (await _row(pass_id)).lease_expires_at is None+++async def test_a_closing_transition_empties_the_lease_and_the_bundle(owner):+ """Written by the statement's ``clear``: the sync route's own end drops+ the lease and the bundle, the outputs stay; an update that clears+ nothing leaves them."""+ pass_id = await _new_pass_with_a_bundle(owner)++ assert await update_dream_pass(+ pass_id, outcome(DreamPassResult(user_id=owner, pass_id=pass_id), None)+ )++ row = await _row(pass_id)+ assert row.status is DreamPassStatus.COMPLETE+ assert (row.lease_token, row.lease_expires_at, row.input_bundle) == (+ None,+ None,+ None,+ )+ assert row.cleanup_pending_at is None+ assert row.phase_outputs.consolidate == ConsolidationOutput()+++async def test_a_close_that_leaves_a_cleanup_marks_the_row_and_keeps_its_lease(+ owner,+):+ """A cancel marks the row for the cleanup after the pass, drops the+ bundle and keeps the lease that cleanup needs, until the write that says+ it finished empties the mark and the lease."""+ pass_id = await _new_pass_with_a_bundle(owner)++ assert await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))++ row = await _row(pass_id)+ assert row.status is DreamPassStatus.CANCELLED+ assert (row.lease_token, row.lease_expires_at) == ("tok", _NOW)+ assert (row.input_bundle, row.cleanup_pending_at is not None) == (None, True)+ assert pass_id in [r.id for r in await list_dream_pass_cleanups(_NOW)]++ assert await update_dream_pass(pass_id, cleanup_finished())++ done = await _row(pass_id)+ assert (done.cleanup_pending_at, done.lease_token, done.lease_expires_at) == (+ None,+ None,+ None,+ )+ assert pass_id not in [r.id for r in await list_dream_pass_cleanups(_NOW)]+++async def test_an_update_that_clears_nothing_leaves_the_lease(owner):+ pass_id = await _new_pass(owner, lease_expires_at=_NOW)++ await update_dream_pass(pass_id, DreamPassUpdate(phase=DreamPassPhase.CONSOLIDATE))++ row = await _row(pass_id)+ assert (row.lease_token, row.lease_expires_at) == ("tok", _NOW)+++async def test_the_reaper_lists_open_passes_whose_lease_lapsed_oldest_first(owner):+ oldest = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ older = await _new_pass(owner, lease_expires_at=_LONG_AGO + timedelta(days=1))+ await _new_pass(owner, lease_expires_at=_NOW + timedelta(hours=1))+ await _new_pass(owner, lease_expires_at=None)+ closed = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ await update_dream_pass(closed, expired("stale", not_updated_since=None))+ # A closed row marked for its cleanup still carries its lapsed lease.+ assert (await _row(closed)).lease_expires_at == _LONG_AGO++ rows = await list_expired_dream_passes(_NOW, limit=1000)+ mine = [row.id for row in rows if row.user_id == owner]+ assert mine == [oldest, older]++ first = await list_expired_dream_passes(_NOW, limit=1)+ assert [row.id for row in first] == [oldest]+++async def test_the_reapers_scan_of_open_passes_goes_on_from_where_it_stopped(+ owner,+):+ """Keyset pages in lapse order, then by id: rows that share a lapse are+ neither skipped nor listed twice across the pages."""+ lapse = _LONG_AGO - timedelta(days=1)+ tied = sorted([await _new_pass(owner, lease_expires_at=lapse) for _ in range(3)])+ later = await _new_pass(owner, lease_expires_at=lapse + timedelta(seconds=1))++ first = await list_expired_dream_passes(_NOW, limit=2)+ assert [row.id for row in first] == tied[:2]++ after = (first[-1].lease_expires_at, first[-1].id)+ assert after[0] == lapse+ second = await list_expired_dream_passes(+ _NOW, limit=2, after_lapse=after[0], after_id=after[1]+ )+ assert [row.id for row in second] == [tied[2], later]+++async def test_the_reaper_scans_an_index_on_status_and_lease_expiry():+ indexes = await query_raw_with_schema(+ "SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'DreamPass'"+ )++ by_name = {row["indexname"]: row["indexdef"] for row in indexes}+ assert (+ '(status, "leaseExpiresAt")' in by_name["DreamPass_status_leaseExpiresAt_idx"]+ )+++async def test_the_reapers_expiry_marks_the_row_until_its_cleanup_finishes(owner):+ pass_id = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ row = await _row(pass_id)+ assert not await update_dream_pass(pass_id, cleanup_finished())++ assert await update_dream_pass(+ pass_id, reaped("lapsed", not_updated_since=row.updated_at)+ )++ closed = await _row(pass_id)+ assert (closed.status, closed.error) == (DreamPassStatus.EXPIRED, "lapsed")+ assert (closed.lease_token, closed.lease_expires_at) == ("tok", None)+ assert closed.cleanup_pending_at is not None+ assert closed.cancel_generation == 1+ listed = await list_dream_pass_cleanups(_NOW, limit=1000)+ assert pass_id in [r.id for r in listed]++ assert await update_dream_pass(pass_id, cleanup_finished())++ done = await _row(pass_id)+ assert (done.cleanup_pending_at, done.lease_token) == (None, None)+ assert (done.status, done.error, done.cancel_generation) == (+ DreamPassStatus.EXPIRED,+ "lapsed",+ 1,+ )+ cleanups = await list_dream_pass_cleanups(_NOW, limit=1000)+ assert pass_id not in [r.id for r in cleanups]+++async def test_the_cleanup_scan_lists_closed_marked_rows_longest_pending_first(+ owner,+):+ marked = []+ for minutes in (5, 50):+ pass_id = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ row = await _row(pass_id)+ assert await update_dream_pass(+ pass_id, reaped("lapsed", not_updated_since=row.updated_at)+ )+ await PrismaDreamPass.prisma().update(+ where={"id": pass_id},+ data={"cleanupPendingAt": _LONG_AGO + timedelta(minutes=minutes)},+ )+ marked.append(pass_id)+ unmarked = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ await update_dream_pass(+ unmarked, outcome(DreamPassResult(user_id=owner, pass_id=unmarked), None)+ )+ await _new_pass(owner, lease_expires_at=_LONG_AGO)++ rows = await list_dream_pass_cleanups(_NOW, limit=1000)++ assert [r.id for r in rows if r.user_id == owner] == marked+ first = await list_dream_pass_cleanups(_NOW, limit=1)+ assert [r.id for r in first] == marked[:1]+++async def test_the_reapers_scan_of_cleanups_goes_on_from_where_it_stopped(+ owner,+):+ """Keyset pages in mark order, then by id, up to a mark: rows that share+ a mark are neither skipped nor listed twice, and one marked after the+ bound waits."""+ mark = _LONG_AGO - timedelta(days=1)+ tied = []+ for _ in range(3):+ pass_id = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ await PrismaDreamPass.prisma().update(+ where={"id": pass_id}, data={"cleanupPendingAt": mark}+ )+ tied.append(pass_id)+ tied.sort()+ later = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ await update_dream_pass(later, cancelled("testing", owner_user_id=owner))+ await PrismaDreamPass.prisma().update(+ where={"id": later}, data={"cleanupPendingAt": mark + timedelta(seconds=1)}+ )++ first = await list_dream_pass_cleanups(_NOW, limit=2, marked_through=mark)+ assert [row.id for row in first] == tied[:2]+ rest = await list_dream_pass_cleanups(+ _NOW,+ limit=10,+ marked_through=mark,+ after_mark=first[-1].cleanup_pending_at,+ after_id=first[-1].id,+ )+ assert [row.id for row in rest] == tied[2:]+ unbounded = await list_dream_pass_cleanups(+ _NOW, limit=2, after_mark=mark, after_id=tied[1]+ )+ assert [row.id for row in unbounded] == [tied[2], later]+++async def test_the_cleanup_scan_lists_a_row_once_its_cleanup_is_due(owner):+ """Due once the row was marked, or its lease lapsed, before the cutoff,+ or it holds no lease: a pass stopped while it ran has had its grace."""+ due_before = _NOW - timedelta(minutes=30)+ shapes = {+ "marked long ago": (_LONG_AGO + timedelta(hours=1), _NOW + timedelta(days=1)),+ "lease lapsed": (_NOW, _LONG_AGO),+ "no lease": (_NOW, None),+ "marked just now, lease fresh": (_NOW, _NOW + timedelta(days=1)),+ }+ ids = {}+ for name, (marked, lease_expires_at) in shapes.items():+ pass_id = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ await PrismaDreamPass.prisma().update(+ where={"id": pass_id},+ data={"cleanupPendingAt": marked, "leaseExpiresAt": lease_expires_at},+ )+ ids[pass_id] = name++ rows = await list_dream_pass_cleanups(due_before, limit=1000)++ listed = [ids[r.id] for r in rows if r.id in ids]+ assert listed[0] == "marked long ago"+ assert set(listed) == {"marked long ago", "lease lapsed", "no lease"}+++async def test_the_cleanup_scan_has_its_index():+ indexes = await query_raw_with_schema(+ "SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'DreamPass'"+ )++ by_name = {row["indexname"]: row["indexdef"] for row in indexes}+ assert (+ '(status, "cleanupPendingAt")'+ in by_name["DreamPass_status_cleanupPendingAt_idx"]+ )+++async def test_retention_keeps_a_closed_row_whose_cleanup_is_pending(owner):+ pending = await _new_pass(owner, lease_expires_at=_LONG_AGO)+ row = await _row(pending)+ assert await update_dream_pass(+ pending, reaped("lapsed", not_updated_since=row.updated_at)+ )+ await PrismaDreamPass.prisma().update(+ where={"id": pending}, data={"createdAt": _LONG_AGO}+ )++ assert await delete_old_dream_passes(_LONG_AGO + timedelta(days=1), limit=10) == 0+ assert await update_dream_pass(pending, cleanup_finished())+ assert await delete_old_dream_passes(_LONG_AGO + timedelta(days=1), limit=10) == 1+++async def test_retention_deletes_closed_passes_past_their_time_a_batch_at_a_time(+ owner,+):+ old_closed = [await _new_pass(owner) for _ in range(3)]+ old_open = await _new_pass(owner)+ recent_closed = await _new_pass(owner)+ for pass_id in [*old_closed, recent_closed]:+ await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ await update_dream_pass(pass_id, cleanup_finished())+ for pass_id in [*old_closed, old_open]:+ await PrismaDreamPass.prisma().update(+ where={"id": pass_id}, data={"createdAt": _LONG_AGO}+ )+ cutoff = _LONG_AGO + timedelta(days=1)++ assert await delete_old_dream_passes(cutoff, limit=2) == 2+ assert await delete_old_dream_passes(cutoff, limit=2) == 1+ assert await delete_old_dream_passes(cutoff, limit=2) == 0++ left = {row.id for row in await list_dream_passes(owner, limit=10)}+ assert left == {old_open, recent_closed}+++async def test_a_users_passes_list_the_open_ones_only_when_asked(owner):+ open_pass = await _new_pass(owner)+ closed = await _new_pass(owner)+ await update_dream_pass(closed, cancelled("testing", owner_user_id=owner))++ everything = await list_dream_passes(owner, limit=10)+ only_open = await list_dream_passes(owner, limit=10, open_only=True)++ assert {row.id for row in everything} == {closed, open_pass}+ assert [row.id for row in only_open] == [open_pass]+++async def _new_pass_with_a_bundle(owner: str) -> str:+ """A leased pass with its input bundle and consolidate's output."""+ pass_id = await _new_pass(owner, lease_expires_at=_NOW)+ await update_dream_pass(+ pass_id,+ DreamPassUpdate(+ input_bundle=DreamInput(+ user_id=owner,+ group_id=f"user_{owner}",+ window_start=_NOW,+ window_end=_NOW,+ ),+ phase=DreamPassPhase.RECOMBINE,+ phase_outputs=DreamPhaseOutputs(consolidate=ConsolidationOutput()),+ ),+ )+ assert (await _row(pass_id)).input_bundle is not None+ return pass_id+++async def _new_pass(owner: str, *, lease_expires_at: datetime | None = None) -> str:+ pass_id = str(uuid.uuid4())+ await create_dream_pass(+ DreamPassDraft(+ id=pass_id,+ user_id=owner,+ scope_key=owner,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ trigger=DreamPassTrigger.CRON,+ started_at=_NOW,+ lease_token="tok" if lease_expires_at else None,+ lease_expires_at=lease_expires_at,+ )+ )+ return pass_id+++async def _row(pass_id: str) -> DreamPassRecord:+ row = await get_dream_pass(pass_id)+ assert row is not None+ return rowdiff --git a/autogpt_platform/backend/backend/data/dream_pass_models.py b/autogpt_platform/backend/backend/data/dream_pass_models.pyindex 28310d6c5c..1d2f1ec5d8 100644--- a/autogpt_platform/backend/backend/data/dream_pass_models.py+++ b/autogpt_platform/backend/backend/data/dream_pass_models.py@@ -8,6 +8,7 @@ How an update lands on a row, only ever moving it forward, is"""from datetime import datetime+from typing import Any, Literalimport prisma.modelsfrom prisma.enums import (@@ -30,18 +31,38 @@ from backend.copilot.dream.schemas import ()# The statuses a row can still move on from. Any other status is terminal: the-# row is final and no update writes it.+# row is final, and only the cleanup after its pass finishing writes it again+# (``DreamPassUpdate.closed_row``).OPEN_STATUSES: tuple[DreamPassStatus, ...] = (DreamPassStatus.QUEUED,DreamPassStatus.RUNNING,DreamPassStatus.SUBMITTED,DreamPassStatus.APPLYING,)+CLOSED_STATUSES: tuple[DreamPassStatus, ...] = tuple(+ status for status in DreamPassStatus if status not in OPEN_STATUSES+)# A new row's cancelGeneration (the column default). Only a stop from outside# the pass moves it, so a pass that reads anything else was stopped.INITIAL_CANCEL_GENERATION = 0+# The nullable columns an update may empty, by their field names.+ClearableColumn = Literal[+ "lease_token", "lease_expires_at", "input_bundle", "cleanup_pending_at"+]++# What a row drops as it closes: a finished pass holds no lease, and nothing+# resumes it from its input bundle, the bulk of an open batch row. Its phase+# outputs, operations and usage stay.+CLOSED_ROW_CLEARS: frozenset[ClearableColumn] = frozenset(+ {"lease_token", "lease_expires_at", "input_bundle"}+)+# What a row marked for a cleanup drops as it closes: the bundle only. It+# keeps its lease until the cleanup has finished: the token to release the+# pass's lock with, and the expiry that says whether the pass may still run.+MARKED_ROW_CLEARS: frozenset[ClearableColumn] = frozenset({"input_bundle"})+class DreamPhaseOutputs(BaseModel):"""Each phase's validated output, as the next phase and apply read it."""@@ -76,7 +97,9 @@ class DreamPassOperations(BaseModel):class DreamPassDraft(BaseModel):- """A new row: whose pass it is, how it runs, what started it."""+ """A new row: whose pass it is, how it runs, what started it, the lease+ it starts with (the token of the lock it is about to take), and the usage+ attribution it runs under."""id: str = Field(min_length=1)user_id: str = Field(min_length=1)@@ -87,6 +110,12 @@ class DreamPassDraft(BaseModel):status: DreamPassStatus = DreamPassStatus.RUNNINGphase: DreamPassPhase = DreamPassPhase.GATHERstarted_at: datetime | None = None+ lease_token: str | None = None+ lease_expires_at: datetime | None = None+ # The pass's ``TrialCostContext`` as JSON (#15208), captured when it+ # started: every charge of its phases goes by it, and a row without one+ # is never charged (``copilot/dream/batch_costs.py``).+ cost_context: dict[str, Any] | None = Noneclass DreamPassUpdate(BaseModel):@@ -102,6 +131,16 @@ class DreamPassUpdate(BaseModel):``bump_cancel_generation``, and may make the write conditional on morethan the row being open: ``owner_user_id`` (the row is that user's) and``not_updated_since`` (nothing has written the row after that instant).++ ``clear`` names the nullable columns the update empties, which a ``None``+ field cannot say; every transition that closes a row clears+ ``CLOSED_ROW_CLEARS``, or ``MARKED_ROW_CLEARS`` when it marks the row for+ a cleanup (``cleanup_pending_at``). A column is given a value or cleared,+ not both.++ ``closed_row`` is the one kind of write a closed row takes: the cleanup+ after its pass saying it has finished. It applies to a closed row only,+ and may only clear columns."""status: DreamPassStatus | None = None@@ -123,6 +162,9 @@ class DreamPassUpdate(BaseModel):bump_cancel_generation: bool = Falseowner_user_id: str | None = Nonenot_updated_since: datetime | None = None+ cleanup_pending_at: datetime | None = None+ clear: frozenset[ClearableColumn] = frozenset()+ closed_row: bool = False@model_validator(mode="after")def _batch_names_its_phase(self) -> "DreamPassUpdate":@@ -130,6 +172,32 @@ class DreamPassUpdate(BaseModel):raise ValueError("a provider batch id needs the phase it runs")return self+ @model_validator(mode="after")+ def _set_or_cleared(self) -> "DreamPassUpdate":+ given: dict[ClearableColumn, bool] = {+ "lease_token": self.lease_token is not None,+ "lease_expires_at": self.lease_expires_at is not None,+ "input_bundle": self.input_bundle is not None,+ "cleanup_pending_at": self.cleanup_pending_at is not None,+ }+ both = sorted(column for column in self.clear if given[column])+ if both:+ raise ValueError(f"set and cleared at once: {', '.join(both)}")+ return self++ @model_validator(mode="after")+ def _closed_row_only_clears(self) -> "DreamPassUpdate":+ if not self.closed_row:+ return self+ written = self.model_dump(+ exclude={"clear", "closed_row"}, exclude_defaults=True+ )+ if written:+ raise ValueError(+ f"a closed row only has columns cleared: {sorted(written)}"+ )+ return self+class DreamPassRecord(BaseModel):"""One ``DreamPass`` row."""@@ -147,6 +215,8 @@ class DreamPassRecord(BaseModel):provider_batch_id: str | Nonelease_token: str | Nonelease_expires_at: datetime | None+ cleanup_pending_at: datetime | None = None+ cost_context: dict[str, Any] | None = Noneinput_bundle: DreamInput | Nonephase_outputs: DreamPhaseOutputsoperations: DreamPassOperations@@ -177,6 +247,10 @@ class DreamPassRecord(BaseModel):provider_batch_id=row.providerBatchId,lease_token=row.leaseToken,lease_expires_at=row.leaseExpiresAt,+ cleanup_pending_at=row.cleanupPendingAt,+ cost_context=(+ dict(row.costContext) if row.costContext is not None else None+ ),input_bundle=(input_bundle_from_dict(dict(row.inputBundle))if row.inputBundle is not Nonediff --git a/autogpt_platform/backend/backend/data/dream_pass_test.py b/autogpt_platform/backend/backend/data/dream_pass_test.pyindex 236aa5e8bf..f90836b706 100644--- a/autogpt_platform/backend/backend/data/dream_pass_test.py+++ b/autogpt_platform/backend/backend/data/dream_pass_test.py@@ -104,6 +104,23 @@ async def test_a_new_row_reads_back_running_and_gathering(make_user):assert fetched.phase_outputs == DreamPhaseOutputs()assert fetched.operations == DreamPassOperations()assert fetched.input_bundle is None and fetched.usage is None+ assert fetched.cost_context is None+++async def test_a_new_row_keeps_the_usage_attribution_its_pass_started_under(+ make_user,+):+ """The attribution every charge of the pass goes by, as written; no+ transition writes it again."""+ owner = await make_user()+ attribution = {"user_id": owner, "trial_id": "trial-1", "generation": None}+ draft = _draft(owner, cost_context=attribution)++ await create_dream_pass(draft)+ await update_dream_pass(draft.id, DreamPassUpdate(error="moved on"))++ fetched = await get_dream_pass(draft.id)+ assert fetched is not None and fetched.cost_context == attributionasync def test_an_update_writes_only_the_columns_it_sets(make_user):diff --git a/autogpt_platform/backend/backend/data/dream_pass_update.py b/autogpt_platform/backend/backend/data/dream_pass_update.pyindex 058f333b01..08772461d8 100644--- a/autogpt_platform/backend/backend/data/dream_pass_update.py+++ b/autogpt_platform/backend/backend/data/dream_pass_update.py@@ -16,12 +16,23 @@ statement against the row as it stands when the statement runs:* ``cancelGeneration`` goes up by one when the update stops the pass fromoutside, in the same statement that closes the row, so of two stopsracing for one open row exactly one lands and bumps it;+ * ``leaseExpiresAt`` only moves later: a renewal written late (a sync+ renewal the store gave up on, landing after the batch submit's longer+ lease) never shortens the lease the row holds. Every renewal is now+ plus its route's TTL, so none needs to move it earlier;+ * ``leaseToken``, ``leaseExpiresAt``, ``inputBundle`` and+ ``cleanupPendingAt`` become NULL when the update names them in ``clear``+ (a closing row drops its bundle, and its lease unless the close marks+ it for a cleanup, whose end then clears the lease with the mark): the+ only way a lease is dropped or cut short;* every other column takes the new value when the update gives one.Nothing is written to a row that has reached a terminal status, nor to onethat fails the update's own conditions: another user's row when it names the-owner, a row written since the instant it names. The statement text is-constant; every value is a bound parameter.+owner, a row written since the instant it names. The one exception is the+update marked ``closed_row`` (the cleanup after the pass saying it has+finished), which applies to a closed row only and only clears. The statement+text is constant; every value is a bound parameter."""from typing import Any@@ -29,9 +40,22 @@ from typing import Anyfrom pydantic import BaseModelfrom backend.copilot.dream.input_bundle import input_bundle_to_dict-from backend.data.dream_pass_models import OPEN_STATUSES, DreamPassUpdate+from backend.data.dream_pass_models import (+ CLOSED_STATUSES,+ OPEN_STATUSES,+ ClearableColumn,+ DreamPassUpdate,+)from backend.util.json import dumps, sanitize_json, sanitize_string+# The column each clearable field names in ``TRANSITION_SQL``.+_CLEARABLE_COLUMNS: dict[ClearableColumn, str] = {+ "lease_token": "leaseToken",+ "lease_expires_at": "leaseExpiresAt",+ "input_bundle": "inputBundle",+ "cleanup_pending_at": "cleanupPendingAt",+}+# ``{schema_prefix}`` is filled in by ``db.execute_raw_with_schema``; the# doubled braces are a literal empty JSON object once it has.TRANSITION_SQL = """@@ -45,15 +69,17 @@ UPDATE {schema_prefix}"DreamPass" SETTHEN $4::text ELSE "providerBatchId" END,"skipReason" = COALESCE($5::text, "skipReason"),"error" = COALESCE($6::text, "error"),- "leaseToken" = COALESCE($7::text, "leaseToken"),- "leaseExpiresAt" = COALESCE(- $8::timestamptz AT TIME ZONE 'UTC', "leaseExpiresAt"),+ "leaseToken" = CASE WHEN 'leaseToken' = ANY($22::text[]) THEN NULL+ ELSE COALESCE($7::text, "leaseToken") END,+ "leaseExpiresAt" = CASE WHEN 'leaseExpiresAt' = ANY($22::text[]) THEN NULL+ ELSE GREATEST($8::timestamptz AT TIME ZONE 'UTC', "leaseExpiresAt") END,"windowStart" = COALESCE($9::timestamptz AT TIME ZONE 'UTC', "windowStart"),"windowEnd" = COALESCE($10::timestamptz AT TIME ZONE 'UTC', "windowEnd"),"submittedAt" = COALESCE($11::timestamptz AT TIME ZONE 'UTC', "submittedAt"),"appliedAt" = COALESCE($12::timestamptz AT TIME ZONE 'UTC', "appliedAt"),"completedAt" = COALESCE($13::timestamptz AT TIME ZONE 'UTC', "completedAt"),- "inputBundle" = COALESCE($14::jsonb, "inputBundle"),+ "inputBundle" = CASE WHEN 'inputBundle' = ANY($22::text[]) THEN NULL+ ELSE COALESCE($14::jsonb, "inputBundle") END,"usage" = COALESCE($15::jsonb, "usage"),"phaseOutputs" = CASE WHEN $16::jsonb IS NULL THEN "phaseOutputs"ELSE COALESCE("phaseOutputs", '{{}}'::jsonb) || $16::jsonb END,@@ -61,6 +87,8 @@ UPDATE {schema_prefix}"DreamPass" SETELSE COALESCE("operations", '{{}}'::jsonb) || $17::jsonb END,"cancelGeneration" = "cancelGeneration"+ CASE WHEN $19::boolean THEN 1 ELSE 0 END,+ "cleanupPendingAt" = CASE WHEN 'cleanupPendingAt' = ANY($22::text[]) THEN NULL+ ELSE COALESCE($23::timestamptz AT TIME ZONE 'UTC', "cleanupPendingAt") END,"updatedAt" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'WHERE "id" = $1 AND "status"::text = ANY($18::text[])AND ($20::text IS NULL OR "userId" = $20::text)@@ -95,10 +123,15 @@ def transition_args(pass_id: str, update: DreamPassUpdate) -> list[Any]:_json(update.usage.model_dump(mode="json")) if update.usage else None,_merge_part(update.phase_outputs),_merge_part(update.operations),- [status.value for status in OPEN_STATUSES],+ [+ status.value+ for status in (CLOSED_STATUSES if update.closed_row else OPEN_STATUSES)+ ],update.bump_cancel_generation,update.owner_user_id,update.not_updated_since,+ sorted(_CLEARABLE_COLUMNS[column] for column in update.clear),+ update.cleanup_pending_at,]diff --git a/autogpt_platform/backend/backend/data/dream_pass_update_test.py b/autogpt_platform/backend/backend/data/dream_pass_update_test.pyindex 607550c5bc..ede084432b 100644--- a/autogpt_platform/backend/backend/data/dream_pass_update_test.py+++ b/autogpt_platform/backend/backend/data/dream_pass_update_test.py@@ -229,7 +229,9 @@ async def test_timestamps_land_as_the_same_instant(owner):row = await _row(pass_id)assert row.lease_expires_at == datetime(2026, 9, 26, 3, 0, tzinfo=timezone.utc)assert row.window_start == _NOW- assert row.updated_at >= before.updated_at+ # The insert stamps updatedAt with the client's clock and the update+ # with the database's, which can trail it by a few milliseconds.+ assert row.updated_at >= before.updated_at - timedelta(seconds=1)async def test_deleting_an_expert_finds_its_passes_by_index():diff --git a/autogpt_platform/backend/backend/executor/batch_executor.py b/autogpt_platform/backend/backend/executor/batch_executor.pyindex 9b46b813ad..be4d14a9bb 100644--- a/autogpt_platform/backend/backend/executor/batch_executor.py+++ b/autogpt_platform/backend/backend/executor/batch_executor.py@@ -15,6 +15,13 @@ arrive asynchronously up to ~24h later. This service owns:``callback_namespace`` (e.g. ``"dream_pass"``). Handlers run theapply step, write cost-log rows, and update whatever JobStatusrow the caller cares about.+ * **Dropping**: a namespace may also register a ``should_dispatch``+ check, asked before each poll of a due entry, and an ``on_drop`` hook.+ The check only answers; ``False`` drops the entry unpolled, claimed+ like a dispatch (tombstone + HDEL) so it never comes back, and only the+ walker whose claim took it off the queue then runs the hook (the dream+ pass cancels a stopped pass's batch and ends the pass). A walker that+ finds the entry already claimed does nothing.Lives at the same architectural level as ``Scheduler`` — own process,own Redis connection, own service-port for health checks. Subscribing@@ -71,6 +78,11 @@ DISPATCHED_TTL_SECONDS = 7 * 24 * 60 * 60# entry failed even if the provider still says ``processing``.MAX_BATCH_LIFETIME_SECONDS = 24 * 60 * 60+# How long a namespace's ``should_dispatch`` check may take before the walk+# goes on and treats the entry as due (its handler keeps its own checks), and+# how long its ``on_drop`` hook may take once a drop has been claimed.+DISPATCH_CHECK_TIMEOUT_SECONDS = 60+# Poll cadence (exponential backoff). The BatchExecutor walks the queue# every ``WALK_INTERVAL_SECONDS`` but each individual entry is only# polled when its ``next_poll_at`` has elapsed.@@ -168,7 +180,8 @@ async def remove_pending(provider_batch_id: str) -> None:atomically tombstones + HDELs in one Lua script so concurrentwalkers (or a crash-replay) can never double-dispatch. The walkloop's only callers are the refused-claim zombie cleanups on the- ended, failed, and lifetime-timeout paths — clearing a pending row+ ended, failed, and lifetime-timeout paths, and a drop whose claim+ another walker already took (``_drop``) — each clearing a pending rowwhose batch already carries a dispatch tombstone."""from backend.data.redis_client import get_redis_async@@ -211,23 +224,46 @@ async def _claim_dispatch(provider_batch_id: str) -> bool:# It is the handler's responsibility to log costs, update any caller-# supplied job status, and decide what to do with errored rows.BatchResultHandler = Callable[[PendingEntry, list[BatchResultRow]], Awaitable[None]]+# A namespace's check before an entry is polled: whether it is still wanted.+# It only answers; whatever a drop needs done is its drop hook's.+DispatchCheck = Callable[[PendingEntry], Awaitable[bool]]+# What a dropped entry needs done, run by the walker whose claim dropped it.+DropHook = Callable[[PendingEntry], Awaitable[None]]_HANDLERS: dict[str, BatchResultHandler] = {}+_DISPATCH_CHECKS: dict[str, DispatchCheck] = {}+_DROP_HOOKS: dict[str, DropHook] = {}-def register_handler(namespace: str, handler: BatchResultHandler) -> None:- """Register a result handler for one namespace.+def register_handler(+ namespace: str,+ handler: BatchResultHandler,+ *,+ should_dispatch: DispatchCheck | None = None,+ on_drop: DropHook | None = None,+) -> None:+ """Register a result handler for one namespace, and optionally the check+ asked before each poll of one of its entries (``False`` drops it) and+ the hook the walker that drops one runs.Called at module-import time from the caller's ``__init__.py``(or from the BatchExecutor's bootstrap). Re-registering a- namespace overwrites — last registration wins.+ namespace overwrites — last registration wins, check and hook included."""_HANDLERS[namespace] = handler+ _DISPATCH_CHECKS.pop(namespace, None)+ _DROP_HOOKS.pop(namespace, None)+ if should_dispatch is not None:+ _DISPATCH_CHECKS[namespace] = should_dispatch+ if on_drop is not None:+ _DROP_HOOKS[namespace] = on_dropdef clear_handlers_for_test() -> None:"""Used by tests to reset the registry between cases."""_HANDLERS.clear()+ _DISPATCH_CHECKS.clear()+ _DROP_HOOKS.clear()# ---------------------------------------------------------------------------@@ -277,11 +313,16 @@ async def _walk_entry(api_key_for: Callable[[ProviderLiteral], str | None],) -> None:"""Process a single pending entry — gates, poll, then dispatch."""- if (now - entry.submitted_at).total_seconds() > MAX_BATCH_LIFETIME_SECONDS:- await _handle_timeout(entry)+ timed_out = (now - entry.submitted_at).total_seconds() > MAX_BATCH_LIFETIME_SECONDS+ if not timed_out and entry.next_poll_at > now:+ return++ if not await _dispatch_wanted(entry):+ await _drop(entry)return- if entry.next_poll_at > now:+ if timed_out:+ await _handle_timeout(entry)returnif entry.callback_namespace not in _HANDLERS:@@ -403,6 +444,67 @@ async def _handle_timeout(entry: PendingEntry) -> None:)+async def _dispatch_wanted(entry: PendingEntry) -> bool:+ """The namespace's ``should_dispatch`` answer for a due *entry*; ``True``+ when it registered none, or when the check raises or takes longer than+ ``DISPATCH_CHECK_TIMEOUT_SECONDS``: an entry is never dropped on a check+ that could not answer."""+ check = _DISPATCH_CHECKS.get(entry.callback_namespace)+ if check is None:+ return True+ try:+ return await asyncio.wait_for(+ check(entry), timeout=DISPATCH_CHECK_TIMEOUT_SECONDS+ )+ except Exception:+ logger.warning(+ "Dispatch check for namespace=%s failed on batch %s — polling it",+ entry.callback_namespace,+ entry.provider_batch_id,+ exc_info=True,+ )+ return True+++async def _drop(entry: PendingEntry) -> None:+ """Take *entry* off the queue unpolled and undispatched, claimed like a+ dispatch (tombstone + HDEL) so a re-enqueue never brings it back, then+ run the namespace's drop hook. Claimed first: of walkers dropping the+ same entry, only the one whose claim took it runs the hook; one that+ finds it claimed already only clears the leftover row."""+ if not await _claim_dispatch(entry.provider_batch_id):+ await remove_pending(entry.provider_batch_id)+ logger.info(+ "Batch %s for namespace=%s already claimed; its drop is the claimant's",+ entry.provider_batch_id,+ entry.callback_namespace,+ )+ return+ logger.info(+ "Batch %s for namespace=%s dropped by its dispatch check",+ entry.provider_batch_id,+ entry.callback_namespace,+ )+ await _run_drop_hook(entry)+++async def _run_drop_hook(entry: PendingEntry) -> None:+ """The namespace's drop hook for a claimed *entry*, bounded by+ ``DISPATCH_CHECK_TIMEOUT_SECONDS``; a failure is logged, never raised,+ and not retried: the entry is off the queue."""+ hook = _DROP_HOOKS.get(entry.callback_namespace)+ if hook is None:+ return+ try:+ await asyncio.wait_for(hook(entry), timeout=DISPATCH_CHECK_TIMEOUT_SECONDS)+ except Exception:+ logger.exception(+ "Drop hook for namespace=%s failed on batch %s",+ entry.callback_namespace,+ entry.provider_batch_id,+ )++async def _push_back(entry: PendingEntry) -> None:"""Re-queue *entry* with exponential backoff for a later walk."""entry.poll_delay_seconds = _bump_delay(entry.poll_delay_seconds)diff --git a/autogpt_platform/backend/backend/executor/batch_executor_test.py b/autogpt_platform/backend/backend/executor/batch_executor_test.pyindex 4cb76dcb4c..7fe0b11517 100644--- a/autogpt_platform/backend/backend/executor/batch_executor_test.py+++ b/autogpt_platform/backend/backend/executor/batch_executor_test.py@@ -2,6 +2,7 @@from __future__ import annotations+import asyncioimport jsonfrom datetime import datetime, timedelta, timezonefrom typing import Any@@ -407,6 +408,182 @@ class TestWalkOnceDispatch:assert entries[0].poll_delay_seconds == 60+class TestShouldDispatch:+ """A namespace's check, asked before each poll of a due entry: ``False``+ drops the entry unpolled and undispatched, for good, claimed before the+ namespace's drop hook runs, so only the walker whose claim took it runs+ the hook."""++ @pytest.mark.asyncio+ async def test_a_check_that_says_no_drops_the_entry_for_good(self, fake_redis):+ handler = AsyncMock()+ check = AsyncMock(return_value=False)+ pending_at_drop: list[int] = []++ async def on_drop(entry: PendingEntry) -> None:+ pending_at_drop.append(len(await list_pending()))++ register_handler("dream_pass", handler, should_dispatch=check, on_drop=on_drop)+ await enqueue_pending(_entry())+ poll, download = AsyncMock(return_value="ended"), AsyncMock(return_value=[])++ with patch("backend.executor.batch_executor.poll_batch", poll), patch(+ "backend.executor.batch_executor.download_batch_results", download+ ):+ await walk_once(api_key_for=lambda p: "sk-ant-test")+ # A producer re-enqueues it: the claim it was dropped under+ # refuses, and it is cleared without a dispatch.+ check.return_value = True+ await enqueue_pending(_entry())+ await walk_once(api_key_for=lambda p: "sk-ant-test")++ check.assert_awaited()+ assert check.await_args_list[0].args[0].provider_batch_id == "msgbatch_1"+ handler.assert_not_awaited()+ # The hook ran once, for the walk that dropped the entry, and only+ # after its claim had taken it off the queue.+ assert pending_at_drop == [0]+ assert await list_pending() == []++ @pytest.mark.asyncio+ async def test_a_walker_that_finds_the_drop_claimed_does_nothing(self, fake_redis):+ """A walker that read the entry before another claimed its drop+ (here: the same entry back on the queue after its drop): the claim+ refuses, the leftover row goes, and the hook is not run again."""+ on_drop = AsyncMock()+ register_handler(+ "dream_pass",+ AsyncMock(),+ should_dispatch=AsyncMock(return_value=False),+ on_drop=on_drop,+ )+ await enqueue_pending(_entry())+ await walk_once(api_key_for=lambda p: "sk-ant-test")+ on_drop.assert_awaited_once()++ await enqueue_pending(_entry())+ await walk_once(api_key_for=lambda p: "sk-ant-test")++ on_drop.assert_awaited_once()+ assert await list_pending() == []++ @pytest.mark.asyncio+ @pytest.mark.parametrize("fails", ["raises", "hangs"])+ async def test_a_drop_hook_that_fails_leaves_the_entry_dropped(+ self, fake_redis, fails+ ):+ async def hang(entry):+ await asyncio.Event().wait()++ on_drop = AsyncMock(side_effect=RuntimeError("cleanup failed"))+ if fails == "hangs":+ on_drop = AsyncMock(side_effect=hang)+ register_handler(+ "dream_pass",+ AsyncMock(),+ should_dispatch=AsyncMock(return_value=False),+ on_drop=on_drop,+ )+ await enqueue_pending(_entry())++ with patch(+ "backend.executor.batch_executor.DISPATCH_CHECK_TIMEOUT_SECONDS", 0.05+ ):+ await walk_once(api_key_for=lambda p: "sk-ant-test")++ on_drop.assert_awaited_once()+ assert await list_pending() == []++ @pytest.mark.asyncio+ async def test_a_check_that_says_yes_polls_and_dispatches_as_before(+ self, fake_redis+ ):+ handler = AsyncMock()+ check = AsyncMock(return_value=True)+ register_handler("dream_pass", handler, should_dispatch=check)+ await enqueue_pending(_entry())++ with patch(+ "backend.executor.batch_executor.poll_batch",+ new=AsyncMock(return_value="ended"),+ ), patch(+ "backend.executor.batch_executor.download_batch_results",+ new=AsyncMock(return_value=[]),+ ):+ await walk_once(api_key_for=lambda p: "sk-ant-test")++ check.assert_awaited_once()+ handler.assert_awaited_once()++ @pytest.mark.asyncio+ @pytest.mark.parametrize("fails", ["raises", "hangs"])+ async def test_a_check_that_cannot_answer_polls_as_before(self, fake_redis, fails):+ async def hang(entry):+ await asyncio.Event().wait()++ check = AsyncMock(side_effect=RuntimeError("store down"))+ if fails == "hangs":+ check = AsyncMock(side_effect=hang)+ register_handler("dream_pass", AsyncMock(), should_dispatch=check)+ await enqueue_pending(_entry())+ poll = AsyncMock(return_value="processing")++ with patch("backend.executor.batch_executor.poll_batch", poll), patch(+ "backend.executor.batch_executor.DISPATCH_CHECK_TIMEOUT_SECONDS", 0.05+ ):+ await walk_once(api_key_for=lambda p: "sk-ant-test")++ poll.assert_awaited_once()+ assert len(await list_pending()) == 1++ @pytest.mark.asyncio+ async def test_a_timed_out_entry_is_checked_before_its_timeout_dispatch(+ self, fake_redis+ ):+ handler = AsyncMock()+ check = AsyncMock(return_value=False)+ register_handler("dream_pass", handler, should_dispatch=check)+ long_ago = datetime.now(timezone.utc) - timedelta(+ seconds=MAX_BATCH_LIFETIME_SECONDS + 1+ )+ await enqueue_pending(_entry(submitted_at=long_ago))++ await walk_once(api_key_for=lambda p: "sk-ant-test")++ check.assert_awaited_once()+ handler.assert_not_awaited()+ assert await list_pending() == []++ @pytest.mark.asyncio+ async def test_an_entry_not_yet_due_is_not_checked(self, fake_redis):+ check = AsyncMock(return_value=False)+ register_handler("dream_pass", AsyncMock(), should_dispatch=check)+ later = datetime.now(timezone.utc) + timedelta(minutes=5)+ await enqueue_pending(_entry(next_poll_at=later))++ await walk_once(api_key_for=lambda p: "sk-ant-test")++ check.assert_not_awaited()+ assert len(await list_pending()) == 1++ @pytest.mark.asyncio+ async def test_registering_again_without_a_check_drops_the_old_one(+ self, fake_redis+ ):+ check = AsyncMock(return_value=False)+ register_handler("dream_pass", AsyncMock(), should_dispatch=check)+ register_handler("dream_pass", AsyncMock())+ await enqueue_pending(_entry())++ with patch(+ "backend.executor.batch_executor.poll_batch",+ new=AsyncMock(return_value="processing"),+ ):+ await walk_once(api_key_for=lambda p: "sk-ant-test")++ check.assert_not_awaited()++class TestUnknownNamespace:@pytest.mark.asyncioasync def test_no_handler_leaves_entry_pending_for_recovery(self, fake_redis):diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex a88fbf8bbf..40d3be1d1d 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -14,7 +14,7 @@ from concurrent.futures import Futurefrom contextlib import nullcontextfrom datetime import datetime, timedelta, timezonefrom enum import Enum-from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Union+from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Protocol, Unionfrom urllib.parse import parse_qs, urlencode, urlparse, urlunparsefrom zoneinfo import ZoneInfoNotFoundError@@ -2331,6 +2331,79 @@ def execution_accuracy_alerts():return report_execution_accuracy_alerts()+# The dream modules import the batch executor, whose package imports this+# module, so the dream jobs below import them where they run, as the other+# dream job bodies in this module do.+++def execute_dream_pass_reaper() -> None:+ """Close the dream passes of every user that outlived their lease+ (``copilot/dream/reaper.py``). The run bounds itself, and that budget is+ what stops it: ``run_async``'s timeout frees the job thread should the+ event loop stall, but does not cancel the run."""+ from backend.copilot.dream.reaper import REAPER_BUDGET_SECONDS, reap_expired_passes++ run_async(reap_expired_passes(), timeout=REAPER_BUDGET_SECONDS + 30)+++def execute_dream_pass_retention() -> None:+ """Delete the closed dream passes past their retention+ (``copilot/dream/retention.py``), bounded the same way."""+ from backend.copilot.dream.retention import (+ RETENTION_BUDGET_SECONDS,+ delete_expired_records,+ )++ run_async(+ delete_expired_records(config.dream_pass_retention_days),+ timeout=RETENTION_BUDGET_SECONDS + 60,+ )+++class _SystemJobs(Protocol):+ """The call the dream pass jobs make on the scheduler, typed:+ ``BackgroundScheduler.add_job`` itself is unannotated."""++ def add_job(+ self,+ func: Callable[[], None],+ trigger: str | CronTrigger,+ *,+ id: str,+ replace_existing: bool,+ max_instances: int,+ jobstore: str,+ coalesce: bool = ...,+ minutes: int = ...,+ ) -> JobObj: ...+++def _register_dream_pass_jobs(scheduler: _SystemJobs) -> None:+ """The two jobs over every user's DreamPass rows, registered once at+ start like the other system jobs: the reaper every+ ``REAPER_INTERVAL_MINUTES``, retention weekly (Sunday 05:30 UTC)."""+ from backend.copilot.dream.reaper import REAPER_INTERVAL_MINUTES++ scheduler.add_job(+ execute_dream_pass_reaper,+ id="dream_pass_reaper",+ trigger="interval",+ minutes=REAPER_INTERVAL_MINUTES,+ replace_existing=True,+ max_instances=1,+ coalesce=True,+ jobstore=Jobstores.EXECUTION.value,+ )+ scheduler.add_job(+ execute_dream_pass_retention,+ CronTrigger(day_of_week="sun", hour=5, minute=30, timezone="UTC"),+ id="dream_pass_retention",+ replace_existing=True,+ max_instances=1,+ jobstore=Jobstores.EXECUTION.value,+ )++def ensure_embeddings_coverage():"""Ensure all content types (store agents, blocks, docs) have embeddings for search.@@ -3103,6 +3176,10 @@ class Scheduler(AppService):jobstore=Jobstores.EXECUTION.value,)+ # Dream passes: close those that outlived their lease, and delete+ # the closed ones past their retention.+ _register_dream_pass_jobs(self.scheduler)+# Schedule Index Reconcile - Every 24 hours# Safety net for the best-effort index writes: re-adds rows a# failed write missed and clears rows for jobs APSchedulerdiff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.pyindex 0de780e4be..2ed7ae3df7 100644--- a/autogpt_platform/backend/backend/executor/scheduler_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py@@ -12,6 +12,8 @@ from apscheduler.triggers.cron import CronTriggerfrom backend.api.model import CreateGraphfrom backend.copilot.dream import deadline, scope_crons+from backend.copilot.dream.reaper import REAPER_BUDGET_SECONDS+from backend.copilot.dream.retention import RETENTION_BUDGET_SECONDSfrom backend.copilot.dream.scheduling import DREAM_SYSTEM_JOBSfrom backend.copilot.graphiti.scope import MemoryScopefrom backend.copilot.learning.dispositions import SkillLearningResult@@ -26,6 +28,9 @@ from backend.executor.scheduler import (_execute_graph,_memory_scope_may_fire,_normalize_cron_day_of_week,+ _register_dream_pass_jobs,+ execute_dream_pass_reaper,+ execute_dream_pass_retention,execute_dream_pass_with_status,execute_skill_learning_sync,)@@ -1432,6 +1437,63 @@ def _run_dream_wrapper(result, **kwargs) -> tuple[MagicMock, MagicMock]:return execute, errored+# ---------------------------------------------------------------------------+# The dream pass reaper and retention: system jobs over every user's passes+# ---------------------------------------------------------------------------+++class TestDreamPassReaperAndRetentionJobs:+ def test_both_are_registered_once_for_every_user_not_per_user(self) -> None:+ scheduler = MagicMock()++ _register_dream_pass_jobs(scheduler)++ jobs = {c.kwargs["id"]: c for c in scheduler.add_job.call_args_list}+ assert set(jobs) == {"dream_pass_reaper", "dream_pass_retention"}+ reaper = jobs["dream_pass_reaper"]+ assert reaper.args[0] is execute_dream_pass_reaper+ assert (reaper.kwargs["trigger"], reaper.kwargs["minutes"]) == ("interval", 10)+ retention = jobs["dream_pass_retention"]+ assert retention.args[0] is execute_dream_pass_retention+ trigger = repr(retention.args[1])+ assert "day_of_week='sun'" in trigger and "hour='5'" in trigger+ for job in jobs.values():+ assert job.kwargs["max_instances"] == 1+ assert job.kwargs["replace_existing"] is True+ assert job.kwargs["jobstore"] == Jobstores.EXECUTION.value+ assert "kwargs" not in job.kwargs++ def test_the_reaper_runs_on_the_shared_loop_bounded_by_its_budget(self) -> None:+ sentinel = object()+ with (+ patch("backend.executor.scheduler.run_async") as run_async,+ patch(+ "backend.copilot.dream.reaper.reap_expired_passes",+ new=MagicMock(return_value=sentinel),+ ),+ ):+ execute_dream_pass_reaper()++ run_async.assert_called_once_with(sentinel, timeout=REAPER_BUDGET_SECONDS + 30)++ def test_retention_keeps_the_configured_number_of_days(self, monkeypatch) -> None:+ monkeypatch.setattr(+ "backend.executor.scheduler.config.dream_pass_retention_days", 30+ )+ sentinel = object()+ delete = MagicMock(return_value=sentinel)+ with (+ patch("backend.executor.scheduler.run_async") as run_async,+ patch("backend.copilot.dream.retention.delete_expired_records", new=delete),+ ):+ execute_dream_pass_retention()++ delete.assert_called_once_with(30)+ run_async.assert_called_once_with(+ sentinel, timeout=RETENTION_BUDGET_SECONDS + 60+ )++# ---------------------------------------------------------------------------# JobStatus transitions for the admin-triggered community rebuild wrapper# ---------------------------------------------------------------------------diff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.pyindex b78248f8b0..c027a9a070 100644--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py@@ -1985,6 +1985,17 @@ def test_reconcile_stripe_tiers_interval_follows_config_setting(monkeypatch):assert match.kwargs["seconds"] == 12 * 3600+def test_dream_pass_reaper_and_retention_are_registered_at_start(monkeypatch):+ ids = [c.kwargs["id"] for c in _registered_jobs(monkeypatch, 6).add_job_calls]++ assert ids.count("dream_pass_reaper") == 1+ assert ids.count("dream_pass_retention") == 1+++def test_dream_pass_retention_defaults_to_ninety_days():+ assert Config.model_fields["dream_pass_retention_days"].default == 90++@pytest.mark.parametrize("enabled", [False, True])def test_orphan_sweep_registered_only_when_switched_on(monkeypatch, enabled):monkeypatch.setattr(diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex 4b246ede88..51dbc9a156 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -848,6 +848,16 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):description="Hours between platform link token cleanup runs (1-24 hours)",)+ dream_pass_retention_days: int = Field(+ default=90,+ ge=1,+ le=3650,+ description=(+ "Days a finished dream pass's record is kept before the weekly "+ "retention job deletes it (1-3650 days)"+ ),+ )+stripe_tier_reconcile_interval_hours: int = Field(default=6,ge=1,diff --git a/autogpt_platform/backend/migrations/20260926200000_add_dream_pass_lease_index/migration.sql b/autogpt_platform/backend/migrations/20260926200000_add_dream_pass_lease_index/migration.sqlnew file mode 100644index 0000000000..e164e0c25d--- /dev/null+++ b/autogpt_platform/backend/migrations/20260926200000_add_dream_pass_lease_index/migration.sql@@ -0,0 +1,16 @@+-- The dream pass reaper (backend/copilot/dream/reaper.py) lists, across every+-- user, the open passes whose lease lapsed and the closed passes whose cleanup+-- has not finished, oldest first. cleanupPendingAt marks the second kind: set+-- by every close that leaves a cleanup behind (a cancel, an expiry, a batch+-- pass's end, the reaper's own close), cleared once that cleanup has finished.+-- The two indexes keep both scans a range scan per status. A nullable column+-- and two new indexes on an existing table; no row is rewritten.++-- AlterTable+ALTER TABLE "DreamPass" ADD COLUMN "cleanupPendingAt" TIMESTAMP(3);++-- CreateIndex+CREATE INDEX "DreamPass_status_leaseExpiresAt_idx" ON "DreamPass"("status", "leaseExpiresAt");++-- CreateIndex+CREATE INDEX "DreamPass_status_cleanupPendingAt_idx" ON "DreamPass"("status", "cleanupPendingAt");diff --git a/autogpt_platform/backend/migrations/20261015130000_dream_pass_cost_context/migration.sql b/autogpt_platform/backend/migrations/20261015130000_dream_pass_cost_context/migration.sqlnew file mode 100644index 0000000000..eb22e87bda--- /dev/null+++ b/autogpt_platform/backend/migrations/20261015130000_dream_pass_cost_context/migration.sql@@ -0,0 +1,11 @@+-- The usage attribution (#15208) a dream pass started under: the trial or+-- paid generation its phases are charged to, captured once when the pass+-- starts. Every path that charges a batch pass's landed phases reads it here,+-- whatever process runs it and however late: the result callbacks, a failed+-- pass's end, the drop of a closed pass's batch and the reaper's cleanup. A+-- row without one is never charged: one written before this column, or one+-- whose capture failed. A nullable column on an existing table; no row is+-- rewritten.++-- AlterTable+ALTER TABLE "DreamPass" ADD COLUMN "costContext" JSONB;diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prismaindex abea153e31..3c761fccb5 100644--- a/autogpt_platform/backend/schema.prisma+++ b/autogpt_platform/backend/schema.prisma@@ -606,12 +606,19 @@ model DreamPass {// The provider batch of the phase in flight (batch route only).providerBatchId String?- // The dream lock's ownership token, and when the lock lapses unless renewed.+ // The dream lock's ownership token, and when the lock lapses unless renewed:+ // written with the row, renewed at each step, emptied when the row closes,+ // or, for a row closed with a cleanup pending, once that cleanup is done.leaseToken String?leaseExpiresAt DateTime?+ // Set by a close that leaves a cleanup behind (a stop from outside, a batch+ // pass's end, the reaper's) and cleared once every step of that cleanup has+ // finished, so the reaper resumes whatever the pass left undone.+ cleanupPendingAt DateTime?+// The pass's DreamInput, in the format the batch path already persists to- // Redis (batch route only).+ // Redis (batch route only), emptied when the row closes.inputBundle Json?// Each phase's validated output, keyed by phase.phaseOutputs Json?@@ -619,6 +626,9 @@ model DreamPass {operations Json?// DreamPassUsage: tokens and cost, per phase and in total.usage Json?+ // The usage attribution (TrialCostContext) the pass started under, which+ // every charge of its phases goes by. A row without one is never charged.+ costContext Json?// The episode window the pass gathered.windowStart DateTime?@@ -636,6 +646,10 @@ model DreamPass {@@index([userId, createdAt])// Deleting an expert cascades here; without it that scans every pass.@@index([expertId])+ // The reaper's scans: open passes whose lease lapsed, oldest first, and+ // closed passes whose cleanup has not finished.+ @@index([status, leaseExpiresAt])+ @@index([status, cleanupPendingAt])}////////////////////////////////////////////////////////////diff --git a/autogpt_platform/backend/scripts/gen_prisma_types_stub.py b/autogpt_platform/backend/scripts/gen_prisma_types_stub.pyindex 63e6afd246..54a12f52f6 100644--- a/autogpt_platform/backend/scripts/gen_prisma_types_stub.py+++ b/autogpt_platform/backend/scripts/gen_prisma_types_stub.py@@ -41,6 +41,11 @@ def _is_safe_type_alias(node: ast.Assign) -> bool:- Literal types (don't cause type budget issues)- Simple type references (SortMode, SortOrder, etc.)- TypeVar definitions+ - Unions of the above; not a Union naming a private type, which the stub+ leaves out: each ``*OrderByInput`` union would point at types that do+ not exist there and read as Unknown, leaving every ``find_many`` and+ ``find_first`` partially unknown under strict checking. Collapsed like+ the rest, it reads as ``dict[str, Any]``."""if not node.value:return False@@ -55,14 +60,16 @@ def _is_safe_type_alias(node: ast.Assign) -> bool:return True# Union types are safe (e.g. Serializable = Union[...])if base_name == "Union":- return True+ return not _names_a_private_type(node.value.slice)# TypeVar is safeif base_name == "TypeVar":return Trueelif isinstance(node.value.value, ast.Attribute):# Handle typing_extensions.Literal etc.- if node.value.value.attr in ("Literal", "Union"):+ if node.value.value.attr == "Literal":return True+ if node.value.value.attr == "Union":+ return not _names_a_private_type(node.value.slice)# Check if it's a simple Name reference (like SortMode = _types.SortMode)if isinstance(node.value, ast.Attribute):@@ -77,6 +84,22 @@ def _is_safe_type_alias(node: ast.Assign) -> bool:return False+def _names_a_private_type(members: ast.expr) -> bool:+ """Whether a Union's *members* name a private type, directly or as a+ string forward reference."""+ for node in ast.walk(members):+ if isinstance(node, ast.Name) and _is_private(node.id):+ return True+ if (+ isinstance(node, ast.Constant)+ and isinstance(node.value, str)+ and node.value.isidentifier()+ and _is_private(node.value)+ ):+ return True+ return False++def collect_top_level_symbols(tree: ast.Module, source_lines: list[str]) -> tuple[Set[str], Set[str], list[str], Set[str]]:diff --git a/autogpt_platform/backend/scripts/gen_prisma_types_stub_test.py b/autogpt_platform/backend/scripts/gen_prisma_types_stub_test.pynew file mode 100644index 0000000000..9d9effbbb6--- /dev/null+++ b/autogpt_platform/backend/scripts/gen_prisma_types_stub_test.py@@ -0,0 +1,49 @@+"""Tests for the Prisma types stub generator: which aliases keep their type+and which collapse to ``dict[str, Any]``."""++from pathlib import Path++from scripts.gen_prisma_types_stub import generate_stub++_TYPES = """+from typing import Union, List+from typing_extensions import Literal++SortOrder = _types.SortOrder+DreamPassScalarFieldKeys = Literal['id', 'status']+Serializable = Union[None, bool, str, List['Serializable']]+DreamPassOrderByInput = Union[+ '_DreamPass_id_OrderByInput',+ '_DreamPass_status_OrderByInput',+]+++class _DreamPass_id_OrderByInput(dict):+ pass+++class DreamPassWhereInput(dict):+ pass+"""+++def test_a_union_of_private_types_collapses_and_the_rest_keep_their_type(+ tmp_path: Path,+) -> None:+ """A Union naming the private types the stub leaves out would read as+ Unknown (every ``*OrderByInput``, so every ``find_many``); it collapses+ like the classes do. Literals, public Unions and module references+ keep their type."""+ source = tmp_path / "types.py"+ source.write_text(_TYPES, encoding="utf-8")+ stub = tmp_path / "types.pyi"++ generate_stub(source, stub)++ lines = stub.read_text(encoding="utf-8").splitlines()+ assert "DreamPassOrderByInput = _PrismaDict" in lines+ assert "DreamPassWhereInput = _PrismaDict" in lines+ assert "SortOrder = _types.SortOrder" in lines+ assert "DreamPassScalarFieldKeys = Literal['id', 'status']" in lines+ assert any(line.startswith("Serializable = Union[") for line in lines)+ assert not any("_DreamPass_id_OrderByInput" in line for line in lines)diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex 45f7a5385b..1df843f884 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -1527,6 +1527,78 @@}}},+ "/api/admin/memory/{user_id}/dream/passes": {+ "get": {+ "tags": ["v2", "admin", "admin", "memory"],+ "summary": "List Dream Pass Records",+ "description": "List a user's dream passes, across the account and every expert,\nnewest first: the durable records, each as ``GET .../record`` returns\nit. A finished pass keeps its phase outputs, operations and usage; its\ninput bundle went when it closed, and its lease with it, or, for a pass\nwhose close left a cleanup pending, once that cleanup finished.",+ "operationId": "getV2ListDreamPassRecords",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "user_id",+ "in": "path",+ "required": true,+ "schema": {+ "type": "string",+ "description": "User id or 'me'",+ "title": "User Id"+ },+ "description": "User id or 'me'"+ },+ {+ "name": "status",+ "in": "query",+ "required": false,+ "schema": {+ "enum": ["open", "all"],+ "type": "string",+ "description": "Only the passes still running, or all of them",+ "default": "all",+ "title": "Status"+ },+ "description": "Only the passes still running, or all of them"+ },+ {+ "name": "limit",+ "in": "query",+ "required": false,+ "schema": {+ "type": "integer",+ "maximum": 100,+ "minimum": 1,+ "description": "How many passes, newest first",+ "default": 20,+ "title": "Limit"+ },+ "description": "How many passes, newest first"+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/DreamPassListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },"/api/admin/memory/{user_id}/dream/{job_id}": {"get": {"tags": ["v2", "admin", "admin", "memory"],@@ -1585,7 +1657,7 @@"post": {"tags": ["v2", "admin", "admin", "memory"],"summary": "Cancel Dream Pass Run",- "description": "Cancel an open dream pass and return its record, now CANCELLED.\n\nThe row closes at once; the pass itself stops at its next check (a phase\nboundary, its batch submit, or just before apply, on either route)\nwithout applying. A batch pass waiting on its provider stops when that\nbatch lands. A pass already past its last check still applies, once, and\nwhatever a pass applied stays. 404 for a pass that does not exist or\nbelongs to another user, alike; 409 once the pass has ended.",+ "description": "Cancel an open dream pass and return its record, now CANCELLED.\n\nThe row closes at once; the pass itself stops at its next check (a phase\nboundary, its batch submit, or just before apply, on either route)\nwithout applying. A batch pass waiting on its provider has that batch\ncancelled too, and the batch executor drops it at its next poll, which\nreleases the pass's lock. A pass already past its last check still\napplies, once, and whatever a pass applied stays. 404 for a pass that\ndoes not exist or belongs to another user, alike; 409 once the pass has\nended.","operationId": "postV2CancelDreamPassRun","security": [{ "HTTPBearerJWT": [] }],"parameters": [@@ -23544,6 +23616,19 @@"title": "DreamPassCancelRequest","description": "Why an admin cancels a dream pass, kept as the pass's error."},+ "DreamPassListResponse": {+ "properties": {+ "passes": {+ "items": { "$ref": "#/components/schemas/DreamPassRecord" },+ "type": "array",+ "title": "Passes"+ }+ },+ "type": "object",+ "required": ["passes"],+ "title": "DreamPassListResponse",+ "description": "A user's dream passes, every scope, newest first."+ },"DreamPassOperations": {"properties": {"planned": {@@ -23611,6 +23696,20 @@],"title": "Lease Expires At"},+ "cleanup_pending_at": {+ "anyOf": [+ { "type": "string", "format": "date-time" },+ { "type": "null" }+ ],+ "title": "Cleanup Pending At"+ },+ "cost_context": {+ "anyOf": [+ { "additionalProperties": true, "type": "object" },+ { "type": "null" }+ ],+ "title": "Cost Context"+ },"input_bundle": {"anyOf": [{ "$ref": "#/components/schemas/DreamInput" },