Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend/copilot): one guard against overlapping dream passes and a stop path built on the DreamPass record (#14970)

Nicholas Tindle9a0414c822fa
diff --git a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py
index 0ef257c645..b8b2367ebf 100644
--- a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py
+++ b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py
@@ -19,10 +19,11 @@ from autogpt_libs.auth import get_user_id, requires_admin_user
from autogpt_libs.auth.jwt_utils import get_jwt_payload
from fastapi import APIRouter, Depends, HTTPException, Path, Query, Request, Security
from fastapi.responses import JSONResponse
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
from redis.exceptions import ResponseError
from backend.api.features.experts import experts_db
+from backend.copilot.dream.cancel import cancel_dream_pass
from backend.copilot.dream.job_status import (
JobKind,
JobState,
@@ -1019,6 +1020,19 @@ class DreamPassRecordResponse(BaseModel):
result: DreamPassResult
+# What a cancelled pass's record says when the admin gave no reason.
+ADMIN_CANCEL_REASON = "cancelled by an admin"
+
+
+class DreamPassCancelRequest(BaseModel):
+ """Why an admin cancels a dream pass, kept as the pass's error."""
+
+ # Stripped before the length check, so a blank reason is refused.
+ model_config = ConfigDict(str_strip_whitespace=True)
+
+ reason: str | None = Field(default=None, min_length=1, max_length=500)
+
+
@router.post(
"/{user_id}/dream",
response_model=JobTriggerResponse,
@@ -1029,12 +1043,24 @@ async def trigger_dream_pass(
user_id: Annotated[str, Path(description="User id or 'me'")],
caller_id: Annotated[str, Depends(get_user_id)],
jwt_payload: Annotated[dict, Security(get_jwt_payload)],
+ force: Annotated[
+ bool,
+ Query(
+ description=(
+ "Expire a fresh open dream pass of the account first instead "
+ "of skipping behind it"
+ )
+ ),
+ ] = False,
) -> JSONResponse:
"""Fire a dream pass and return 202 + job_id immediately.
Frontend polls ``GET /api/admin/memory/{user_id}/dream/{job_id}``
for progress. Runs ONLY the dream pass submitter — for the full
- nightly fan-out use ``POST /{user_id}/nightly``.
+ nightly fan-out use ``POST /{user_id}/nightly``. The pass goes through
+ the same guard as the nightly one: it skips behind a fresh open pass of
+ the account unless ``force`` expires that pass first. ``force`` does not
+ take a lock another pass holds.
"""
target = _resolve_user_id(user_id, caller_id)
await _resolve_and_audit_memory_scope(
@@ -1052,7 +1078,7 @@ async def trigger_dream_pass(
try:
await get_scheduler_client().schedule_immediate_dream_pass(
- user_id=target, job_id=job_id
+ user_id=target, job_id=job_id, force=force
)
except Exception as exc:
logger.warning(
@@ -1119,8 +1145,9 @@ async def get_dream_pass_record(
The job status above is a Redis row that lapses after six hours and, for
a batch pass, carries no usage; the record outlives the pass and holds
- what its phases used. 404 for a pass that does not exist or belongs to
- another user, alike.
+ what its phases used, unless a cancel or a newer pass closed it first
+ (the cost log still has that usage). 404 for a pass that does not exist
+ or belongs to another user, alike.
"""
target = _resolve_user_id(user_id, caller_id)
_audit_cross_user_access(
@@ -1137,6 +1164,46 @@ async def get_dream_pass_record(
)
+ "/{user_id}/dream/{pass_id}/cancel",
+ response_model=DreamPassRecordResponse,
+)
+async def cancel_dream_pass_run(
+ request: Request,
+ user_id: Annotated[str, Path(description="User id or 'me'")],
+ pass_id: Annotated[str, Path(description="The dream pass id")],
+ caller_id: Annotated[str, Depends(get_user_id)],
+ jwt_payload: Annotated[dict[str, Any], Security(get_jwt_payload)],
+ body: DreamPassCancelRequest | None = None,
+) -> DreamPassRecordResponse:
+ """Cancel an open dream pass and return its record, now CANCELLED.
+
+ The row closes at once; the pass itself stops at its next check (a phase
+ boundary, its batch submit, or just before apply, on either route)
+ without applying. A batch pass waiting on its provider stops when that
+ batch lands. A pass already past its last check still applies, once, and
+ whatever a pass applied stays. 404 for a pass that does not exist or
+ belongs to another user, alike; 409 once the pass has ended.
+ """
+ target = _resolve_user_id(user_id, caller_id)
+ _audit_cross_user_access(
+ request=request,
+ caller_id=caller_id,
+ target_id=target,
+ jwt_payload=jwt_payload,
+ )
+ reason = (body.reason if body else None) or ADMIN_CANCEL_REASON
+ cancel = await cancel_dream_pass(pass_id, user_id=target, reason=reason)
+ if cancel.record is None:
+ raise HTTPException(status_code=404, detail="dream pass not found")
+ if not cancel.cancelled:
+ ended = cancel.record.status.value.lower()
+ raise HTTPException(status_code=409, detail=f"dream pass already {ended}")
+ return DreamPassRecordResponse(
+ record=cancel.record, result=dream_pass_result_from_row(cancel.record)
+ )
+
+
@router.post("/{user_id}/ratification", response_model=RatificationResult)
async def trigger_ratification_pass(
request: Request,
diff --git a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes_test.py b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes_test.py
index 594e3997e6..98047893a6 100644
--- a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes_test.py
+++ b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes_test.py
@@ -17,6 +17,7 @@ from prisma.enums import (
from redis.exceptions import ResponseError
from backend.api.features.experts.models import PROTECTED_SOUL_RULES, Expert
+from backend.copilot.dream.cancel import DreamPassCancel
from backend.copilot.dream.schemas import DreamPassUsage, IngestionDrainStatus
from backend.copilot.graphiti.client import derive_memory_group_id
from backend.copilot.graphiti.recall import live_fact_predicate
@@ -27,6 +28,7 @@ from backend.data.dream_pass_models import (
DreamPhaseOutputs,
)
+from .memory_admin_routes import ADMIN_CANCEL_REASON
from .memory_admin_routes import router as memory_admin_router
app = fastapi.FastAPI()
@@ -958,6 +960,169 @@ class TestDreamPassRecord:
read.assert_not_awaited()
+class TestTriggerDreamPass:
+ """``POST /{user_id}/dream`` hands ``force`` to the scheduler, for the
+ pass's guard to expire a fresh open pass instead of skipping behind it."""
+
+ def _post(self, url: str):
+ scheduler = MagicMock()
+ scheduler.schedule_immediate_dream_pass = AsyncMock(
+ return_value={"scheduled": True, "job_id": "x", "kind": "dream_pass"}
+ )
+ with (
+ patch(f"{_MOCK_MODULE}.get_scheduler_client", return_value=scheduler),
+ patch(
+ f"{_MOCK_MODULE}.write_initial_status",
+ new=_make_fake_initial_status("dream_pass"),
+ ),
+ ):
+ resp = client.post(url)
+ return resp, scheduler.schedule_immediate_dream_pass
+
+ def test_force_reaches_the_scheduler(self) -> None:
+ resp, schedule = self._post("/admin/memory/abc/dream?force=true")
+
+ assert resp.status_code == 202
+ schedule.assert_awaited_once_with(
+ user_id="abc", job_id=resp.json()["job_id"], force=True
+ )
+
+ def test_force_is_off_unless_asked_for(self) -> None:
+ resp, schedule = self._post("/admin/memory/abc/dream")
+
+ assert resp.status_code == 202
+ assert schedule.call_args.kwargs["force"] is False
+
+ def test_a_force_that_is_not_a_boolean_is_422(self) -> None:
+ resp, schedule = self._post("/admin/memory/abc/dream?force=maybe")
+
+ assert resp.status_code == 422
+ schedule.assert_not_awaited()
+
+
+def _cancelled_row(user_id: str) -> DreamPassRecord:
+ return _dream_pass_row(user_id).model_copy(
+ update={
+ "status": DreamPassStatus.CANCELLED,
+ "phase": DreamPassPhase.RECOMBINE,
+ "cancel_generation": 1,
+ "error": "testing",
+ "operations": DreamPassOperations(),
+ "usage": None,
+ }
+ )
+
+
+class TestCancelDreamPass:
+ """``POST /{user_id}/dream/{pass_id}/cancel`` closes an open pass's record
+ CANCELLED and returns it; the pass stops at its next check."""
+
+ def test_cancels_an_open_pass_and_returns_its_record(self) -> None:
+ cancel = AsyncMock(
+ return_value=DreamPassCancel(cancelled=True, record=_cancelled_row("abc"))
+ )
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = client.post(
+ "/admin/memory/abc/dream/p1/cancel", json={"reason": "testing"}
+ )
+
+ assert resp.status_code == 200
+ cancel.assert_awaited_once_with("p1", user_id="abc", reason="testing")
+ body = resp.json()
+ assert (body["record"]["status"], body["record"]["cancel_generation"]) == (
+ "CANCELLED",
+ 1,
+ )
+ assert body["result"]["error"] == "testing"
+ assert body["result"]["skipped"] is False
+
+ def test_without_a_reason_the_admin_default_is_recorded(self) -> None:
+ cancel = AsyncMock(
+ return_value=DreamPassCancel(cancelled=True, record=_cancelled_row("abc"))
+ )
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = client.post("/admin/memory/abc/dream/p1/cancel")
+
+ assert resp.status_code == 200
+ cancel.assert_awaited_once_with("p1", user_id="abc", reason=ADMIN_CANCEL_REASON)
+
+ def test_a_reason_is_kept_without_its_surrounding_whitespace(self) -> None:
+ cancel = AsyncMock(
+ return_value=DreamPassCancel(cancelled=True, record=_cancelled_row("abc"))
+ )
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = client.post(
+ "/admin/memory/abc/dream/p1/cancel", json={"reason": " testing\n"}
+ )
+
+ assert resp.status_code == 200
+ cancel.assert_awaited_once_with("p1", user_id="abc", reason="testing")
+
+ def test_me_cancels_the_callers_own_pass(self, mock_jwt_admin) -> None:
+ caller = mock_jwt_admin["user_id"]
+ cancel = AsyncMock(
+ return_value=DreamPassCancel(cancelled=True, record=_cancelled_row(caller))
+ )
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = client.post("/admin/memory/me/dream/p1/cancel")
+
+ assert resp.status_code == 200
+ assert cancel.call_args.kwargs["user_id"] == caller
+
+ def test_a_missing_or_another_users_pass_is_404(self) -> None:
+ # The cancel is owner-scoped: another user's pass reads back as None,
+ # exactly like a pass that does not exist.
+ cancel = AsyncMock(return_value=DreamPassCancel(cancelled=False, record=None))
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = client.post("/admin/memory/abc/dream/p1/cancel")
+
+ assert resp.status_code == 404
+
+ def test_a_pass_that_already_ended_is_409(self) -> None:
+ cancel = AsyncMock(
+ return_value=DreamPassCancel(cancelled=False, record=_dream_pass_row("abc"))
+ )
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = client.post("/admin/memory/abc/dream/p1/cancel")
+
+ assert resp.status_code == 409
+ assert resp.json()["detail"] == "dream pass already complete"
+
+ @pytest.mark.parametrize(
+ "reason",
+ ["", " ", "\t\n", "x" * 501],
+ ids=["empty", "space", "whitespace", "overlong"],
+ )
+ def test_a_blank_or_overlong_reason_is_422_before_the_cancel(
+ self, reason: str
+ ) -> None:
+ cancel = AsyncMock()
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = client.post(
+ "/admin/memory/abc/dream/p1/cancel", json={"reason": reason}
+ )
+
+ assert resp.status_code == 422
+ cancel.assert_not_awaited()
+
+ def test_non_admin_gets_403_before_the_cancel(self, mock_jwt_user) -> None:
+ app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]
+ cancel = AsyncMock()
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = client.post("/admin/memory/abc/dream/p1/cancel")
+
+ assert resp.status_code == 403
+ cancel.assert_not_awaited()
+
+ def test_a_store_that_does_not_answer_is_500(self) -> None:
+ cancel = AsyncMock(side_effect=TimeoutError())
+ quiet_client = fastapi.testclient.TestClient(app, raise_server_exceptions=False)
+ with patch(f"{_MOCK_MODULE}.cancel_dream_pass", new=cancel):
+ resp = quiet_client.post("/admin/memory/abc/dream/p1/cancel")
+
+ assert resp.status_code == 500
+
+
class TestAdminGating:
"""Non-admin callers must get 403 on every route."""
diff --git a/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py b/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py
new file mode 100644
index 0000000000..1e14cd2c35
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py
@@ -0,0 +1,398 @@
+"""The lock check right before a dream pass applies, on both routes, with the
+real dream lock, callbacks and orchestrator over the in-memory Redis and
+store: a pass whose lock lapsed, and was maybe taken by a newer pass, never
+applies, so at most one pass of a scope applies. Only apply's graph writes and
+the models are stubbed."""
+
+import asyncio
+from datetime import datetime, timedelta, timezone
+from unittest.mock import AsyncMock
+
+import pytest
+from prisma.enums import (
+ DreamPassPhase,
+ DreamPassRoute,
+ DreamPassStatus,
+ DreamPassTrigger,
+)
+from pydantic import BaseModel
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.copilot.inference.complete import StructuredCompletion
+from backend.copilot.inference.context import InferenceUsage, RouteDecision
+from backend.data.dream_pass_models import DreamPassDraft
+from backend.executor.batch_executor import PendingEntry
+from backend.util.llm.providers import BatchResultRow
+
+from . import batch_callbacks as batch_callbacks_mod
+from . import batch_outcome as batch_outcome_mod
+from . import cancel as cancel_mod
+from . import job_status
+from . import locks as locks_mod
+from . import orchestrator as orchestrator_mod
+from .batch_callbacks import handle_dream_batch_result
+from .batch_state import state_key, write_phase_to_state
+from .batch_submit import input_bundle_key, persist_input_bundle
+from .cancel import LOCK_LOST_ERROR, cancel_dream_pass
+from .fetch import DreamInput, EpisodeRow
+from .schemas import (
+ ConsolidatedFact,
+ ConsolidationOutput,
+ DreamOperations,
+ DreamPhase,
+ RecombinationOutput,
+)
+
+_SCOPE = MemoryScope.for_user("u1")
+_LOCK_KEY = _SCOPE.redis_key("dream_lock")
+_PHASE_MODELS = {
+ "consolidate": "claude-sonnet-5",
+ "recombine": "claude-opus-5-5",
+ "sanitize": "claude-sonnet-5",
+}
+_OPS = DreamOperations(
+ writes=[ConsolidatedFact(content="Nick ships on Fridays", confidence=0.9)],
+ summary_for_user="ok",
+)
+_BILLED = InferenceUsage(
+ model="m",
+ input_tokens=100,
+ output_tokens=20,
+ cost_usd=0.001,
+ cost_source="provider",
+ payer="platform_allowance",
+)
+
+
+def apply(mocker) -> AsyncMock:
+ """Apply on both routes: one mock, so the calls show which pass applied."""
+ applied = AsyncMock(return_value={"session_id": "s", "consolidated_count": 1})
+ mocker.patch.object(orchestrator_mod, "apply_operations", applied)
+ mocker.patch("backend.copilot.dream.apply.apply_operations", applied)
+ return applied
+
+
+def charges(mocker) -> AsyncMock:
+ return mocker.patch(
+ "backend.copilot.dream.batch_costs.record_phase_cost", AsyncMock()
+ )
+
+
+def sync_pass(mocker) -> None:
+ """A sync pass for the same scope, its three phases answering in turn."""
+ mocker.patch.object(orchestrator_mod, "resolve_route", side_effect=_route)
+ mocker.patch.object(
+ orchestrator_mod, "resolve_dream_execution_path", return_value="sync_baseline"
+ )
+ mocker.patch.object(
+ orchestrator_mod, "is_feature_enabled", AsyncMock(return_value=False)
+ )
+ mocker.patch.object(
+ orchestrator_mod, "check_dream_budget", AsyncMock(return_value=(True, None))
+ )
+ mocker.patch.object(
+ orchestrator_mod,
+ "record_phase_cost",
+ AsyncMock(side_effect=lambda ctx, usage: usage),
+ )
+ mocker.patch.object(
+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_input())
+ )
+ answers = (ConsolidationOutput(), RecombinationOutput(), _OPS)
+ mocker.patch.object(
+ orchestrator_mod,
+ "structured_complete",
+ AsyncMock(
+ side_effect=[StructuredCompletion(value=a, usage=_BILLED) for a in answers]
+ ),
+ )
+
+
+class TestTheBatchRoute:
+ async def test_a_callback_paused_at_its_gate_never_applies_after_its_replacement(
+ self, fake_dream_db, fake_dream_redis, apply, charges, sync_pass, mocker
+ ):
+ """Codex's interleaving: the last callback has checked its row and
+ waits on the apply gate; its lock lapses; a forced admin pass takes
+ the scope, expires the old row and applies. The old callback then
+ claims its gate, finds the lock gone and does not apply."""
+ await _seed_batch_pass(fake_dream_db, fake_dream_redis)
+ at_gate, resume = asyncio.Event(), asyncio.Event()
+ claim = batch_callbacks_mod.claim_apply_gate
+
+ async def paused_claim(pass_id: str):
+ at_gate.set()
+ await resume.wait()
+ return await claim(pass_id)
+
+ mocker.patch.object(batch_callbacks_mod, "claim_apply_gate", paused_claim)
+ old = asyncio.create_task(_deliver_sanitize())
+ await asyncio.wait_for(at_gate.wait(), 5)
+ await fake_dream_redis.delete(_LOCK_KEY)
+
+ newer = await orchestrator_mod.execute_dream_pass(
+ "u1", trigger="admin", force=True
+ )
+ resume.set()
+ await asyncio.wait_for(old, 5)
+
+ assert newer.error is None and not newer.skipped
+ assert [call.args[1] for call in apply.await_args_list] == [newer.pass_id]
+ error = f"expired: forced by an admin-triggered dream pass {newer.pass_id}"
+ await _assert_ended_without_applying(fake_dream_db, fake_dream_redis, error)
+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.EXPIRED
+ assert fake_dream_db.rows[newer.pass_id]["status"] is DreamPassStatus.COMPLETE
+ assert _charged(charges) == ["consolidate", "recombine", "sanitize"]
+
+ async def test_a_callback_whose_lock_a_newer_pass_holds_leaves_it_alone(
+ self, fake_dream_db, fake_dream_redis, apply, charges, mocker
+ ):
+ await _seed_batch_pass(fake_dream_db, fake_dream_redis)
+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"
+ release = mocker.spy(batch_outcome_mod, "release_dream_lock")
+
+ await _deliver_sanitize()
+
+ apply.assert_not_awaited()
+ release.assert_not_called()
+ await _assert_ended_without_applying(
+ fake_dream_db, fake_dream_redis, LOCK_LOST_ERROR
+ )
+ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"
+ row = fake_dream_db.rows["p1"]
+ assert (row["status"], row["error"]) == (
+ DreamPassStatus.ERRORED,
+ LOCK_LOST_ERROR,
+ )
+ assert _charged(charges) == ["consolidate", "recombine", "sanitize"]
+
+ async def test_a_lock_that_cannot_be_read_after_the_claim_fails_closed(
+ self, fake_dream_db, fake_dream_redis, apply, charges, mocker
+ ):
+ """The pass does not apply on a lock it cannot confirm, and releases
+ it by compare-and-delete, which only ever deletes its own."""
+ await _seed_batch_pass(fake_dream_db, fake_dream_redis)
+ mocker.patch.object(
+ cancel_mod, "dream_lock_held_by", AsyncMock(side_effect=TimeoutError())
+ )
+
+ await _deliver_sanitize()
+
+ apply.assert_not_awaited()
+ await _assert_ended_without_applying(
+ fake_dream_db, fake_dream_redis, LOCK_LOST_ERROR
+ )
+ assert _LOCK_KEY not in fake_dream_redis.store
+
+ async def test_a_cancel_after_the_last_check_is_too_late_and_applies_once(
+ self, fake_dream_db, fake_dream_redis, apply, charges, mocker
+ ):
+ """The documented window: a cancel that lands after the pass's last
+ check, while it claims its gate, is like one landing during apply.
+ The pass still holds its lock, so it applies, once, and nothing
+ else does; the row keeps CANCELLED."""
+ await _seed_batch_pass(fake_dream_db, fake_dream_redis)
+ claim = batch_callbacks_mod.claim_apply_gate
+
+ async def cancelled_while_claiming(pass_id: str):
+ assert (
+ await cancel_dream_pass(pass_id, user_id="u1", reason="late")
+ ).cancelled
+ return await claim(pass_id)
+
+ mocker.patch.object(
+ batch_callbacks_mod, "claim_apply_gate", cancelled_while_claiming
+ )
+
+ await _deliver_sanitize()
+
+ assert [call.args[1] for call in apply.await_args_list] == ["p1"]
+ row = fake_dream_db.rows["p1"]
+ assert (row["status"], row["cancel_generation"]) == (
+ DreamPassStatus.CANCELLED,
+ 1,
+ )
+ status = await job_status.read_status(kind="dream_pass", job_id="j1")
+ assert status is not None and status.state == "complete"
+ assert _LOCK_KEY not in fake_dream_redis.store
+
+
+class TestTheSyncRoute:
+ async def test_a_pass_whose_lock_lapsed_before_apply_does_not_apply(
+ self, fake_dream_db, fake_dream_redis, apply, sync_pass, mocker
+ ):
+ """The lock lapsed during sanitize and a newer pass took it: the pass
+ ends before apply with its phases' usage, and the newer holder's lock
+ stays."""
+ await _run_with_lock_taken_during_sanitize(mocker, fake_dream_redis)
+ result = await orchestrator_mod.execute_dream_pass("u1")
+
+ assert result.error == LOCK_LOST_ERROR
+ assert result.usage is not None
+ assert [p.phase for p in result.usage.phases] == [
+ "consolidate",
+ "recombine",
+ "sanitize",
+ ]
+ apply.assert_not_awaited()
+ assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"
+ row = fake_dream_db.rows[result.pass_id]
+ assert (row["status"], row["error"]) == (
+ DreamPassStatus.ERRORED,
+ LOCK_LOST_ERROR,
+ )
+
+ async def test_a_pass_that_cannot_read_its_lock_before_apply_does_not_apply(
+ self, fake_dream_db, fake_dream_redis, apply, sync_pass, mocker
+ ):
+ mocker.patch.object(
+ locks_mod.DreamLockHandle,
+ "held",
+ AsyncMock(side_effect=ConnectionError("redis down")),
+ )
+
+ result = await orchestrator_mod.execute_dream_pass("u1")
+
+ assert result.error == LOCK_LOST_ERROR
+ apply.assert_not_awaited()
+ assert _LOCK_KEY not in fake_dream_redis.store
+
+ async def test_a_pass_that_still_holds_its_lock_applies(
+ self, fake_dream_db, fake_dream_redis, apply, sync_pass
+ ):
+ result = await orchestrator_mod.execute_dream_pass("u1")
+
+ assert result.error is None
+ assert [call.args[1] for call in apply.await_args_list] == [result.pass_id]
+
+
+def _route(scope, job, *, config=None) -> RouteDecision:
+ return RouteDecision(
+ engine="provider_sync",
+ auth_provider="platform",
+ provider="open_router",
+ model=f"{job.tier}-model",
+ payer="platform_allowance",
+ execution_path="sync_baseline",
+ cost_log_provider="open_router",
+ reason="test",
+ )
+
+
+def _input() -> DreamInput:
+ now = datetime.now(timezone.utc)
+ episode = EpisodeRow(
+ uuid="e1",
+ name=None,
+ content="I ship on Fridays",
+ source_description=None,
+ valid_at=None,
+ created_at=None,
+ )
+ return DreamInput(
+ user_id="u1",
+ group_id=_SCOPE.group_id,
+ window_start=now - timedelta(days=14),
+ window_end=now,
+ episodes=[episode],
+ )
+
+
+async def _run_with_lock_taken_during_sanitize(mocker, fake_dream_redis) -> None:
+ """Make the sanitize call hand the scope's lock to a newer pass."""
+ answers = iter((ConsolidationOutput(), RecombinationOutput(), _OPS))
+
+ async def answer(*_args, **_kwargs) -> StructuredCompletion:
+ value = next(answers)
+ if value is _OPS:
+ fake_dream_redis.store[_LOCK_KEY] = "newer-token"
+ return StructuredCompletion(value=value, usage=_BILLED)
+
+ mocker.patch.object(
+ orchestrator_mod, "structured_complete", AsyncMock(side_effect=answer)
+ )
+
+
+async def _seed_batch_pass(fake_dream_db, fake_dream_redis) -> None:
+ """Batch pass p1 at its last phase: its lock held under the token its
+ bundle carries, consolidate and recombine landed, its job and row open."""
+ now = datetime.now(timezone.utc)
+ await persist_input_bundle(
+ "p1",
+ DreamInput(
+ user_id="u1", group_id=_SCOPE.group_id, window_start=now, window_end=now
+ ),
+ lock_token="our-token",
+ )
+ fake_dream_redis.store[_LOCK_KEY] = "our-token"
+ await job_status.write_initial_status(kind="dream_pass", job_id="j1", user_id="u1")
+ landed: tuple[tuple[DreamPhase, BaseModel], ...] = (
+ ("consolidate", ConsolidationOutput()),
+ ("recombine", RecombinationOutput()),
+ )
+ for phase, output in landed:
+ await write_phase_to_state(
+ pass_id="p1", phase=phase, row=_landed(phase, output)
+ )
+ fake_dream_db.seed(
+ DreamPassDraft(
+ id="p1",
+ user_id="u1",
+ scope_key=_SCOPE.scope_key,
+ route=DreamPassRoute.ANTHROPIC_BATCH,
+ trigger=DreamPassTrigger.CRON,
+ status=DreamPassStatus.SUBMITTED,
+ phase=DreamPassPhase.SANITIZE,
+ ),
+ provider_batch_id="msgbatch_sanitize",
+ lease_token="our-token",
+ )
+
+
+async def _deliver_sanitize() -> None:
+ now = datetime.now(timezone.utc)
+ entry = PendingEntry(
+ provider="anthropic",
+ provider_batch_id="msgbatch_sanitize",
+ callback_namespace="dream_pass",
+ submitted_at=now,
+ next_poll_at=now,
+ payload={
+ "user_id": "u1",
+ "cost_context": {"user_id": "u1", "trial_id": None, "generation": None},
+ "pass_id": "p1",
+ "job_id": "j1",
+ "phase": "sanitize",
+ "phase_models": _PHASE_MODELS,
+ },
+ )
+ await handle_dream_batch_result(entry, [_landed("sanitize", _OPS)])
+
+
+def _landed(phase: str, output: BaseModel) -> BatchResultRow:
+ return BatchResultRow(
+ custom_id=f"p1:{phase}",
+ content=output.model_dump_json(),
+ input_tokens=10,
+ output_tokens=20,
+ )
+
+
+async def _assert_ended_without_applying(
+ fake_dream_db, fake_dream_redis, error: str
+) -> None:
+ """Ended as ``fail_pass`` ends a pass: its job errored with *error*, its
+ batch state and bundle cleaned, its apply gate spent."""
+ status = await job_status.read_status(kind="dream_pass", job_id="j1")
+ assert status is not None and (status.state, status.error) == ("errored", error)
+ assert state_key("p1") not in fake_dream_redis.hashes
+ assert input_bundle_key("p1") not in fake_dream_redis.store
+ assert fake_dream_redis.store["dream:applied:p1"] == "1"
+ assert DreamPassStatus.COMPLETE not in fake_dream_db.statuses("p1")
+
+
+def _charged(charges: AsyncMock) -> list[str]:
+ return [call.args[0].job.phase for call in charges.await_args_list]
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
index 6ab45831d1..f4b85c8bbd 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
@@ -20,7 +20,9 @@ The pass's Redis state and its at-most-once gates are ``batch_state.py``;
how the pass ends (JobStatus, the durable ``DreamPass`` record, the lock)
is ``batch_outcome.py``; what its phases cost and used is ``batch_costs.py``. Each
callback also advances the record: the phase that landed and its output,
-the next batch, the apply, and the end.
+the next batch, the apply, and the end. It ends a pass cancelled or expired
+meanwhile before it chains or claims the apply gate, and one that lost its
+scope's lock after the claim, before it applies (``cancel.py``).
"""
from __future__ import annotations
@@ -54,9 +56,16 @@ from .batch_state import (
read_state,
write_phase_to_state,
)
-from .batch_submit import PHASE_RESPONSE_MODELS, read_input_bundle, submit_phase
+from .batch_submit import (
+ PHASE_RESPONSE_MODELS,
+ read_input_bundle,
+ read_lock_token,
+ submit_phase,
+)
+from .cancel import end_batch_pass_if_lock_lost, end_batch_pass_if_stopped, pass_closed
from .clamp import clamp_operations
from .llm import parse_json_with_prose_fallback
+from .provider_batch import anthropic_api_key
from .schemas import DreamOperations, DreamPhase, IngestionDrainStatus
from .store import (
record_applying,
@@ -196,6 +205,9 @@ async def _handle_guarded(
scoped = bp
try:
input_bundle = await read_input_bundle(bp.pass_id)
+ if input_bundle is None and await pass_closed(bp.pass_id):
+ logger.debug("Dream pass %s ended; late delivery ignored", bp.pass_id)
+ return
if input_bundle is None:
logger.error(
"Dream batch input missing; refusing payload-only scope for pass=%s",
@@ -295,15 +307,17 @@ async def _landed_output(
async def _chain_next_phase(
bp: BatchPass, input_bundle: DreamInput, next_phase: DreamPhase
) -> None:
- """Submit the next phase in the chain.
+ """Submit the next phase in the chain, unless the pass was stopped.
Uses the validated ``DreamInput`` + accumulated prior phase outputs
from Redis, builds the next phase's prompt, fires another
batch submission. On any failure to submit, marks the JobStatus
errored — silent submission failures are unrecoverable.
"""
+ if await end_batch_pass_if_stopped(bp):
+ return
state = await read_state(bp.pass_id)
- api_key = _anthropic_api_key()
+ api_key = anthropic_api_key()
if api_key is None:
await fail_pass(bp, f"{next_phase}: no Anthropic API key configured")
return
@@ -400,14 +414,17 @@ async def _finalize_complete(bp: BatchPass, input_bundle: DreamInput) -> None:
async def _claim_apply(
bp: BatchPass, state: dict[str, dict[str, Any]], ops: DreamOperations
) -> bool:
- """Record APPLYING, then claim the apply gate; ``False`` once a duplicate
- delivery or an unreadable gate has ended this one.
-
- The record write comes first, so a delivery that stalls or dies on it
- has not claimed the gate and a redelivery still applies. Once the gate
- is claimed, apply follows with nothing awaited in between.
+ """Record APPLYING, check for a stop, claim the apply gate, check the
+ lock; ``False`` once a stop, a duplicate, an unreadable gate or a lost lock
+ has ended this delivery. A delivery that stalls or dies before the claim
+ has not claimed it, so a redelivery still applies. Between the claim and
+ apply only the lock is read (one GET), so a newer pass that took the scope
+ while this one waited is never applied over.
"""
await record_applying(bp.pass_id, ops)
+ lock_token = await read_lock_token(bp.pass_id)
+ if await end_batch_pass_if_stopped(bp):
+ return False
gate = await claim_apply_gate(bp.pass_id)
if gate == "error":
# We cannot tell first-vs-duplicate apart, and "complete with no
@@ -419,7 +436,7 @@ async def _claim_apply(
if gate == "duplicate":
await _finish_duplicate(bp, state, ops)
return False
- return True
+ return not await end_batch_pass_if_lock_lost(bp, lock_token)
async def _terminal_ops(
@@ -512,31 +529,6 @@ async def _finish_applied(
await best_effort_cleanup(bp.pass_id)
-def _anthropic_api_key() -> str | None:
- """Look up the Anthropic key from the copilot config first, then
- fall back to the shared settings key.
-
- Mirrors what ``backend/executor/batch_executor.py::_default_api_key_for``
- does — callbacks live in their own subprocess so they need to
- re-resolve the key on their own.
- """
- try:
- from backend.copilot.config import ChatConfig
-
- cfg = ChatConfig()
- if cfg.direct_anthropic_api_key:
- return cfg.direct_anthropic_api_key
- except Exception:
- logger.debug("ChatConfig unavailable during dream batch callback")
- try:
- from backend.util.settings import Settings
-
- key = Settings().secrets.anthropic_api_key
- return key or None
- except Exception:
- return None
-
-
def _register() -> None:
try:
from backend.executor.batch_executor import register_handler
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
index 7ead9f1f77..61b9d1f8ba 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
@@ -10,6 +10,8 @@ from __future__ import annotations
import asyncio
import json
+import logging
+from collections.abc import Awaitable, Callable
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
@@ -23,16 +25,19 @@ from prisma.enums import (
from backend.copilot.dream import job_status
from backend.copilot.dream.batch_callbacks import handle_dream_batch_result
-from backend.copilot.dream.batch_state import write_phase_to_state
+from backend.copilot.dream.batch_state import state_key, write_phase_to_state
from backend.copilot.dream.batch_submit import input_bundle_key, persist_input_bundle
+from backend.copilot.dream.cancel import cancel_dream_pass
from backend.copilot.dream.fetch import DreamInput
from backend.copilot.dream.job_status import read_status, write_initial_status
-from backend.copilot.dream.pass_record import dream_pass_result_from_row
+from backend.copilot.dream.pass_record import dream_pass_result_from_row, expired
from backend.copilot.dream.schemas import (
DreamOperations,
DreamOperationsSnapshot,
+ DreamPhase,
IngestionDrainStatus,
)
+from backend.copilot.dream.store import write_stop
from backend.copilot.graphiti.scope import MemoryScope
from backend.copilot.trial_cost_context import get_trial_cost_context
from backend.copilot.usage_activation import UsageActivationUnavailable
@@ -355,7 +360,7 @@ class TestPhaseChaining:
"backend.copilot.dream.job_status.update_status_phase", update_status
),
patch(
- "backend.copilot.dream.batch_callbacks._anthropic_api_key",
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
return_value="sk-ant-test",
),
):
@@ -403,7 +408,7 @@ class TestPhaseChaining:
with (
patch("backend.copilot.dream.batch_callbacks.submit_phase", submit_phase),
patch(
- "backend.copilot.dream.batch_callbacks._anthropic_api_key",
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
return_value="sk-ant-test",
),
patch("backend.copilot.dream.job_status.update_status_phase", AsyncMock()),
@@ -544,6 +549,9 @@ class TestPhaseChaining:
from backend.copilot.dream.batch_submit import persist_input_bundle
from backend.copilot.dream.fetch import DreamInput
+ _, _, string_store = fake_redis
+ expert_lock = MemoryScope.for_expert("u1", "expert-1").redis_key("dream_lock")
+ string_store[expert_lock] = "tok-expert"
now = datetime.now(timezone.utc)
await persist_input_bundle(
"p-expert",
@@ -613,6 +621,8 @@ class TestPhaseChaining:
from backend.copilot.dream.batch_submit import persist_input_bundle
from backend.copilot.dream.fetch import DreamInput
+ _, _, string_store = fake_redis
+ string_store["dream:inflight:u1"] = "tok-u1"
now = datetime.now(timezone.utc)
await persist_input_bundle(
"p1",
@@ -926,7 +936,7 @@ class TestErrorPaths:
patch("backend.copilot.dream.job_status.mark_errored", mark_errored),
patch("backend.copilot.dream.job_status.update_status_phase", AsyncMock()),
patch(
- "backend.copilot.dream.batch_callbacks._anthropic_api_key",
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
return_value="sk-ant-test",
),
):
@@ -1178,6 +1188,8 @@ class TestLockTokenWiring:
not route through the crash guard to fail_pass — both keys carry
24h TTLs, so cleanup is best-effort and the completed job stays
completed."""
+ _, _, string_store = fake_redis
+ string_store["dream:inflight:u1"] = "tok-u1"
await self._seed_terminal_pass()
mark_complete = AsyncMock()
@@ -1345,7 +1357,7 @@ class TestDreamPassRecord:
with (
patch("backend.copilot.dream.batch_callbacks.submit_phase", submit_phase),
patch(
- "backend.copilot.dream.batch_callbacks._anthropic_api_key",
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
return_value="sk-ant-test",
),
patch("backend.copilot.dream.job_status.update_status_phase", AsyncMock()),
@@ -1440,6 +1452,8 @@ class TestDreamPassRecord:
async def test_an_apply_crash_closes_the_row_errored_after_applying(
self, fake_redis, fake_dream_db
):
+ _, _, string_store = fake_redis
+ string_store["dream:inflight:u1"] = "tok-u1"
await _persist_autopilot_bundle()
_seed_submitted_pass(fake_dream_db)
with (
@@ -1545,7 +1559,7 @@ class TestDreamPassRecord:
with (
patch("backend.copilot.dream.batch_callbacks.submit_phase", submit_phase),
patch(
- "backend.copilot.dream.batch_callbacks._anthropic_api_key",
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
return_value="sk-ant-test",
),
):
@@ -1569,6 +1583,9 @@ class TestDreamPassRecord:
):
"""A delivery that dies while its APPLYING write is in flight has not
claimed the apply gate yet, so the next delivery still applies."""
+ fake_dream_redis.store[MemoryScope.for_user("u1").redis_key("dream_lock")] = (
+ "tok-u1"
+ )
await _persist_autopilot_bundle()
_seed_submitted_pass(fake_dream_db)
for phase, content in (
@@ -1666,7 +1683,7 @@ class TestDreamPassRecord:
),
patch("backend.copilot.dream.batch_callbacks.submit_phase", submit_phase),
patch(
- "backend.copilot.dream.batch_callbacks._anthropic_api_key",
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
return_value="sk-ant-test",
),
patch("backend.copilot.dream.job_status.update_status_phase", AsyncMock()),
@@ -1678,14 +1695,17 @@ class TestDreamPassRecord:
"p1",
"p2",
}
- # Each callback's two writes (its output, its next batch), each
- # abandoned at the deadline.
- assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (4, 4)
+ # Each callback's two writes (its output, its next batch) and its
+ # stop check's read, each abandoned at the deadline; the check that
+ # got no answer let the chain go on.
+ assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (6, 6)
@pytest.mark.asyncio
async def test_a_store_outage_never_fails_a_batch_pass(
self, fake_redis, fake_dream_db
):
+ _, _, string_store = fake_redis
+ string_store["dream:inflight:u1"] = "tok-u1"
await _persist_autopilot_bundle()
_seed_submitted_pass(fake_dream_db)
fake_dream_db.fail = True
@@ -1717,6 +1737,293 @@ class TestDreamPassRecord:
assert fake_dream_db.writes == []
+async def _cancel_p1() -> str:
+ assert (await cancel_dream_pass("p1", user_id="u1", reason="testing")).cancelled
+ return "cancelled: testing"
+
+
+async def _expire_p1() -> str:
+ assert await write_stop("p1", expired("lease lapsed", not_updated_since=None))
+ return "expired: lease lapsed"
+
+
+class TestAStoppedPass:
+ """A pass whose row was cancelled, or expired by a newer pass's guard,
+ while its batch was in flight: the callback that lands next ends it before
+ it chains the next phase or claims the apply gate, through ``fail_pass``,
+ after asking the provider to cancel the batch the row names."""
+
+ @pytest.fixture
+ def anthropic(self):
+ """The Anthropic client the provider cancel builds."""
+ client = MagicMock()
+ client.messages.batches.cancel = AsyncMock()
+ with (
+ patch(
+ "backend.copilot.dream.provider_batch.anthropic_api_key",
+ return_value="sk-ant-test",
+ ),
+ patch(
+ "backend.util.llm.providers.anthropic.AsyncAnthropic",
+ return_value=client,
+ ),
+ ):
+ yield client
+
+ @pytest.fixture
+ def charges(self):
+ charge = AsyncMock()
+ with patch("backend.copilot.dream.batch_costs.record_phase_cost", charge):
+ yield charge
+
+ async def _in_flight(
+ self, fake_dream_db, fake_dream_redis, landed: tuple[DreamPhase, ...]
+ ) -> str:
+ """A batch pass on batch ``msgbatch_live``: its bundle with the lock
+ token, the lock it holds, its admin job row and the phases that
+ landed before this callback. Returns the lock's key."""
+ now = datetime.now(timezone.utc)
+ await persist_input_bundle(
+ "p1",
+ DreamInput(
+ user_id="u1", group_id="user_u1", window_start=now, window_end=now
+ ),
+ lock_token="tok-u1",
+ )
+ lock_key = MemoryScope.for_user("u1").redis_key("dream_lock")
+ fake_dream_redis.store[lock_key] = "tok-u1"
+ await job_status.write_initial_status(
+ kind="dream_pass", job_id="j1", user_id="u1"
+ )
+ contents = {
+ "consolidate": _CONSOLIDATE_CONTENT,
+ "recombine": _RECOMBINE_CONTENT,
+ }
+ for phase in landed:
+ await write_phase_to_state(
+ pass_id="p1",
+ phase=phase,
+ row=_row(custom_id=f"p1:{phase}", content=contents[phase]),
+ )
+ fake_dream_db.seed(
+ DreamPassDraft(
+ id="p1",
+ user_id="u1",
+ scope_key="u1",
+ route=DreamPassRoute.ANTHROPIC_BATCH,
+ trigger=DreamPassTrigger.CRON,
+ status=DreamPassStatus.SUBMITTED,
+ phase=DreamPassPhase.CONSOLIDATE,
+ ),
+ provider_batch_id="msgbatch_live",
+ )
+ return lock_key
+
+ async def _assert_ended(
+ self, fake_dream_db, fake_dream_redis, lock_key: str, error: str
+ ) -> None:
+ """Ended as ``fail_pass`` ends a pass: the job errored with the stop,
+ the lock released, the batch state and bundle cleaned, and the row
+ left closed as the stop closed it."""
+ status = await job_status.read_status(kind="dream_pass", job_id="j1")
+ assert status is not None and (status.state, status.error) == (
+ "errored",
+ error,
+ )
+ assert lock_key not in fake_dream_redis.store
+ assert state_key("p1") not in fake_dream_redis.hashes
+ assert input_bundle_key("p1") not in fake_dream_redis.store
+ assert fake_dream_db.rows["p1"]["cancel_generation"] == 1
+ assert DreamPassStatus.ERRORED not in fake_dream_db.statuses("p1")
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "stop", [_cancel_p1, _expire_p1], ids=["cancelled", "expired"]
+ )
+ async def test_it_does_not_chain_its_next_phase(
+ self,
+ fake_dream_db,
+ fake_dream_redis,
+ anthropic,
+ charges,
+ stop: Callable[[], Awaitable[str]],
+ ):
+ lock_key = await self._in_flight(fake_dream_db, fake_dream_redis, landed=())
+ error = await stop()
+ submit_phase = AsyncMock()
+
+ with (
+ patch("backend.copilot.dream.batch_callbacks.submit_phase", submit_phase),
+ patch(
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
+ return_value="sk-ant-test",
+ ),
+ ):
+ await handle_dream_batch_result(
+ _entry(phase="consolidate"),
+ [_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT)],
+ )
+
+ submit_phase.assert_not_awaited()
+ anthropic.messages.batches.cancel.assert_awaited_once_with("msgbatch_live")
+ # The phase that landed was billed at the provider, so it is charged.
+ assert [call.args[0].job.phase for call in charges.await_args_list] == [
+ "consolidate"
+ ]
+ await self._assert_ended(fake_dream_db, fake_dream_redis, lock_key, error)
+
+ @pytest.mark.asyncio
+ async def test_an_unattributed_callback_ends_it_charging_nothing(
+ self, fake_dream_db, fake_dream_redis, anthropic, charges
+ ):
+ """A cancelled pass whose next callback carries no usage attribution
+ (#15208) ends as any failed pass ends, through ``fail_pass``, but
+ charges nothing, not even the phase that landed: it has no usage
+ context of its own to charge it under. The row stays as the cancel
+ closed it."""
+ lock_key = await self._in_flight(
+ fake_dream_db, fake_dream_redis, landed=("consolidate",)
+ )
+ await _cancel_p1()
+ entry = _entry(phase="recombine")
+ del entry.payload["cost_context"]
+
+ with pytest.raises(UsageActivationUnavailable):
+ await handle_dream_batch_result(
+ entry, [_row(custom_id="p1:recombine", content=_RECOMBINE_CONTENT)]
+ )
+
+ charges.assert_not_awaited()
+ anthropic.messages.batches.cancel.assert_not_awaited()
+ await self._assert_ended(
+ fake_dream_db,
+ fake_dream_redis,
+ lock_key,
+ "batch usage attribution unavailable",
+ )
+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.CANCELLED
+
+ @pytest.mark.asyncio
+ async def test_a_replayed_callback_keeps_the_job_it_ended(
+ self, fake_dream_db, fake_dream_redis, anthropic, charges
+ ):
+ """The same consolidate batch delivered again after the stop ended the
+ pass and dropped its bundle: the job keeps the stop's error, nothing
+ is charged or cancelled twice."""
+ await self._in_flight(fake_dream_db, fake_dream_redis, landed=())
+ error = await _cancel_p1()
+ entry = _entry(phase="consolidate")
+ rows = [_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT)]
+ with (
+ patch("backend.copilot.dream.batch_callbacks.submit_phase", AsyncMock()),
+ patch(
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
+ return_value="sk-ant-test",
+ ),
+ ):
+ await handle_dream_batch_result(entry, rows)
+ await handle_dream_batch_result(entry, rows)
+
+ status = await job_status.read_status(kind="dream_pass", job_id="j1")
+ assert status is not None and (status.state, status.error) == (
+ "errored",
+ error,
+ )
+ anthropic.messages.batches.cancel.assert_awaited_once_with("msgbatch_live")
+ assert [call.args[0].job.phase for call in charges.await_args_list] == [
+ "consolidate"
+ ]
+ assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.CANCELLED
+
+ @pytest.mark.asyncio
+ async def test_it_does_not_claim_the_apply_gate(
+ self, fake_dream_db, fake_dream_redis, anthropic, charges
+ ):
+ lock_key = await self._in_flight(
+ fake_dream_db, fake_dream_redis, landed=("consolidate", "recombine")
+ )
+ error = await _cancel_p1()
+ apply = AsyncMock()
+
+ with patch("backend.copilot.dream.apply.apply_operations", apply):
+ await handle_dream_batch_result(
+ _entry(phase="sanitize"),
+ [_row(custom_id="p1:sanitize", content=_SANITIZE_CONTENT)],
+ )
+
+ apply.assert_not_awaited()
+ assert "dream:applied:p1" not in fake_dream_redis.store
+ anthropic.messages.batches.cancel.assert_awaited_once_with("msgbatch_live")
+ assert [call.args[0].job.phase for call in charges.await_args_list] == [
+ "consolidate",
+ "recombine",
+ "sanitize",
+ ]
+ await self._assert_ended(fake_dream_db, fake_dream_redis, lock_key, error)
+
+ @pytest.mark.asyncio
+ async def test_a_cancel_landing_as_applying_is_recorded_is_caught_before_the_gate(
+ self, fake_dream_db, fake_dream_redis, anthropic, charges
+ ):
+ """The check reads the row after the APPLYING write, so a cancel that
+ lands while that write is in flight still stops the pass."""
+ lock_key = await self._in_flight(
+ fake_dream_db, fake_dream_redis, landed=("consolidate", "recombine")
+ )
+ record = fake_dream_db.update_dream_pass
+
+ async def cancelled_while_applying(pass_id, update):
+ written = await record(pass_id, update)
+ if update.status is DreamPassStatus.APPLYING:
+ await _cancel_p1()
+ return written
+
+ apply = AsyncMock()
+ with (
+ patch.object(fake_dream_db, "update_dream_pass", cancelled_while_applying),
+ patch("backend.copilot.dream.apply.apply_operations", apply),
+ ):
+ await handle_dream_batch_result(
+ _entry(phase="sanitize"),
+ [_row(custom_id="p1:sanitize", content=_SANITIZE_CONTENT)],
+ )
+
+ apply.assert_not_awaited()
+ assert "dream:applied:p1" not in fake_dream_redis.store
+ assert fake_dream_db.statuses("p1") == [
+ DreamPassStatus.APPLYING,
+ DreamPassStatus.CANCELLED,
+ ]
+ await self._assert_ended(
+ fake_dream_db, fake_dream_redis, lock_key, "cancelled: testing"
+ )
+
+ @pytest.mark.asyncio
+ async def test_a_provider_cancel_that_fails_is_logged_and_the_pass_ends(
+ self, fake_dream_db, fake_dream_redis, anthropic, charges, caplog
+ ):
+ lock_key = await self._in_flight(fake_dream_db, fake_dream_redis, landed=())
+ error = await _cancel_p1()
+ anthropic.messages.batches.cancel.side_effect = RuntimeError(
+ "batch has already ended"
+ )
+ submit_phase = AsyncMock()
+
+ with (
+ caplog.at_level(logging.WARNING),
+ patch("backend.copilot.dream.batch_callbacks.submit_phase", submit_phase),
+ ):
+ await handle_dream_batch_result(
+ _entry(phase="consolidate"),
+ [_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT)],
+ )
+
+ anthropic.messages.batches.cancel.assert_awaited_once_with("msgbatch_live")
+ assert "did not cancel dream batch msgbatch_live" in caplog.text
+ submit_phase.assert_not_awaited()
+ await self._assert_ended(fake_dream_db, fake_dream_redis, lock_key, error)
+
+
@pytest.mark.asyncio
async def test_callback_restores_original_trial_generation_for_late_result():
entry = _entry()
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py b/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py
index 7385f4d4b6..94a25eec47 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py
@@ -7,6 +7,11 @@ The BatchExecutor polls; ``batch_callbacks`` submits phases 2 (recombine) and
The dream lock goes with the pass: extended to the batch window here and
released by the callbacks when the pass ends.
+A pass whose row was cancelled (or expired) while it gathered submits
+nothing; one whose row refuses its submit (closed meanwhile, or gone) cancels
+that batch here and keeps its lock to release on the way out, rather than
+handing both to callbacks that would only stop.
+
Any caller can take this route (it depends on the flag and the deployment's
key, not the caller). ``status_id`` ties the callbacks' updates to the
JobStatus row an admin trigger created; without one the callbacks skip the
@@ -18,20 +23,26 @@ import logging
from backend.copilot.config import ChatConfig
from backend.executor.batch_executor import remove_pending
+from .batch_state import best_effort_cleanup
from .batch_submit import (
delete_input_bundle,
persist_input_bundle,
phase_models_for_config,
submit_phase,
)
+from .cancel import stopped_error
from .fetch import DreamInput
from .locks import BATCH_LOCK_TTL_SECONDS, DreamLockHandle
from .pass_run import DreamPassRun
+from .provider_batch import cancel_provider_batch
from .schemas import DreamPassResult
from .store import record_submitted
logger = logging.getLogger(__name__)
+# How a pass ends whose row refused its submit and cannot say why.
+REFUSED_SUBMIT_ERROR = "stopped: row refused the submitted write"
+
async def submit_dream_pass_batch(
run: DreamPassRun,
@@ -47,6 +58,9 @@ async def submit_dream_pass_batch(
# Shouldn't happen — routing.py only picks anthropic_batch when
# the key is present. Guard for type-narrowing + future safety.
return run.failure("anthropic_batch: no Anthropic API key (routing bug)")
+ stopped = await stopped_error(run.pass_id)
+ if stopped is not None:
+ return run.failure(stopped)
# Persist DreamInput so the per-phase callbacks can rebuild the
# next phase's prompt without re-fetching from Postgres + FalkorDB.
@@ -81,7 +95,8 @@ async def _hand_off(
) -> DreamPassResult:
"""Phase 1 is enqueued — hand the dream lock to the batch callback so it
spans the full async lifetime (apply runs hours later). Extend the TTL to
- the batch window first; the callback releases it on terminal/failure.
+ the batch window and record the submit first; the callback releases the
+ lock on terminal/failure.
A failed extend means the lock expired before the handoff — a newer pass
may already own the graph, so the just-submitted batch must never be
@@ -89,6 +104,10 @@ async def _hand_off(
callback chain) and drop the input bundle. The provider batch is orphaned;
its results are discarded. The lock is NOT disowned, so the context
manager's compare-and-delete release stays a safe no-op.
+
+ A row that refuses the submit (a stop closed it meanwhile, or it is gone)
+ gets the same revoke, the provider batch cancelled too, and keeps the lock
+ to release on the way out.
"""
logger.info(
"Dream pass %s submitted via Anthropic batch=%s (phase=consolidate)",
@@ -101,12 +120,63 @@ async def _hand_off(
return run.failure(
"anthropic_batch: dream lock lost before handoff — batch revoked"
)
+ stopped = await _record_submit(run, lock_handle, provider_batch_id, input_bundle)
+ if stopped is not None:
+ await _revoke_stopped(run.pass_id, provider_batch_id)
+ return run.failure(stopped)
lock_handle.disown()
- await record_submitted(
+ return run.handed_off()
+
+
+async def _record_submit(
+ run: DreamPassRun,
+ lock_handle: DreamLockHandle,
+ provider_batch_id: str,
+ input_bundle: DreamInput,
+) -> str | None:
+ """Record the submit on the pass's row; why the pass must not be handed
+ off, else ``None``.
+
+ A refused write is authoritative: the row has closed or is gone, so the
+ pass never hands off, its error the stop that closed the row or, when the
+ row cannot say, ``REFUSED_SUBMIT_ERROR``. A write that failed or ran out
+ of time says nothing about the row: the pass hands off, logged, unless its
+ row reads stopped."""
+ recorded = await record_submitted(
run.pass_id,
input_bundle=input_bundle,
provider_batch_id=provider_batch_id,
lease_token=lock_handle.token,
lease_ttl_seconds=BATCH_LOCK_TTL_SECONDS,
)
- return run.handed_off()
+ if recorded:
+ return None
+ stopped = await stopped_error(run.pass_id)
+ if stopped is not None:
+ return stopped
+ if recorded is False:
+ logger.warning(
+ f"Dream pass {run.pass_id}: its row refused batch {provider_batch_id} "
+ "and does not say why; not handing it off"
+ )
+ return REFUSED_SUBMIT_ERROR
+ logger.warning(
+ f"Dream pass {run.pass_id}: batch {provider_batch_id} is not on its "
+ "record; handing it to the callbacks anyway"
+ )
+ return None
+
+
+async def _revoke_stopped(pass_id: str, provider_batch_id: str) -> None:
+ """Take back a stopped pass's just-submitted batch: off the executor's
+ queue, cancelled at the provider (best-effort), its bundle dropped."""
+ logger.info(f"Dream pass {pass_id} stopped at its handoff; revoking batch")
+ try:
+ await remove_pending(provider_batch_id)
+ except Exception:
+ logger.warning(
+ f"Dream pass {pass_id}: could not revoke batch {provider_batch_id}",
+ exc_info=True,
+ )
+ await cancel_provider_batch(provider_batch_id)
+ await best_effort_cleanup(pass_id)
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py
new file mode 100644
index 0000000000..9c6bf42bee
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py
@@ -0,0 +1,233 @@
+"""The batch route's handoff meeting a stop, through the sync entry point with
+the real dream lock over the in-memory Redis and the pass's row in the
+in-memory store: a pass cancelled while it gathered submits nothing, and one
+cancelled while it submitted takes its batch back and releases its lock
+instead of handing both to callbacks that would only stop."""
+
+import logging
+from datetime import datetime, timedelta, timezone
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from prisma.enums import DreamPassStatus
+
+from backend.copilot.graphiti.scope import MemoryScope
+
+from . import batch_handoff as batch_handoff_mod
+from . import orchestrator as orchestrator_mod
+from .batch_submit import input_bundle_key
+from .cancel import cancel_dream_pass
+from .fetch import DreamInput, EpisodeRow
+from .locks import BATCH_LOCK_TTL_SECONDS
+
+_SCOPE = MemoryScope.for_user("u")
+_LOCK_KEY = _SCOPE.redis_key("dream_lock")
+
+
+def gather(mocker) -> AsyncMock:
+ """A pass on the batch route, everything up to the provider stubbed; its
+ gather returned so a test can cancel the pass while it gathers."""
+ mocker.patch.object(
+ orchestrator_mod, "resolve_dream_execution_path", return_value="anthropic_batch"
+ )
+ config = MagicMock()
+ config.direct_anthropic_api_key = "key"
+ mocker.patch.object(orchestrator_mod, "ChatConfig", return_value=config)
+ mocker.patch.object(
+ orchestrator_mod, "is_feature_enabled", AsyncMock(return_value=False)
+ )
+ mocker.patch.object(
+ orchestrator_mod, "check_dream_budget", AsyncMock(return_value=(True, None))
+ )
+ mocker.patch.object(batch_handoff_mod, "phase_models_for_config", return_value={})
+ return mocker.patch.object(
+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_input())
+ )
+
+
+def provider(mocker) -> MagicMock:
+ """The provider side: the submit, the executor's queue and the cancel."""
+ sides = MagicMock()
+ sides.submit = mocker.patch.object(
+ batch_handoff_mod,
+ "submit_phase",
+ AsyncMock(return_value=MagicMock(provider_batch_id="batch-1")),
+ )
+ sides.revoke = mocker.patch.object(batch_handoff_mod, "remove_pending", AsyncMock())
+ sides.cancel = mocker.patch.object(
+ batch_handoff_mod, "cancel_provider_batch", AsyncMock()
+ )
+ return sides
+
+
+async def test_a_pass_cancelled_while_it_gathered_submits_nothing(
+ fake_dream_db, fake_dream_redis, gather, provider
+):
+ async def cancelled_gather(scope: MemoryScope) -> DreamInput:
+ await _cancel_the_pass(fake_dream_db)
+ return _input()
+
+ gather.side_effect = cancelled_gather
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ provider.submit.assert_not_awaited()
+ assert (result.error, result.execution_path) == (
+ "cancelled: testing",
+ "anthropic_batch",
+ )
+ assert input_bundle_key(result.pass_id) not in fake_dream_redis.store
+ assert _LOCK_KEY not in fake_dream_redis.store
+ _assert_cancelled(fake_dream_db, result.pass_id)
+
+
+async def test_a_pass_cancelled_while_it_submitted_takes_its_batch_back(
+ fake_dream_db, fake_dream_redis, gather, provider
+):
+ """The batch was submitted before the cancel reached the row, which then
+ refuses the submit: the batch is revoked from the executor and cancelled
+ at the provider, the bundle dropped, the lock released on the way out."""
+
+ async def submitted_then_cancelled(**_kwargs) -> MagicMock:
+ await _cancel_the_pass(fake_dream_db)
+ return MagicMock(provider_batch_id="batch-1")
+
+ provider.submit.side_effect = submitted_then_cancelled
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ provider.submit.assert_awaited_once()
+ provider.revoke.assert_awaited_once_with("batch-1")
+ provider.cancel.assert_awaited_once_with("batch-1")
+ assert result.error == "cancelled: testing"
+ assert input_bundle_key(result.pass_id) not in fake_dream_redis.store
+ assert _LOCK_KEY not in fake_dream_redis.store
+ _assert_cancelled(fake_dream_db, result.pass_id)
+ assert fake_dream_db.rows[result.pass_id].get("provider_batch_id") is None
+
+
+async def test_a_pass_nobody_stopped_records_its_batch_then_hands_off_its_lock(
+ fake_dream_db, fake_dream_redis, gather, provider
+):
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error is None and not result.skipped
+ row = fake_dream_db.rows[result.pass_id]
+ assert (row["status"], row["provider_batch_id"]) == (
+ DreamPassStatus.SUBMITTED,
+ "batch-1",
+ )
+ assert fake_dream_redis.ttls[_LOCK_KEY] == BATCH_LOCK_TTL_SECONDS
+ provider.revoke.assert_not_awaited()
+ provider.cancel.assert_not_awaited()
+
+
+async def test_a_submit_write_that_fails_on_an_open_row_still_hands_off(
+ fake_dream_db, fake_dream_redis, gather, provider, mocker, caplog
+):
+ record = fake_dream_db.update_dream_pass
+
+ async def submit_write_fails(pass_id, update):
+ if update.status is DreamPassStatus.SUBMITTED:
+ raise ConnectionError("dream pass database unreachable")
+ return await record(pass_id, update)
+
+ mocker.patch.object(fake_dream_db, "update_dream_pass", submit_write_fails)
+
+ with caplog.at_level(logging.WARNING, logger=batch_handoff_mod.logger.name):
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error is None
+ assert "batch batch-1 is not on its record" in caplog.text
+ assert fake_dream_redis.ttls[_LOCK_KEY] == BATCH_LOCK_TTL_SECONDS
+ provider.cancel.assert_not_awaited()
+
+
+async def test_a_refused_submit_whose_row_cannot_be_read_is_not_handed_off(
+ fake_dream_db, fake_dream_redis, gather, provider, mocker, caplog
+):
+ """The row refused the submit (a cancel closed it meanwhile) and the
+ read that would say why fails: a refusal is authoritative, so the pass
+ stops as if the row had said so, and the reason it could not read is
+ logged."""
+
+ async def submitted_then_cancelled_unreadably(**_kwargs) -> MagicMock:
+ await _cancel_the_pass(fake_dream_db)
+ mocker.patch.object(
+ fake_dream_db,
+ "get_dream_pass",
+ AsyncMock(side_effect=ConnectionError("dream pass database unreachable")),
+ )
+ return MagicMock(provider_batch_id="batch-1")
+
+ provider.submit.side_effect = submitted_then_cancelled_unreadably
+
+ with caplog.at_level(logging.WARNING, logger=batch_handoff_mod.logger.name):
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error == batch_handoff_mod.REFUSED_SUBMIT_ERROR
+ assert "refused batch batch-1 and does not say why" in caplog.text
+ _assert_revoked(provider, fake_dream_redis, result.pass_id)
+ _assert_cancelled(fake_dream_db, result.pass_id)
+
+
+async def test_a_pass_whose_row_is_gone_is_not_handed_off(
+ fake_dream_db, fake_dream_redis, gather, provider, mocker
+):
+ """A row that was never inserted (or was deleted with its user or
+ expert) refuses the submit too: the pass has nothing that could stop it
+ later, so it stops here."""
+ mocker.patch.object(
+ fake_dream_db,
+ "create_dream_pass",
+ AsyncMock(side_effect=ConnectionError("dream pass database unreachable")),
+ )
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error == batch_handoff_mod.REFUSED_SUBMIT_ERROR
+ _assert_revoked(provider, fake_dream_redis, result.pass_id)
+
+
+def _assert_revoked(provider: MagicMock, fake_dream_redis, pass_id: str) -> None:
+ """The batch taken back: off the executor's queue and cancelled at the
+ provider, the bundle dropped, the lock released on the way out."""
+ provider.revoke.assert_awaited_once_with("batch-1")
+ provider.cancel.assert_awaited_once_with("batch-1")
+ assert input_bundle_key(pass_id) not in fake_dream_redis.store
+ assert _LOCK_KEY not in fake_dream_redis.store
+
+
+async def _cancel_the_pass(fake_dream_db) -> None:
+ [pass_id] = list(fake_dream_db.rows)
+ assert (await cancel_dream_pass(pass_id, user_id="u", reason="testing")).cancelled
+
+
+def _assert_cancelled(fake_dream_db, pass_id: str) -> None:
+ row = fake_dream_db.rows[pass_id]
+ assert (row["status"], row["error"], row["cancel_generation"]) == (
+ DreamPassStatus.CANCELLED,
+ "testing",
+ 1,
+ )
+
+
+def _input() -> DreamInput:
+ now = datetime.now(timezone.utc)
+ episode = EpisodeRow(
+ uuid="e1",
+ name=None,
+ content="hello",
+ source_description=None,
+ valid_at=None,
+ created_at=None,
+ )
+ return DreamInput(
+ user_id="u",
+ group_id=_SCOPE.group_id,
+ window_start=now - timedelta(days=14),
+ window_end=now,
+ episodes=[episode],
+ )
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py b/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py
index c5cd0e84df..d0d710e259 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py
@@ -4,7 +4,9 @@ to its durable DreamPass record, and the lock it gives back.
A batch pass ends in a callback, in another process, long after it started:
``fail_pass`` for every failure, ``record_completion`` once apply has run.
Either way the record gets the usage of every phase that landed, read off the
-pass's Redis state and priced in ``batch_costs.py``.
+pass's Redis state and priced in ``batch_costs.py``, unless a stop closed the
+record first: a closed record refuses the write, and that usage lives on only
+in the cost log.
"""
from __future__ import annotations
@@ -61,17 +63,20 @@ class BatchPass(BaseModel):
)
-async def fail_pass(bp: BatchPass, error: str, *, charge: bool = True) -> None:
+async def fail_pass(
+ bp: BatchPass, error: str, *, holds_lock: bool = True, charge: bool = True
+) -> None:
"""Close the pass errored: its admin job and its record first, then
- charge the phases that landed, release the lock and clean up per-pass
+ charge the phases that landed, release the lock (unless the pass lost it,
+ *holds_lock* false: it is then another pass's) and clean up per-pass
state. Each step is best-effort on its own, so one that fails never
stops the ones after it. A pass that cannot be charged under its own
usage attribution passes ``charge=False``: its record still shows what
the landed phases used, and nothing is charged under another one.
The record carries the usage of the phases that landed, read off the
- pass's state; a state that cannot be read leaves that usage unknown
- rather than the record open.
+ pass's state, unless a stop already closed it; a state that cannot be
+ read leaves that usage unknown rather than the record open.
We incurred the provider tokens for completed phases regardless of
whether the whole pass landed, so they're recorded against the
@@ -87,8 +92,8 @@ async def fail_pass(bp: BatchPass, error: str, *, charge: bool = True) -> None:
)
if state and charge:
await _charge_landed_phases(bp, state)
- # Release the dream lock the batch path disowned to this callback.
- await release_lock(bp)
+ if holds_lock:
+ await release_lock(bp)
await best_effort_cleanup(bp.pass_id)
diff --git a/autogpt_platform/backend/backend/copilot/dream/cancel.py b/autogpt_platform/backend/backend/copilot/dream/cancel.py
new file mode 100644
index 0000000000..48a7948e30
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/cancel.py
@@ -0,0 +1,190 @@
+"""The stop path of a dream pass: cancel it, and notice a stop while it runs.
+
+``cancel_dream_pass`` closes the pass's row CANCELLED in one conditional
+transition that also bumps its cancel generation, and only while the row is
+open and the caller's own. It does not reach into the running pass; the pass
+reads its row at its next check and stops itself:
+
+ * a sync pass at each phase boundary and once more just before apply
+ (``stop_if_stopped``, ``stop_before_apply``) ends with its failure result:
+ the usage of the phases billed so far kept, nothing applied, its lock
+ released on the way out like any other failure;
+ * a pass about to submit its first batch submits nothing, and one whose row
+ refuses its submit cancels that batch instead of handing it on
+ (``batch_handoff``);
+ * a batch pass in its callback, before it chains the next phase and before
+ it claims the apply gate (``end_batch_pass_if_stopped``), cancels its
+ provider batch best-effort and ends through ``fail_pass``: its landed
+ phases charged, its lock released, its batch state and bundle cleaned.
+
+A cancel that lands after a pass's last check, while it claims apply or
+applies, is too late: that apply runs, once, and the row stays CANCELLED.
+
+A row a newer pass's guard expired (``guard.py``) has its generation bumped the
+same way, so its pass stops at the same checks. And right before apply each
+route checks that it still holds its scope's lock (``stop_before_apply``,
+``end_batch_pass_if_lock_lost``): a newer pass takes the scope only once
+this one's lock has lapsed, so a pass that lost it never applies over the
+newer one. A closed row takes no other write, so the failure each route
+records on the way out leaves it CANCELLED (or EXPIRED). The stop checks read
+the row under the store's deadline and never stop a pass on a read that fails:
+the store being down must not end work that nobody cancelled. The lock check
+fails closed: a pass that cannot confirm its lock does not apply.
+"""
+
+import logging
+from collections.abc import Awaitable
+
+from pydantic import BaseModel
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.dream_pass_models import OPEN_STATUSES, DreamPassRecord
+
+from .batch_outcome import BatchPass, fail_pass
+from .locks import DreamLockHandle, dream_lock_held_by
+from .pass_record import cancelled, stop_error
+from .pass_run import DreamPassRun, PassEnded
+from .provider_batch import cancel_provider_batch
+from .store import read_dream_pass, read_open_passes, read_pass, write_stop
+
+logger = logging.getLogger(__name__)
+
+# How a pass that lost its scope's lock before apply ends.
+LOCK_LOST_ERROR = "apply: dream lock lost before apply"
+
+
+class DreamPassCancel(BaseModel):
+ """What a cancel did: ``record`` is the pass's row after it (``None`` when
+ the user has no such pass) and ``cancelled`` whether this call closed it;
+ a row that is there but was not closed had already ended."""
+
+ cancelled: bool
+ record: DreamPassRecord | None
+
+
+async def cancel_dream_pass(
+ pass_id: str, *, user_id: str, reason: str
+) -> DreamPassCancel:
+ """Cancel *user_id*'s pass *pass_id* while it is open, then read its row
+ back. Unlike a pass's own writes, a store call that fails or runs out of
+ time raises: the caller is answering a request, or about to erase the
+ memory the pass would write."""
+ closed = await write_stop(pass_id, cancelled(reason, owner_user_id=user_id))
+ record = await read_dream_pass(pass_id, user_id=user_id)
+ if closed:
+ logger.info(f"Dream pass {pass_id} cancelled: {reason}")
+ return DreamPassCancel(cancelled=closed, record=record)
+
+
+async def cancel_open_passes(
+ scope: MemoryScope, *, user_id: str, reason: str
+) -> list[DreamPassCancel]:
+ """Cancel every open pass of *scope*, one ``cancel_dream_pass`` each, for
+ the wipe to call before it erases the scope's memory. Raises when the
+ store cannot list or cancel them, so the wipe does not go on unsure.
+
+ This closes the rows; it is not a barrier. Each pass stops at its next
+ check, and one already past its last check still applies, so a wipe must
+ also wait for the scope's lock before it erases."""
+ rows = await read_open_passes(scope)
+ return [
+ await cancel_dream_pass(row.id, user_id=user_id, reason=reason) for row in rows
+ ]
+
+
+async def stopped_error(pass_id: str) -> str | None:
+ """Why the pass's row says it was stopped from outside; ``None`` when it
+ was not, or when the store cannot say in time and the pass goes on."""
+ row = await _read_row(pass_id)
+ return stop_error(row) if row is not None else None
+
+
+async def pass_closed(pass_id: str) -> bool:
+ """Whether the pass's row has ended; ``False`` when there is no row or the
+ store cannot say in time."""
+ row = await _read_row(pass_id)
+ return row is not None and row.status not in OPEN_STATUSES
+
+
+async def stop_if_stopped(run: DreamPassRun) -> None:
+ """Raise ``PassEnded`` with the run's failure when its row says it was
+ stopped from outside; the sync pass's check before each phase."""
+ error = await stopped_error(run.pass_id)
+ if error is None:
+ return
+ logger.info(f"Dream pass {run.pass_id} stops: {error}")
+ raise PassEnded(run.failure(error))
+
+
+async def stop_before_apply(run: DreamPassRun, lock_handle: DreamLockHandle) -> None:
+ """The sync pass's last checks before apply: its row shows no stop and it
+ still holds its scope's lock; else raise ``PassEnded`` with its failure."""
+ await stop_if_stopped(run)
+ if not await _lock_held(lock_handle.held(), run.pass_id):
+ raise PassEnded(run.failure(LOCK_LOST_ERROR))
+
+
+async def end_batch_pass_if_stopped(bp: BatchPass) -> bool:
+ """End the batch pass when its row says it was stopped from outside, and
+ say whether it did; the callback's check before it chains the next phase
+ and before it claims the apply gate.
+
+ The batch the row names is cancelled at the provider first. By the time a
+ callback runs, that is usually the batch that just ended, which refuses;
+ it matters when the row names a later one still in flight."""
+ row = await _read_row(bp.pass_id)
+ error = stop_error(row) if row is not None else None
+ if row is None or error is None:
+ return False
+ logger.info(f"Dream batch pass {bp.pass_id} stops: {error}")
+ if row.provider_batch_id:
+ await cancel_provider_batch(row.provider_batch_id)
+ await fail_pass(bp, error)
+ return True
+
+
+async def end_batch_pass_if_lock_lost(bp: BatchPass, lock_token: str | None) -> bool:
+ """The batch pass's check once its apply gate is claimed, with nothing
+ durable awaited before it: end the pass and say so when it no longer
+ holds its scope's lock under *lock_token*.
+
+ Its landed phases are charged and its state and bundle cleaned. A lock
+ that is no longer the pass's is left alone; one that could not be read is
+ released by compare-and-delete, which only ever deletes the pass's own.
+ The job and the record say why: the stop that closed the row (a newer
+ pass's expiry), else that the lock was lost."""
+ scope = MemoryScope.build(bp.user_id, bp.expert_id)
+ held = await _lock_held(dream_lock_held_by(scope, lock_token), bp.pass_id)
+ if held:
+ return False
+ error = await stopped_error(bp.pass_id) or LOCK_LOST_ERROR
+ logger.warning(f"Dream batch pass {bp.pass_id} does not apply: {error}")
+ await fail_pass(bp, error, holds_lock=held is None)
+ return True
+
+
+async def _lock_held(check: Awaitable[bool], pass_id: str) -> bool | None:
+ """*check*'s answer, or ``None`` when Redis cannot give it in time: a pass
+ that cannot confirm its lock does not apply."""
+ try:
+ return await check
+ except Exception:
+ logger.warning(
+ f"Dream pass {pass_id}: could not confirm it still holds its dream lock",
+ exc_info=True,
+ )
+ return None
+
+
+async def _read_row(pass_id: str) -> DreamPassRecord | None:
+ """The pass's row for a stop check, or ``None`` when there is none or the
+ store cannot say in time, and the pass goes on."""
+ try:
+ return await read_pass(pass_id)
+ except Exception:
+ logger.warning(
+ f"Dream pass {pass_id}: could not read its row to check for a stop; "
+ "going on",
+ exc_info=True,
+ )
+ return None
diff --git a/autogpt_platform/backend/backend/copilot/dream/cancel_test.py b/autogpt_platform/backend/backend/copilot/dream/cancel_test.py
new file mode 100644
index 0000000000..54b6dc90d0
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/cancel_test.py
@@ -0,0 +1,299 @@
+"""The stop path with the store mocked the way ``store_test.py`` mocks it: the
+cancel's one transition and what it reads back, cancelling a scope's open
+passes for the wipe, and the checks a running pass makes, which stop it on a
+cancelled or expired row and never on a store that does not answer."""
+
+import asyncio
+import logging
+from datetime import datetime, timezone
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from prisma.enums import (
+ DreamPassPhase,
+ DreamPassRoute,
+ DreamPassStatus,
+ DreamPassTrigger,
+)
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.dream_pass_models import (
+ DreamPassOperations,
+ DreamPassRecord,
+ DreamPhaseOutputs,
+)
+
+from . import cancel, store
+from .batch_outcome import BatchPass
+from .cancel import (
+ DreamPassCancel,
+ cancel_dream_pass,
+ cancel_open_passes,
+ end_batch_pass_if_stopped,
+ stop_if_stopped,
+)
+from .pass_record import MAX_ERROR_CHARS
+from .pass_run import DreamPassRun, PassEnded
+from .schemas import PhaseUsage
+
+_NOW = datetime.now(timezone.utc)
+_SCOPE = MemoryScope.for_user("u1")
+_BATCH_PASS = BatchPass(
+ user_id="u1", expert_id=None, pass_id="p1", job_id="j1", phase_models={}
+)
+
+
+def db(mocker) -> MagicMock:
+ """The accessor's target, every write landing and no row to read."""
+ database = MagicMock()
+ database.update_dream_pass = AsyncMock(return_value=True)
+ database.get_dream_pass = AsyncMock(return_value=None)
+ database.get_dream_pass_for_user = AsyncMock(return_value=None)
+ database.list_open_dream_passes = AsyncMock(return_value=[])
+ mocker.patch.object(store, "dream_db", return_value=database)
+ return database
+
+
+async def _hang(*_args) -> None:
+ await asyncio.Event().wait()
+
+
+def _row(pass_id: str = "p1", **overrides) -> DreamPassRecord:
+ fields = {
+ "id": pass_id,
+ "user_id": "u1",
+ "expert_id": None,
+ "scope_key": _SCOPE.scope_key,
+ "route": DreamPassRoute.SYNC,
+ "trigger": DreamPassTrigger.CRON,
+ "phase": DreamPassPhase.RECOMBINE,
+ "status": DreamPassStatus.RUNNING,
+ "skip_reason": None,
+ "cancel_generation": 0,
+ "provider_batch_id": None,
+ "lease_token": None,
+ "lease_expires_at": None,
+ "input_bundle": None,
+ "phase_outputs": DreamPhaseOutputs(),
+ "operations": DreamPassOperations(),
+ "usage": None,
+ "window_start": None,
+ "window_end": None,
+ "error": None,
+ "created_at": _NOW,
+ "started_at": _NOW,
+ "submitted_at": None,
+ "applied_at": None,
+ "completed_at": None,
+ "updated_at": _NOW,
+ }
+ return DreamPassRecord(**{**fields, **overrides})
+
+
+def _cancelled(pass_id: str = "p1", **overrides) -> DreamPassRecord:
+ return _row(
+ pass_id,
+ status=DreamPassStatus.CANCELLED,
+ cancel_generation=1,
+ error="testing",
+ completed_at=_NOW,
+ **overrides,
+ )
+
+
+class TestCancelDreamPass:
+ async def test_an_open_pass_is_closed_cancelled_and_read_back(self, db):
+ db.get_dream_pass_for_user.return_value = _cancelled()
+
+ result = await cancel_dream_pass("p1", user_id="u1", reason="testing")
+
+ assert result == DreamPassCancel(cancelled=True, record=_cancelled())
+ pass_id, update = db.update_dream_pass.await_args.args
+ assert pass_id == "p1"
+ assert (update.status, update.error, update.owner_user_id) == (
+ DreamPassStatus.CANCELLED,
+ "testing",
+ "u1",
+ )
+ assert update.bump_cancel_generation is True
+ assert update.completed_at is not None
+ assert update.not_updated_since is None
+ db.get_dream_pass_for_user.assert_awaited_once_with("p1", "u1")
+
+ async def test_a_pass_that_already_ended_is_left_as_it_was(self, db):
+ finished = _row(status=DreamPassStatus.COMPLETE, phase=DreamPassPhase.DONE)
+ db.update_dream_pass.return_value = False
+ db.get_dream_pass_for_user.return_value = finished
+
+ result = await cancel_dream_pass("p1", user_id="u1", reason="testing")
+
+ assert result == DreamPassCancel(cancelled=False, record=finished)
+
+ async def test_a_missing_or_another_users_pass_reads_as_none(self, db):
+ db.update_dream_pass.return_value = False
+
+ result = await cancel_dream_pass("p1", user_id="u2", reason="testing")
+
+ assert result == DreamPassCancel(cancelled=False, record=None)
+ _, update = db.update_dream_pass.await_args.args
+ assert update.owner_user_id == "u2"
+
+ async def test_a_long_reason_is_capped_like_any_error(self, db):
+ await cancel_dream_pass("p1", user_id="u1", reason="x" * (MAX_ERROR_CHARS * 2))
+
+ _, update = db.update_dream_pass.await_args.args
+ assert update.error == "x" * MAX_ERROR_CHARS
+
+ @pytest.mark.parametrize(
+ "stalled",
+ [lambda db: db.update_dream_pass, lambda db: db.get_dream_pass_for_user],
+ ids=["the_cancel", "the_read_back"],
+ )
+ async def test_a_store_that_does_not_answer_raises_at_its_deadline(
+ self, db, monkeypatch, stalled
+ ):
+ monkeypatch.setattr(store, "RECORD_WRITE_TIMEOUT_SECONDS", 0.05)
+ stalled(db).side_effect = _hang
+ loop = asyncio.get_running_loop()
+ started = loop.time()
+
+ with pytest.raises(TimeoutError):
+ await asyncio.wait_for(
+ cancel_dream_pass("p1", user_id="u1", reason="testing"), 5
+ )
+
+ # The store's deadline, not this test's own guard, gave up.
+ assert loop.time() - started < 2
+
+ async def test_a_failed_read_back_raises(self, db):
+ db.get_dream_pass_for_user.side_effect = ConnectionError("db down")
+
+ with pytest.raises(ConnectionError):
+ await cancel_dream_pass("p1", user_id="u1", reason="testing")
+
+
+class TestCancelOpenPasses:
+ async def test_every_open_pass_of_the_scope_is_cancelled(self, db):
+ db.list_open_dream_passes.return_value = [_row("a"), _row("b")]
+ db.get_dream_pass_for_user.side_effect = [_cancelled("a"), _cancelled("b")]
+
+ results = await cancel_open_passes(_SCOPE, user_id="u1", reason="wiped")
+
+ assert results == [
+ DreamPassCancel(cancelled=True, record=_cancelled("a")),
+ DreamPassCancel(cancelled=True, record=_cancelled("b")),
+ ]
+ # Every open row, however many: the wipe must not leave one running.
+ db.list_open_dream_passes.assert_awaited_once_with(_SCOPE.scope_key, limit=None)
+ sent = [call.args for call in db.update_dream_pass.await_args_list]
+ assert [pass_id for pass_id, _ in sent] == ["a", "b"]
+ assert {(u.error, u.owner_user_id) for _, u in sent} == {("wiped", "u1")}
+
+ async def test_a_scope_the_store_cannot_list_raises(self, db):
+ db.list_open_dream_passes.side_effect = ConnectionError("db down")
+
+ with pytest.raises(ConnectionError):
+ await cancel_open_passes(_SCOPE, user_id="u1", reason="wiped")
+
+
+class TestTheSyncCheck:
+ async def test_a_cancelled_row_ends_the_pass_with_its_usage(self, db):
+ run = DreamPassRun.begin("u1", "sync_baseline")
+ run.phases.append(PhaseUsage(phase="consolidate", model="m", input_tokens=10))
+ db.get_dream_pass.return_value = _cancelled(run.pass_id)
+
+ with pytest.raises(PassEnded) as ended:
+ await stop_if_stopped(run)
+
+ result = ended.value.result
+ assert result.error == "cancelled: testing"
+ assert result.skipped is False
+ assert result.usage is not None and result.usage.phases == run.phases
+ db.get_dream_pass.assert_awaited_once_with(run.pass_id)
+
+ async def test_an_expired_row_ends_the_pass_too(self, db):
+ run = DreamPassRun.begin("u1", "sync_baseline")
+ db.get_dream_pass.return_value = _row(
+ run.pass_id,
+ status=DreamPassStatus.EXPIRED,
+ cancel_generation=1,
+ error="no progress since then; expired by dream pass p9",
+ )
+
+ with pytest.raises(PassEnded) as ended:
+ await stop_if_stopped(run)
+
+ assert ended.value.result.error == (
+ "expired: no progress since then; expired by dream pass p9"
+ )
+
+ @pytest.mark.parametrize(
+ "row",
+ [_row(), _row(status=DreamPassStatus.APPLYING), None],
+ ids=["running", "applying", "no_row"],
+ )
+ async def test_a_row_nobody_stopped_lets_the_pass_go_on(self, db, row):
+ db.get_dream_pass.return_value = row
+
+ await stop_if_stopped(DreamPassRun.begin("u1", "sync_baseline"))
+
+ async def test_a_check_the_store_cannot_answer_lets_the_pass_go_on(
+ self, db, monkeypatch, caplog
+ ):
+ monkeypatch.setattr(store, "RECORD_WRITE_TIMEOUT_SECONDS", 0.05)
+ db.get_dream_pass.side_effect = _hang
+
+ with caplog.at_level(logging.WARNING, logger=cancel.logger.name):
+ await asyncio.wait_for(
+ stop_if_stopped(DreamPassRun.begin("u1", "sync_baseline")), 5
+ )
+
+ assert "check for a stop" in caplog.text
+
+
+class TestTheBatchCheck:
+ @pytest.fixture
+ def ends(self, mocker) -> tuple[AsyncMock, AsyncMock]:
+ provider = mocker.patch.object(cancel, "cancel_provider_batch", AsyncMock())
+ fail = mocker.patch.object(cancel, "fail_pass", AsyncMock())
+ return provider, fail
+
+ async def test_a_stopped_pass_cancels_its_batch_and_ends_through_fail_pass(
+ self, db, ends
+ ):
+ provider, fail = ends
+ db.get_dream_pass.return_value = _cancelled(
+ route=DreamPassRoute.ANTHROPIC_BATCH, provider_batch_id="msgbatch_1"
+ )
+
+ assert await end_batch_pass_if_stopped(_BATCH_PASS) is True
+
+ provider.assert_awaited_once_with("msgbatch_1")
+ fail.assert_awaited_once_with(_BATCH_PASS, "cancelled: testing")
+
+ async def test_a_stopped_row_without_a_batch_skips_the_provider(self, db, ends):
+ provider, fail = ends
+ db.get_dream_pass.return_value = _cancelled()
+
+ assert await end_batch_pass_if_stopped(_BATCH_PASS) is True
+
+ provider.assert_not_awaited()
+ fail.assert_awaited_once()
+
+ async def test_a_pass_nobody_stopped_goes_on(self, db, ends):
+ provider, fail = ends
+ db.get_dream_pass.return_value = _row(status=DreamPassStatus.SUBMITTED)
+
+ assert await end_batch_pass_if_stopped(_BATCH_PASS) is False
+
+ provider.assert_not_awaited()
+ fail.assert_not_awaited()
+
+ async def test_a_check_the_store_cannot_answer_goes_on(self, db, ends):
+ provider, fail = ends
+ db.get_dream_pass.side_effect = ConnectionError("db down")
+
+ assert await end_batch_pass_if_stopped(_BATCH_PASS) is False
+
+ fail.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/copilot/dream/conftest.py b/autogpt_platform/backend/backend/copilot/dream/conftest.py
index 4c80a41677..e30e6d6b71 100644
--- a/autogpt_platform/backend/backend/copilot/dream/conftest.py
+++ b/autogpt_platform/backend/backend/copilot/dream/conftest.py
@@ -14,7 +14,7 @@ in-memory stand-in by default. Tests that need their own fake keep patching
No Postgres either, and the DatabaseManager RPC client behind ``dream_db()``
would stall a write until the store's deadline, so the dream store writes
each pass's record to an in-memory ``FakeDreamDb`` that tests can read the
-transitions back from.
+transitions back from. The master flag the guard reads answers on.
"""
from __future__ import annotations
@@ -23,10 +23,12 @@ import asyncio
from datetime import datetime, timezone
from enum import Enum
from typing import Any
+from unittest.mock import AsyncMock
import pytest
import pytest_asyncio
+from backend.copilot import usage_activation_fixtures
from backend.data.dream_pass_models import (
OPEN_STATUSES,
DreamPassDraft,
@@ -187,10 +189,12 @@ class FakeDreamDb:
Applies an update by the rules of ``backend/data/dream_pass_update.py``:
a ``None`` field leaves its column, status and phase only move forward,
a batch id lands only while the row has not moved past the update's
- phase, ``phase_outputs`` and ``operations`` merge one field at a time,
- and a terminal row is final. ``writes`` holds every draft and update
- that reached an open row, in order, as sent. ``fail`` makes every call
- raise, as an unreachable database would.
+ phase, ``phase_outputs`` and ``operations`` merge one field at a time, a
+ stop bumps the cancel generation, and a terminal row, another user's row
+ (for an update naming the owner) or a row written after the update's
+ instant is not written. ``writes`` holds every draft and update that was
+ written, in order, as sent. ``fail`` makes every call raise, as an
+ unreachable database would.
"""
def __init__(self) -> None:
@@ -203,12 +207,12 @@ class FakeDreamDb:
if draft.id in self.rows:
raise ValueError(f"duplicate dream pass {draft.id}")
self.writes.append((draft.id, draft))
- self.rows[draft.id] = {**dict(draft), "phase_outputs": {}, "operations": {}}
+ self.seed(draft)
async def update_dream_pass(self, pass_id: str, update: DreamPassUpdate) -> bool:
self._raise_if_down()
row = self.rows.get(pass_id)
- if row is None or row["status"] not in OPEN_STATUSES:
+ if row is None or not _writable(row, update):
return False
self.writes.append((pass_id, update))
if update.provider_batch_id is not None and _order(row["phase"]) <= _order(
@@ -220,7 +224,7 @@ class FakeDreamDb:
if update.phase is not None and _order(update.phase) > _order(row["phase"]):
row["phase"] = update.phase
for field, value in dict(update).items():
- if value is None or field in ("status", "phase", "provider_batch_id"):
+ if value is None or field in _NOT_WRITTEN_AS_IS:
continue
if field in ("phase_outputs", "operations"):
row[field].update(
@@ -228,11 +232,44 @@ class FakeDreamDb:
)
else:
row[field] = value
+ row["cancel_generation"] += int(update.bump_cancel_generation)
+ row["updated_at"] = datetime.now(timezone.utc)
return True
- def seed(self, draft: DreamPassDraft) -> None:
- """A row as an earlier step (another process) would have left it."""
- self.rows[draft.id] = {**dict(draft), "phase_outputs": {}, "operations": {}}
+ async def get_dream_pass(self, pass_id: str) -> DreamPassRecord | None:
+ self._raise_if_down()
+ return self.record(pass_id) if pass_id in self.rows else None
+
+ async def get_dream_pass_for_user(
+ self, pass_id: str, user_id: str
+ ) -> DreamPassRecord | None:
+ row = await self.get_dream_pass(pass_id)
+ return row if row is not None and row.user_id == user_id else None
+
+ async def list_open_dream_passes(
+ self, scope_key: str, limit: int | None = None
+ ) -> list[DreamPassRecord]:
+ """Open rows of the scope, the last inserted first, at most *limit*."""
+ self._raise_if_down()
+ return [
+ self.record(pass_id)
+ for pass_id, row in reversed(self.rows.items())
+ if row["scope_key"] == scope_key and row["status"] in OPEN_STATUSES
+ ][:limit]
+
+ def seed(self, draft: DreamPassDraft, **columns: Any) -> None:
+ """A row as an earlier step (another process) would have left it;
+ *columns* set the rest of it (a lease, when it was last written)."""
+ now = datetime.now(timezone.utc)
+ self.rows[draft.id] = {
+ **dict(draft),
+ "phase_outputs": {},
+ "operations": {},
+ "cancel_generation": 0,
+ "created_at": now,
+ "updated_at": now,
+ **columns,
+ }
def statuses(self, pass_id: str) -> list[Any]:
return [w.status for pid, w in self.writes if pid == pass_id and w.status]
@@ -245,13 +282,9 @@ class FakeDreamDb:
def record(self, pass_id: str) -> DreamPassRecord:
"""The row as ``get_dream_pass`` would read it back."""
row = self.rows[pass_id]
- stamp = row.get("started_at") or datetime.now(timezone.utc)
return DreamPassRecord.model_validate(
{
**{name: row.get(name) for name in DreamPassRecord.model_fields},
- "cancel_generation": 0,
- "created_at": stamp,
- "updated_at": stamp,
"phase_outputs": DreamPhaseOutputs(**row["phase_outputs"]),
"operations": DreamPassOperations(**row["operations"]),
}
@@ -262,6 +295,33 @@ class FakeDreamDb:
raise ConnectionError("dream pass database unreachable")
+# Update fields that are not a column written as sent: the forward-only
+# columns, the generation bump and the conditions.
+_NOT_WRITTEN_AS_IS = frozenset(
+ {
+ "status",
+ "phase",
+ "provider_batch_id",
+ "bump_cancel_generation",
+ "owner_user_id",
+ "not_updated_since",
+ }
+)
+
+
+def _writable(row: dict[str, Any], update: DreamPassUpdate) -> bool:
+ """Whether *update* may write *row*: open, the owner's when it names one,
+ not written after its instant when it names one."""
+ return (
+ row["status"] in OPEN_STATUSES
+ and update.owner_user_id in (None, row["user_id"])
+ and (
+ update.not_updated_since is None
+ or row["updated_at"] <= update.not_updated_since
+ )
+ )
+
+
def _order(value: Enum | None) -> int:
"""Where *value* sits in its enum's declared order, as Postgres compares
enums."""
@@ -269,6 +329,12 @@ def _order(value: Enum | None) -> int:
return list(type(value)).index(value)
+# A pass captures its usage attribution before its first provider call
+# (#15208's ``attributed_usage``); every dream test gets the ready, untrialed
+# usage state unless it sets its own.
+usage_snapshot = usage_activation_fixtures.usage_snapshot
+
+
@pytest.fixture(autouse=True)
def fake_dream_db(monkeypatch: pytest.MonkeyPatch) -> FakeDreamDb:
"""Give every dream test an in-memory pass record behind the store."""
@@ -278,8 +344,8 @@ def fake_dream_db(monkeypatch: pytest.MonkeyPatch) -> FakeDreamDb:
class StalledDreamDb:
- """A DatabaseManager that never answers: every record write hangs until
- the store's deadline gives up on it and cancels it."""
+ """A DatabaseManager that never answers: every record write and read
+ hangs until the store's deadline gives up on it and cancels it."""
def __init__(self) -> None:
self.started = 0
@@ -292,6 +358,16 @@ class StalledDreamDb:
await self._hang()
return True
+ async def get_dream_pass(self, pass_id: str) -> DreamPassRecord | None:
+ await self._hang()
+ return None
+
+ async def list_open_dream_passes(
+ self, scope_key: str, limit: int | None = None
+ ) -> list[DreamPassRecord]:
+ await self._hang()
+ return []
+
async def _hang(self) -> None:
self.started += 1
try:
@@ -313,6 +389,16 @@ def stalled_dream_db(monkeypatch: pytest.MonkeyPatch) -> StalledDreamDb:
return stalled
[email protected](autouse=True)
+def dream_pass_flag(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:
+ """The master flag answers on, authoritatively, at every pass's guard;
+ without this the read would reach the configured flag vendor. Tests of
+ the flag set the mock's return value or side effect."""
+ flag = AsyncMock(return_value=(True, True))
+ monkeypatch.setattr("backend.copilot.dream.guard.evaluate_feature_flag", flag)
+ return flag
+
+
@pytest.fixture(autouse=True)
def fake_dream_redis(monkeypatch: pytest.MonkeyPatch) -> FakeAsyncRedis:
"""Give every dream test an in-memory Redis unless it patches its own."""
diff --git a/autogpt_platform/backend/backend/copilot/dream/guard.py b/autogpt_platform/backend/backend/copilot/dream/guard.py
new file mode 100644
index 0000000000..d452aa17f3
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/guard.py
@@ -0,0 +1,177 @@
+"""One guard against overlapping dream passes of a scope.
+
+Every pass runs it once, after it has taken the scope's lock and before it
+gathers, so before the batch route submits anything; the admin and eval
+triggers go through it like the nightly job. The lock alone does not keep two
+passes apart: it lapses under a sync pass that outlives it and under a batch
+pass whose last phase lands after its lease, and a row can stay open behind a
+free lock when the pass's last write was lost. So the guard reads the scope's
+open ``DreamPass`` rows, newest first, up to ``GUARD_ROW_LIMIT`` other than the
+pass's own:
+
+ master flag off (an authoritative answer) skip, ``disabled``
+ no other open row go on
+ an open row, fresh skip, ``pass_in_progress``
+ an open row, fresh, ``force`` expire it, go on
+ an open row, stale expire it, go on
+
+A row is fresh while its lease has not lapsed or, without a lease, while it was
+written within the pass lock's TTL. Expiring is one conditional transition that
+bumps the row's cancel generation, so its pass stops at its next check; it
+lands only while the row is still open. A stale row's expiry is also a
+compare-and-set on the row's last write: it lands only if nothing has written
+the row since the guard read it, and a row that moved in between is alive and
+blocks after all. An admin's forced expiry of a fresh row skips that
+compare-and-set: that row's pass no longer holds the lock (the forcing pass
+took it), and it stops at its next check or at its lock check before apply.
+Rows past the limit, and rows no newer pass looks at, are left for a reaper.
+
+Two passes triggered together can both skip: the one that lost the lock race
+records ``lock_held`` while its row is still open, and the winner's guard sees
+that fresh row and skips as ``pass_in_progress``. That is a deliberate
+liveness trade-off, never two passes at once but sometimes none: an admin
+retry or the scope's next trigger runs the pass.
+
+The guard never blocks on the store or the flag service: a read or write that
+fails or runs out of time is logged at warning and the pass goes on as it did
+before the guard existed, and the guard as a whole gets
+``GUARD_BUDGET_SECONDS``, after which the pass goes on unguarded. The master
+flag is read here, once per pass; the phases never read it.
+"""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta, timezone
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.dream_pass_models import OPEN_STATUSES, DreamPassRecord
+from backend.util.feature_flag import Flag, evaluate_feature_flag
+
+from .locks import DEFAULT_LOCK_TTL_SECONDS
+from .pass_record import expired
+from .pass_run import DreamPassRun, PassEnded
+from .store import read_open_passes, read_pass, write_stop
+
+logger = logging.getLogger(__name__)
+
+# How long the master-flag read may take before the pass goes on without it.
+FLAG_READ_TIMEOUT_SECONDS = 10.0
+# The whole guard's budget, however many reads and expiries it makes.
+GUARD_BUDGET_SECONDS = 15.0
+# How many of the scope's other open passes, newest first, the guard weighs.
+GUARD_ROW_LIMIT = 5
+
+
+async def guard_dream_pass(run: DreamPassRun, scope: MemoryScope) -> None:
+ """Raise ``PassEnded`` with the skip that keeps *run* from starting, or
+ return and let it gather, the pass going on unguarded once the guard has
+ used ``GUARD_BUDGET_SECONDS``."""
+ try:
+ async with asyncio.timeout(GUARD_BUDGET_SECONDS):
+ await _guard(run, scope)
+ except TimeoutError:
+ logger.warning(
+ f"Dream pass {run.pass_id}: the guard used up its "
+ f"{GUARD_BUDGET_SECONDS:g}s budget; going on unguarded"
+ )
+
+
+async def _guard(run: DreamPassRun, scope: MemoryScope) -> None:
+ if not await _dream_pass_enabled(run):
+ raise PassEnded(run.skipped("disabled"))
+ for row in await _other_open_passes(run, scope):
+ if await _holds_scope(run, row):
+ logger.info(
+ f"Dream pass {run.pass_id} skipped: pass {row.id} of the same "
+ f"scope is in progress ({row.status.value} at {row.phase.value})"
+ )
+ raise PassEnded(run.skipped("pass_in_progress"))
+
+
+async def _dream_pass_enabled(run: DreamPassRun) -> bool:
+ """The master flag. Off only on an authoritative answer: a read that
+ fails, runs out of time or falls back to its default lets the pass go on,
+ as the cron entry that fired it (if any) already let it through."""
+ flag = Flag.DREAM_PASS_ENABLED.value
+ try:
+ enabled, authoritative = await asyncio.wait_for(
+ evaluate_feature_flag(Flag.DREAM_PASS_ENABLED, run.user_id),
+ timeout=FLAG_READ_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ logger.warning(
+ f"Dream pass {run.pass_id}: could not read {flag}; going on",
+ exc_info=True,
+ )
+ return True
+ if not authoritative:
+ logger.warning(f"Dream pass {run.pass_id}: {flag} unanswered; going on")
+ return True
+ return enabled
+
+
+async def _other_open_passes(
+ run: DreamPassRun, scope: MemoryScope
+) -> list[DreamPassRecord]:
+ """The scope's newest open passes other than *run*, at most
+ ``GUARD_ROW_LIMIT``; none when the store cannot say in time, and the pass
+ goes on unguarded. One more row is read for the pass's own."""
+ try:
+ rows = await read_open_passes(scope, limit=GUARD_ROW_LIMIT + 1)
+ except Exception:
+ logger.warning(
+ f"Dream pass {run.pass_id}: could not read the scope's open passes; "
+ "going on unguarded",
+ exc_info=True,
+ )
+ return []
+ return [row for row in rows if row.id != run.pass_id][:GUARD_ROW_LIMIT]
+
+
+async def _holds_scope(run: DreamPassRun, row: DreamPassRecord) -> bool:
+ """Whether *row*'s pass still holds the scope: a fresh row does, unless
+ the run is forced; a stale or forced row is expired first and holds it
+ only if it moved before the expiry could land."""
+ stale = _is_stale(row, datetime.now(timezone.utc))
+ if not stale and not run.force:
+ return True
+ return not await _expire(run, row, stale=stale)
+
+
+def _is_stale(row: DreamPassRecord, now: datetime) -> bool:
+ """A row with a lease is stale once the lease has lapsed; one without,
+ once nothing has written it for the pass lock's TTL."""
+ if row.lease_expires_at is not None:
+ return row.lease_expires_at <= now
+ return row.updated_at <= now - timedelta(seconds=DEFAULT_LOCK_TTL_SECONDS)
+
+
+async def _expire(run: DreamPassRun, row: DreamPassRecord, *, stale: bool) -> bool:
+ """Expire *row* and say whether it no longer holds the scope: expired now,
+ closed by its own pass meanwhile, or unknown because the store did not
+ answer in time (the pass goes on)."""
+ reason = _expiry_reason(run, row, stale=stale)
+ unchanged_since = row.updated_at if stale else None
+ try:
+ if await write_stop(row.id, expired(reason, not_updated_since=unchanged_since)):
+ logger.warning(f"Dream pass {run.pass_id} expired pass {row.id}: {reason}")
+ return True
+ current = await read_pass(row.id)
+ except Exception:
+ logger.warning(
+ f"Dream pass {run.pass_id}: could not expire pass {row.id}; going on",
+ exc_info=True,
+ )
+ return True
+ return current is None or current.status not in OPEN_STATUSES
+
+
+def _expiry_reason(run: DreamPassRun, row: DreamPassRecord, *, stale: bool) -> str:
+ """What the expired row's error says happened."""
+ if not stale:
+ return f"forced by an admin-triggered dream pass {run.pass_id}"
+ if row.lease_expires_at is not None:
+ lapsed = f"lease lapsed at {row.lease_expires_at.isoformat()}"
+ else:
+ lapsed = f"no progress since {row.updated_at.isoformat()}"
+ return f"{lapsed}; expired by dream pass {run.pass_id}"
diff --git a/autogpt_platform/backend/backend/copilot/dream/guard_test.py b/autogpt_platform/backend/backend/copilot/dream/guard_test.py
new file mode 100644
index 0000000000..01c35ae34b
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/guard_test.py
@@ -0,0 +1,320 @@
+"""The guard against overlapping dream passes, with the store mocked the way
+``store_test.py`` mocks it: which open row skips a pass, which one it expires
+and goes on past, and that neither the store nor the flag service can hold a
+pass up."""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta, timezone
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from prisma.enums import (
+ DreamPassPhase,
+ DreamPassRoute,
+ DreamPassStatus,
+ DreamPassTrigger,
+)
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.data.dream_pass_models import (
+ DreamPassOperations,
+ DreamPassRecord,
+ DreamPhaseOutputs,
+)
+from backend.util.feature_flag import Flag
+
+from . import guard, store
+from .locks import DEFAULT_LOCK_TTL_SECONDS
+from .pass_run import DreamPassRun, PassEnded
+
+_NOW = datetime.now(timezone.utc)
+_SCOPE = MemoryScope.for_user("u1")
+_STALE = _NOW - timedelta(seconds=DEFAULT_LOCK_TTL_SECONDS + 60)
+
+
+def db(mocker) -> MagicMock:
+ """The accessor's target: no open pass, and every write landing."""
+ database = MagicMock()
+ database.list_open_dream_passes = AsyncMock(return_value=[])
+ database.update_dream_pass = AsyncMock(return_value=True)
+ database.get_dream_pass = AsyncMock(return_value=None)
+ mocker.patch.object(store, "dream_db", return_value=database)
+ return database
+
+
+async def _hang(*_args) -> None:
+ await asyncio.Event().wait()
+
+
+def _run(*, force: bool = False) -> DreamPassRun:
+ return DreamPassRun.begin("u1", "sync_baseline", force=force)
+
+
+def _row(pass_id: str = "other", **overrides) -> DreamPassRecord:
+ """Another pass of the scope, open and written a minute ago."""
+ fields = {
+ "id": pass_id,
+ "user_id": "u1",
+ "expert_id": None,
+ "scope_key": _SCOPE.scope_key,
+ "route": DreamPassRoute.SYNC,
+ "trigger": DreamPassTrigger.CRON,
+ "phase": DreamPassPhase.RECOMBINE,
+ "status": DreamPassStatus.RUNNING,
+ "skip_reason": None,
+ "cancel_generation": 0,
+ "provider_batch_id": None,
+ "lease_token": None,
+ "lease_expires_at": None,
+ "input_bundle": None,
+ "phase_outputs": DreamPhaseOutputs(),
+ "operations": DreamPassOperations(),
+ "usage": None,
+ "window_start": None,
+ "window_end": None,
+ "error": None,
+ "created_at": _NOW - timedelta(minutes=5),
+ "started_at": _NOW - timedelta(minutes=5),
+ "submitted_at": None,
+ "applied_at": None,
+ "completed_at": None,
+ "updated_at": _NOW - timedelta(minutes=1),
+ }
+ return DreamPassRecord(**{**fields, **overrides})
+
+
+async def _skip_reason(run: DreamPassRun) -> str | None:
+ with pytest.raises(PassEnded) as ended:
+ await guard.guard_dream_pass(run, _SCOPE)
+ assert ended.value.result.skipped is True
+ assert ended.value.result.pass_id == run.pass_id
+ return ended.value.result.skip_reason
+
+
+class TestOpenPasses:
+ async def test_a_scope_with_only_the_pass_own_row_goes_on(self, db):
+ run = _run()
+ db.list_open_dream_passes.return_value = [_row(run.pass_id)]
+
+ await guard.guard_dream_pass(run, _SCOPE)
+
+ db.list_open_dream_passes.assert_awaited_once_with(
+ _SCOPE.scope_key, limit=guard.GUARD_ROW_LIMIT + 1
+ )
+ db.update_dream_pass.assert_not_awaited()
+
+ async def test_a_fresh_open_pass_skips_this_one(self, db, caplog):
+ run = _run()
+ db.list_open_dream_passes.return_value = [_row(run.pass_id), _row()]
+
+ with caplog.at_level(logging.INFO, logger=guard.logger.name):
+ assert await _skip_reason(run) == "pass_in_progress"
+
+ db.update_dream_pass.assert_not_awaited()
+ [record] = [r for r in caplog.records if "in progress" in r.getMessage()]
+ assert record.levelno == logging.INFO
+ assert run.pass_id in record.getMessage() and "other" in record.getMessage()
+
+ async def test_a_leased_pass_is_fresh_until_its_lease_lapses(self, db):
+ """Written a day ago, but its lease still runs: a batch pass waiting
+ on its provider."""
+ db.list_open_dream_passes.return_value = [
+ _row(
+ status=DreamPassStatus.SUBMITTED,
+ updated_at=_NOW - timedelta(days=1),
+ lease_expires_at=_NOW + timedelta(hours=1),
+ )
+ ]
+
+ assert await _skip_reason(_run()) == "pass_in_progress"
+ db.update_dream_pass.assert_not_awaited()
+
+ async def test_a_stale_pass_is_expired_and_this_one_goes_on(self, db, caplog):
+ run = _run()
+ stale = _row(updated_at=_STALE)
+ db.list_open_dream_passes.return_value = [stale]
+
+ with caplog.at_level(logging.WARNING, logger=guard.logger.name):
+ await guard.guard_dream_pass(run, _SCOPE)
+
+ pass_id, update = db.update_dream_pass.await_args.args
+ assert pass_id == "other"
+ assert update.status is DreamPassStatus.EXPIRED
+ assert update.bump_cancel_generation is True
+ assert update.not_updated_since == stale.updated_at
+ assert update.owner_user_id is None
+ assert update.completed_at is not None
+ assert update.error is not None
+ assert "no progress since" in update.error and run.pass_id in update.error
+ [record] = [r for r in caplog.records if "expired pass other" in r.getMessage()]
+ assert record.levelno == logging.WARNING
+
+ async def test_a_lapsed_lease_is_stale_however_recently_written(self, db):
+ leased = _row(
+ status=DreamPassStatus.SUBMITTED,
+ updated_at=_NOW - timedelta(minutes=1),
+ lease_expires_at=_NOW - timedelta(minutes=1),
+ )
+ db.list_open_dream_passes.return_value = [leased]
+
+ await guard.guard_dream_pass(_run(), _SCOPE)
+
+ _, update = db.update_dream_pass.await_args.args
+ assert update.status is DreamPassStatus.EXPIRED
+ assert update.not_updated_since == leased.updated_at
+ assert update.error is not None and "lease lapsed" in update.error
+
+ async def test_a_stale_pass_that_moved_before_its_expiry_still_blocks(self, db):
+ """The expiry is refused because the row was written after the guard
+ read it: that pass is alive."""
+ stale = _row(updated_at=_STALE)
+ db.list_open_dream_passes.return_value = [stale]
+ db.update_dream_pass.return_value = False
+ db.get_dream_pass.return_value = stale.model_copy(
+ update={"updated_at": _NOW, "phase": DreamPassPhase.SANITIZE}
+ )
+
+ assert await _skip_reason(_run()) == "pass_in_progress"
+ db.get_dream_pass.assert_awaited_once_with("other")
+
+ async def test_a_stale_pass_that_ended_meanwhile_does_not_block(self, db):
+ stale = _row(updated_at=_STALE)
+ db.list_open_dream_passes.return_value = [stale]
+ db.update_dream_pass.return_value = False
+ db.get_dream_pass.return_value = stale.model_copy(
+ update={"status": DreamPassStatus.COMPLETE}
+ )
+
+ await guard.guard_dream_pass(_run(), _SCOPE)
+
+ async def test_force_expires_a_fresh_pass_whatever_wrote_it_last(self, db):
+ run = _run(force=True)
+ db.list_open_dream_passes.return_value = [_row()]
+
+ await guard.guard_dream_pass(run, _SCOPE)
+
+ pass_id, update = db.update_dream_pass.await_args.args
+ assert pass_id == "other"
+ assert update.status is DreamPassStatus.EXPIRED
+ assert update.bump_cancel_generation is True
+ assert update.not_updated_since is None
+ assert update.error == f"forced by an admin-triggered dream pass {run.pass_id}"
+
+ async def test_stale_passes_are_expired_before_a_fresh_one_skips(self, db):
+ db.list_open_dream_passes.return_value = [
+ _row("stale", updated_at=_STALE),
+ _row("fresh"),
+ ]
+
+ assert await _skip_reason(_run()) == "pass_in_progress"
+
+ [call] = db.update_dream_pass.await_args_list
+ assert call.args[0] == "stale"
+
+
+class TestTheGuardsBounds:
+ async def test_the_guard_weighs_at_most_its_row_limit(self, db):
+ """However many stale rows the store lists, the guard expires at most
+ ``GUARD_ROW_LIMIT`` of them, in the (newest first) order listed."""
+ rows = [_row(f"stale-{i}", updated_at=_STALE) for i in range(8)]
+ db.list_open_dream_passes.return_value = rows
+
+ await guard.guard_dream_pass(_run(), _SCOPE)
+
+ expired = [call.args[0] for call in db.update_dream_pass.await_args_list]
+ assert expired == [f"stale-{i}" for i in range(guard.GUARD_ROW_LIMIT)]
+
+ async def test_the_guard_gives_up_once_its_budget_is_spent(
+ self, db, monkeypatch, caplog
+ ):
+ """Five stale rows whose expiries each stall to the store's deadline
+ would hold the pass for five deadlines; the guard's own budget ends
+ it first, and the pass goes on."""
+ monkeypatch.setattr(store, "RECORD_WRITE_TIMEOUT_SECONDS", 0.05)
+ monkeypatch.setattr(guard, "GUARD_BUDGET_SECONDS", 0.12)
+ db.list_open_dream_passes.return_value = [
+ _row(f"stale-{i}", updated_at=_STALE) for i in range(5)
+ ]
+ db.update_dream_pass.side_effect = _hang
+ loop = asyncio.get_running_loop()
+ started = loop.time()
+
+ with caplog.at_level(logging.WARNING, logger=guard.logger.name):
+ await asyncio.wait_for(guard.guard_dream_pass(_run(), _SCOPE), 5)
+
+ assert loop.time() - started < 5 * 0.05
+ assert db.update_dream_pass.await_count < 5
+ assert "used up its 0.12s budget" in caplog.text
+
+
+class TestTheStoreNeverHoldsThePassUp:
+ async def test_a_read_that_runs_out_of_time_lets_the_pass_go_on(
+ self, db, monkeypatch, caplog
+ ):
+ monkeypatch.setattr(store, "RECORD_WRITE_TIMEOUT_SECONDS", 0.05)
+ db.list_open_dream_passes.side_effect = _hang
+
+ with caplog.at_level(logging.WARNING, logger=guard.logger.name):
+ await asyncio.wait_for(guard.guard_dream_pass(_run(), _SCOPE), 5)
+
+ [record] = [r for r in caplog.records if "open passes" in r.getMessage()]
+ assert record.levelno == logging.WARNING
+ db.update_dream_pass.assert_not_awaited()
+
+ async def test_a_read_that_fails_lets_the_pass_go_on(self, db):
+ db.list_open_dream_passes.side_effect = ConnectionError("db down")
+
+ await guard.guard_dream_pass(_run(), _SCOPE)
+
+ async def test_an_expiry_that_runs_out_of_time_lets_the_pass_go_on(
+ self, db, monkeypatch, caplog
+ ):
+ monkeypatch.setattr(store, "RECORD_WRITE_TIMEOUT_SECONDS", 0.05)
+ db.list_open_dream_passes.return_value = [_row(updated_at=_STALE)]
+ db.update_dream_pass.side_effect = _hang
+
+ with caplog.at_level(logging.WARNING, logger=guard.logger.name):
+ await asyncio.wait_for(guard.guard_dream_pass(_run(), _SCOPE), 5)
+
+ assert "could not expire pass other" in caplog.text
+
+
+class TestTheMasterFlag:
+ async def test_the_flag_off_skips_the_pass_as_disabled(self, db, dream_pass_flag):
+ dream_pass_flag.return_value = (False, True)
+ run = _run()
+
+ assert await _skip_reason(run) == "disabled"
+
+ dream_pass_flag.assert_awaited_once_with(Flag.DREAM_PASS_ENABLED, "u1")
+ db.list_open_dream_passes.assert_not_awaited()
+
+ async def test_force_does_not_get_past_the_flag(self, db, dream_pass_flag):
+ dream_pass_flag.return_value = (False, True)
+
+ assert await _skip_reason(_run(force=True)) == "disabled"
+
+ async def test_a_flag_read_without_an_answer_lets_the_pass_go_on(
+ self, db, dream_pass_flag, caplog
+ ):
+ """The vendor fell back to the default: a blip, not an "off"."""
+ dream_pass_flag.return_value = (False, False)
+
+ with caplog.at_level(logging.WARNING, logger=guard.logger.name):
+ await guard.guard_dream_pass(_run(), _SCOPE)
+
+ assert "unanswered" in caplog.text
+ db.list_open_dream_passes.assert_awaited_once()
+
+ async def test_a_flag_read_that_runs_out_of_time_lets_the_pass_go_on(
+ self, db, dream_pass_flag, monkeypatch, caplog
+ ):
+ monkeypatch.setattr(guard, "FLAG_READ_TIMEOUT_SECONDS", 0.05)
+ dream_pass_flag.side_effect = _hang
+
+ with caplog.at_level(logging.WARNING, logger=guard.logger.name):
+ await asyncio.wait_for(guard.guard_dream_pass(_run(), _SCOPE), 5)
+
+ assert "could not read dream-pass-enabled" in caplog.text
diff --git a/autogpt_platform/backend/backend/copilot/dream/job_status.py b/autogpt_platform/backend/backend/copilot/dream/job_status.py
index 87f76a72c2..84bb48f813 100644
--- a/autogpt_platform/backend/backend/copilot/dream/job_status.py
+++ b/autogpt_platform/backend/backend/copilot/dream/job_status.py
@@ -225,9 +225,10 @@ async def mark_complete(
async def mark_errored(
- *, kind: JobKind, job_id: str, error: str
+ *, kind: JobKind, job_id: str, error: str, result: BaseModel | None = None
) -> JobStatus[Any] | None:
- """Transition to ``state='errored'`` with a short error string.
+ """Transition to ``state='errored'`` with a short error string, and the
+ work body's *result* when it returned one (a failed dream pass's usage).
Refuses to overwrite a job already in ``state='complete'`` — the
batch tail runs cleanup after ``mark_complete``, and a transient
@@ -258,6 +259,7 @@ async def mark_errored(
"updated_at": now,
"completed_at": now,
"error": error[:2000], # cap; error strings can be huge stack traces
+ "result": result.model_dump() if result is not None else existing.result,
}
)
await _persist(updated)
diff --git a/autogpt_platform/backend/backend/copilot/dream/job_status_test.py b/autogpt_platform/backend/backend/copilot/dream/job_status_test.py
index e73b71c2b9..b4fd265838 100644
--- a/autogpt_platform/backend/backend/copilot/dream/job_status_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/job_status_test.py
@@ -17,6 +17,7 @@ from backend.copilot.dream.job_status import (
update_status_phase,
write_initial_status,
)
+from backend.copilot.dream.schemas import DreamPassResult, DreamPassUsage
@pytest.fixture
@@ -205,6 +206,27 @@ class TestMarkErrored:
assert result.state == "complete"
assert result.result == {"n": 1}
+ @pytest.mark.asyncio
+ async def test_keeps_the_result_a_failed_work_body_returned(self, fake_redis):
+ """A stopped dream pass returns its failure with the usage of the
+ phases it was billed for; the errored job keeps that result."""
+ await write_initial_status(kind="dream_pass", job_id="j1", user_id="u")
+ failure = DreamPassResult(
+ user_id="u",
+ pass_id="p1",
+ error="cancelled: testing",
+ usage=DreamPassUsage(total_input_tokens=100),
+ )
+
+ await mark_errored(
+ kind="dream_pass", job_id="j1", error=failure.error or "", result=failure
+ )
+
+ status = await read_status(kind="dream_pass", job_id="j1")
+ assert status is not None and status.state == "errored"
+ assert status.result is not None
+ assert DreamPassResult.model_validate(status.result) == failure
+
class TestReadStatus:
@pytest.mark.asyncio
diff --git a/autogpt_platform/backend/backend/copilot/dream/locks.py b/autogpt_platform/backend/backend/copilot/dream/locks.py
index 0380a01e98..8708948112 100644
--- a/autogpt_platform/backend/backend/copilot/dream/locks.py
+++ b/autogpt_platform/backend/backend/copilot/dream/locks.py
@@ -19,10 +19,15 @@ transactions.
The sync-path TTL is 1800 s (30 min), sized to the scheduler's job
timeout; the batch path extends it (see ``BATCH_LOCK_TTL_SECONDS``).
+
+Right before it applies, a pass reads the key once more (``held``,
+``dream_lock_held_by``): a lock that lapsed may already be a newer pass's,
+and only the holder may write the scope's graph.
"""
from __future__ import annotations
+import asyncio
import logging
import uuid
from contextlib import asynccontextmanager
@@ -43,6 +48,9 @@ DEFAULT_LOCK_TTL_SECONDS = 1800
# lock can't expire before the executor times the batch out.
BATCH_LOCK_TTL_SECONDS = 24 * 60 * 60 + 600
+# How long the ownership read right before apply may take.
+LOCK_CHECK_TIMEOUT_SECONDS = 2.0
+
# Compare-and-delete: only the holder whose token still matches the stored
# value may delete the key. Single-key Lua routes on Redis Cluster. Graphiti's
# per-graph write lock releases and renews with these two scripts as well
@@ -95,7 +103,7 @@ class DreamLockHandle:
its per-pass state (the input bundle) for the callback to read back hours
later."""
- def __init__(self, redis, key: str, user_id: str, token: str) -> None:
+ def __init__(self, redis: Any, key: str, user_id: str, token: str) -> None:
self._redis = redis
self._key = key
self.user_id = user_id
@@ -134,6 +142,14 @@ class DreamLockHandle:
)
return bool(extended)
+ async def held(self) -> bool:
+ """Whether the key still holds this handle's token: one GET, raising
+ when Redis does not answer within ``LOCK_CHECK_TIMEOUT_SECONDS``."""
+ current = await asyncio.wait_for(
+ self._redis.get(self._key), timeout=LOCK_CHECK_TIMEOUT_SECONDS
+ )
+ return _as_text(current) == self.token
+
@asynccontextmanager
async def dream_lock(
@@ -204,12 +220,25 @@ async def read_dream_lock_token(scope: MemoryScope) -> str | None:
from backend.data.redis_client import get_redis_async
redis = await get_redis_async()
- raw = await redis.get(scope.redis_key("dream_lock"))
+ return _as_text(await redis.get(scope.redis_key("dream_lock")))
+
+
+async def dream_lock_held_by(scope: MemoryScope, token: str | None) -> bool:
+ """Whether the scope's lock still holds *token* (``False`` for no token):
+ the batch callback's check right before apply. One GET, raising when
+ Redis does not answer within ``LOCK_CHECK_TIMEOUT_SECONDS``."""
+ if token is None:
+ return False
+ current = await asyncio.wait_for(
+ read_dream_lock_token(scope), timeout=LOCK_CHECK_TIMEOUT_SECONDS
+ )
+ return current == token
+
+
+def _as_text(raw: Any) -> str | None:
if raw is None:
return None
- if isinstance(raw, bytes):
- return raw.decode("utf-8")
- return str(raw)
+ return raw.decode("utf-8") if isinstance(raw, bytes) else str(raw)
async def release_dream_lock(scope: MemoryScope, token: str | None) -> None:
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
index 0927139d75..b274d6d7ae 100644
--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
@@ -16,7 +16,9 @@ Every pass, on either route, gets a durable ``DreamPass`` row
(``store.py``): inserted at the start, advanced after each step, and
closed with how the pass ended, a write attempted before the pass releases
its lock. Like every record write it is best-effort: one that fails or
-times out leaves the row open behind a free lock until a reaper closes it.
+times out leaves the row open behind a free lock until a later pass's guard
+or a reaper closes it. A pass holding the lock runs ``guard.py`` first, and
+``cancel.py``'s checks before each phase, before its batch submit and apply.
The orchestrator never raises out — every failure becomes a
``DreamPassResult`` with ``error`` set and, on the sync route, the usage
@@ -27,7 +29,7 @@ so the admin trigger always gets a structured response back.
from __future__ import annotations
import logging
-from collections.abc import Awaitable
+from collections.abc import Awaitable, Callable
from datetime import datetime, timezone
from typing import TypeVar
@@ -49,6 +51,7 @@ from backend.util.feature_flag import Flag, is_feature_enabled
from .apply import apply_operations, drain_status_from_stats
from .batch_handoff import submit_dream_pass_batch
from .billing import PhaseChargeError, check_dream_budget, record_phase_cost
+from .cancel import stop_before_apply, stop_if_stopped
from .clamp import clamp_operations
from .fetch import (
DreamInput,
@@ -57,9 +60,10 @@ from .fetch import (
is_dream_authored_episode,
parse_episode_timestamp,
)
+from .guard import guard_dream_pass
from .locks import DEFAULT_LOCK_TTL_SECONDS, DreamLockHandle, DreamLockHeld, dream_lock
from .pass_record import DreamTrigger
-from .pass_run import DreamPassRun
+from .pass_run import DreamPassRun, PassEnded
from .phase_jobs import phase_job
from .prompts import (
build_consolidate_prompt,
@@ -366,6 +370,7 @@ async def execute_dream_pass(
status_id: str | None = None,
expert_id: str | None = None,
trigger: DreamTrigger = "cron",
+ force: bool = False,
) -> DreamPassResult:
"""Public async entry point used by the scheduler + admin trigger.
@@ -379,10 +384,10 @@ async def execute_dream_pass(
``trigger`` is what started the pass, as its ``DreamPass`` row records
it: ``cron`` for the nightly job, ``admin`` for the admin triggers,
- ``eval`` for an eval run.
+ ``eval`` for an eval run. ``force`` is the admin trigger's, see ``guard.py``.
"""
return await _execute_dream_pass_async(
- user_id, status_id=status_id, expert_id=expert_id, trigger=trigger
+ user_id, status_id=status_id, expert_id=expert_id, trigger=trigger, force=force
)
@@ -394,9 +399,10 @@ async def _execute_dream_pass_async(
config: ChatConfig | None = None,
status_id: str | None = None,
trigger: DreamTrigger = "cron",
+ force: bool = False,
) -> DreamPassResult:
config = config or ChatConfig()
- run = DreamPassRun.begin(user_id, await _route_for(user_id, config))
+ run = DreamPassRun.begin(user_id, await _route_for(user_id, config), force=force)
try:
scope = MemoryScope.build(user_id, expert_id)
await start_pass(
@@ -480,10 +486,11 @@ async def _dream(
config: ChatConfig,
status_id: str | None,
) -> DreamPassResult:
- """Gather, then hand the pass to the batch route or run its three phases
- and apply them. A pass that ends early (a skip, a billing or phase
- failure) comes back as the result it ended with."""
+ """Guard, gather, then hand the pass to the batch route or run its three
+ phases and apply them. A pass that ends early (a skip, a stop, a billing
+ or phase failure) comes back as the result it ended with."""
try:
+ await guard_dream_pass(run, scope)
input_bundle = await _gather(run, scope, config=config, status_id=status_id)
if run.execution_path == "anthropic_batch":
# Phase 1 goes to Anthropic's Messages Batches API with the
@@ -505,9 +512,9 @@ async def _dream(
config=config,
)
sanitized = await _run_phases(run, inference, input_bundle)
- except _PassEnded as ended:
+ return await _apply(run, scope, lock_handle, sanitized, input_bundle)
+ except PassEnded as ended:
return ended.result
- return await _apply(run, scope, lock_handle, sanitized, input_bundle)
async def _gather(
@@ -521,20 +528,20 @@ async def _gather(
The billing check runs inside the lock so a paywalled user doesn't burn
the slot for an eligible concurrent pass on a shared FalkorDB. Raises
- ``_PassEnded`` with the skip or failure that ends the pass early.
+ ``PassEnded`` with the skip or failure that ends the pass early.
"""
budget_ok, budget_skip = await check_dream_budget(run.user_id, config=config)
if not budget_ok:
if budget_skip == "rate_limit_unavailable":
- raise _PassEnded(run.failure(f"billing: {budget_skip}"))
- raise _PassEnded(run.skipped(budget_skip or "insufficient_credits"))
+ raise PassEnded(run.failure(f"billing: {budget_skip}"))
+ raise PassEnded(run.skipped(budget_skip or "insufficient_credits"))
input_bundle = await gather_dream_input(scope)
if not input_bundle.episodes and not input_bundle.facts:
# Nothing to consolidate — skipped so the admin UI can render
# "nothing to dream about yet".
- raise _PassEnded(run.skipped("no_input"))
+ raise PassEnded(run.skipped("no_input"))
if await _nothing_new_since_last_pass(run, scope, input_bundle, status_id):
- raise _PassEnded(run.skipped("no_new_activity"))
+ raise PassEnded(run.skipped("no_new_activity"))
await record_gathered(run.pass_id, input_bundle)
return input_bundle
@@ -579,32 +586,33 @@ async def _run_phases(
"""Consolidate, recombine and sanitize in turn, recording each phase's
usage on the run and its output on the record as it lands."""
consolidated = await _phase(
- run, "consolidate", _run_consolidate(inference, input_bundle)
+ run, "consolidate", lambda: _run_consolidate(inference, input_bundle)
)
recombined = await _phase(
- run, "recombine", _run_recombine(inference, input_bundle, consolidated)
+ run, "recombine", lambda: _run_recombine(inference, input_bundle, consolidated)
)
return await _phase(
run,
"sanitize",
- _run_sanitize(inference, input_bundle, consolidated, recombined),
+ lambda: _run_sanitize(inference, input_bundle, consolidated, recombined),
)
async def _phase(
run: DreamPassRun,
phase: DreamPhase,
- call: Awaitable[tuple[_Output, PhaseUsage]],
+ call: Callable[[], Awaitable[tuple[_Output, PhaseUsage]]],
) -> _Output:
"""One phase's output. A phase with no usable answer ends the pass
- (``_PassEnded``) with the completed phases' usage. Failed inference
+ (``PassEnded``) with the completed phases' usage. Failed inference
attempts are absorbed. A completed phase whose charge failed
(``PhaseChargeError``) joins that usage too, then its error ends the
pass as a crash does."""
+ await stop_if_stopped(run)
try:
- output, usage = await call
+ output, usage = await call()
except InferenceError as exc:
- raise _PassEnded(run.failure(f"{phase}: {exc}")) from exc
+ raise PassEnded(run.failure(f"{phase}: {exc}")) from exc
except PhaseChargeError as exc:
run.phases.append(phase_usage(phase, exc.usage))
raise
@@ -620,13 +628,14 @@ async def _apply(
sanitized: DreamOperations,
input_bundle: DreamInput,
) -> DreamPassResult:
- """Clamp the sanitized operations, apply them and stamp the marker."""
+ """Clamp the operations, make the last checks, apply and stamp the marker."""
ops = clamp_operations(
sanitized,
len(input_bundle.facts),
known_fact_uuids=input_bundle.known_fact_uuids,
)
await record_applying(run.pass_id, ops)
+ await stop_before_apply(run, lock_handle)
apply_stats = await apply_operations(
scope,
run.pass_id,
@@ -683,11 +692,3 @@ def _applied_result(
),
usage=run.usage(),
)
-
-
-class _PassEnded(Exception):
- """A pass that ends before apply, carrying the result it ends with."""
-
- def __init__(self, result: DreamPassResult) -> None:
- super().__init__(result.error or result.skip_reason)
- self.result = result
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator_stop_test.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator_stop_test.py
new file mode 100644
index 0000000000..a3d863b2d6
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator_stop_test.py
@@ -0,0 +1,311 @@
+"""The guard and the stop path through the sync entry point, with the real
+dream lock over the in-memory Redis and the pass's row in the in-memory
+store: a pass that finds its scope taken, the admin trigger that forces past
+it, the master flag turned off after the cron fired, and a pass cancelled at
+each of its checks."""
+
+from collections.abc import Awaitable, Callable
+from datetime import datetime, timedelta, timezone
+from unittest.mock import AsyncMock
+
+import pytest
+from prisma.enums import (
+ DreamPassPhase,
+ DreamPassRoute,
+ DreamPassStatus,
+ DreamPassTrigger,
+)
+
+from backend.copilot.graphiti.scope import MemoryScope
+from backend.copilot.inference.complete import StructuredCompletion
+from backend.copilot.inference.context import InferenceUsage, RouteDecision
+from backend.data.dream_pass_models import DreamPassDraft
+
+from . import orchestrator as orchestrator_mod
+from .cancel import cancel_dream_pass
+from .fetch import DreamInput, EpisodeRow
+from .locks import DEFAULT_LOCK_TTL_SECONDS
+from .pass_record import expired
+from .schemas import ConsolidationOutput, DreamOperations, RecombinationOutput
+from .store import write_stop
+
+_SCOPE = MemoryScope.for_user("u")
+_LOCK_KEY = _SCOPE.redis_key("dream_lock")
+_BILLED = InferenceUsage(
+ model="m",
+ input_tokens=100,
+ output_tokens=20,
+ cost_usd=0.001,
+ cost_source="provider",
+ payer="platform_allowance",
+)
+_ANSWERS = (
+ ConsolidationOutput(facts=[]),
+ RecombinationOutput(proposals=[]),
+ DreamOperations(summary_for_user="ok"),
+)
+
+
+def _route(scope, job, *, config=None) -> RouteDecision:
+ return RouteDecision(
+ engine="provider_sync",
+ auth_provider="platform",
+ provider="open_router",
+ model=f"{job.tier}-model",
+ payer="platform_allowance",
+ execution_path="sync_baseline",
+ cost_log_provider="open_router",
+ reason="test",
+ )
+
+
+def _input() -> DreamInput:
+ now = datetime.now(timezone.utc)
+ episode = EpisodeRow(
+ uuid="e1",
+ name=None,
+ content="hello",
+ source_description=None,
+ valid_at=None,
+ created_at=None,
+ )
+ return DreamInput(
+ user_id="u",
+ group_id=_SCOPE.group_id,
+ window_start=now - timedelta(days=14),
+ window_end=now,
+ episodes=[episode],
+ )
+
+
[email protected](autouse=True)
+def budget(mocker) -> AsyncMock:
+ """A sync pass whose budget (returned), route and charges are stubbed;
+ the lock, the marker and the record run for real over the in-memory
+ stand-ins."""
+ mocker.patch.object(orchestrator_mod, "resolve_route", side_effect=_route)
+ mocker.patch.object(
+ orchestrator_mod, "resolve_dream_execution_path", return_value="sync_baseline"
+ )
+ mocker.patch.object(
+ orchestrator_mod, "is_feature_enabled", AsyncMock(return_value=False)
+ )
+ mocker.patch.object(
+ orchestrator_mod,
+ "record_phase_cost",
+ AsyncMock(side_effect=lambda ctx, usage: usage),
+ )
+ return mocker.patch.object(
+ orchestrator_mod, "check_dream_budget", AsyncMock(return_value=(True, None))
+ )
+
+
+def gather(mocker) -> AsyncMock:
+ return mocker.patch.object(
+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_input())
+ )
+
+
+def apply(mocker) -> AsyncMock:
+ return mocker.patch.object(
+ orchestrator_mod,
+ "apply_operations",
+ AsyncMock(return_value={"session_id": "s", "consolidated_count": 0}),
+ )
+
+
+def _phases(mocker, during: int | None = None, stop=None) -> AsyncMock:
+ """Each phase answers, billed; *stop* runs while call *during* is in
+ flight, as a cancel landing mid-phase would."""
+ calls = 0
+
+ async def answer(*_args, **_kwargs) -> StructuredCompletion:
+ nonlocal calls
+ if calls == during and stop is not None:
+ await stop()
+ calls += 1
+ return StructuredCompletion(value=_ANSWERS[calls - 1], usage=_BILLED)
+
+ return mocker.patch.object(
+ orchestrator_mod, "structured_complete", AsyncMock(side_effect=answer)
+ )
+
+
+def _seed_other_pass(fake_dream_db, **columns) -> str:
+ """Another pass of the scope, open at recombine, written a minute ago."""
+ draft = DreamPassDraft(
+ id="other",
+ user_id="u",
+ scope_key=_SCOPE.scope_key,
+ route=DreamPassRoute.SYNC,
+ trigger=DreamPassTrigger.CRON,
+ phase=DreamPassPhase.RECOMBINE,
+ )
+ written = datetime.now(timezone.utc) - timedelta(minutes=1)
+ fake_dream_db.seed(draft, **{"updated_at": written, **columns})
+ return draft.id
+
+
+def _cancel_own_pass(fake_dream_db) -> Callable[[], Awaitable[None]]:
+ async def cancel() -> None:
+ [pass_id] = list(fake_dream_db.rows)
+ result = await cancel_dream_pass(pass_id, user_id="u", reason="testing")
+ assert result.cancelled
+
+ return cancel
+
+
+class TestTheGuard:
+ @pytest.mark.parametrize("trigger", ["cron", "admin", "eval"])
+ async def test_a_pass_behind_a_fresh_open_pass_is_recorded_skipped(
+ self, fake_dream_db, fake_dream_redis, gather, trigger
+ ):
+ other = _seed_other_pass(fake_dream_db)
+
+ result = await orchestrator_mod.execute_dream_pass(
+ "u", status_id="job-1" if trigger == "admin" else None, trigger=trigger
+ )
+
+ assert (result.skipped, result.skip_reason) == (True, "pass_in_progress")
+ gather.assert_not_awaited()
+ row = fake_dream_db.rows[result.pass_id]
+ assert (row["status"], row["skip_reason"]) == (
+ DreamPassStatus.SKIPPED,
+ "pass_in_progress",
+ )
+ assert fake_dream_db.rows[other]["status"] is DreamPassStatus.RUNNING
+ assert fake_dream_db.rows[other]["cancel_generation"] == 0
+ assert _LOCK_KEY not in fake_dream_redis.store
+
+ async def test_the_admin_trigger_with_force_expires_it_and_runs(
+ self, mocker, fake_dream_db, gather, apply
+ ):
+ other = _seed_other_pass(fake_dream_db)
+ _phases(mocker)
+
+ result = await orchestrator_mod.execute_dream_pass(
+ "u", status_id="job-1", trigger="admin", force=True
+ )
+
+ assert result.error is None and result.skipped is False
+ apply.assert_awaited_once()
+ row = fake_dream_db.rows[other]
+ assert row["status"] is DreamPassStatus.EXPIRED
+ assert row["cancel_generation"] == 1
+ assert row["error"] == (
+ f"forced by an admin-triggered dream pass {result.pass_id}"
+ )
+ assert fake_dream_db.rows[result.pass_id]["status"] is (
+ DreamPassStatus.COMPLETE
+ )
+
+ async def test_a_stale_open_pass_is_expired_and_the_pass_runs(
+ self, mocker, fake_dream_db, gather, apply
+ ):
+ stale_at = datetime.now(timezone.utc) - timedelta(
+ seconds=DEFAULT_LOCK_TTL_SECONDS + 60
+ )
+ other = _seed_other_pass(fake_dream_db, updated_at=stale_at)
+ _phases(mocker)
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error is None and result.skipped is False
+ apply.assert_awaited_once()
+ assert fake_dream_db.rows[other]["status"] is DreamPassStatus.EXPIRED
+
+ async def test_a_flag_turned_off_after_the_cron_fired_skips_as_disabled(
+ self, fake_dream_db, fake_dream_redis, gather, budget, dream_pass_flag
+ ):
+ dream_pass_flag.return_value = (False, True)
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert (result.skipped, result.skip_reason) == (True, "disabled")
+ gather.assert_not_awaited()
+ budget.assert_not_awaited()
+ row = fake_dream_db.rows[result.pass_id]
+ assert (row["status"], row["skip_reason"]) == (
+ DreamPassStatus.SKIPPED,
+ "disabled",
+ )
+ assert _LOCK_KEY not in fake_dream_redis.store
+
+
+class TestTheStopChecks:
+ async def test_a_pass_reads_its_row_before_each_phase_and_before_apply(
+ self, mocker, fake_dream_db, gather, apply
+ ):
+ _phases(mocker)
+ reads = mocker.spy(fake_dream_db, "get_dream_pass")
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error is None
+ assert [call.args for call in reads.call_args_list] == [(result.pass_id,)] * 4
+
+ @pytest.mark.parametrize(
+ "during, billed",
+ [
+ (None, []),
+ (0, ["consolidate"]),
+ (1, ["consolidate", "recombine"]),
+ (2, ["consolidate", "recombine", "sanitize"]),
+ ],
+ ids=["before_consolidate", "before_recombine", "before_sanitize", "pre_apply"],
+ )
+ async def test_a_cancelled_pass_stops_at_its_next_check(
+ self, mocker, fake_dream_db, fake_dream_redis, gather, apply, during, billed
+ ):
+ cancel = _cancel_own_pass(fake_dream_db)
+ if during is None:
+ gather.side_effect = _gather_then(cancel)
+ phases = _phases(mocker, during, cancel)
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error == "cancelled: testing"
+ assert result.usage is not None
+ assert [p.phase for p in result.usage.phases] == billed
+ assert phases.await_count == len(billed)
+ apply.assert_not_awaited()
+ assert _LOCK_KEY not in fake_dream_redis.store
+ assert _SCOPE.redis_key("last_completed") not in fake_dream_redis.store
+ row = fake_dream_db.rows[result.pass_id]
+ assert (row["status"], row["error"], row["cancel_generation"]) == (
+ DreamPassStatus.CANCELLED,
+ "testing",
+ 1,
+ )
+ assert DreamPassStatus.ERRORED not in fake_dream_db.statuses(result.pass_id)
+
+ async def test_a_pass_a_newer_one_expired_stops_the_same_way(
+ self, mocker, fake_dream_db, gather, apply
+ ):
+ async def expire() -> None:
+ [pass_id] = list(fake_dream_db.rows)
+ await write_stop(pass_id, expired("forced", not_updated_since=None))
+
+ _phases(mocker, 1, expire)
+
+ result = await orchestrator_mod.execute_dream_pass("u")
+
+ assert result.error == "expired: forced"
+ assert result.usage is not None
+ assert [p.phase for p in result.usage.phases] == ["consolidate", "recombine"]
+ apply.assert_not_awaited()
+ assert fake_dream_db.rows[result.pass_id]["status"] is DreamPassStatus.EXPIRED
+
+
+def _gather_then(
+ stop: Callable[[], Awaitable[None]],
+) -> Callable[[MemoryScope], Awaitable[DreamInput]]:
+ """A gather during which *stop* lands."""
+
+ async def gather(scope: MemoryScope) -> DreamInput:
+ await stop()
+ return _input()
+
+ return gather
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
index 6cc7d77c0b..719f4d20af 100644
--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
@@ -161,9 +161,24 @@ def _build_input(*, episodes=1, facts=1) -> DreamInput:
)
+class _HeldLock:
+ """A dream lock handle whose lock is still held whenever a pass looks."""
+
+ token = "tok"
+
+ async def held(self) -> bool:
+ return True
+
+ async def extend(self, ttl_seconds: int) -> bool:
+ return True
+
+ def disown(self) -> None:
+ return None
+
+
@asynccontextmanager
async def _noop_lock(*args, **kwargs):
- yield
+ yield _HeldLock()
@pytest.fixture(autouse=True)
@@ -394,7 +409,7 @@ async def test_held_dream_lock_handle_is_threaded_into_apply(mocker):
"""apply renews the dream lock before the drain + demotions, which it can
only do with the handle the orchestrator holds. Dropping that kwarg would
silently reinstate the lock-expiry-during-drain window, so pin it."""
- sentinel_handle = object()
+ sentinel_handle = _HeldLock()
@asynccontextmanager
async def _handle_lock(*args, **kwargs):
@@ -2702,7 +2717,7 @@ async def test_a_late_submit_write_keeps_the_first_callbacks_progress(
AsyncMock(return_value=MagicMock(provider_batch_id="batch-recombine")),
)
mocker.patch(
- "backend.copilot.dream.batch_callbacks._anthropic_api_key",
+ "backend.copilot.dream.batch_callbacks.anthropic_api_key",
return_value="sk-ant-test",
)
entered, release = asyncio.Event(), asyncio.Event()
@@ -2745,8 +2760,11 @@ async def test_a_stalled_record_store_costs_each_write_only_its_deadline(
assert result.error is None and result.dream_session_id == "s"
apply_mock.assert_awaited_once()
- # The insert and six updates, each abandoned at the deadline.
- assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (7, 7)
+ # The insert, the guard's read of the scope's open passes, six updates
+ # and the four stop checks (three phases, apply), each abandoned at the
+ # deadline; neither the guard nor a check held the pass on a store that
+ # did not answer.
+ assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (12, 12)
@pytest.mark.asyncio
diff --git a/autogpt_platform/backend/backend/copilot/dream/pass_record.py b/autogpt_platform/backend/backend/copilot/dream/pass_record.py
index cabb82a1fc..1722ffbee7 100644
--- a/autogpt_platform/backend/backend/copilot/dream/pass_record.py
+++ b/autogpt_platform/backend/backend/copilot/dream/pass_record.py
@@ -2,8 +2,10 @@
the row reads back as.
Pure: how a route, a trigger and a step are named on the row, the insert or
-update each transition makes (``store.py`` writes them, bounded), and the
-``DreamPassResult`` a row describes, for the admin API and the eval driver.
+update each transition makes (``store.py`` writes them, bounded), including
+the two that stop a pass from outside (a cancel, an expiry), and what a row
+reads back as: the ``DreamPassResult`` it describes, for the admin API and the
+eval driver, and whether it says its pass was stopped.
"""
from datetime import datetime, timedelta, timezone
@@ -19,6 +21,7 @@ from pydantic import BaseModel
from backend.copilot.graphiti.scope import MemoryScope
from backend.data.dream_pass_models import (
+ INITIAL_CANCEL_GENERATION,
DreamPassApplied,
DreamPassDraft,
DreamPassOperations,
@@ -179,6 +182,43 @@ def failed(
)
+def cancelled(reason: str, *, owner_user_id: str) -> DreamPassUpdate:
+ """A cancel: the owner's open pass closes CANCELLED with *reason* as its
+ error, its cancel generation bumped so the running pass stops at its next
+ check. Written only while the row is open and the owner's."""
+ return DreamPassUpdate(
+ status=DreamPassStatus.CANCELLED,
+ error=reason[:MAX_ERROR_CHARS],
+ completed_at=datetime.now(timezone.utc),
+ bump_cancel_generation=True,
+ owner_user_id=owner_user_id,
+ )
+
+
+def expired(reason: str, *, not_updated_since: datetime | None) -> DreamPassUpdate:
+ """A newer pass's guard closing an open pass EXPIRED, its cancel
+ generation bumped like a cancel's. Written only while the row is open and,
+ given *not_updated_since* (the stale row as the guard read it), only if
+ nothing has written it since; an admin's forced expiry gives ``None``."""
+ return DreamPassUpdate(
+ status=DreamPassStatus.EXPIRED,
+ error=reason[:MAX_ERROR_CHARS],
+ completed_at=datetime.now(timezone.utc),
+ bump_cancel_generation=True,
+ not_updated_since=not_updated_since,
+ )
+
+
+def stop_error(row: DreamPassRecord) -> str | None:
+ """Why the row says its pass was stopped from outside, as the pass's
+ error (``cancelled: <reason>``, ``expired: <reason>``); ``None`` while its
+ cancel generation is still the one a new row starts at."""
+ if row.cancel_generation == INITIAL_CANCEL_GENERATION:
+ return None
+ how = row.status.value.lower()
+ return f"{how}: {row.error}" if row.error else how
+
+
def dream_pass_result_from_row(row: DreamPassRecord) -> DreamPassResult:
"""The ``DreamPassResult`` a pass's row describes, on either route.
diff --git a/autogpt_platform/backend/backend/copilot/dream/pass_run.py b/autogpt_platform/backend/backend/copilot/dream/pass_run.py
index 430983704c..d0cdf15694 100644
--- a/autogpt_platform/backend/backend/copilot/dream/pass_run.py
+++ b/autogpt_platform/backend/backend/copilot/dream/pass_run.py
@@ -2,14 +2,16 @@
when it started, and what its phases have used so far.
Built once when a pass starts and handed down, so every result the pass can
-end with (a skip, a failure at any step, an unexpected error, the hand-off to
-the batch route) carries the same identity and timing, and every failure
-carries the usage of every phase billed before it.
+end with (a skip, a failure at any step, a stop from outside, an unexpected
+error, the hand-off to the batch route) carries the same identity and timing,
+and every failure carries the usage of every phase billed before it. A step
+that ends the pass early raises ``PassEnded`` with the result it ends with.
"""
import asyncio
import uuid
from datetime import datetime, timezone
+from typing import Literal
from pydantic import BaseModel, Field
@@ -17,6 +19,18 @@ from .routing import ExecutionPath
from .schemas import DreamPassResult, DreamPassUsage, PhaseUsage
from .usage import aggregate_usage
+# Why a pass was skipped rather than run: its scope was taken (the lock held,
+# or another of its passes still open and fresh), the master flag was off, the
+# budget said no, or there was nothing (new) to dream about.
+DreamSkipReason = Literal[
+ "lock_held",
+ "pass_in_progress",
+ "disabled",
+ "insufficient_credits",
+ "no_input",
+ "no_new_activity",
+]
+
class DreamPassRun(BaseModel):
user_id: str
@@ -24,16 +38,20 @@ class DreamPassRun(BaseModel):
started_at: datetime
monotonic_start: float
execution_path: ExecutionPath
+ force: bool = False
phases: list[PhaseUsage] = Field(default_factory=list)
@classmethod
- def begin(cls, user_id: str, execution_path: ExecutionPath) -> "DreamPassRun":
+ def begin(
+ cls, user_id: str, execution_path: ExecutionPath, *, force: bool = False
+ ) -> "DreamPassRun":
return cls(
user_id=user_id,
pass_id=str(uuid.uuid4()),
started_at=datetime.now(timezone.utc),
monotonic_start=asyncio.get_event_loop().time(),
execution_path=execution_path,
+ force=force,
)
def usage(self) -> DreamPassUsage:
@@ -44,15 +62,15 @@ class DreamPassRun(BaseModel):
return asyncio.get_event_loop().time() - self.monotonic_start
def failure(self, error: str) -> DreamPassResult:
- """The pass failed. On the sync route it still carries the usage of
- the phases completed before the error. The batch route's phases bill at the provider and
+ """The pass failed or was stopped. Sync usage covers completed phases.
+ The batch route's phases bill at the provider and
reach the callbacks, never this side (a batch submitted before the
failure still runs), so its usage here is unknown."""
if self.execution_path == "anthropic_batch":
return self._ended(error=error)
return self._ended(error=error, usage=self.usage())
- def skipped(self, reason: str) -> DreamPassResult:
+ def skipped(self, reason: DreamSkipReason) -> DreamPassResult:
return self._ended(skipped=True, skip_reason=reason)
def handed_off(self) -> DreamPassResult:
@@ -81,3 +99,11 @@ class DreamPassRun(BaseModel):
skip_reason=skip_reason,
usage=usage,
)
+
+
+class PassEnded(Exception):
+ """A pass that ends before apply, carrying the result it ends with."""
+
+ def __init__(self, result: DreamPassResult) -> None:
+ super().__init__(result.error or result.skip_reason)
+ self.result = result
diff --git a/autogpt_platform/backend/backend/copilot/dream/provider_batch.py b/autogpt_platform/backend/backend/copilot/dream/provider_batch.py
new file mode 100644
index 0000000000..f1f5b1bb5c
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/provider_batch.py
@@ -0,0 +1,64 @@
+"""Anthropic's side of a batch dream pass: the key the callbacks submit and
+cancel with, and cancelling one of the pass's batches.
+
+The callbacks run in the BatchExecutor, a process of their own, so they look
+the key up themselves the way ``batch_executor._default_api_key_for`` does:
+the copilot config's direct Anthropic key first, then the shared settings key.
+"""
+
+import asyncio
+import logging
+
+from backend.copilot.config import ChatConfig
+from backend.util.llm.providers import cancel_batch
+from backend.util.settings import Settings
+
+logger = logging.getLogger(__name__)
+
+# How long a best-effort cancel may take, the client's retries included.
+PROVIDER_CANCEL_TIMEOUT_SECONDS = 10.0
+
+
+def anthropic_api_key() -> str | None:
+ """The copilot config's direct Anthropic key, else the shared settings
+ key; ``None`` when neither is set or can be read."""
+ try:
+ key = ChatConfig().direct_anthropic_api_key
+ if key:
+ return key
+ except Exception:
+ logger.debug("ChatConfig unavailable during dream batch callback")
+ try:
+ return Settings().secrets.anthropic_api_key or None
+ except Exception:
+ return None
+
+
+async def cancel_provider_batch(provider_batch_id: str) -> None:
+ """Ask Anthropic to cancel *provider_batch_id* through the client's
+ ``messages.batches.cancel``. Best-effort: a missing key, a refusal (a
+ batch that has already ended cannot be cancelled) or the deadline is
+ logged, never raised, so it never stands in the way of the stop that
+ asked for it."""
+ api_key = anthropic_api_key()
+ if api_key is None:
+ logger.warning(f"No Anthropic key to cancel dream batch {provider_batch_id}")
+ return
+ try:
+ acknowledged = await asyncio.wait_for(
+ cancel_batch(
+ provider="anthropic",
+ provider_batch_id=provider_batch_id,
+ api_key=api_key,
+ ),
+ timeout=PROVIDER_CANCEL_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ logger.warning(
+ f"Cancelling dream batch {provider_batch_id} failed", exc_info=True
+ )
+ return
+ if acknowledged:
+ logger.info(f"Anthropic is cancelling dream batch {provider_batch_id}")
+ else:
+ logger.warning(f"Anthropic did not cancel dream batch {provider_batch_id}")
diff --git a/autogpt_platform/backend/backend/copilot/dream/provider_batch_test.py b/autogpt_platform/backend/backend/copilot/dream/provider_batch_test.py
new file mode 100644
index 0000000000..e547967665
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/provider_batch_test.py
@@ -0,0 +1,80 @@
+"""Cancelling a dream pass's batch at Anthropic: the SDK call it makes, and
+that nothing about it (no key, a refusal, a hang) reaches the stop that asked
+for it."""
+
+import asyncio
+import logging
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from . import provider_batch
+from .provider_batch import anthropic_api_key, cancel_provider_batch
+
+
+def client(mocker) -> MagicMock:
+ """The Anthropic client ``cancel_batch`` builds, with the key it gets."""
+ mocker.patch.object(provider_batch, "anthropic_api_key", return_value="sk-test")
+ anthropic_client = MagicMock()
+ anthropic_client.messages.batches.cancel = AsyncMock()
+ mocker.patch(
+ "backend.util.llm.providers.anthropic.AsyncAnthropic",
+ return_value=anthropic_client,
+ )
+ return anthropic_client
+
+
+async def test_the_batch_is_cancelled_through_the_anthropic_client(client):
+ await cancel_provider_batch("msgbatch_1")
+
+ client.messages.batches.cancel.assert_awaited_once_with("msgbatch_1")
+
+
+async def test_a_refused_cancel_is_logged_not_raised(client, caplog):
+ client.messages.batches.cancel.side_effect = RuntimeError("batch has ended")
+
+ with caplog.at_level(logging.WARNING, logger=provider_batch.logger.name):
+ await cancel_provider_batch("msgbatch_1")
+
+ assert "did not cancel dream batch msgbatch_1" in caplog.text
+
+
+async def test_a_cancel_that_hangs_is_abandoned_at_its_deadline(
+ client, monkeypatch, caplog
+):
+ monkeypatch.setattr(provider_batch, "PROVIDER_CANCEL_TIMEOUT_SECONDS", 0.05)
+
+ async def hang(*_args, **_kwargs) -> None:
+ await asyncio.Event().wait()
+
+ client.messages.batches.cancel.side_effect = hang
+
+ with caplog.at_level(logging.WARNING, logger=provider_batch.logger.name):
+ await asyncio.wait_for(cancel_provider_batch("msgbatch_1"), 5)
+
+ assert "Cancelling dream batch msgbatch_1 failed" in caplog.text
+
+
+async def test_without_a_key_nothing_is_sent(mocker, caplog):
+ mocker.patch.object(provider_batch, "anthropic_api_key", return_value=None)
+ anthropic_cls = mocker.patch("backend.util.llm.providers.anthropic.AsyncAnthropic")
+
+ with caplog.at_level(logging.WARNING, logger=provider_batch.logger.name):
+ await cancel_provider_batch("msgbatch_1")
+
+ anthropic_cls.assert_not_called()
+ assert "No Anthropic key" in caplog.text
+
+
+def test_the_key_is_the_copilot_configs_before_the_shared_one(mocker):
+ config = mocker.patch.object(provider_batch, "ChatConfig")
+ config.return_value.direct_anthropic_api_key = "sk-config"
+ settings = mocker.patch.object(provider_batch, "Settings")
+ settings.return_value.secrets.anthropic_api_key = "sk-shared"
+
+ assert anthropic_api_key() == "sk-config"
+ config.return_value.direct_anthropic_api_key = None
+ assert anthropic_api_key() == "sk-shared"
+ settings.return_value.secrets.anthropic_api_key = ""
+ assert anthropic_api_key() is None
diff --git a/autogpt_platform/backend/backend/copilot/dream/store.py b/autogpt_platform/backend/backend/copilot/dream/store.py
index 821e99b76b..17790ec7c5 100644
--- a/autogpt_platform/backend/backend/copilot/dream/store.py
+++ b/autogpt_platform/backend/backend/copilot/dream/store.py
@@ -22,6 +22,9 @@ timing: the pass now waits on these bounded writes before it locks, inside the
lock, before it chains a batch or applies, and before it cleans up.
``read_dream_pass`` is the read side, for the admin API and the eval driver.
+The guard, the stop checks and the cancel read and write here too, under the
+same deadline, but a failure reaches them: each decides for itself whether to
+go on (``guard.py``, ``cancel.py``).
"""
import asyncio
@@ -103,8 +106,10 @@ async def record_submitted(
provider_batch_id: str,
lease_token: str,
lease_ttl_seconds: int,
-) -> None:
- await _write(
+) -> bool | None:
+ """The batch submit, and whether it landed (see ``_write``): the handoff
+ never hands the lock on once the row has refused it."""
+ return await _write(
pass_id,
"the batch submit",
lambda: submitted(
@@ -153,13 +158,40 @@ async def record_batch_failed(
async def read_dream_pass(pass_id: str, *, user_id: str) -> DreamPassRecord | None:
"""The pass's row when *user_id* owns it, else ``None``. Unlike the
- writes, a failed read raises: its caller is answering a request."""
- return await dream_db().get_dream_pass_for_user(pass_id, user_id)
+ writes, a read that fails or runs out of time raises: its caller is
+ answering a request."""
+ return await _bounded(dream_db().get_dream_pass_for_user(pass_id, user_id))
+
+
+async def read_pass(pass_id: str) -> DreamPassRecord | None:
+ """The pass's row, or ``None`` when it was never inserted. Raises when
+ the read fails or runs out of time."""
+ return await _bounded(dream_db().get_dream_pass(pass_id))
+
+async def read_open_passes(
+ scope: MemoryScope, *, limit: int | None = None
+) -> list[DreamPassRecord]:
+ """The scope's passes that are still open, newest first, at most *limit*
+ when given. Raises when the read fails or runs out of time."""
+ return await _bounded(
+ dream_db().list_open_dream_passes(scope.scope_key, limit=limit)
+ )
+
+
+async def write_stop(pass_id: str, update: DreamPassUpdate) -> bool:
+ """Write a stop from outside the pass (a cancel, an expiry) and say
+ whether it landed, which its caller acts on, unlike a pass's own writes.
+ Raises when the write fails or runs out of time."""
+ return await _bounded(dream_db().update_dream_pass(pass_id, update))
-async def _write(pass_id: str, step: str, build: Callable[[], DreamPassUpdate]) -> None:
+
+async def _write(
+ pass_id: str, step: str, build: Callable[[], DreamPassUpdate]
+) -> bool | None:
"""Write one transition, bounded, building it inside the guard so no part
- of it can fail the pass."""
+ of it can fail the pass: ``True`` when it landed, ``False`` when the row
+ refused it, ``None`` when it failed or ran out of time."""
try:
written = await _bounded(dream_db().update_dream_pass(pass_id, build()))
except Exception:
@@ -167,12 +199,13 @@ async def _write(pass_id: str, step: str, build: Callable[[], DreamPassUpdate])
f"Dream pass {pass_id}: could not record {step}; the pass goes on",
exc_info=True,
)
- return
+ return None
if not written:
# Expected, not a fault: the row is closed (a late or repeated
# delivery, which the transition rules refuse by design) or was
# never inserted (that failure was logged when it happened).
logger.debug("Dream pass %s: no open record to write %s to", pass_id, step)
+ return written
async def _bounded(call: Awaitable[_T]) -> _T:
diff --git a/autogpt_platform/backend/backend/data/db_manager_test.py b/autogpt_platform/backend/backend/data/db_manager_test.py
index 354cd63b63..dbe58107de 100644
--- a/autogpt_platform/backend/backend/data/db_manager_test.py
+++ b/autogpt_platform/backend/backend/data/db_manager_test.py
@@ -10,6 +10,7 @@ from prisma.enums import (
from pydantic import TypeAdapter
from backend.copilot.dream.fetch import DreamInput
+from backend.copilot.dream.pass_record import cancelled, expired
from backend.copilot.dream.schemas import (
ConsolidatedFact,
ConsolidationOutput,
@@ -107,6 +108,12 @@ def test_dream_pass_models_survive_the_rpc_round_trip() -> None:
body = inspect.signature(endpoint).parameters["body"].annotation
sent = body.model_validate(to_dict({"pass_id": "p1", "update": update}))
assert sent.update == update
+ for stop in (
+ expired("stale", not_updated_since=now.replace(microsecond=123000)),
+ cancelled("testing", owner_user_id="u1"),
+ ):
+ sent = body.model_validate(to_dict({"pass_id": "p1", "update": stop}))
+ assert sent.update == stop
record = DreamPassRecord(
id="p1",
diff --git a/autogpt_platform/backend/backend/data/dream_pass.py b/autogpt_platform/backend/backend/data/dream_pass.py
index f2fe752192..9a9ccbfacb 100644
--- a/autogpt_platform/backend/backend/data/dream_pass.py
+++ b/autogpt_platform/backend/backend/data/dream_pass.py
@@ -44,10 +44,11 @@ async def create_dream_pass(draft: DreamPassDraft) -> DreamPassRecord:
async def update_dream_pass(pass_id: str, update: DreamPassUpdate) -> bool:
"""Apply one transition to the pass's row, in a single statement.
- ``False`` when there is no such row or it has reached a terminal status.
- ``True`` means the row was open, not that every field moved: a status or
- phase behind the row's, or a batch for a phase the row has left, is
- dropped in the statement.
+ ``False`` when there is no such row, it has reached a terminal status, or
+ it fails the update's owner or not-updated-since condition. ``True``
+ means the row was written, not that every field moved: a status or phase
+ behind the row's, or a batch for a phase the row has left, is dropped in
+ the statement.
"""
written = await execute_raw_with_schema(
TRANSITION_SQL, *transition_args(pass_id, update)
@@ -69,11 +70,15 @@ async def get_dream_pass_for_user(pass_id: str, user_id: str) -> DreamPassRecord
return DreamPassRecord.from_db(row) if row else None
-async def list_open_dream_passes(scope_key: str) -> list[DreamPassRecord]:
- """The scope's passes that have not reached a terminal status, oldest first."""
+async def list_open_dream_passes(
+ scope_key: str, limit: int | None = None
+) -> list[DreamPassRecord]:
+ """The scope's passes that have not reached a terminal status, newest
+ first; at most *limit* of them when given."""
rows = await prisma.models.DreamPass.prisma().find_many(
where={"scopeKey": scope_key, "status": {"in": list(OPEN_STATUSES)}},
- order={"createdAt": "asc"},
+ order={"createdAt": "desc"},
+ take=limit,
)
return [DreamPassRecord.from_db(row) for row in rows]
diff --git a/autogpt_platform/backend/backend/data/dream_pass_models.py b/autogpt_platform/backend/backend/data/dream_pass_models.py
index 4bb2c42c26..28310d6c5c 100644
--- a/autogpt_platform/backend/backend/data/dream_pass_models.py
+++ b/autogpt_platform/backend/backend/data/dream_pass_models.py
@@ -38,6 +38,10 @@ OPEN_STATUSES: tuple[DreamPassStatus, ...] = (
DreamPassStatus.APPLYING,
)
+# A new row's cancelGeneration (the column default). Only a stop from outside
+# the pass moves it, so a pass that reads anything else was stopped.
+INITIAL_CANCEL_GENERATION = 0
+
class DreamPhaseOutputs(BaseModel):
"""Each phase's validated output, as the next phase and apply read it."""
@@ -93,6 +97,11 @@ class DreamPassUpdate(BaseModel):
``phase_outputs`` and ``operations`` merge into what the row holds, one
top-level field at a time, so a transition sends only the phase or the
part it produced.
+
+ A stop from outside the pass (a cancel, an expiry) sets
+ ``bump_cancel_generation``, and may make the write conditional on more
+ than the row being open: ``owner_user_id`` (the row is that user's) and
+ ``not_updated_since`` (nothing has written the row after that instant).
"""
status: DreamPassStatus | None = None
@@ -111,6 +120,9 @@ class DreamPassUpdate(BaseModel):
submitted_at: datetime | None = None
applied_at: datetime | None = None
completed_at: datetime | None = None
+ bump_cancel_generation: bool = False
+ owner_user_id: str | None = None
+ not_updated_since: datetime | None = None
@model_validator(mode="after")
def _batch_names_its_phase(self) -> "DreamPassUpdate":
diff --git a/autogpt_platform/backend/backend/data/dream_pass_stop_test.py b/autogpt_platform/backend/backend/data/dream_pass_stop_test.py
new file mode 100644
index 0000000000..5b356a7375
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/dream_pass_stop_test.py
@@ -0,0 +1,265 @@
+"""The two transitions that stop a dream pass from outside, against a real
+database: a cancel lands only on the owner's open row and bumps its
+generation exactly once however many race for it, and an expiry lands only on
+an open row nothing has written since the instant it names."""
+
+import asyncio
+import uuid
+from collections.abc import AsyncIterator
+from datetime import datetime, timedelta, timezone
+
+import pytest
+from prisma.enums import (
+ DreamPassPhase,
+ DreamPassRoute,
+ DreamPassStatus,
+ DreamPassTrigger,
+)
+from prisma.models import User
+
+from backend.copilot.dream.pass_record import cancelled, expired, failed
+from backend.copilot.dream.schemas import ConsolidationOutput
+from backend.data.db import execute_raw_with_schema
+from backend.util.json import SafeJson
+
+from .dream_pass import (
+ create_dream_pass,
+ get_dream_pass,
+ list_open_dream_passes,
+ update_dream_pass,
+)
+from .dream_pass_models import (
+ DreamPassDraft,
+ DreamPassRecord,
+ DreamPassUpdate,
+ DreamPhaseOutputs,
+)
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+async def owner() -> AsyncIterator[str]:
+ """A throwaway user; deleting it afterwards cascades to its passes."""
+ user_id = str(uuid.uuid4())
+ await User.prisma().create(
+ data={
+ "id": user_id,
+ "email": f"dream-pass-stop-{user_id}@example.com",
+ "topUpConfig": SafeJson({}),
+ "timezone": "UTC",
+ }
+ )
+ yield user_id
+ await User.prisma().delete(where={"id": user_id})
+
+
+async def _new_pass(owner: str) -> str:
+ pass_id = str(uuid.uuid4())
+ await create_dream_pass(
+ DreamPassDraft(
+ id=pass_id,
+ user_id=owner,
+ scope_key=owner,
+ route=DreamPassRoute.ANTHROPIC_BATCH,
+ trigger=DreamPassTrigger.CRON,
+ started_at=datetime.now(timezone.utc),
+ )
+ )
+ return pass_id
+
+
+async def _row(pass_id: str) -> DreamPassRecord:
+ row = await get_dream_pass(pass_id)
+ assert row is not None
+ return row
+
+
+async def test_a_cancel_closes_an_open_row_and_bumps_its_generation(owner):
+ pass_id = await _new_pass(owner)
+
+ assert await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))
+
+ row = await _row(pass_id)
+ assert (row.status, row.error, row.cancel_generation) == (
+ DreamPassStatus.CANCELLED,
+ "testing",
+ 1,
+ )
+ assert row.completed_at is not None
+
+
+async def test_a_cancel_reaches_only_an_open_row(owner):
+ pass_id = await _new_pass(owner)
+ await update_dream_pass(
+ pass_id,
+ DreamPassUpdate(
+ status=DreamPassStatus.COMPLETE,
+ phase=DreamPassPhase.DONE,
+ completed_at=datetime.now(timezone.utc),
+ ),
+ )
+
+ assert not await update_dream_pass(
+ pass_id, cancelled("testing", owner_user_id=owner)
+ )
+
+ row = await _row(pass_id)
+ assert (row.status, row.error, row.cancel_generation) == (
+ DreamPassStatus.COMPLETE,
+ None,
+ 0,
+ )
+
+
+async def test_a_cancel_reaches_only_the_owners_row(owner):
+ pass_id = await _new_pass(owner)
+
+ assert not await update_dream_pass(
+ pass_id, cancelled("testing", owner_user_id=str(uuid.uuid4()))
+ )
+
+ row = await _row(pass_id)
+ assert (row.status, row.cancel_generation) == (DreamPassStatus.RUNNING, 0)
+
+
+async def test_nothing_moves_a_cancelled_row(owner):
+ pass_id = await _new_pass(owner)
+ await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))
+ later = datetime.now(timezone.utc)
+
+ for update in (
+ failed("cancelled: testing", None, later),
+ cancelled("again", owner_user_id=owner),
+ expired("stale", not_updated_since=None),
+ DreamPassUpdate(
+ phase_outputs=DreamPhaseOutputs(consolidate=ConsolidationOutput())
+ ),
+ ):
+ assert not await update_dream_pass(pass_id, update)
+
+ row = await _row(pass_id)
+ assert (row.status, row.error, row.cancel_generation) == (
+ DreamPassStatus.CANCELLED,
+ "testing",
+ 1,
+ )
+ assert row.phase_outputs == DreamPhaseOutputs()
+
+
+async def test_racing_cancels_bump_the_generation_exactly_once(owner):
+ for _ in range(5):
+ pass_id = await _new_pass(owner)
+
+ landed = await asyncio.gather(
+ *(
+ update_dream_pass(
+ pass_id, cancelled(f"cancel {n}", owner_user_id=owner)
+ )
+ for n in range(8)
+ )
+ )
+
+ assert landed.count(True) == 1
+ row = await _row(pass_id)
+ assert (row.status, row.cancel_generation) == (DreamPassStatus.CANCELLED, 1)
+ assert row.error == f"cancel {landed.index(True)}"
+
+
+async def test_a_cancel_and_an_expiry_racing_close_the_row_once(owner):
+ for _ in range(5):
+ pass_id = await _new_pass(owner)
+
+ landed = await asyncio.gather(
+ update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner)),
+ update_dream_pass(pass_id, expired("stale", not_updated_since=None)),
+ )
+
+ assert landed.count(True) == 1
+ row = await _row(pass_id)
+ closed_by = DreamPassStatus.CANCELLED if landed[0] else DreamPassStatus.EXPIRED
+ assert (row.status, row.cancel_generation) == (closed_by, 1)
+
+
+async def test_an_expiry_lands_only_if_nothing_wrote_the_row_since(owner):
+ pass_id = await _new_pass(owner)
+ await update_dream_pass(pass_id, DreamPassUpdate(phase=DreamPassPhase.CONSOLIDATE))
+ seen = await _row(pass_id)
+ await asyncio.sleep(0.02)
+ await update_dream_pass(pass_id, DreamPassUpdate(phase=DreamPassPhase.RECOMBINE))
+
+ assert not await update_dream_pass(
+ pass_id, expired("stale", not_updated_since=seen.updated_at)
+ )
+ assert (await _row(pass_id)).status is DreamPassStatus.RUNNING
+
+ latest = await _row(pass_id)
+ assert await update_dream_pass(
+ pass_id, expired("stale", not_updated_since=latest.updated_at)
+ )
+ row = await _row(pass_id)
+ assert (row.status, row.error, row.cancel_generation) == (
+ DreamPassStatus.EXPIRED,
+ "stale",
+ 1,
+ )
+ assert row.completed_at is not None
+
+
+async def test_an_expiry_with_a_stale_cutoff_lands_only_on_an_old_row(owner):
+ pass_id = await _new_pass(owner)
+ await update_dream_pass(pass_id, DreamPassUpdate(phase=DreamPassPhase.CONSOLIDATE))
+ cutoff = datetime.now(timezone.utc) - timedelta(minutes=30)
+
+ assert not await update_dream_pass(
+ pass_id, expired("stale", not_updated_since=cutoff)
+ )
+
+ await execute_raw_with_schema(
+ 'UPDATE {schema_prefix}"DreamPass" '
+ """SET "updatedAt" = "updatedAt" - interval '1 hour' WHERE "id" = $1""",
+ pass_id,
+ )
+ assert await update_dream_pass(pass_id, expired("stale", not_updated_since=cutoff))
+ assert (await _row(pass_id)).status is DreamPassStatus.EXPIRED
+
+
+async def test_a_forced_expiry_lands_on_a_fresh_row(owner):
+ pass_id = await _new_pass(owner)
+ await update_dream_pass(pass_id, DreamPassUpdate(phase=DreamPassPhase.CONSOLIDATE))
+
+ assert await update_dream_pass(pass_id, expired("forced", not_updated_since=None))
+
+ row = await _row(pass_id)
+ assert (row.status, row.cancel_generation) == (DreamPassStatus.EXPIRED, 1)
+
+
+async def test_open_passes_list_newest_first_and_capped(owner):
+ """The guard weighs a scope's newest open passes, a handful at most."""
+ pass_ids = []
+ for _ in range(3):
+ pass_ids.append(await _new_pass(owner))
+ await asyncio.sleep(0.01)
+ await update_dream_pass(
+ pass_ids[1], DreamPassUpdate(status=DreamPassStatus.COMPLETE)
+ )
+
+ newest = await list_open_dream_passes(owner, limit=1)
+ every = await list_open_dream_passes(owner)
+
+ assert [row.id for row in newest] == [pass_ids[2]]
+ assert [row.id for row in every] == [pass_ids[2], pass_ids[0]]
+
+
+async def test_the_times_the_guard_compares_read_back_in_utc(owner):
+ """The guard compares a row's lease and last write with an aware "now"."""
+ pass_id = await _new_pass(owner)
+ lease = datetime.now(timezone.utc) + timedelta(hours=24)
+ await update_dream_pass(pass_id, DreamPassUpdate(lease_expires_at=lease))
+
+ row = await _row(pass_id)
+
+ assert row.updated_at.tzinfo is not None
+ assert row.lease_expires_at is not None
+ assert row.lease_expires_at.tzinfo is not None
+ assert abs(row.lease_expires_at - lease) < timedelta(milliseconds=1)
diff --git a/autogpt_platform/backend/backend/data/dream_pass_update.py b/autogpt_platform/backend/backend/data/dream_pass_update.py
index d7f7005723..058f333b01 100644
--- a/autogpt_platform/backend/backend/data/dream_pass_update.py
+++ b/autogpt_platform/backend/backend/data/dream_pass_update.py
@@ -13,10 +13,15 @@ statement against the row as it stands when the statement runs:
earlier phase never replaces a later phase's batch;
* ``phaseOutputs`` and ``operations`` merge key by key with jsonb ``||``, so
two callbacks writing different phases both land;
+ * ``cancelGeneration`` goes up by one when the update stops the pass from
+ outside, in the same statement that closes the row, so of two stops
+ racing for one open row exactly one lands and bumps it;
* every other column takes the new value when the update gives one.
-Nothing is written to a row that has reached a terminal status. The statement
-text is constant; every value is a bound parameter.
+Nothing is written to a row that has reached a terminal status, nor to one
+that fails the update's own conditions: another user's row when it names the
+owner, a row written since the instant it names. The statement text is
+constant; every value is a bound parameter.
"""
from typing import Any
@@ -54,8 +59,13 @@ UPDATE {schema_prefix}"DreamPass" SET
ELSE COALESCE("phaseOutputs", '{{}}'::jsonb) || $16::jsonb END,
"operations" = CASE WHEN $17::jsonb IS NULL THEN "operations"
ELSE COALESCE("operations", '{{}}'::jsonb) || $17::jsonb END,
+ "cancelGeneration" = "cancelGeneration"
+ + CASE WHEN $19::boolean THEN 1 ELSE 0 END,
"updatedAt" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
WHERE "id" = $1 AND "status"::text = ANY($18::text[])
+ AND ($20::text IS NULL OR "userId" = $20::text)
+ AND ($21::timestamptz IS NULL
+ OR "updatedAt" <= $21::timestamptz AT TIME ZONE 'UTC')
"""
@@ -86,6 +96,9 @@ def transition_args(pass_id: str, update: DreamPassUpdate) -> list[Any]:
_merge_part(update.phase_outputs),
_merge_part(update.operations),
[status.value for status in OPEN_STATUSES],
+ update.bump_cancel_generation,
+ update.owner_user_id,
+ update.not_updated_since,
]
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index 814b08ad93..a88fbf8bbf 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -1948,8 +1948,11 @@ def execute_nightly_batch_with_status(user_id: str, job_id: str):
)
-def execute_dream_pass_with_status(user_id: str, job_id: str):
- """Run the dream pass in isolation and record JobStatus transitions."""
+def execute_dream_pass_with_status(user_id: str, job_id: str, force: bool = False):
+ """Run the dream pass in isolation and record JobStatus transitions.
+
+ ``force`` has the pass's guard expire a fresh open pass of the account
+ instead of skipping behind it (the admin trigger's ``?force=true``)."""
from backend.copilot.dream.job_status import (
mark_complete,
mark_errored,
@@ -1972,7 +1975,7 @@ def execute_dream_pass_with_status(user_id: str, job_id: str):
try:
result = run_async(
- execute_dream_pass(user_id, status_id=job_id, trigger="admin"),
+ execute_dream_pass(user_id, status_id=job_id, trigger="admin", force=force),
timeout=SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS,
)
except Exception as exc:
@@ -2034,7 +2037,9 @@ def execute_dream_pass_with_status(user_id: str, job_id: str):
# a failed dream as a successful one.
try:
run_async(
- mark_errored(kind="dream_pass", job_id=job_id, error=result.error),
+ mark_errored(
+ kind="dream_pass", job_id=job_id, error=result.error, result=result
+ ),
timeout=10,
)
except Exception:
@@ -4741,14 +4746,17 @@ class Scheduler(AppService):
return {"scheduled": True, "job_id": job_id, "kind": "nightly"}
@expose
- def schedule_immediate_dream_pass(self, user_id: str, job_id: str) -> dict:
- """Schedule a one-shot dream pass run keyed by ``job_id``."""
+ def schedule_immediate_dream_pass(
+ self, user_id: str, job_id: str, force: bool = False
+ ) -> dict[str, Any]:
+ """Schedule a one-shot dream pass run keyed by ``job_id``; ``force``
+ goes to the pass's guard (see ``execute_dream_pass_with_status``)."""
import datetime as _dt
scheduler_job_id = f"adhoc_dream_{job_id}"
self.scheduler.add_job(
execute_dream_pass_with_status,
- kwargs={"user_id": user_id, "job_id": job_id},
+ kwargs={"user_id": user_id, "job_id": job_id, "force": force},
trigger="date",
run_date=_dt.datetime.now(_dt.timezone.utc),
id=scheduler_job_id,
diff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.py
index d7cac20d8f..0de780e4be 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py
@@ -26,6 +26,7 @@ from backend.executor.scheduler import (
_execute_graph,
_memory_scope_may_fire,
_normalize_cron_day_of_week,
+ execute_dream_pass_with_status,
execute_skill_learning_sync,
)
from backend.usecases.sample import create_test_graph, create_test_user
@@ -1356,6 +1357,81 @@ class TestExecuteNightlyBatchWithStatus:
errored_mock.assert_not_called()
+# ---------------------------------------------------------------------------
+# The admin dream trigger's ``force``, from the scheduled job to the pass
+# ---------------------------------------------------------------------------
+
+
+class TestAdminDreamPassForce:
+ """``force`` rides the one-shot job's kwargs to the pass, whose guard then
+ expires a fresh open pass instead of skipping behind it."""
+
+ @pytest.mark.parametrize("force", [True, False])
+ def test_the_one_shot_job_carries_force(self, force: bool) -> None:
+ s = _stub_scheduler()
+ scheduler = MagicMock()
+ s.scheduler = scheduler
+
+ s.schedule_immediate_dream_pass(user_id="abc", job_id="job-1", force=force)
+
+ kwargs = scheduler.add_job.call_args.kwargs
+ assert kwargs["kwargs"] == {"user_id": "abc", "job_id": "job-1", "force": force}
+ assert kwargs["id"] == "adhoc_dream_job-1"
+
+ def test_a_job_persisted_before_force_existed_runs_unforced(self) -> None:
+ execute = self._run_wrapper(kwargs={})
+
+ execute.assert_called_once_with(
+ "abc", status_id="job-1", trigger="admin", force=False
+ )
+
+ def test_force_reaches_the_pass(self) -> None:
+ execute = self._run_wrapper(kwargs={"force": True})
+
+ execute.assert_called_once_with(
+ "abc", status_id="job-1", trigger="admin", force=True
+ )
+
+ def _run_wrapper(self, kwargs: dict) -> MagicMock:
+ """Run the wrapper with the pass and the status writers mocked;
+ ``run_async`` hands back a clean result for the pass only."""
+ execute, _ = _run_dream_wrapper(_dream_result(), **kwargs)
+ return execute
+
+
+class TestExecuteDreamPassWithStatus:
+ def test_a_failed_pass_keeps_its_result_on_the_job(self) -> None:
+ """A pass stopped by a cancel returns its failure with the usage of
+ the phases it was billed for; the errored job keeps that result."""
+ failed = _dream_result(error="cancelled: testing")
+
+ _, errored = _run_dream_wrapper(failed)
+
+ errored.assert_called_once_with(
+ kind="dream_pass", job_id="job-1", error="cancelled: testing", result=failed
+ )
+
+
+def _run_dream_wrapper(result, **kwargs) -> tuple[MagicMock, MagicMock]:
+ """Run the dream pass wrapper with the pass returning *result* and the
+ status writers mocked; the pass mock and ``mark_errored``."""
+ sentinel = object()
+ execute = MagicMock(return_value=sentinel)
+
+ def fake_run_async(coro, timeout=None):
+ return result if coro is sentinel else None
+
+ with (
+ patch("backend.executor.scheduler.run_async", side_effect=fake_run_async),
+ patch("backend.copilot.dream.orchestrator.execute_dream_pass", new=execute),
+ patch("backend.copilot.dream.job_status.mark_complete"),
+ patch("backend.copilot.dream.job_status.mark_errored") as errored,
+ patch("backend.copilot.dream.job_status.update_status_phase"),
+ ):
+ execute_dream_pass_with_status("abc", "job-1", **kwargs)
+ return execute, errored
+
+
# ---------------------------------------------------------------------------
# JobStatus transitions for the admin-triggered community rebuild wrapper
# ---------------------------------------------------------------------------
diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prisma
index 8d916cca4d..abea153e31 100644
--- a/autogpt_platform/backend/schema.prisma
+++ b/autogpt_platform/backend/schema.prisma
@@ -599,8 +599,9 @@ model DreamPass {
phase DreamPassPhase @default(GATHER)
status DreamPassStatus @default(QUEUED)
skipReason String?
- // Bumped by a cancel, so a step that started under an older generation
- // knows to stop. Nothing cancels yet.
+ // Bumped by a stop from outside the pass (a cancel, or an expiry by a newer
+ // pass's guard) in the statement that closes the row, so a step that started
+ // under an older generation knows to stop.
cancelGeneration Int @default(0)
// The provider batch of the phase in flight (batch route only).
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index b9721c1b80..45f7a5385b 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -1476,7 +1476,7 @@
"post": {
"tags": ["v2", "admin", "admin", "memory"],
"summary": "Trigger Dream Pass",
- "description": "Fire a dream pass and return 202 + job_id immediately.\n\nFrontend polls ``GET /api/admin/memory/{user_id}/dream/{job_id}``\nfor progress. Runs ONLY the dream pass submitter — for the full\nnightly fan-out use ``POST /{user_id}/nightly``.",
+ "description": "Fire a dream pass and return 202 + job_id immediately.\n\nFrontend polls ``GET /api/admin/memory/{user_id}/dream/{job_id}``\nfor progress. Runs ONLY the dream pass submitter — for the full\nnightly fan-out use ``POST /{user_id}/nightly``. The pass goes through\nthe same guard as the nightly one: it skips behind a fresh open pass of\nthe account unless ``force`` expires that pass first. ``force`` does not\ntake a lock another pass holds.",
"operationId": "postV2TriggerDreamPass",
"security": [{ "HTTPBearerJWT": [] }],
"parameters": [
@@ -1490,6 +1490,18 @@
"title": "User Id"
},
"description": "User id or 'me'"
+ },
+ {
+ "name": "force",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "boolean",
+ "description": "Expire a fresh open dream pass of the account first instead of skipping behind it",
+ "default": false,
+ "title": "Force"
+ },
+ "description": "Expire a fresh open dream pass of the account first instead of skipping behind it"
}
],
"responses": {
@@ -1569,11 +1581,80 @@
}
}
},
+ "/api/admin/memory/{user_id}/dream/{pass_id}/cancel": {
+ "post": {
+ "tags": ["v2", "admin", "admin", "memory"],
+ "summary": "Cancel Dream Pass Run",
+ "description": "Cancel an open dream pass and return its record, now CANCELLED.\n\nThe row closes at once; the pass itself stops at its next check (a phase\nboundary, its batch submit, or just before apply, on either route)\nwithout applying. A batch pass waiting on its provider stops when that\nbatch lands. A pass already past its last check still applies, once, and\nwhatever a pass applied stays. 404 for a pass that does not exist or\nbelongs to another user, alike; 409 once the pass has ended.",
+ "operationId": "postV2CancelDreamPassRun",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "user_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "description": "User id or 'me'",
+ "title": "User Id"
+ },
+ "description": "User id or 'me'"
+ },
+ {
+ "name": "pass_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "description": "The dream pass id",
+ "title": "Pass Id"
+ },
+ "description": "The dream pass id"
+ }
+ ],
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ { "$ref": "#/components/schemas/DreamPassCancelRequest" },
+ { "type": "null" }
+ ],
+ "title": "Body"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/DreamPassRecordResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/admin/memory/{user_id}/dream/{pass_id}/record": {
"get": {
"tags": ["v2", "admin", "admin", "memory"],
"summary": "Get Dream Pass Record",
- "description": "Read one dream pass's durable record, on either route.\n\nThe job status above is a Redis row that lapses after six hours and, for\na batch pass, carries no usage; the record outlives the pass and holds\nwhat its phases used. 404 for a pass that does not exist or belongs to\nanother user, alike.",
+ "description": "Read one dream pass's durable record, on either route.\n\nThe job status above is a Redis row that lapses after six hours and, for\na batch pass, carries no usage; the record outlives the pass and holds\nwhat its phases used, unless a cancel or a newer pass closed it first\n(the cost log still has that usage). 404 for a pass that does not exist\nor belongs to another user, alike.",
"operationId": "getV2GetDreamPassRecord",
"security": [{ "HTTPBearerJWT": [] }],
"parameters": [
@@ -23449,6 +23530,20 @@
"title": "DreamPassApplied",
"description": "What apply reported: the counts and narrative a ``DreamPassResult``\ncarries, and the per-operation snapshot."
},
+ "DreamPassCancelRequest": {
+ "properties": {
+ "reason": {
+ "anyOf": [
+ { "type": "string", "maxLength": 500, "minLength": 1 },
+ { "type": "null" }
+ ],
+ "title": "Reason"
+ }
+ },
+ "type": "object",
+ "title": "DreamPassCancelRequest",
+ "description": "Why an admin cancels a dream pass, kept as the pass's error."
+ },
"DreamPassOperations": {
"properties": {
"planned": {